From 980d6d3f23f70097803d41055ffb9ef5e53466a2 Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 08:59:41 +0300
Subject: [PATCH 01/32] issue-2330 fixed

Fix x86 BLSI carry flag semantics.

Issue: https://github.com/unicorn-engine/unicorn/issues/2330
---
 qemu/target/i386/cc_helper.c          | 18 +++++++++++++++++
 qemu/target/i386/cc_helper_template.h | 18 +++++++++++++++++
 qemu/target/i386/cpu.h                |  5 +++++
 qemu/target/i386/translate.c          | 19 +++++++++++++++++-
 tests/unit/test_x86.c                 | 29 +++++++++++++++++++++++++++
 5 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/qemu/target/i386/cc_helper.c b/qemu/target/i386/cc_helper.c
index 159dda4605..5110eb8dab 100644
--- a/qemu/target/i386/cc_helper.c
+++ b/qemu/target/i386/cc_helper.c
@@ -185,6 +185,13 @@ target_ulong helper_cc_compute_all(target_ulong dst, target_ulong src1,
     case CC_OP_BMILGL:
         return compute_all_bmilgl(dst, src1);
 
+    case CC_OP_BLSIB:
+        return compute_all_blsib(dst, src1);
+    case CC_OP_BLSIW:
+        return compute_all_blsiw(dst, src1);
+    case CC_OP_BLSIL:
+        return compute_all_blsil(dst, src1);
+
     case CC_OP_ADCX:
         return compute_all_adcx(dst, src1, src2);
     case CC_OP_ADOX:
@@ -215,6 +222,8 @@ target_ulong helper_cc_compute_all(target_ulong dst, target_ulong src1,
         return compute_all_sarq(dst, src1);
     case CC_OP_BMILGQ:
         return compute_all_bmilgq(dst, src1);
+    case CC_OP_BLSIQ:
+        return compute_all_blsiq(dst, src1);
 #endif
     }
 }
@@ -307,6 +316,13 @@ target_ulong helper_cc_compute_c(target_ulong dst, target_ulong src1,
     case CC_OP_BMILGL:
         return compute_c_bmilgl(dst, src1);
 
+    case CC_OP_BLSIB:
+        return compute_c_blsib(dst, src1);
+    case CC_OP_BLSIW:
+        return compute_c_blsiw(dst, src1);
+    case CC_OP_BLSIL:
+        return compute_c_blsil(dst, src1);
+
 #ifdef TARGET_X86_64
     case CC_OP_ADDQ:
         return compute_c_addq(dst, src1);
@@ -320,6 +336,8 @@ target_ulong helper_cc_compute_c(target_ulong dst, target_ulong src1,
         return compute_c_shlq(dst, src1);
     case CC_OP_BMILGQ:
         return compute_c_bmilgq(dst, src1);
+    case CC_OP_BLSIQ:
+        return compute_c_blsiq(dst, src1);
 #endif
     }
 }
diff --git a/qemu/target/i386/cc_helper_template.h b/qemu/target/i386/cc_helper_template.h
index dc34d0d4c4..f161f343be 100644
--- a/qemu/target/i386/cc_helper_template.h
+++ b/qemu/target/i386/cc_helper_template.h
@@ -235,6 +235,24 @@ static int glue(compute_c_bmilg, SUFFIX)(DATA_TYPE dst, DATA_TYPE src1)
     return src1 == 0;
 }
 
+static int glue(compute_all_blsi, SUFFIX)(DATA_TYPE dst, DATA_TYPE src1)
+{
+    int cf, pf, af, zf, sf, of;
+
+    cf = (src1 != 0);
+    pf = 0; /* undefined */
+    af = 0; /* undefined */
+    zf = (dst == 0) * CC_Z;
+    sf = lshift(dst, 8 - DATA_BITS) & CC_S;
+    of = 0;
+    return cf | pf | af | zf | sf | of;
+}
+
+static int glue(compute_c_blsi, SUFFIX)(DATA_TYPE dst, DATA_TYPE src1)
+{
+    return src1 != 0;
+}
+
 #undef DATA_BITS
 #undef SIGN_MASK
 #undef DATA_TYPE
diff --git a/qemu/target/i386/cpu.h b/qemu/target/i386/cpu.h
index 10d93b89ac..fbb2706d20 100644
--- a/qemu/target/i386/cpu.h
+++ b/qemu/target/i386/cpu.h
@@ -1081,6 +1081,11 @@ typedef enum {
     CC_OP_BMILGL,
     CC_OP_BMILGQ,
 
+    CC_OP_BLSIB, /* Z,S via CC_DST, C = SRC!=0; O=0; P,A undefined */
+    CC_OP_BLSIW,
+    CC_OP_BLSIL,
+    CC_OP_BLSIQ,
+
     CC_OP_ADCX, /* CC_DST = C, CC_SRC = rest.  */
     CC_OP_ADOX, /* CC_DST = O, CC_SRC = rest.  */
     CC_OP_ADCOX, /* CC_DST = C, CC_SRC2 = O, CC_SRC = rest.  */
diff --git a/qemu/target/i386/translate.c b/qemu/target/i386/translate.c
index 8278774a58..3b87776dd9 100644
--- a/qemu/target/i386/translate.c
+++ b/qemu/target/i386/translate.c
@@ -313,6 +313,11 @@ static const uint8_t cc_op_live[CC_OP_NB] = {
     [CC_OP_BMILGL] = USES_CC_DST | USES_CC_SRC,
     [CC_OP_BMILGQ] = USES_CC_DST | USES_CC_SRC,
 
+    [CC_OP_BLSIB] = USES_CC_DST | USES_CC_SRC,
+    [CC_OP_BLSIW] = USES_CC_DST | USES_CC_SRC,
+    [CC_OP_BLSIL] = USES_CC_DST | USES_CC_SRC,
+    [CC_OP_BLSIQ] = USES_CC_DST | USES_CC_SRC,
+
     [CC_OP_ADCX] = USES_CC_DST | USES_CC_SRC,
     [CC_OP_ADOX] = USES_CC_SRC | USES_CC_SRC2,
     [CC_OP_ADCOX] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
@@ -960,6 +965,14 @@ static CCPrepare gen_prepare_eflags_c(DisasContext *s, TCGv reg)
         t0 = gen_ext_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_src, size, false);
         return (CCPrepare) { .cond = TCG_COND_EQ, .reg = t0, .mask = -1 };
 
+    case CC_OP_BLSIB:
+    case CC_OP_BLSIW:
+    case CC_OP_BLSIL:
+    case CC_OP_BLSIQ:
+        size = s->cc_op - CC_OP_BLSIB;
+        t0 = gen_ext_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_src, size, false);
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = t0, .mask = -1 };
+
     case CC_OP_ADCX:
     case CC_OP_ADCOX:
         return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_dst,
@@ -4393,7 +4406,11 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 }
                 tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
                 gen_op_mov_reg_v(s, ot, s->vex_v, s->T0);
-                set_cc_op(s, CC_OP_BMILGB + ot);
+                if ((reg & 7) == 3) {
+                    set_cc_op(s, CC_OP_BLSIB + ot);
+                } else {
+                    set_cc_op(s, CC_OP_BMILGB + ot);
+                }
                 break;
 
             default:
diff --git a/tests/unit/test_x86.c b/tests/unit/test_x86.c
index 8eedc31ee2..647334917f 100644
--- a/tests/unit/test_x86.c
+++ b/tests/unit/test_x86.c
@@ -1155,6 +1155,34 @@ static void test_x86_eflags_reserved_bit(void)
     OK(uc_close(uc));
 }
 
+static void test_x86_blsi_cf_case(uint64_t src, uint64_t expected_dst,
+                                  bool expected_cf, bool expected_zf)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe2\xf8\xf3\xdb"; /* blsi rax, rbx */
+    uint64_t rax;
+    uint64_t rflags;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &src));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_read(uc, UC_X86_REG_RFLAGS, &rflags));
+
+    TEST_CHECK(rax == expected_dst);
+    TEST_CHECK((bool)(rflags & 1) == expected_cf);
+    TEST_CHECK((bool)(rflags & 0x40) == expected_zf);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_blsi_cf(void)
+{
+    test_x86_blsi_cf_case(1, 1, true, false);
+    test_x86_blsi_cf_case(0, 0, false, true);
+}
+
 static void test_x86_nested_uc_emu_start_exits_cb(uc_engine *uc, uint64_t addr,
                                                   size_t size, void *data)
 {
@@ -2234,6 +2262,7 @@ TEST_LIST = {
     {"test_x86_nested_emu_stop", test_x86_nested_emu_stop},
     {"test_x86_64_nested_emu_start_error", test_x86_64_nested_emu_start_error},
     {"test_x86_eflags_reserved_bit", test_x86_eflags_reserved_bit},
+    {"test_x86_blsi_cf", test_x86_blsi_cf},
     {"test_x86_nested_uc_emu_start_exits", test_x86_nested_uc_emu_start_exits},
     {"test_x86_clear_count_cache", test_x86_clear_count_cache},
     {"test_x86_correct_address_in_small_jump_hook",

From 17ea6c603914194e9844281620c1c59d8cfaec86 Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 09:08:44 +0300
Subject: [PATCH 02/32] issue-2331 fixed

Fix x86 BZHI index boundary semantics.

Issue: https://github.com/unicorn-engine/unicorn/issues/2331
---
 qemu/target/i386/translate.c | 13 +++++++++----
 tests/unit/test_x86.c        | 31 +++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/qemu/target/i386/translate.c b/qemu/target/i386/translate.c
index 3b87776dd9..bb078578c9 100644
--- a/qemu/target/i386/translate.c
+++ b/qemu/target/i386/translate.c
@@ -4186,16 +4186,21 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 tcg_gen_ext8u_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
                 {
                     TCGv bound = tcg_const_tl(tcg_ctx, ot == MO_64 ? 63 : 31);
+                    TCGv zero = tcg_const_tl(tcg_ctx, 0);
                     /* Note that since we're using BMILG (in order to get O
                        cleared) we need to store the inverse into C.  */
-                    tcg_gen_setcond_tl(tcg_ctx, TCG_COND_LT, tcg_ctx->cpu_cc_src,
+                    tcg_gen_setcond_tl(tcg_ctx, TCG_COND_LEU, tcg_ctx->cpu_cc_src,
                                        s->T1, bound);
-                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_GT, s->T1, s->T1,
+                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_GTU, s->T1, s->T1,
                                        bound, bound, s->T1);
                     tcg_temp_free(tcg_ctx, bound);
+                    tcg_gen_movi_tl(tcg_ctx, s->A0, -1);
+                    tcg_gen_shl_tl(tcg_ctx, s->A0, s->A0, s->T1);
+                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, s->A0,
+                                       tcg_ctx->cpu_cc_src, zero,
+                                       zero, s->A0);
+                    tcg_temp_free(tcg_ctx, zero);
                 }
-                tcg_gen_movi_tl(tcg_ctx, s->A0, -1);
-                tcg_gen_shl_tl(tcg_ctx, s->A0, s->A0, s->T1);
                 tcg_gen_andc_tl(tcg_ctx, s->T0, s->T0, s->A0);
                 gen_op_mov_reg_v(s, ot, reg, s->T0);
                 gen_op_update1_cc(s);
diff --git a/tests/unit/test_x86.c b/tests/unit/test_x86.c
index 647334917f..ea8093c792 100644
--- a/tests/unit/test_x86.c
+++ b/tests/unit/test_x86.c
@@ -1183,6 +1183,36 @@ static void test_x86_blsi_cf(void)
     test_x86_blsi_cf_case(0, 0, false, true);
 }
 
+static void test_x86_bzhi_index_case(uint64_t index, uint64_t expected_dst,
+                                     bool expected_cf, bool expected_sf)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe2\xf0\xf5\xc3"; /* bzhi rax, rbx, rcx */
+    uint64_t rax;
+    uint64_t rflags;
+    uint64_t src = 0xffffffffffffffffULL;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &src));
+    OK(uc_reg_write(uc, UC_X86_REG_RCX, &index));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_read(uc, UC_X86_REG_RFLAGS, &rflags));
+
+    TEST_CHECK(rax == expected_dst);
+    TEST_CHECK((bool)(rflags & 1) == expected_cf);
+    TEST_CHECK((bool)(rflags & 0x80) == expected_sf);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_bzhi_index_boundary(void)
+{
+    test_x86_bzhi_index_case(63, 0x7fffffffffffffffULL, false, false);
+    test_x86_bzhi_index_case(255, 0xffffffffffffffffULL, true, true);
+}
+
 static void test_x86_nested_uc_emu_start_exits_cb(uc_engine *uc, uint64_t addr,
                                                   size_t size, void *data)
 {
@@ -2263,6 +2293,7 @@ TEST_LIST = {
     {"test_x86_64_nested_emu_start_error", test_x86_64_nested_emu_start_error},
     {"test_x86_eflags_reserved_bit", test_x86_eflags_reserved_bit},
     {"test_x86_blsi_cf", test_x86_blsi_cf},
+    {"test_x86_bzhi_index_boundary", test_x86_bzhi_index_boundary},
     {"test_x86_nested_uc_emu_start_exits", test_x86_nested_uc_emu_start_exits},
     {"test_x86_clear_count_cache", test_x86_clear_count_cache},
     {"test_x86_correct_address_in_small_jump_hook",

From 936aa67fafe97ad097ece0bf44d74d37cf4f28bb Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 09:15:23 +0300
Subject: [PATCH 03/32] issue-2332 fixed

Fix x86 CMPXCHG accumulator update semantics.

Issue: https://github.com/unicorn-engine/unicorn/issues/2332
---
 qemu/target/i386/translate.c | 40 +++++++++++++++++++++++++++---
 tests/unit/test_x86.c        | 47 ++++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/qemu/target/i386/translate.c b/qemu/target/i386/translate.c
index bb078578c9..37fba47106 100644
--- a/qemu/target/i386/translate.c
+++ b/qemu/target/i386/translate.c
@@ -493,6 +493,38 @@ static void gen_op_mov_reg_v(DisasContext *s, MemOp ot, int reg, TCGv t0)
     }
 }
 
+static void gen_op_update_cmpxchg_acc(DisasContext *s, MemOp ot, TCGv oldv,
+                                      TCGv cmpv)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv new_acc = tcg_temp_new(tcg_ctx);
+
+    switch(ot) {
+    case MO_8:
+        tcg_gen_deposit_tl(tcg_ctx, new_acc, tcg_ctx->cpu_regs[R_EAX],
+                           oldv, 0, 8);
+        break;
+    case MO_16:
+        tcg_gen_deposit_tl(tcg_ctx, new_acc, tcg_ctx->cpu_regs[R_EAX],
+                           oldv, 0, 16);
+        break;
+    case MO_32:
+        tcg_gen_ext32u_tl(tcg_ctx, new_acc, oldv);
+        break;
+#ifdef TARGET_X86_64
+    case MO_64:
+        tcg_gen_mov_tl(tcg_ctx, new_acc, oldv);
+        break;
+#endif
+    default:
+        tcg_abort();
+    }
+
+    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, tcg_ctx->cpu_regs[R_EAX],
+                       oldv, cmpv, tcg_ctx->cpu_regs[R_EAX], new_acc);
+    tcg_temp_free(tcg_ctx, new_acc);
+}
+
 static inline
 void gen_op_mov_v_reg(DisasContext *s, MemOp ot, TCGv t0, int reg)
 {
@@ -5761,7 +5793,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 gen_lea_modrm(env, s, modrm);
                 tcg_gen_atomic_cmpxchg_tl(tcg_ctx, oldv, s->A0, cmpv, newv,
                                           s->mem_index, ot | MO_LE);
-                gen_op_mov_reg_v(s, ot, R_EAX, oldv);
+                gen_extu(tcg_ctx, ot, oldv);
+                gen_extu(tcg_ctx, ot, cmpv);
+                gen_op_update_cmpxchg_acc(s, ot, oldv, cmpv);
             } else {
                 if (mod == 3) {
                     rm = (modrm & 7) | REX_B(s);
@@ -5776,7 +5810,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 /* store value = (old == cmp ? new : old);  */
                 tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, newv, oldv, cmpv, newv, oldv);
                 if (mod == 3) {
-                    gen_op_mov_reg_v(s, ot, R_EAX, oldv);
+                    gen_op_update_cmpxchg_acc(s, ot, oldv, cmpv);
                     gen_op_mov_reg_v(s, ot, rm, newv);
                 } else {
                     /* Perform an unconditional store cycle like physical cpu;
@@ -5784,7 +5818,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                        idempotency if the store faults and the instruction
                        is restarted */
                     gen_op_st_v(s, ot, newv, s->A0);
-                    gen_op_mov_reg_v(s, ot, R_EAX, oldv);
+                    gen_op_update_cmpxchg_acc(s, ot, oldv, cmpv);
                 }
             }
             tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, oldv);
diff --git a/tests/unit/test_x86.c b/tests/unit/test_x86.c
index ea8093c792..78e7016c59 100644
--- a/tests/unit/test_x86.c
+++ b/tests/unit/test_x86.c
@@ -1043,6 +1043,52 @@ static void test_x86_cmpxchg(void)
     OK(uc_close(uc));
 }
 
+static void test_x86_cmpxchg32_acc_case(uint64_t initial_rax,
+                                        uint64_t initial_mem,
+                                        uint64_t expected_rax,
+                                        uint64_t expected_mem,
+                                        bool expected_zf)
+{
+    uc_engine *uc;
+    char code[] = "\x41\x0f\xb1\x18"; /* cmpxchg dword ptr [r8], ebx */
+    uint64_t data_address = 0x2000000;
+    uint64_t rax = initial_rax;
+    uint64_t rbx = 0;
+    uint64_t r8 = data_address;
+    uint64_t rflags;
+    uint64_t mem;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_mem_map(uc, data_address, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, data_address, &initial_mem, sizeof(initial_mem)));
+    OK(uc_reg_write(uc, UC_X86_REG_R8, &r8));
+    OK(uc_reg_write(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &rbx));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_read(uc, UC_X86_REG_RFLAGS, &rflags));
+    OK(uc_mem_read(uc, data_address, &mem, sizeof(mem)));
+
+    TEST_CHECK(rax == expected_rax);
+    TEST_CHECK(mem == expected_mem);
+    TEST_CHECK((bool)(rflags & 0x40) == expected_zf);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_cmpxchg32_accumulator(void)
+{
+    test_x86_cmpxchg32_acc_case(0xffffffffffffffffULL,
+                                0xffffffffffffffffULL,
+                                0xffffffffffffffffULL,
+                                0xffffffff00000000ULL, true);
+    test_x86_cmpxchg32_acc_case(0xffffffff00000000ULL,
+                                0xffffffffffffffffULL,
+                                0x00000000ffffffffULL,
+                                0xffffffffffffffffULL, false);
+}
+
 static void test_x86_nested_emu_start_cb(uc_engine *uc, uint64_t addr,
                                          size_t size, void *data)
 {
@@ -2288,6 +2334,7 @@ TEST_LIST = {
     {"test_x86_clear_empty_tb", test_x86_clear_empty_tb},
     {"test_x86_hook_tcg_op", test_x86_hook_tcg_op},
     {"test_x86_cmpxchg", test_x86_cmpxchg},
+    {"test_x86_cmpxchg32_accumulator", test_x86_cmpxchg32_accumulator},
     {"test_x86_nested_emu_start", test_x86_nested_emu_start},
     {"test_x86_nested_emu_stop", test_x86_nested_emu_stop},
     {"test_x86_64_nested_emu_start_error", test_x86_64_nested_emu_start_error},

From a7a3c1e9dd09b32502f099c7b1e1a5bdfd615cb0 Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 09:22:56 +0300
Subject: [PATCH 04/32] issue-2336 fixed

Fix x86 RET imm16 stack adjustment semantics.

Issue: https://github.com/unicorn-engine/unicorn/issues/2336
---
 qemu/target/i386/translate.c |  4 ++--
 tests/unit/test_x86.c        | 23 +++++++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/qemu/target/i386/translate.c b/qemu/target/i386/translate.c
index 37fba47106..6a99ed0506 100644
--- a/qemu/target/i386/translate.c
+++ b/qemu/target/i386/translate.c
@@ -7159,7 +7159,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         /************************/
         /* control */
     case 0xc2: /* ret im */
-        val = x86_ldsw_code(env, s);
+        val = x86_lduw_code(env, s);
         ot = gen_pop_T0(s);
         gen_stack_update(s, val + (1 << ot));
         /* Note that gen_pop_T0 uses a zero-extending load.  */
@@ -7176,7 +7176,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         gen_jr(s, s->T0);
         break;
     case 0xca: /* lret im */
-        val = x86_ldsw_code(env, s);
+        val = x86_lduw_code(env, s);
     do_lret:
         if (s->pe && !s->vm86) {
             gen_update_cc_op(s);
diff --git a/tests/unit/test_x86.c b/tests/unit/test_x86.c
index 78e7016c59..a4ac7f5d08 100644
--- a/tests/unit/test_x86.c
+++ b/tests/unit/test_x86.c
@@ -1089,6 +1089,28 @@ static void test_x86_cmpxchg32_accumulator(void)
                                 0xffffffffffffffffULL, false);
 }
 
+static void test_x86_ret_imm16_unsigned(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc2\x00\xff"; /* ret 0xff00 */
+    uint64_t stack_address = 0x2000000;
+    uint64_t return_address = code_start + sizeof(code) - 1;
+    uint64_t rsp = stack_address;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_mem_map(uc, stack_address, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, stack_address, &return_address,
+                    sizeof(return_address)));
+    OK(uc_reg_write(uc, UC_X86_REG_RSP, &rsp));
+
+    OK(uc_emu_start(uc, code_start, return_address, 0, 1));
+    OK(uc_reg_read(uc, UC_X86_REG_RSP, &rsp));
+
+    TEST_CHECK(rsp == stack_address + 8 + 0xff00);
+
+    OK(uc_close(uc));
+}
+
 static void test_x86_nested_emu_start_cb(uc_engine *uc, uint64_t addr,
                                          size_t size, void *data)
 {
@@ -2335,6 +2357,7 @@ TEST_LIST = {
     {"test_x86_hook_tcg_op", test_x86_hook_tcg_op},
     {"test_x86_cmpxchg", test_x86_cmpxchg},
     {"test_x86_cmpxchg32_accumulator", test_x86_cmpxchg32_accumulator},
+    {"test_x86_ret_imm16_unsigned", test_x86_ret_imm16_unsigned},
     {"test_x86_nested_emu_start", test_x86_nested_emu_start},
     {"test_x86_nested_emu_stop", test_x86_nested_emu_stop},
     {"test_x86_64_nested_emu_start_error", test_x86_64_nested_emu_start_error},

From a902ec971cb01c7a4daebf10bbd73fd928e06abd Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 09:31:31 +0300
Subject: [PATCH 05/32] issue-2335 fixed

Fix x86 RIP-relative addressing with trailing immediates.

Issue: https://github.com/unicorn-engine/unicorn/issues/2335
---
 qemu/target/i386/translate.c |  4 +++
 tests/unit/test_x86.c        | 60 ++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)

diff --git a/qemu/target/i386/translate.c b/qemu/target/i386/translate.c
index 6a99ed0506..b97b00d0cb 100644
--- a/qemu/target/i386/translate.c
+++ b/qemu/target/i386/translate.c
@@ -4659,6 +4659,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     goto illegal_op;
                 }
                 ot = mo_64_32(s->dflag);
+                s->rip_offset = 1;
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
                 b = x86_ldub_code(env, s);
                 if (ot == MO_64) {
@@ -6361,6 +6362,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         rm = (modrm & 7) | REX_B(s);
         reg = ((modrm >> 3) & 7) | rex_r;
         if (mod != 3) {
+            if (shift) {
+                s->rip_offset = 1;
+            }
             gen_lea_modrm(env, s, modrm);
             opreg = OR_TMP0;
         } else {
diff --git a/tests/unit/test_x86.c b/tests/unit/test_x86.c
index a4ac7f5d08..6068f8cc7c 100644
--- a/tests/unit/test_x86.c
+++ b/tests/unit/test_x86.c
@@ -1111,6 +1111,63 @@ static void test_x86_ret_imm16_unsigned(void)
     OK(uc_close(uc));
 }
 
+static void test_x86_rorx_rip_relative_imm(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe3\x7b\xf0\x05\xf6\x14\x00\x00\x00";
+    uint64_t expected_address = code_start + sizeof(code) - 1 + 0x14f6;
+    uint8_t data[] = {0xaa, 0x11, 0x22, 0x33, 0x44};
+    uint64_t rax;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_mem_write(uc, expected_address - 1, data, sizeof(data)));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+
+    TEST_CHECK(rax == 0x44332211);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_shiftd_rip_relative_imm(const char *code,
+                                             size_t code_size, uint64_t rbx,
+                                             uint16_t expected_value)
+{
+    uc_engine *uc;
+    uint64_t expected_address = code_start + code_size + 0x14f7;
+    uint8_t data[] = {0xaa, 0x11, 0x22, 0x33, 0x44};
+    uint8_t previous;
+    uint16_t mem;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, code_size);
+    OK(uc_mem_write(uc, expected_address - 1, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &rbx));
+
+    OK(uc_emu_start(uc, code_start, code_start + code_size, 0, 1));
+    OK(uc_mem_read(uc, expected_address - 1, &previous, sizeof(previous)));
+    OK(uc_mem_read(uc, expected_address, &mem, sizeof(mem)));
+
+    TEST_CHECK(previous == 0xaa);
+    TEST_CHECK(mem == expected_value);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_shld_rip_relative_imm(void)
+{
+    char code[] = "\x66\x0f\xa4\x1d\xf7\x14\x00\x00\x01";
+
+    test_x86_shiftd_rip_relative_imm(code, sizeof(code) - 1, 0x8000, 0x4423);
+}
+
+static void test_x86_shrd_rip_relative_imm(void)
+{
+    char code[] = "\x66\x0f\xac\x1d\xf7\x14\x00\x00\x01";
+
+    test_x86_shiftd_rip_relative_imm(code, sizeof(code) - 1, 1, 0x9108);
+}
+
 static void test_x86_nested_emu_start_cb(uc_engine *uc, uint64_t addr,
                                          size_t size, void *data)
 {
@@ -2358,6 +2415,9 @@ TEST_LIST = {
     {"test_x86_cmpxchg", test_x86_cmpxchg},
     {"test_x86_cmpxchg32_accumulator", test_x86_cmpxchg32_accumulator},
     {"test_x86_ret_imm16_unsigned", test_x86_ret_imm16_unsigned},
+    {"test_x86_rorx_rip_relative_imm", test_x86_rorx_rip_relative_imm},
+    {"test_x86_shld_rip_relative_imm", test_x86_shld_rip_relative_imm},
+    {"test_x86_shrd_rip_relative_imm", test_x86_shrd_rip_relative_imm},
     {"test_x86_nested_emu_start", test_x86_nested_emu_start},
     {"test_x86_nested_emu_stop", test_x86_nested_emu_stop},
     {"test_x86_64_nested_emu_start_error", test_x86_64_nested_emu_start_error},

From 84515fd17fb2d0672eb2ca782dcb2d3a085b6929 Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 09:38:52 +0300
Subject: [PATCH 06/32] issue-2334 fixed

Fix x86 PDEP 32-bit mask width handling.

Issue: https://github.com/unicorn-engine/unicorn/issues/2334
---
 qemu/target/i386/translate.c |  4 ++--
 tests/unit/test_x86.c        | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/qemu/target/i386/translate.c b/qemu/target/i386/translate.c
index b97b00d0cb..f410803e12 100644
--- a/qemu/target/i386/translate.c
+++ b/qemu/target/i386/translate.c
@@ -4275,12 +4275,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 }
                 ot = mo_64_32(s->dflag);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                /* Note that by zero-extending the source operand, we
-                   automatically handle zero-extending the result.  */
                 if (ot == MO_64) {
                     tcg_gen_mov_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
                 } else {
+                    /* Keep the helper within the 32-bit operand size. */
                     tcg_gen_ext32u_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
+                    tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
                 }
                 gen_helper_pdep(tcg_ctx, tcg_ctx->cpu_regs[reg], s->T1, s->T0);
                 break;
diff --git a/tests/unit/test_x86.c b/tests/unit/test_x86.c
index 6068f8cc7c..1a99c8b9e5 100644
--- a/tests/unit/test_x86.c
+++ b/tests/unit/test_x86.c
@@ -1168,6 +1168,27 @@ static void test_x86_shrd_rip_relative_imm(void)
     test_x86_shiftd_rip_relative_imm(code, sizeof(code) - 1, 1, 0x9108);
 }
 
+static void test_x86_pdep32_zero_extend(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe2\x63\xf5\xc1"; /* pdep eax, ebx, ecx */
+    uint64_t rax = 0xffffffffffffffffULL;
+    uint64_t rbx = 0xffffffffffffff00ULL;
+    uint64_t rcx = 0xffffffffffffff00ULL;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &rbx));
+    OK(uc_reg_write(uc, UC_X86_REG_RCX, &rcx));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+
+    TEST_CHECK(rax == 0x00000000ffff0000ULL);
+
+    OK(uc_close(uc));
+}
+
 static void test_x86_nested_emu_start_cb(uc_engine *uc, uint64_t addr,
                                          size_t size, void *data)
 {
@@ -2418,6 +2439,7 @@ TEST_LIST = {
     {"test_x86_rorx_rip_relative_imm", test_x86_rorx_rip_relative_imm},
     {"test_x86_shld_rip_relative_imm", test_x86_shld_rip_relative_imm},
     {"test_x86_shrd_rip_relative_imm", test_x86_shrd_rip_relative_imm},
+    {"test_x86_pdep32_zero_extend", test_x86_pdep32_zero_extend},
     {"test_x86_nested_emu_start", test_x86_nested_emu_start},
     {"test_x86_nested_emu_stop", test_x86_nested_emu_stop},
     {"test_x86_64_nested_emu_start_error", test_x86_64_nested_emu_start_error},

From d5e72b42619159ccd9db8503472c7674e5e8d1a9 Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 09:54:12 +0300
Subject: [PATCH 07/32] issue-2314 fixed

Fix RISC-V single-precision NaN-boxing semantics.

Issue: https://github.com/unicorn-engine/unicorn/issues/2314
---
 qemu/target/riscv/fpu_helper.c               | 114 +++++++++++++++----
 qemu/target/riscv/insn_trans/trans_rvf.inc.c |  95 +++++++++++-----
 tests/unit/test_riscv.c                      |  90 +++++++++++++++
 3 files changed, 253 insertions(+), 46 deletions(-)

diff --git a/qemu/target/riscv/fpu_helper.c b/qemu/target/riscv/fpu_helper.c
index 3fb6684b16..9efdb5d4da 100644
--- a/qemu/target/riscv/fpu_helper.c
+++ b/qemu/target/riscv/fpu_helper.c
@@ -23,6 +23,22 @@
 #include "exec/helper-proto.h"
 #include "fpu/softfloat.h"
 
+#define RISCV_NANBOX32_MASK UINT64_C(0xffffffff00000000)
+
+static uint64_t nanbox_s(float32 f)
+{
+    return (uint64_t)f | RISCV_NANBOX32_MASK;
+}
+
+static float32 check_nanbox_s(uint64_t f)
+{
+    if ((f & RISCV_NANBOX32_MASK) == RISCV_NANBOX32_MASK) {
+        return (uint32_t)f;
+    }
+
+    return 0x7fc00000u;
+}
+
 target_ulong riscv_cpu_get_fflags(CPURISCVState *env)
 {
     int soft = get_float_exception_flags(&env->fp_status);
@@ -83,7 +99,11 @@ void helper_set_rounding_mode(CPURISCVState *env, uint32_t rm)
 uint64_t helper_fmadd_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                         uint64_t frs3)
 {
-    return float32_muladd(frs1, frs2, frs3, 0, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+    frs3 = check_nanbox_s(frs3);
+
+    return nanbox_s(float32_muladd(frs1, frs2, frs3, 0, &env->fp_status));
 }
 
 uint64_t helper_fmadd_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
@@ -95,8 +115,12 @@ uint64_t helper_fmadd_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
 uint64_t helper_fmsub_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                         uint64_t frs3)
 {
-    return float32_muladd(frs1, frs2, frs3, float_muladd_negate_c,
-                          &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+    frs3 = check_nanbox_s(frs3);
+
+    return nanbox_s(float32_muladd(frs1, frs2, frs3,
+                                   float_muladd_negate_c, &env->fp_status));
 }
 
 uint64_t helper_fmsub_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
@@ -109,8 +133,13 @@ uint64_t helper_fmsub_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
 uint64_t helper_fnmsub_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                          uint64_t frs3)
 {
-    return float32_muladd(frs1, frs2, frs3, float_muladd_negate_product,
-                          &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+    frs3 = check_nanbox_s(frs3);
+
+    return nanbox_s(float32_muladd(frs1, frs2, frs3,
+                                   float_muladd_negate_product,
+                                   &env->fp_status));
 }
 
 uint64_t helper_fnmsub_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
@@ -123,8 +152,14 @@ uint64_t helper_fnmsub_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
 uint64_t helper_fnmadd_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                          uint64_t frs3)
 {
-    return float32_muladd(frs1, frs2, frs3, float_muladd_negate_c |
-                          float_muladd_negate_product, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+    frs3 = check_nanbox_s(frs3);
+
+    return nanbox_s(float32_muladd(frs1, frs2, frs3,
+                                   float_muladd_negate_c |
+                                   float_muladd_negate_product,
+                                   &env->fp_status));
 }
 
 uint64_t helper_fnmadd_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
@@ -136,101 +171,138 @@ uint64_t helper_fnmadd_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
 
 uint64_t helper_fadd_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float32_add(frs1, frs2, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
+    return nanbox_s(float32_add(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fsub_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float32_sub(frs1, frs2, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
+    return nanbox_s(float32_sub(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fmul_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float32_mul(frs1, frs2, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
+    return nanbox_s(float32_mul(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fdiv_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float32_div(frs1, frs2, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
+    return nanbox_s(float32_div(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fmin_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float32_minnum(frs1, frs2, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
+    return nanbox_s(float32_minnum(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fmax_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float32_maxnum(frs1, frs2, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
+    return nanbox_s(float32_maxnum(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fsqrt_s(CPURISCVState *env, uint64_t frs1)
 {
-    return float32_sqrt(frs1, &env->fp_status);
+    frs1 = check_nanbox_s(frs1);
+
+    return nanbox_s(float32_sqrt(frs1, &env->fp_status));
 }
 
 target_ulong helper_fle_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
     return float32_le(frs1, frs2, &env->fp_status);
 }
 
 target_ulong helper_flt_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
     return float32_lt(frs1, frs2, &env->fp_status);
 }
 
 target_ulong helper_feq_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
+    frs1 = check_nanbox_s(frs1);
+    frs2 = check_nanbox_s(frs2);
+
     return float32_eq_quiet(frs1, frs2, &env->fp_status);
 }
 
 target_ulong helper_fcvt_w_s(CPURISCVState *env, uint64_t frs1)
 {
+    frs1 = check_nanbox_s(frs1);
+
     return float32_to_int32(frs1, &env->fp_status);
 }
 
 target_ulong helper_fcvt_wu_s(CPURISCVState *env, uint64_t frs1)
 {
+    frs1 = check_nanbox_s(frs1);
+
     return (int32_t)float32_to_uint32(frs1, &env->fp_status);
 }
 
 #if defined(TARGET_RISCV64)
 uint64_t helper_fcvt_l_s(CPURISCVState *env, uint64_t frs1)
 {
+    frs1 = check_nanbox_s(frs1);
+
     return float32_to_int64(frs1, &env->fp_status);
 }
 
 uint64_t helper_fcvt_lu_s(CPURISCVState *env, uint64_t frs1)
 {
+    frs1 = check_nanbox_s(frs1);
+
     return float32_to_uint64(frs1, &env->fp_status);
 }
 #endif
 
 uint64_t helper_fcvt_s_w(CPURISCVState *env, target_ulong rs1)
 {
-    return int32_to_float32((int32_t)rs1, &env->fp_status);
+    return nanbox_s(int32_to_float32((int32_t)rs1, &env->fp_status));
 }
 
 uint64_t helper_fcvt_s_wu(CPURISCVState *env, target_ulong rs1)
 {
-    return uint32_to_float32((uint32_t)rs1, &env->fp_status);
+    return nanbox_s(uint32_to_float32((uint32_t)rs1, &env->fp_status));
 }
 
 #if defined(TARGET_RISCV64)
 uint64_t helper_fcvt_s_l(CPURISCVState *env, uint64_t rs1)
 {
-    return int64_to_float32(rs1, &env->fp_status);
+    return nanbox_s(int64_to_float32(rs1, &env->fp_status));
 }
 
 uint64_t helper_fcvt_s_lu(CPURISCVState *env, uint64_t rs1)
 {
-    return uint64_to_float32(rs1, &env->fp_status);
+    return nanbox_s(uint64_to_float32(rs1, &env->fp_status));
 }
 #endif
 
 target_ulong helper_fclass_s(uint64_t frs1)
 {
-    float32 f = frs1;
+    float32 f = check_nanbox_s(frs1);
     bool sign = float32_is_neg(f);
 
     if (float32_is_infinity(f)) {
@@ -279,11 +351,13 @@ uint64_t helper_fmax_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 
 uint64_t helper_fcvt_s_d(CPURISCVState *env, uint64_t rs1)
 {
-    return float64_to_float32(rs1, &env->fp_status);
+    return nanbox_s(float64_to_float32(rs1, &env->fp_status));
 }
 
 uint64_t helper_fcvt_d_s(CPURISCVState *env, uint64_t rs1)
 {
+    rs1 = check_nanbox_s(rs1);
+
     return float32_to_float64(rs1, &env->fp_status);
 }
 
diff --git a/qemu/target/riscv/insn_trans/trans_rvf.inc.c b/qemu/target/riscv/insn_trans/trans_rvf.inc.c
index de044bfeb9..be05639eb2 100644
--- a/qemu/target/riscv/insn_trans/trans_rvf.inc.c
+++ b/qemu/target/riscv/insn_trans/trans_rvf.inc.c
@@ -23,6 +23,29 @@
         return false;                       \
 } while (0)
 
+#define RISCV_NANBOX32_MASK UINT64_C(0xffffffff00000000)
+
+static void gen_nanbox_s(TCGContext *tcg_ctx, TCGv_i64 ret, TCGv_i64 value)
+{
+    tcg_gen_ori_i64(tcg_ctx, ret, value, RISCV_NANBOX32_MASK);
+}
+
+static void gen_check_nanbox_s(TCGContext *tcg_ctx, TCGv_i64 ret,
+                               TCGv_i64 value)
+{
+    TCGv_i64 boxed = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 mask = tcg_const_i64(tcg_ctx, RISCV_NANBOX32_MASK);
+    TCGv_i64 qnan = tcg_const_i64(tcg_ctx, 0x7fc00000u);
+
+    tcg_gen_andi_i64(tcg_ctx, boxed, value, RISCV_NANBOX32_MASK);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_EQ, ret, boxed, mask, value, qnan);
+    tcg_gen_andi_i64(tcg_ctx, ret, ret, UINT32_MAX);
+
+    tcg_temp_free_i64(tcg_ctx, qnan);
+    tcg_temp_free_i64(tcg_ctx, mask);
+    tcg_temp_free_i64(tcg_ctx, boxed);
+}
+
 static bool trans_flw(DisasContext *ctx, arg_flw *a)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -174,13 +197,20 @@ static bool trans_fsgnj_s(DisasContext *ctx, arg_fsgnj_s *a)
     REQUIRE_FPU;
     REQUIRE_EXT(ctx, RVF);
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
-
-    if (a->rs1 == a->rs2) { /* FMOV */
-        tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rs1]);
-    } else { /* FSGNJ */
-        tcg_gen_deposit_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rs2], tcg_ctx->cpu_fpr[a->rs1],
-                            0, 31);
-    }
+    TCGv_i64 frs1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 frs2 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 sign = tcg_temp_new_i64(tcg_ctx);
+
+    gen_check_nanbox_s(tcg_ctx, frs1, tcg_ctx->cpu_fpr[a->rs1]);
+    gen_check_nanbox_s(tcg_ctx, frs2, tcg_ctx->cpu_fpr[a->rs2]);
+    tcg_gen_andi_i64(tcg_ctx, sign, frs2, INT32_MIN);
+    tcg_gen_andi_i64(tcg_ctx, frs1, frs1, ~INT32_MIN);
+    tcg_gen_or_i64(tcg_ctx, frs1, frs1, sign);
+    gen_nanbox_s(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], frs1);
+
+    tcg_temp_free_i64(tcg_ctx, sign);
+    tcg_temp_free_i64(tcg_ctx, frs2);
+    tcg_temp_free_i64(tcg_ctx, frs1);
     mark_fs_dirty(ctx);
     return true;
 }
@@ -190,15 +220,21 @@ static bool trans_fsgnjn_s(DisasContext *ctx, arg_fsgnjn_s *a)
     REQUIRE_FPU;
     REQUIRE_EXT(ctx, RVF);
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
-
-    if (a->rs1 == a->rs2) { /* FNEG */
-        tcg_gen_xori_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rs1], INT32_MIN);
-    } else {
-        TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);
-        tcg_gen_not_i64(tcg_ctx, t0, tcg_ctx->cpu_fpr[a->rs2]);
-        tcg_gen_deposit_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], t0, tcg_ctx->cpu_fpr[a->rs1], 0, 31);
-        tcg_temp_free_i64(tcg_ctx, t0);
-    }
+    TCGv_i64 frs1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 frs2 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 sign = tcg_temp_new_i64(tcg_ctx);
+
+    gen_check_nanbox_s(tcg_ctx, frs1, tcg_ctx->cpu_fpr[a->rs1]);
+    gen_check_nanbox_s(tcg_ctx, frs2, tcg_ctx->cpu_fpr[a->rs2]);
+    tcg_gen_andi_i64(tcg_ctx, sign, frs2, INT32_MIN);
+    tcg_gen_xori_i64(tcg_ctx, sign, sign, INT32_MIN);
+    tcg_gen_andi_i64(tcg_ctx, frs1, frs1, ~INT32_MIN);
+    tcg_gen_or_i64(tcg_ctx, frs1, frs1, sign);
+    gen_nanbox_s(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], frs1);
+
+    tcg_temp_free_i64(tcg_ctx, sign);
+    tcg_temp_free_i64(tcg_ctx, frs2);
+    tcg_temp_free_i64(tcg_ctx, frs1);
     mark_fs_dirty(ctx);
     return true;
 }
@@ -208,15 +244,21 @@ static bool trans_fsgnjx_s(DisasContext *ctx, arg_fsgnjx_s *a)
     REQUIRE_FPU;
     REQUIRE_EXT(ctx, RVF);
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
-
-    if (a->rs1 == a->rs2) { /* FABS */
-        tcg_gen_andi_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rs1], ~INT32_MIN);
-    } else {
-        TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);
-        tcg_gen_andi_i64(tcg_ctx, t0, tcg_ctx->cpu_fpr[a->rs2], INT32_MIN);
-        tcg_gen_xor_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rs1], t0);
-        tcg_temp_free_i64(tcg_ctx, t0);
-    }
+    TCGv_i64 frs1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 frs2 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 sign = tcg_temp_new_i64(tcg_ctx);
+
+    gen_check_nanbox_s(tcg_ctx, frs1, tcg_ctx->cpu_fpr[a->rs1]);
+    gen_check_nanbox_s(tcg_ctx, frs2, tcg_ctx->cpu_fpr[a->rs2]);
+    tcg_gen_xor_i64(tcg_ctx, sign, frs1, frs2);
+    tcg_gen_andi_i64(tcg_ctx, sign, sign, INT32_MIN);
+    tcg_gen_andi_i64(tcg_ctx, frs1, frs1, ~INT32_MIN);
+    tcg_gen_or_i64(tcg_ctx, frs1, frs1, sign);
+    gen_nanbox_s(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], frs1);
+
+    tcg_temp_free_i64(tcg_ctx, sign);
+    tcg_temp_free_i64(tcg_ctx, frs2);
+    tcg_temp_free_i64(tcg_ctx, frs1);
     mark_fs_dirty(ctx);
     return true;
 }
@@ -398,10 +440,11 @@ static bool trans_fmv_w_x(DisasContext *ctx, arg_fmv_w_x *a)
     gen_get_gpr(tcg_ctx, t0, a->rs1);
 
 #if defined(TARGET_RISCV64)
-    tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], t0);
+    tcg_gen_ext32u_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], t0);
 #else
     tcg_gen_extu_i32_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], t0);
 #endif
+    gen_nanbox_s(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rd]);
 
     mark_fs_dirty(ctx);
     tcg_temp_free(tcg_ctx, t0);
diff --git a/tests/unit/test_riscv.c b/tests/unit/test_riscv.c
index a9f0fc7f6a..2f8ffd98dc 100644
--- a/tests/unit/test_riscv.c
+++ b/tests/unit/test_riscv.c
@@ -372,6 +372,92 @@ static void test_riscv64_fp_move_to_int(void)
     uc_close(uc);
 }
 
+static void test_riscv64_fclass_s_nanboxing(void)
+{
+    uc_engine *uc;
+    char code[] = "\xd3\x95\x07\xe0"; /* fclass.s a1, f15 */
+    uint64_t f15 = 0xffffffff7f800000ULL;
+    uint64_t a1 = 0;
+    uint64_t mstatus = 0x6000;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F15, &f15));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A1, &a1));
+    TEST_CHECK(a1 == 0x80);
+
+    f15 = 0x62bf9dd562bf9dd5ULL;
+    a1 = 0;
+    OK(uc_reg_write(uc, UC_RISCV_REG_F15, &f15));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A1, &a1));
+    TEST_CHECK(a1 == 0x200);
+
+    f15 = 0xbc26093bbc260929ULL;
+    a1 = 0;
+    OK(uc_reg_write(uc, UC_RISCV_REG_F15, &f15));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A1, &a1));
+    TEST_CHECK(a1 == 0x200);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_fclass_s_produced_nanbox(void)
+{
+    uc_engine *uc;
+    char code[] = "\xd3\x07\x05\xd0\xd3\x95\x07\xe0";
+    uint64_t a0 = 1;
+    uint64_t a1 = 0;
+    uint64_t f15 = 0;
+    uint64_t mstatus = 0x6000;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F15, &f15));
+
+    TEST_CHECK(a1 == 0x40);
+    TEST_CHECK(f15 == 0xffffffff3f800000ULL);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_fmv_w_x_nanbox(void)
+{
+    uc_engine *uc;
+    char code[] = "\xd3\x07\x05\xf0\xd3\x95\x07\xe0";
+    uint64_t a0 = 0x7f800000;
+    uint64_t a1 = 0;
+    uint64_t f15 = 0;
+    uint64_t mstatus = 0x6000;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F15, &f15));
+
+    TEST_CHECK(a1 == 0x80);
+    TEST_CHECK(f15 == 0xffffffff7f800000ULL);
+
+    OK(uc_close(uc));
+}
+
 static void test_riscv64_code_patching(void)
 {
     uc_engine *uc;
@@ -809,6 +895,10 @@ TEST_LIST = {
     {"test_riscv64_fp_move_from_int_reg_write",
      test_riscv64_fp_move_from_int_reg_write},
     {"test_riscv64_fp_move_to_int", test_riscv64_fp_move_to_int},
+    {"test_riscv64_fclass_s_nanboxing", test_riscv64_fclass_s_nanboxing},
+    {"test_riscv64_fclass_s_produced_nanbox",
+     test_riscv64_fclass_s_produced_nanbox},
+    {"test_riscv64_fmv_w_x_nanbox", test_riscv64_fmv_w_x_nanbox},
     {"test_riscv64_ecall", test_riscv64_ecall},
     {"test_riscv32_mmio_map", test_riscv32_mmio_map},
     {"test_riscv64_mmio_map", test_riscv64_mmio_map},

From 874d1d288f08235f61eea9a8aa4fa3f6e0e2c309 Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 10:07:34 +0300
Subject: [PATCH 08/32] issue-1954 fixed

Handle ARM CP15 c15 MRRC/MCRR as RAZ/WI for covered CPU models. Add a regression test for the reported instruction encoding.

Issue: https://github.com/unicorn-engine/unicorn/issues/1954
---
 qemu/target/arm/cpu.c    |  3 +++
 qemu/target/arm/helper.c |  4 ++++
 tests/unit/test_arm.c    | 43 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+)

diff --git a/qemu/target/arm/cpu.c b/qemu/target/arm/cpu.c
index 7613381980..b1a813ba8c 100644
--- a/qemu/target/arm/cpu.c
+++ b/qemu/target/arm/cpu.c
@@ -1628,6 +1628,9 @@ static const ARMCPRegInfo cortexa9_cp_reginfo[] = {
       .fieldoffset = offsetof(CPUARMState, cp15.c15_power_diagnostic) },
     { .name = "NEONBUSY", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .resetvalue = 0, .type = ARM_CP_CONST },
+    { .name = "A9_C15_IMPDEF_64", .cp = 15, .crm = 15, .opc1 = 1,
+      .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_RAW,
+      .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
     /* TLB lockdown control */
     { .name = "TLB_LOCKR", .cp = 15, .crn = 15, .crm = 4, .opc1 = 5, .opc2 = 2,
       .access = PL1_W, .resetvalue = 0, .type = ARM_CP_NOP },
diff --git a/qemu/target/arm/helper.c b/qemu/target/arm/helper.c
index 60c9db9e3e..c616f2ba9c 100644
--- a/qemu/target/arm/helper.c
+++ b/qemu/target/arm/helper.c
@@ -3753,6 +3753,10 @@ static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
       .access = PL1_RW,
       .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
       .resetvalue = 0 },
+    { .name = "C15_IMPDEF_64", .cp = 15, .crm = 15, .opc1 = 1,
+      .access = PL1_RW,
+      .type = ARM_CP_64BIT | ARM_CP_NO_RAW,
+      .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
     REGINFO_SENTINEL
 };
 
diff --git a/tests/unit/test_arm.c b/tests/unit/test_arm.c
index b804892f96..48230437e1 100644
--- a/tests/unit/test_arm.c
+++ b/tests/unit/test_arm.c
@@ -958,6 +958,48 @@ static void test_arm_cp15_c1_c0_2(void)
     OK(uc_close(uc));
 }
 
+static void test_arm_mrrc_cp15_c15_1_cpu(uc_cpu_arm cpu)
+{
+    uc_engine *uc;
+    uc_arm_cp_reg reg = {
+        .cp = 15,
+        .is64 = 1,
+        .sec = 0,
+        .crm = 15,
+        .opc1 = 1,
+        .val = 0x0123456789abcdefULL,
+    };
+    const char code[] = "\x1f\x1f\x40\xec"
+                        "\x1f\x1f\x50\xec";
+    uint32_t r0 = 0x76543210;
+    uint32_t r1 = 0x89abcdef;
+
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, code, sizeof(code) - 1,
+                    cpu);
+
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &r0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_CP_REG, &reg));
+    OK(uc_reg_read(uc, UC_ARM_REG_CP_REG, &reg));
+    TEST_CHECK(reg.val == 0);
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R0, &r0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+
+    TEST_CHECK(r0 == 0);
+    TEST_CHECK(r1 == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm_mrrc_cp15_c15_1(void)
+{
+    test_arm_mrrc_cp15_c15_1_cpu(UC_CPU_ARM_CORTEX_A9);
+    test_arm_mrrc_cp15_c15_1_cpu(UC_CPU_ARM_CORTEX_A15);
+    test_arm_mrrc_cp15_c15_1_cpu(UC_CPU_ARM_MAX);
+}
+
 static bool test_arm_v7_lpae_hook_tlb(uc_engine *uc, uint64_t addr,
                                       uc_mem_type type, uc_tlb_entry *result,
                                       void *user_data)
@@ -1091,6 +1133,7 @@ TEST_LIST = {{"test_arm_nop", test_arm_nop},
              {"test_arm_tcg_opcode_cmp", test_arm_tcg_opcode_cmp},
              {"test_arm_thumb_tcg_opcode_cmn", test_arm_thumb_tcg_opcode_cmn},
              {"test_arm_cp15_c1_c0_2", test_arm_cp15_c1_c0_2},
+             {"test_arm_mrrc_cp15_c15_1", test_arm_mrrc_cp15_c15_1},
              {"test_arm_v7_lpae", test_arm_v7_lpae},
              {"test_arm_svc_hvc_syndrome", test_arm_svc_hvc_syndrome},
              {"test_arm_hook_insn_wfi", test_arm_hook_insn_wfi},

From 7cc40c2490a55f1e0d73ea84fc0f788d5f296f0a Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 10:33:40 +0300
Subject: [PATCH 09/32] issue-2309 fixed

Preserve ARM IT state across continuing memory hooks.

Issue: https://github.com/unicorn-engine/unicorn/issues/2309
---
 qemu/accel/tcg/cputlb.c | 108 ++++++++++++++++++++++++++--------------
 tests/unit/test_arm.c   |  62 +++++++++++++++++++++++
 2 files changed, 132 insertions(+), 38 deletions(-)

diff --git a/qemu/accel/tcg/cputlb.c b/qemu/accel/tcg/cputlb.c
index 3d0bebd33e..3ab0206084 100644
--- a/qemu/accel/tcg/cputlb.c
+++ b/qemu/accel/tcg/cputlb.c
@@ -1483,6 +1483,53 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,
                                 TCGMemOpIdx oi, uintptr_t retaddr);
 
+typedef struct CPUTLBHookState {
+    bool synced;
+#ifdef TARGET_ARM
+    uint32_t condexec_bits;
+    uint32_t restored_condexec_bits;
+#endif
+} CPUTLBHookState;
+
+static inline void tlb_hook_state_init(CPUArchState *env,
+                                       CPUTLBHookState *state)
+{
+    state->synced = false;
+#ifdef TARGET_ARM
+    state->condexec_bits = env->condexec_bits;
+    state->restored_condexec_bits = env->condexec_bits;
+#endif
+}
+
+static inline void tlb_hook_state_sync(CPUArchState *env, uintptr_t retaddr,
+                                       CPUTLBHookState *state)
+{
+    struct uc_struct *uc = env->uc;
+
+    if (!state->synced && !uc->skip_sync_pc_on_exit && retaddr) {
+        cpu_restore_state(uc->cpu, retaddr, false);
+        state->synced = true;
+#ifdef TARGET_ARM
+        state->restored_condexec_bits = env->condexec_bits;
+#endif
+    }
+#ifdef TARGET_ARM
+    else if (state->synced) {
+        env->condexec_bits = state->restored_condexec_bits;
+    }
+#endif
+}
+
+static inline void tlb_hook_state_restore(CPUArchState *env,
+                                          CPUTLBHookState *state)
+{
+#ifdef TARGET_ARM
+    if (state->synced && !env->uc->stop_request) {
+        env->condexec_bits = state->condexec_bits;
+    }
+#endif
+}
+
 static inline uint64_t
 load_memop(const void *haddr, MemOp op)
 {
@@ -1531,7 +1578,9 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
     HOOK_FOREACH_VAR_DECLARE;
     struct uc_struct *uc = env->uc;
     MemoryRegion *mr;
-    bool synced = false;
+    CPUTLBHookState hook_state;
+
+    tlb_hook_state_init(env, &hook_state);
 
     /* Handle CPU specific unaligned behaviour */
     if (addr & ((1 << a_bits) - 1)) {
@@ -1568,10 +1617,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                         continue;
                     if (!HOOK_BOUND_CHECK(hook, paddr))
                         continue;
-                    if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                        cpu_restore_state(uc->cpu, retaddr, false);
-                        synced = true;
-                    }
+                    tlb_hook_state_sync(env, retaddr, &hook_state);
                     JIT_CALLBACK_GUARD_VAR(handled,
                                            ((uc_cb_eventmem_t)hook->callback)(uc, UC_MEM_FETCH_UNMAPPED, paddr, size, 0, hook->user_data));
                     if (handled)
@@ -1589,10 +1635,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                         continue;
                     if (!HOOK_BOUND_CHECK(hook, paddr))
                         continue;
-                    if (!synced &&!uc->skip_sync_pc_on_exit && retaddr) {
-                        cpu_restore_state(uc->cpu, retaddr, false);
-                        synced = true;
-                    }
+                    tlb_hook_state_sync(env, retaddr, &hook_state);
                     JIT_CALLBACK_GUARD_VAR(handled, 
                                            ((uc_cb_eventmem_t)hook->callback)(uc, UC_MEM_READ_UNMAPPED, paddr, size, 0, hook->user_data));
                     if (handled)
@@ -1635,6 +1678,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                 }
                 return 0;
             }
+            tlb_hook_state_restore(env, &hook_state);
         } else {
             uc->invalid_addr = paddr;
             uc->invalid_error = error_code;
@@ -1657,15 +1701,13 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                 continue;
             if (!HOOK_BOUND_CHECK(hook, paddr))
                 continue;
-            if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                cpu_restore_state(uc->cpu, retaddr, false);
-                synced = true;
-            }
+            tlb_hook_state_sync(env, retaddr, &hook_state);
             JIT_CALLBACK_GUARD(((uc_cb_hookmem_t)hook->callback)(env->uc, UC_MEM_READ, paddr, size, 0, hook->user_data));
             // the last callback may already asked to stop emulation
             if (uc->stop_request)
                 break;
         }
+        tlb_hook_state_restore(env, &hook_state);
 
         /* Unicorn: Previous callbacks may invalidate TLB, reload everything.
                     This may have impact on performance but generally fine.
@@ -1689,10 +1731,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                     continue;
                 if (!HOOK_BOUND_CHECK(hook, paddr))
                     continue;
-                if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                    cpu_restore_state(uc->cpu, retaddr, false);
-                    synced = true;
-                }
+                tlb_hook_state_sync(env, retaddr, &hook_state);
                 JIT_CALLBACK_GUARD_VAR(handled, 
                                        ((uc_cb_eventmem_t)hook->callback)(uc, UC_MEM_READ_PROT, paddr, size, 0, hook->user_data));
                 if (handled)
@@ -1717,6 +1756,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                     tlb_addr = code_read ? entry->addr_code : entry->addr_read;
                     tlb_addr &= ~TLB_INVALID_MASK;
                 }
+                tlb_hook_state_restore(env, &hook_state);
             } else {
                 uc->invalid_addr = paddr;
                 uc->invalid_error = UC_ERR_READ_PROT;
@@ -1739,10 +1779,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                     continue;
                 if (!HOOK_BOUND_CHECK(hook, paddr))
                     continue;
-                if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                    cpu_restore_state(uc->cpu, retaddr, false);
-                    synced = true;
-                }
+                tlb_hook_state_sync(env, retaddr, &hook_state);
                 JIT_CALLBACK_GUARD_VAR(handled,
                                        ((uc_cb_eventmem_t)hook->callback)(uc, UC_MEM_FETCH_PROT, paddr, size, 0, hook->user_data));
                 if (handled)
@@ -1755,6 +1792,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
 
             if (handled) {
                 uc->invalid_error = UC_ERR_OK;
+                tlb_hook_state_restore(env, &hook_state);
             } else {
                 uc->invalid_addr = paddr;
                 uc->invalid_error = UC_ERR_FETCH_PROT;
@@ -1853,16 +1891,14 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                     continue;
                 if (!HOOK_BOUND_CHECK(hook, paddr))
                     continue;
-                if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                    cpu_restore_state(uc->cpu, retaddr, false);
-                    synced = true;
-                }
+                tlb_hook_state_sync(env, retaddr, &hook_state);
                 JIT_CALLBACK_GUARD(((uc_cb_hookmem_t)hook->callback)(env->uc, UC_MEM_READ_AFTER, paddr, size, res, hook->user_data));
                 // the last callback may already asked to stop emulation
                 if (uc->stop_request)
                     break;
             }
         }
+        tlb_hook_state_restore(env, &hook_state);
     }
 
     return res;
@@ -2170,7 +2206,9 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
     struct hook *hook;
     bool handled;
     MemoryRegion *mr;
-    bool synced = false;
+    CPUTLBHookState hook_state;
+
+    tlb_hook_state_init(env, &hook_state);
 
     /* Handle CPU specific unaligned behaviour */
     if (addr & ((1 << a_bits) - 1)) {
@@ -2201,15 +2239,13 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
                 continue;
             if (!HOOK_BOUND_CHECK(hook, paddr))
                 continue;
-            if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                cpu_restore_state(uc->cpu, retaddr, false);
-                synced = true;
-            }
+            tlb_hook_state_sync(env, retaddr, &hook_state);
             JIT_CALLBACK_GUARD(((uc_cb_hookmem_t)hook->callback)(uc, UC_MEM_WRITE, paddr, size, val, hook->user_data));
             // the last callback may already asked to stop emulation
             if (uc->stop_request)
                 break;
         }
+        tlb_hook_state_restore(env, &hook_state);
     }
 
     // Unicorn: callback on invalid memory
@@ -2220,10 +2256,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
                 continue;
             if (!HOOK_BOUND_CHECK(hook, paddr))
                 continue;
-            if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                cpu_restore_state(uc->cpu, retaddr, false);
-                synced = true;
-            }
+            tlb_hook_state_sync(env, retaddr, &hook_state);
             JIT_CALLBACK_GUARD_VAR(handled,
                                    ((uc_cb_eventmem_t)hook->callback)(uc, UC_MEM_WRITE_UNMAPPED, paddr, size, val, hook->user_data));
             if (handled)
@@ -2261,6 +2294,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
                 cpu_exit(uc->cpu);
                 return;
             }
+            tlb_hook_state_restore(env, &hook_state);
         }
     }
 
@@ -2273,10 +2307,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
                 continue;
             if (!HOOK_BOUND_CHECK(hook, paddr))
                 continue;
-            if (!synced && !uc->skip_sync_pc_on_exit && retaddr) {
-                cpu_restore_state(uc->cpu, retaddr, false);
-                synced = true;
-            }
+            tlb_hook_state_sync(env, retaddr, &hook_state);
             JIT_CALLBACK_GUARD_VAR(handled,
                                    ((uc_cb_eventmem_t)hook->callback)(uc, UC_MEM_WRITE_PROT, paddr, size, val, hook->user_data));
             if (handled)
@@ -2300,6 +2331,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
                 tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
             }
             uc->invalid_error = UC_ERR_OK;
+            tlb_hook_state_restore(env, &hook_state);
         } else {
             uc->invalid_addr = paddr;
             uc->invalid_error = UC_ERR_WRITE_PROT;
diff --git a/tests/unit/test_arm.c b/tests/unit/test_arm.c
index 48230437e1..b2b299b06c 100644
--- a/tests/unit/test_arm.c
+++ b/tests/unit/test_arm.c
@@ -867,6 +867,67 @@ static void test_arm_mem_hook_read_write(void)
     OK(uc_close(uc));
 }
 
+static void test_arm_thumb_it_mem_read_cb(uc_engine *uc, uc_mem_type type,
+                                          uint64_t address, int size,
+                                          int64_t value, void *user_data)
+{
+    uint64_t *count = (uint64_t *)user_data;
+
+    (void)uc;
+    (void)type;
+    (void)address;
+    (void)size;
+    (void)value;
+
+    (*count)++;
+}
+
+static void test_arm_thumb_it_mem_hook(void)
+{
+    uc_engine *uc;
+    uc_hook hk;
+    uint8_t code[] = {
+        0x00, 0x28, /* cmp r0, #0 */
+        0x1c, 0xbf, /* itt ne */
+        0x11, 0x68, /* ldrne r1, [r2] */
+        0x00, 0x29, /* cmpne r1, #0 */
+        0x00, 0xe0, /* b.n #0x100c */
+        0x00, 0xbf, /* nop */
+        0x03, 0x2c, /* cmp r4, #3 */
+        0x00, 0xd3, /* bcc #0x1012 */
+        0x01, 0x23, /* movs r3, #1 */
+        0x02, 0x23, /* movs r3, #2 */
+    };
+    uint32_t r0 = 1;
+    uint32_t r1 = 0;
+    uint32_t r2 = code_start + 0x200;
+    uint32_t r3 = 0;
+    uint32_t r4 = 1;
+    uint32_t data = LEINT32(1);
+    uint64_t count = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (char *)code, sizeof(code), UC_CPU_ARM_CORTEX_M7);
+
+    OK(uc_mem_write(uc, r2, &data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &r0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &r3));
+    OK(uc_reg_write(uc, UC_ARM_REG_R4, &r4));
+
+    OK(uc_hook_add(uc, &hk, UC_HOOK_MEM_READ,
+                   test_arm_thumb_it_mem_read_cb, &count, 1, 0));
+
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R3, &r3));
+    TEST_CHECK(r3 == 2);
+    TEST_CHECK(count == 1);
+
+    OK(uc_close(uc));
+}
+
 typedef struct {
     uint64_t v0;
     uint64_t v1;
@@ -1130,6 +1191,7 @@ TEST_LIST = {{"test_arm_nop", test_arm_nop},
              {"test_arm_thumb2", test_arm_thumb2},
              {"test_armeb_be32_thumb2", test_armeb_be32_thumb2},
              {"test_arm_mem_hook_read_write", test_arm_mem_hook_read_write},
+             {"test_arm_thumb_it_mem_hook", test_arm_thumb_it_mem_hook},
              {"test_arm_tcg_opcode_cmp", test_arm_tcg_opcode_cmp},
              {"test_arm_thumb_tcg_opcode_cmn", test_arm_thumb_tcg_opcode_cmn},
              {"test_arm_cp15_c1_c0_2", test_arm_cp15_c1_c0_2},

From bdf158a3bccd08de8585452e56e641af55519a2e Mon Sep 17 00:00:00 2001
From: Marius <marius@users.noreply.github.com>
Date: Mon, 22 Jun 2026 10:49:41 +0300
Subject: [PATCH 10/32] issue-1931 fixed

Stop RISC-V translation at the Unicorn exit address before fetching the next instruction, avoiding a spurious fetch from an unmapped following page.

https://github.com/unicorn-engine/unicorn/issues/1931
---
 qemu/target/riscv/translate.c |  4 +++-
 tests/unit/test_riscv.c       | 27 +++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/qemu/target/riscv/translate.c b/qemu/target/riscv/translate.c
index 792bc12fd0..6aedcb86bc 100644
--- a/qemu/target/riscv/translate.c
+++ b/qemu/target/riscv/translate.c
@@ -844,7 +844,7 @@ static void riscv_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
     struct uc_struct *uc = ctx->uc;
     TCGContext *tcg_ctx = uc->tcg_ctx;
     CPURISCVState *env = cpu->env_ptr;
-    uint16_t opcode16 = translator_lduw(tcg_ctx, env, ctx->base.pc_next);
+    uint16_t opcode16;
     TCGOp *tcg_op, *prev_op = NULL;
     bool insn_hook = false;
 
@@ -853,6 +853,8 @@ static void riscv_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
         // Unicorn: We have to exit current execution here.
         dcbase->is_jmp = DISAS_UC_EXIT;
     } else {
+        opcode16 = translator_lduw(tcg_ctx, env, ctx->base.pc_next);
+
         // Unicorn: trace this instruction on request
         if (HOOK_EXISTS_BOUNDED(uc, UC_HOOK_CODE, ctx->base.pc_next)) {
 
diff --git a/tests/unit/test_riscv.c b/tests/unit/test_riscv.c
index 2f8ffd98dc..86c633eb83 100644
--- a/tests/unit/test_riscv.c
+++ b/tests/unit/test_riscv.c
@@ -217,6 +217,32 @@ static void test_riscv64_3steps_pc_update(void)
     OK(uc_close(uc));
 }
 
+static void test_riscv64_until_at_page_end(void)
+{
+    uc_engine *uc;
+    uint64_t address = 0x6d76d7473ffc;
+    uint64_t page = address & ~0xfffULL;
+    char code[] = "\x13\x81\x00\x7d";
+    uint64_t r_x1 = 0x1234;
+    uint64_t r_x2 = 0;
+    uint64_t r_pc = 0;
+
+    OK(uc_open(UC_ARCH_RISCV, UC_MODE_RISCV64, &uc));
+    OK(uc_mem_map(uc, page, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, address, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X1, &r_x1));
+
+    OK(uc_emu_start(uc, address, address + sizeof(code) - 1, 0, 1));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X2, &r_x2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_PC, &r_pc));
+
+    TEST_CHECK(r_x2 == 0x1a04);
+    TEST_CHECK(r_pc == address + sizeof(code) - 1);
+
+    OK(uc_close(uc));
+}
+
 static void test_riscv32_fp_move(void)
 {
     uc_engine *uc;
@@ -887,6 +913,7 @@ TEST_LIST = {
     {"test_riscv64_nop", test_riscv64_nop},
     {"test_riscv32_3steps_pc_update", test_riscv32_3steps_pc_update},
     {"test_riscv64_3steps_pc_update", test_riscv64_3steps_pc_update},
+    {"test_riscv64_until_at_page_end", test_riscv64_until_at_page_end},
     {"test_riscv32_until_pc_update", test_riscv32_until_pc_update},
     {"test_riscv64_until_pc_update", test_riscv64_until_pc_update},
     {"test_riscv32_fp_move", test_riscv32_fp_move},

From ba6227c48d5d412633f01406cdd89eec5ec725ec Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 02:55:06 +0300
Subject: [PATCH 11/32] update to QEMU 7.2.22

Update the embedded QEMU-derived runtime and target code to the 7.2.22 baseline while preserving Unicorn's reduced integration model.

This brings in the QEMU 7.2 support code needed by the fork: softfloat/int128 and helper compatibility, TCG and translator glue, updated symbol-postfix headers, target feature masks, and generated binding constants.

CPU-visible coverage was expanded across the supported targets: AArch64 SVE/SME/MTE/PMU and M-profile MVE, RISC-V RVV/RVH/bitmanip/crypto/Zfh/Sstc/PMP paths, s390x MIE/VE/Vector-FP/string/crypto paths, PPC POWER10 groups, MIPS public modes and MSA registers, M68K FPU/MSP/ColdFire behavior, SPARC register access, and x86 AVX/AVX2/FMA/BMI/VAES/VPCLMUL validation.

The unit test suite was extended with focused regressions for the migrated behavior, and the local MSVC test tree has been validated with full parallel CTest.
---
 .github/workflows/Nuget-publishing.yml        |    15 +-
 .github/workflows/build-uc2.yml               |    17 +-
 CMakeLists.txt                                |    37 +-
 bindings/dotnet/UnicornEngine/Const/Arm.fs    |     8 +-
 bindings/dotnet/UnicornEngine/Const/Common.fs |     4 +
 bindings/dotnet/UnicornEngine/Const/M68k.fs   |     3 +-
 bindings/dotnet/UnicornEngine/Const/Mips.fs   |     5 +-
 bindings/dotnet/UnicornEngine/Const/Riscv.fs  |    46 +-
 bindings/go/unicorn/arm_const.go              |     8 +-
 bindings/go/unicorn/m68k_const.go             |     3 +-
 bindings/go/unicorn/mips_const.go             |     5 +-
 bindings/go/unicorn/riscv_const.go            |    46 +-
 bindings/go/unicorn/unicorn_const.go          |     4 +
 .../java/src/main/java/unicorn/ArmConst.java  |     8 +-
 .../java/src/main/java/unicorn/M68kConst.java |     3 +-
 .../java/src/main/java/unicorn/MipsConst.java |     5 +-
 .../src/main/java/unicorn/RiscvConst.java     |    46 +-
 .../src/main/java/unicorn/UnicornConst.java   |     4 +
 bindings/pascal/unicorn/ArmConst.pas          |     8 +-
 bindings/pascal/unicorn/M68kConst.pas         |     3 +-
 bindings/pascal/unicorn/MipsConst.pas         |     5 +-
 bindings/pascal/unicorn/RiscvConst.pas        |    48 +-
 bindings/pascal/unicorn/UnicornConst.pas      |     4 +
 bindings/python/unicorn/arm_const.py          |     8 +-
 bindings/python/unicorn/m68k_const.py         |     3 +-
 bindings/python/unicorn/mips_const.py         |     5 +-
 bindings/python/unicorn/riscv_const.py        |    46 +-
 bindings/python/unicorn/unicorn_const.py      |     4 +
 .../lib/unicorn_engine/arm_const.rb           |     8 +-
 .../lib/unicorn_engine/m68k_const.rb          |     3 +-
 .../lib/unicorn_engine/mips_const.rb          |     5 +-
 .../lib/unicorn_engine/riscv_const.rb         |    46 +-
 .../lib/unicorn_engine/unicorn_const.rb       |     4 +
 bindings/vb6/uc_def.bas                       |     6 +-
 bindings/zig/unicorn/arm_const.zig            |     8 +-
 bindings/zig/unicorn/m68k_const.zig           |     3 +-
 bindings/zig/unicorn/mips_const.zig           |     5 +-
 bindings/zig/unicorn/riscv_const.zig          |    46 +-
 bindings/zig/unicorn/unicorn_const.zig        |     4 +
 glib_compat/gmacros.h                         |    12 +-
 include/qemu.h                                |     2 +
 include/uc_priv.h                             |     4 +-
 include/unicorn/arm.h                         |     2 +
 include/unicorn/m68k.h                        |     1 +
 include/unicorn/mips.h                        |     3 +
 include/unicorn/riscv.h                       |    49 +
 include/unicorn/unicorn.h                     |     6 +-
 qemu/VERSION                                  |     2 +-
 qemu/aarch64.h                                |  2296 +++
 qemu/accel/tcg/cpu-exec-common.c              |     1 -
 qemu/accel/tcg/cpu-exec.c                     |    24 +-
 qemu/accel/tcg/cputlb.c                       |    29 +-
 qemu/accel/tcg/tcg-runtime.c                  |    15 +-
 qemu/accel/tcg/tcg-runtime.h                  |     5 +-
 qemu/accel/tcg/translator.c                   |    14 +-
 qemu/arm.h                                    |  1479 ++
 qemu/crypto/aes.c                             |    57 +-
 qemu/crypto/sm4.c                             |    48 +
 qemu/exec.c                                   |     1 +
 qemu/fpu/softfloat-parts-addsub.c.inc         |    62 +
 qemu/fpu/softfloat-parts.c.inc                |  1547 +++
 ...alize.inc.c => softfloat-specialize.c.inc} |   769 +-
 qemu/fpu/softfloat.c                          |  9772 +++++--------
 qemu/hw/core/cpu.c                            |    57 +
 qemu/include/crypto/aes.h                     |     4 -
 qemu/include/crypto/sm4.h                     |     6 +
 qemu/include/exec/cputlb.h                    |     2 +-
 qemu/include/exec/helper-head.h               |    36 +-
 qemu/include/exec/hwaddr.h                    |     5 +
 qemu/include/exec/ioport.h                    |     2 +-
 qemu/include/exec/memattrs.h                  |     9 +
 qemu/include/exec/memop.h                     |    46 +-
 qemu/include/exec/poison.h                    |    26 +-
 qemu/include/exec/translator.h                |    12 +
 qemu/include/fpu/softfloat-helpers.h          |    48 +-
 qemu/include/fpu/softfloat-macros.h           |   351 +-
 qemu/include/fpu/softfloat-types.h            |    87 +-
 qemu/include/fpu/softfloat.h                  |   643 +-
 qemu/include/hw/core/cpu.h                    |    32 +
 qemu/include/hw/core/tcg-cpu-ops.h            |   103 +
 qemu/include/hw/mips/cpudevs.h                |     7 -
 qemu/include/libdecnumber/dconfig.h           |     2 +-
 qemu/include/libdecnumber/decNumber.h         |     4 +
 qemu/include/libdecnumber/decNumberLocal.h    |     2 +-
 qemu/include/qemu/atomic128.h                 |     8 +-
 qemu/include/qemu/bitmap.h                    |     9 +
 qemu/include/qemu/compiler.h                  |    51 +
 qemu/include/qemu/cpuid.h                     |    20 +-
 qemu/include/qemu/host-utils.h                |   481 +-
 qemu/include/qemu/int128.h                    |   226 +-
 qemu/include/qemu/log.h                       |    21 +
 qemu/include/qemu/queue.h                     |     7 +-
 qemu/include/qemu/range.h                     |     4 +-
 qemu/include/qemu/xxhash.h                    |    98 +
 qemu/include/sysemu/memory_mapping.h          |     5 +-
 qemu/include/tcg/tcg-gvec-desc.h              |    36 +-
 qemu/include/tcg/tcg-op-gvec.h                |     2 +
 qemu/include/tcg/tcg-op.h                     |     4 +-
 qemu/include/tcg/tcg.h                        |     9 +
 qemu/libdecnumber/decContext.c                |     7 +-
 qemu/libdecnumber/decNumber.c                 |   133 +-
 qemu/libdecnumber/dpd/decimal64.c             |     3 +-
 qemu/m68k.h                                   |   771 +
 qemu/mips.h                                   |   774 ++
 qemu/mips64.h                                 |   774 ++
 qemu/mips64el.h                               |   774 ++
 qemu/mipsel.h                                 |   774 ++
 qemu/ppc.h                                    |   912 ++
 qemu/ppc64.h                                  |   912 ++
 qemu/riscv32.h                                |  1753 +++
 qemu/riscv64.h                                |  1753 +++
 qemu/s390x.h                                  |   771 +
 qemu/softmmu/cpus.c                           |    30 +-
 qemu/softmmu/memory.c                         |    27 +
 qemu/sparc.h                                  |   771 +
 qemu/sparc64.h                                |   771 +
 qemu/target/arm/cpu.c                         |   115 +-
 qemu/target/arm/cpu.h                         |   400 +-
 qemu/target/arm/cpu64.c                       |    50 +-
 qemu/target/arm/crypto_helper.c               |   112 +-
 qemu/target/arm/decode-sve.inc.c              |  2105 ++-
 qemu/target/arm/decode-vfp.inc.c              |    12 +
 qemu/target/arm/helper-a64.c                  |    32 +
 qemu/target/arm/helper-a64.h                  |    15 +
 qemu/target/arm/helper-sme.h                  |   191 +
 qemu/target/arm/helper-sve.h                  |  1002 ++
 qemu/target/arm/helper.c                      |   939 +-
 qemu/target/arm/helper.h                      |   748 +-
 qemu/target/arm/internals.h                   |    61 +
 qemu/target/arm/m_helper.c                    |    15 +-
 qemu/target/arm/mte_helper.c                  |   720 +
 qemu/target/arm/mve_helper.c                  |  4221 ++++++
 qemu/target/arm/sme_helper.c                  |  1010 ++
 qemu/target/arm/sve_helper.c                  | 11476 ++++++++++-----
 qemu/target/arm/tlb_helper.c                  |     6 +-
 qemu/target/arm/translate-a64.c               |   901 +-
 qemu/target/arm/translate-a64.h               |    11 +-
 qemu/target/arm/translate-sme.c               |   824 ++
 qemu/target/arm/translate-sve.c               |  3697 ++++-
 qemu/target/arm/translate-vfp.inc.c           |   792 +-
 qemu/target/arm/translate.c                   |  5176 ++++++-
 qemu/target/arm/translate.h                   |    13 +
 qemu/target/arm/unicorn_arm.c                 |    32 +
 qemu/target/arm/vec_helper.c                  |   505 +-
 qemu/target/arm/vfp_helper.c                  |   117 +-
 qemu/target/i386/TODO                         |     2 +-
 qemu/target/i386/cpu.c                        |   193 +-
 qemu/target/i386/cpu.h                        |   214 +-
 qemu/target/i386/decode-new.c.inc             |  1874 +++
 qemu/target/i386/decode-new.h                 |   250 +
 qemu/target/i386/emit.c.inc                   |  2456 ++++
 qemu/target/i386/fpu_helper.c                 |  2071 ++-
 qemu/target/i386/helper-tcg.h                 |   116 +
 qemu/target/i386/helper.c                     |    23 +-
 qemu/target/i386/helper.h                     |     6 +
 qemu/target/i386/misc_helper.c                |   102 +-
 qemu/target/i386/ops_sse.h                    |  2493 ++--
 qemu/target/i386/ops_sse_header.h             |   415 +-
 qemu/target/i386/seg_helper.c                 |     5 +
 qemu/target/i386/seg_helper.h                 |    66 +
 qemu/target/i386/svm_helper.c                 |     5 +
 qemu/target/i386/translate.c                  |  6186 +++------
 qemu/target/i386/unicorn.c                    |    24 +
 qemu/target/i386/xsave_helper.c               |   297 +-
 qemu/target/m68k/cpu.c                        |    33 +-
 qemu/target/m68k/cpu.h                        |    10 +-
 qemu/target/m68k/fpu_helper.c                 |    71 +-
 qemu/target/m68k/helper.c                     |   201 +-
 qemu/target/m68k/helper.h                     |     1 +
 qemu/target/m68k/op_helper.c                  |    12 +-
 qemu/target/m68k/softfloat.c                  |   241 +-
 qemu/target/m68k/softfloat.h                  |     1 -
 qemu/target/m68k/translate.c                  |   140 +-
 qemu/target/mips/cp0_helper.c                 |    27 +-
 qemu/target/mips/cp0_timer.c                  |    18 +
 qemu/target/mips/cpu.c                        |    11 +-
 qemu/target/mips/cpu.h                        |    26 +
 qemu/target/mips/mips-defs.h                  |    10 +-
 qemu/target/mips/op_helper.c                  |     3 +-
 qemu/target/mips/translate.c                  |   661 +-
 qemu/target/mips/translate_init.inc.c         |   115 +
 qemu/target/mips/unicorn.c                    |    25 +-
 qemu/target/ppc/cpu.h                         |    28 +-
 qemu/target/ppc/dfp_helper.c                  |   126 +
 qemu/target/ppc/excp_helper.c                 |    89 +
 qemu/target/ppc/fpu_helper.c                  |   514 +
 qemu/target/ppc/helper.h                      |   101 +
 qemu/target/ppc/int_helper.c                  |   553 +
 qemu/target/ppc/internal.h                    |    26 +
 qemu/target/ppc/mmu-hash64.c                  |    22 +
 qemu/target/ppc/translate.c                   |   908 +-
 qemu/target/ppc/translate/dfp-impl.inc.c      |    47 +
 qemu/target/ppc/translate/dfp-ops.inc.c       |     2 +
 qemu/target/ppc/translate/fp-impl.inc.c       |   153 +-
 qemu/target/ppc/translate/fp-ops.inc.c        |    16 +-
 qemu/target/ppc/translate/vmx-impl.inc.c      |  2138 ++-
 qemu/target/ppc/translate/vmx-ops.inc.c       |   127 +-
 qemu/target/ppc/translate/vsx-impl.inc.c      |  1434 ++
 qemu/target/ppc/translate/vsx-ops.inc.c       |    40 +
 qemu/target/ppc/translate_init.inc.c          |    32 +-
 qemu/target/riscv/bitmanip_helper.c           |   133 +
 qemu/target/riscv/cpu-param.h                 |    11 +-
 qemu/target/riscv/cpu.c                       |    80 +-
 qemu/target/riscv/cpu.h                       |   111 +-
 qemu/target/riscv/cpu_bits.h                  |    48 +-
 qemu/target/riscv/cpu_helper.c                |   397 +-
 qemu/target/riscv/crypto_helper.c             |   302 +
 qemu/target/riscv/csr.c                       |   358 +-
 qemu/target/riscv/fpu_helper.c                |   262 +-
 qemu/target/riscv/helper.h                    |   983 ++
 .../riscv/insn_trans/trans_privileged.inc.c   |    26 +-
 qemu/target/riscv/insn_trans/trans_rvb.inc.c  |   680 +
 qemu/target/riscv/insn_trans/trans_rvh.inc.c  |   275 +
 qemu/target/riscv/insn_trans/trans_rvi.inc.c  |    14 +
 qemu/target/riscv/insn_trans/trans_rvk.inc.c  |   453 +
 qemu/target/riscv/insn_trans/trans_rvm.inc.c  |    16 +-
 qemu/target/riscv/insn_trans/trans_rvv.inc.c  |  7895 +++++++++++
 .../target/riscv/insn_trans/trans_rvzfh.inc.c |   579 +
 .../riscv/insn_trans/trans_svinval.inc.c      |    47 +
 .../insn_trans/trans_xventanacondops.inc.c    |    59 +
 qemu/target/riscv/instmap.h                   |    41 +
 qemu/target/riscv/op_helper.c                 |    79 +-
 qemu/target/riscv/pmp.c                       |     6 +-
 qemu/target/riscv/riscv32/decode_insn32.inc.c |   642 +-
 qemu/target/riscv/riscv64/decode_insn32.inc.c |   734 +-
 qemu/target/riscv/translate.c                 |   159 +-
 qemu/target/riscv/unicorn.c                   |   197 +
 qemu/target/riscv/vector_helper.c             |  5145 +++++++
 qemu/target/s390x/cc_helper.c                 |    32 +
 qemu/target/s390x/cpu.h                       |     4 +-
 qemu/target/s390x/cpu_models.c                |     1 +
 qemu/target/s390x/excp_helper.c               |    31 +-
 qemu/target/s390x/helper.c                    |     2 +
 qemu/target/s390x/helper.h                    |    85 +-
 qemu/target/s390x/insn-data.def               |   100 +-
 qemu/target/s390x/internal.h                  |    10 +
 qemu/target/s390x/mem_helper.c                |    25 +-
 qemu/target/s390x/translate.c                 |   145 +-
 qemu/target/s390x/translate_vx.inc.c          |  1102 +-
 qemu/target/s390x/vec_fpu_helper.c            |  1134 +-
 qemu/target/s390x/vec_helper.c                |    22 +
 qemu/target/s390x/vec_int_helper.c            |    59 +
 qemu/target/s390x/vec_string_helper.c         |    83 +
 qemu/target/sparc/fop_helper.c                |     4 +-
 qemu/target/sparc/unicorn.c                   |    81 +-
 qemu/target/sparc/unicorn64.c                 |   137 +-
 qemu/target/sparc/vis_helper.c                |     4 +-
 qemu/target/tricore/unicorn.c                 |   127 +-
 qemu/tcg/README                               |    44 +-
 qemu/tcg/aarch64/tcg-target.inc.c             |     4 +-
 qemu/tcg/aarch64/tcg-target.opc.h             |     1 +
 qemu/tcg/arm/tcg-target.h                     |   100 +-
 qemu/tcg/arm/tcg-target.inc.c                 |     4 +-
 qemu/tcg/i386/tcg-target.inc.c                |     4 +-
 qemu/tcg/loongarch64/tcg-target.inc.c         |     4 +-
 qemu/tcg/mips/tcg-target.h                    |    23 +-
 qemu/tcg/mips/tcg-target.inc.c                |     4 +-
 qemu/tcg/ppc/tcg-target.h                     |    22 +-
 qemu/tcg/ppc/tcg-target.inc.c                 |     4 +-
 qemu/tcg/ppc/tcg-target.opc.h                 |     1 -
 qemu/tcg/riscv/tcg-target.h                   |    12 +-
 qemu/tcg/riscv/tcg-target.inc.c               |     4 +-
 qemu/tcg/s390/tcg-target.inc.c                |     4 +-
 qemu/tcg/sparc/tcg-target.inc.c               |     4 +-
 qemu/tcg/tcg-op-gvec.c                        |    38 +-
 qemu/tcg/tcg-op-vec.c                         |    10 +
 qemu/tcg/tcg-op.c                             |     4 +-
 qemu/tcg/tcg.c                                |    13 +
 qemu/tricore.h                                |   771 +
 qemu/util/guest-random.c                      |    20 +-
 qemu/util/host-utils.c                        |   319 +-
 qemu/util/int128.c                            |   147 +
 qemu/util/qemu-timer-common.c                 |     4 +
 qemu/x86_64.h                                 |   921 +-
 symbols.sh                                    |  4280 +++++-
 tests/unit/acutest.h                          |    63 +
 tests/unit/test_arm.c                         | 11369 +++++++++++++++
 tests/unit/test_arm64.c                       | 11587 ++++++++++++++++
 tests/unit/test_m68k.c                        |   438 +
 tests/unit/test_mips.c                        |   876 +-
 tests/unit/test_ppc.c                         |  4947 ++++++-
 tests/unit/test_riscv.c                       | 10619 ++++++++++++++
 tests/unit/test_s390x.c                       |  1515 +-
 tests/unit/test_sparc.c                       |   100 +
 tests/unit/test_tricore.c                     |    93 +-
 tests/unit/test_x86.c                         |  1063 +-
 uc.c                                          |    94 +
 287 files changed, 144176 insertions(+), 19246 deletions(-)
 create mode 100644 qemu/crypto/sm4.c
 create mode 100644 qemu/fpu/softfloat-parts-addsub.c.inc
 create mode 100644 qemu/fpu/softfloat-parts.c.inc
 rename qemu/fpu/{softfloat-specialize.inc.c => softfloat-specialize.c.inc} (55%)
 create mode 100644 qemu/include/crypto/sm4.h
 create mode 100644 qemu/include/hw/core/tcg-cpu-ops.h
 create mode 100644 qemu/target/arm/helper-sme.h
 create mode 100644 qemu/target/arm/mte_helper.c
 create mode 100644 qemu/target/arm/mve_helper.c
 create mode 100644 qemu/target/arm/sme_helper.c
 create mode 100644 qemu/target/arm/translate-sme.c
 create mode 100644 qemu/target/i386/decode-new.c.inc
 create mode 100644 qemu/target/i386/decode-new.h
 create mode 100644 qemu/target/i386/emit.c.inc
 create mode 100644 qemu/target/i386/helper-tcg.h
 create mode 100644 qemu/target/i386/seg_helper.h
 create mode 100644 qemu/target/riscv/bitmanip_helper.c
 create mode 100644 qemu/target/riscv/crypto_helper.c
 create mode 100644 qemu/target/riscv/insn_trans/trans_rvb.inc.c
 create mode 100644 qemu/target/riscv/insn_trans/trans_rvh.inc.c
 create mode 100644 qemu/target/riscv/insn_trans/trans_rvk.inc.c
 create mode 100644 qemu/target/riscv/insn_trans/trans_rvv.inc.c
 create mode 100644 qemu/target/riscv/insn_trans/trans_rvzfh.inc.c
 create mode 100644 qemu/target/riscv/insn_trans/trans_svinval.inc.c
 create mode 100644 qemu/target/riscv/insn_trans/trans_xventanacondops.inc.c
 create mode 100644 qemu/target/riscv/vector_helper.c
 create mode 100644 qemu/util/int128.c

diff --git a/.github/workflows/Nuget-publishing.yml b/.github/workflows/Nuget-publishing.yml
index 681978a410..fe2c0ca2ff 100644
--- a/.github/workflows/Nuget-publishing.yml
+++ b/.github/workflows/Nuget-publishing.yml
@@ -31,6 +31,9 @@ on:
 permissions:
   packages: write
 
+env:
+  CTEST_PARALLEL_LEVEL: 40
+
 jobs:
   Windows:
     runs-on: ${{ matrix.config.os }}
@@ -93,7 +96,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ matrix.config.build_type }}
           cmake --install . --strip --config ${{ matrix.config.build_type }}
-          ctest -VV -C ${{ matrix.config.build_type }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ matrix.config.build_type }}
           mv Release instdir
 
       - name: '🛠️ Win MSVC 32 setup'
@@ -121,7 +124,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ matrix.config.build_type }}
           cmake --install . --strip --config ${{ matrix.config.build_type }}
-          ctest -VV -C ${{ matrix.config.build_type }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ matrix.config.build_type }}
           mv Release instdir
 
       - name: '📦 Pack artifact'
@@ -177,7 +180,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ matrix.config.build_type }}
           cmake --install . --strip
-          ctest -VV -C ${{ matrix.config.build_type }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ matrix.config.build_type }}
 
       - name: '📦 Pack artifact'
         if: always()
@@ -270,7 +273,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ matrix.config.build_type }}
           cmake --install . --strip
-          ctest -VV -C ${{ matrix.config.build_type }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ matrix.config.build_type }}
 
       - name: '🚧 Linux aarch64 build'
         if: contains(matrix.config.arch, 'aarch64')
@@ -288,7 +291,7 @@ jobs:
             -DCMAKE_INSTALL_PREFIX:PATH=instdir
           cmake --build . --config ${{ matrix.config.build_type }}
           cmake --install . --strip
-          ctest -VV -C ${{ matrix.config.build_type }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ matrix.config.build_type }}
 
       - name: '🚧 Linux ppc64le build'
         if: contains(matrix.config.arch, 'ppc64le')
@@ -314,7 +317,7 @@ jobs:
               -DCMAKE_INSTALL_PREFIX:PATH=/instdir
             cmake --build . --config ${{ matrix.config.build_type }}
             cmake --install . --strip
-            ctest -VV -C ${{ matrix.config.build_type }}
+            ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ matrix.config.build_type }}
 
       - name: '📦 Pack artifact'
         if: always()
diff --git a/.github/workflows/build-uc2.yml b/.github/workflows/build-uc2.yml
index c6093e0c76..7d023657a1 100644
--- a/.github/workflows/build-uc2.yml
+++ b/.github/workflows/build-uc2.yml
@@ -31,6 +31,7 @@ on:
 env:
   # Specify build type either according to the tag release or manual override
   BUILD_TYPE: ${{ inputs.buildType != '' && inputs.buildType || startsWith(github.ref, 'refs/tags') && 'Release' || 'Debug' }}
+  CTEST_PARALLEL_LEVEL: 40
 
 jobs:
   Windows:
@@ -167,7 +168,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip --config ${{ env.BUILD_TYPE }}
-          ctest -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
           mv ${{ env.BUILD_TYPE }} instdir
 
       - name: '🛠️ Win MSVC 32 setup'
@@ -194,7 +195,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip --config ${{ env.BUILD_TYPE }}
-          ctest -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
           mv ${{ env.BUILD_TYPE }} instdir
 
       - name: '🚧 Win MINGW build'
@@ -223,7 +224,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
 
       - name: '📦 Pack artifact'
         if: always()
@@ -311,7 +312,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
 
       - name: '🚧 Android x86_64 build'
         if: contains(matrix.config.name, 'android')
@@ -473,7 +474,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
 
       - name: '🚧 Linux aarch64 build'
         if: contains(matrix.config.arch, 'aarch64')
@@ -491,7 +492,7 @@ jobs:
             -DCMAKE_INSTALL_PREFIX:PATH=instdir
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
 
       - name: '🚧 Linux ppc64le build'
         if: contains(matrix.config.arch, 'ppc64le')
@@ -517,7 +518,7 @@ jobs:
               -DCMAKE_INSTALL_PREFIX:PATH=/instdir
             cmake --build . --config ${{ env.BUILD_TYPE }}
             cmake --install . --strip
-            ctest -VV -C ${{ env.BUILD_TYPE }}
+            ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
 
       - name: '📦 Pack artifact'
         if: always()
@@ -619,7 +620,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
 
       - name: '📦 Pack artifact'
         if: always()
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 033db0965d..2bf906beb3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -580,6 +580,7 @@ add_library(arm-softmmu STATIC
     qemu/target/arm/helper.c
     qemu/target/arm/iwmmxt_helper.c
     qemu/target/arm/m_helper.c
+    qemu/target/arm/mve_helper.c
     qemu/target/arm/neon_helper.c
     qemu/target/arm/op_helper.c
     qemu/target/arm/psci.c
@@ -624,14 +625,18 @@ add_library(aarch64-softmmu STATIC
     qemu/target/arm/helper.c
     qemu/target/arm/iwmmxt_helper.c
     qemu/target/arm/m_helper.c
+    qemu/target/arm/mte_helper.c
+    qemu/target/arm/mve_helper.c
     qemu/target/arm/neon_helper.c
     qemu/target/arm/op_helper.c
     qemu/target/arm/pauth_helper.c
     qemu/target/arm/psci.c
+    qemu/target/arm/sme_helper.c
     qemu/target/arm/sve_helper.c
     qemu/target/arm/tlb_helper.c
     qemu/target/arm/translate-a64.c
     qemu/target/arm/translate.c
+    qemu/target/arm/translate-sme.c
     qemu/target/arm/translate-sve.c
     qemu/target/arm/vec_helper.c
     qemu/target/arm/vfp_helper.c
@@ -1024,6 +1029,8 @@ add_library(riscv32-softmmu STATIC
     ${UNICORN_ARCH_COMMON}
 
     qemu/target/riscv/cpu.c
+    qemu/target/riscv/bitmanip_helper.c
+    qemu/target/riscv/crypto_helper.c
     qemu/target/riscv/cpu_helper.c
     qemu/target/riscv/csr.c
     qemu/target/riscv/fpu_helper.c
@@ -1031,6 +1038,7 @@ add_library(riscv32-softmmu STATIC
     qemu/target/riscv/pmp.c
     qemu/target/riscv/translate.c
     qemu/target/riscv/unicorn.c
+    qemu/target/riscv/vector_helper.c
 )
 
 if(MSVC)
@@ -1057,6 +1065,8 @@ add_library(riscv64-softmmu STATIC
     ${UNICORN_ARCH_COMMON}
 
     qemu/target/riscv/cpu.c
+    qemu/target/riscv/bitmanip_helper.c
+    qemu/target/riscv/crypto_helper.c
     qemu/target/riscv/cpu_helper.c
     qemu/target/riscv/csr.c
     qemu/target/riscv/fpu_helper.c
@@ -1064,6 +1074,7 @@ add_library(riscv64-softmmu STATIC
     qemu/target/riscv/pmp.c
     qemu/target/riscv/translate.c
     qemu/target/riscv/unicorn.c
+    qemu/target/riscv/vector_helper.c
 )
 
 if(MSVC)
@@ -1195,6 +1206,7 @@ set(UNICORN_COMMON_SRCS
     qemu/util/getauxval.c
     qemu/util/guest-random.c
     qemu/util/host-utils.c
+    qemu/util/int128.c
     qemu/util/osdep.c
     qemu/util/qdist.c
     qemu/util/qemu-timer.c
@@ -1205,6 +1217,7 @@ set(UNICORN_COMMON_SRCS
     qemu/util/cacheinfo.c
 
     qemu/crypto/aes.c
+    qemu/crypto/sm4.c
 )
 
 # A workaround to avoid circle dependency between unicorn and *-softmmu
@@ -1486,7 +1499,24 @@ if(UNICORN_FUZZ)
 endif()
 
 if(UNICORN_BUILD_TESTS)
+    set(UNICORN_CTEST_PARALLEL_LEVEL "40" CACHE STRING
+        "Default CTest parallel job count")
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17 AND NOT CMAKE_CTEST_ARGUMENTS)
+        set(CMAKE_CTEST_ARGUMENTS
+            "--parallel;${UNICORN_CTEST_PARALLEL_LEVEL};--output-on-failure"
+            CACHE STRING "Default arguments for the CTest build target")
+    endif()
     enable_testing()
+    set(UNICORN_PARALLEL_CTEST_COMMAND
+        ${CMAKE_CTEST_COMMAND}
+        "--parallel" "${UNICORN_CTEST_PARALLEL_LEVEL}"
+        "--output-on-failure")
+    if(CMAKE_CONFIGURATION_TYPES)
+        list(APPEND UNICORN_PARALLEL_CTEST_COMMAND "-C" "$<CONFIG>")
+    endif()
+    add_custom_target(test_parallel
+        COMMAND ${UNICORN_PARALLEL_CTEST_COMMAND}
+        USES_TERMINAL)
     foreach(SAMPLE_FILE ${UNICORN_SAMPLE_FILE})
         add_executable(${SAMPLE_FILE}
             ${CMAKE_CURRENT_SOURCE_DIR}/samples/${SAMPLE_FILE}.c
@@ -1494,9 +1524,9 @@ if(UNICORN_BUILD_TESTS)
         target_link_libraries(${SAMPLE_FILE} PRIVATE
             ${SAMPLES_LIB}
         )
-        endforeach()
+    endforeach()
 
-        foreach(TEST_FILE ${UNICORN_TEST_FILE})
+    foreach(TEST_FILE ${UNICORN_TEST_FILE})
         add_executable(${TEST_FILE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tests/unit/${TEST_FILE}.c
         )
@@ -1516,6 +1546,9 @@ if(UNICORN_BUILD_TESTS)
             target_compile_definitions(${TEST_FILE} PRIVATE TARGET_READ_INLINED)
         endif()
     endforeach()
+    if(UNICORN_TEST_FILE)
+        add_dependencies(test_parallel ${UNICORN_TEST_FILE})
+    endif()
 endif()
 
 
diff --git a/bindings/dotnet/UnicornEngine/Const/Arm.fs b/bindings/dotnet/UnicornEngine/Const/Arm.fs
index 44a0f56081..887e842e03 100644
--- a/bindings/dotnet/UnicornEngine/Const/Arm.fs
+++ b/bindings/dotnet/UnicornEngine/Const/Arm.fs
@@ -42,8 +42,9 @@ module Arm =
     let UC_CPU_ARM_PXA270B1 = 30
     let UC_CPU_ARM_PXA270C0 = 31
     let UC_CPU_ARM_PXA270C5 = 32
-    let UC_CPU_ARM_MAX = 33
-    let UC_CPU_ARM_ENDING = 34
+    let UC_CPU_ARM_CORTEX_M55 = 33
+    let UC_CPU_ARM_MAX = 34
+    let UC_CPU_ARM_ENDING = 35
 
     // ARM registers
 
@@ -188,7 +189,8 @@ module Arm =
     let UC_ARM_REG_XPSR_NZCVQG = 138
     let UC_ARM_REG_CP_REG = 139
     let UC_ARM_REG_ESR = 140
-    let UC_ARM_REG_ENDING = 141
+    let UC_ARM_REG_VPR = 141
+    let UC_ARM_REG_ENDING = 142
 
     // alias registers
     let UC_ARM_REG_R13 = 12
diff --git a/bindings/dotnet/UnicornEngine/Const/Common.fs b/bindings/dotnet/UnicornEngine/Const/Common.fs
index d42495c3f0..4268d115a2 100644
--- a/bindings/dotnet/UnicornEngine/Const/Common.fs
+++ b/bindings/dotnet/UnicornEngine/Const/Common.fs
@@ -79,6 +79,9 @@ module Common =
     let UC_ERR_RESOURCE = 20
     let UC_ERR_EXCEPTION = 21
     let UC_ERR_OVERFLOW = 22
+    let UC_ERR_MMU_READ = 23
+    let UC_ERR_MMU_WRITE = 24
+    let UC_ERR_MMU_FETCH = 25
 
     let UC_PROT_NONE = 0
     let UC_PROT_READ = 1
@@ -155,6 +158,7 @@ module Common =
     let UC_CTL_PAUTH_SIGN = 15
     let UC_CTL_PAUTH_STRIP = 16
     let UC_CTL_PAUTH_AUTH = 17
+    let UC_CTL_INVALID_ADDR = 18
     let UC_CTL_CONTEXT_CPU = 1
     let UC_CTL_CONTEXT_MEMORY = 2
 
diff --git a/bindings/dotnet/UnicornEngine/Const/M68k.fs b/bindings/dotnet/UnicornEngine/Const/M68k.fs
index 77b52c6dae..a264a10c12 100644
--- a/bindings/dotnet/UnicornEngine/Const/M68k.fs
+++ b/bindings/dotnet/UnicornEngine/Const/M68k.fs
@@ -18,7 +18,8 @@ module M68k =
     let UC_CPU_M68K_M5208 = 6
     let UC_CPU_M68K_CFV4E = 7
     let UC_CPU_M68K_ANY = 8
-    let UC_CPU_M68K_ENDING = 9
+    let UC_CPU_M68K_M68010 = 9
+    let UC_CPU_M68K_ENDING = 10
 
     // M68K registers
 
diff --git a/bindings/dotnet/UnicornEngine/Const/Mips.fs b/bindings/dotnet/UnicornEngine/Const/Mips.fs
index 76d0c71494..240043ac44 100644
--- a/bindings/dotnet/UnicornEngine/Const/Mips.fs
+++ b/bindings/dotnet/UnicornEngine/Const/Mips.fs
@@ -42,7 +42,10 @@ module Mips =
     let UC_CPU_MIPS64_LOONGSON_2E = 10
     let UC_CPU_MIPS64_LOONGSON_2F = 11
     let UC_CPU_MIPS64_MIPS64DSPR2 = 12
-    let UC_CPU_MIPS64_ENDING = 13
+    let UC_CPU_MIPS64_OCTEON68XX = 13
+    let UC_CPU_MIPS64_LOONGSON_3A1000 = 14
+    let UC_CPU_MIPS64_LOONGSON_3A4000 = 15
+    let UC_CPU_MIPS64_ENDING = 16
 
     // MIPS registers
 
diff --git a/bindings/dotnet/UnicornEngine/Const/Riscv.fs b/bindings/dotnet/UnicornEngine/Const/Riscv.fs
index 244e5fec45..4dd88237af 100644
--- a/bindings/dotnet/UnicornEngine/Const/Riscv.fs
+++ b/bindings/dotnet/UnicornEngine/Const/Riscv.fs
@@ -6,6 +6,7 @@ open System
 
 [<AutoOpen>]
 module Riscv =
+    let UC_RISCV_VLEN_MAX = 1024
 
     // RISCV32 CPU
 
@@ -223,7 +224,50 @@ module Riscv =
     let UC_RISCV_REG_F31 = 189
     let UC_RISCV_REG_PC = 190
     let UC_RISCV_REG_PRIV = 191
-    let UC_RISCV_REG_ENDING = 192
+
+    // Vector CSRs
+    let UC_RISCV_REG_VSTART = 192
+    let UC_RISCV_REG_VXSAT = 193
+    let UC_RISCV_REG_VXRM = 194
+    let UC_RISCV_REG_VCSR = 195
+    let UC_RISCV_REG_VL = 196
+    let UC_RISCV_REG_VTYPE = 197
+    let UC_RISCV_REG_VLENB = 198
+
+    // Vector registers
+    let UC_RISCV_REG_V0 = 199
+    let UC_RISCV_REG_V1 = 200
+    let UC_RISCV_REG_V2 = 201
+    let UC_RISCV_REG_V3 = 202
+    let UC_RISCV_REG_V4 = 203
+    let UC_RISCV_REG_V5 = 204
+    let UC_RISCV_REG_V6 = 205
+    let UC_RISCV_REG_V7 = 206
+    let UC_RISCV_REG_V8 = 207
+    let UC_RISCV_REG_V9 = 208
+    let UC_RISCV_REG_V10 = 209
+    let UC_RISCV_REG_V11 = 210
+    let UC_RISCV_REG_V12 = 211
+    let UC_RISCV_REG_V13 = 212
+    let UC_RISCV_REG_V14 = 213
+    let UC_RISCV_REG_V15 = 214
+    let UC_RISCV_REG_V16 = 215
+    let UC_RISCV_REG_V17 = 216
+    let UC_RISCV_REG_V18 = 217
+    let UC_RISCV_REG_V19 = 218
+    let UC_RISCV_REG_V20 = 219
+    let UC_RISCV_REG_V21 = 220
+    let UC_RISCV_REG_V22 = 221
+    let UC_RISCV_REG_V23 = 222
+    let UC_RISCV_REG_V24 = 223
+    let UC_RISCV_REG_V25 = 224
+    let UC_RISCV_REG_V26 = 225
+    let UC_RISCV_REG_V27 = 226
+    let UC_RISCV_REG_V28 = 227
+    let UC_RISCV_REG_V29 = 228
+    let UC_RISCV_REG_V30 = 229
+    let UC_RISCV_REG_V31 = 230
+    let UC_RISCV_REG_ENDING = 231
 
     // Alias registers
     let UC_RISCV_REG_ZERO = 1
diff --git a/bindings/go/unicorn/arm_const.go b/bindings/go/unicorn/arm_const.go
index de4ebb18af..842486a775 100644
--- a/bindings/go/unicorn/arm_const.go
+++ b/bindings/go/unicorn/arm_const.go
@@ -37,8 +37,9 @@ const (
 	CPU_ARM_PXA270B1 = 30
 	CPU_ARM_PXA270C0 = 31
 	CPU_ARM_PXA270C5 = 32
-	CPU_ARM_MAX = 33
-	CPU_ARM_ENDING = 34
+	CPU_ARM_CORTEX_M55 = 33
+	CPU_ARM_MAX = 34
+	CPU_ARM_ENDING = 35
 
 // ARM registers
 
@@ -183,7 +184,8 @@ const (
 	ARM_REG_XPSR_NZCVQG = 138
 	ARM_REG_CP_REG = 139
 	ARM_REG_ESR = 140
-	ARM_REG_ENDING = 141
+	ARM_REG_VPR = 141
+	ARM_REG_ENDING = 142
 
 // alias registers
 	ARM_REG_R13 = 12
diff --git a/bindings/go/unicorn/m68k_const.go b/bindings/go/unicorn/m68k_const.go
index 6ed11fac9a..df499fd8ff 100644
--- a/bindings/go/unicorn/m68k_const.go
+++ b/bindings/go/unicorn/m68k_const.go
@@ -13,7 +13,8 @@ const (
 	CPU_M68K_M5208 = 6
 	CPU_M68K_CFV4E = 7
 	CPU_M68K_ANY = 8
-	CPU_M68K_ENDING = 9
+	CPU_M68K_M68010 = 9
+	CPU_M68K_ENDING = 10
 
 // M68K registers
 
diff --git a/bindings/go/unicorn/mips_const.go b/bindings/go/unicorn/mips_const.go
index dfb6ddb066..9c69870b81 100644
--- a/bindings/go/unicorn/mips_const.go
+++ b/bindings/go/unicorn/mips_const.go
@@ -37,7 +37,10 @@ const (
 	CPU_MIPS64_LOONGSON_2E = 10
 	CPU_MIPS64_LOONGSON_2F = 11
 	CPU_MIPS64_MIPS64DSPR2 = 12
-	CPU_MIPS64_ENDING = 13
+	CPU_MIPS64_OCTEON68XX = 13
+	CPU_MIPS64_LOONGSON_3A1000 = 14
+	CPU_MIPS64_LOONGSON_3A4000 = 15
+	CPU_MIPS64_ENDING = 16
 
 // MIPS registers
 
diff --git a/bindings/go/unicorn/riscv_const.go b/bindings/go/unicorn/riscv_const.go
index 08458f77a6..49684c18c6 100644
--- a/bindings/go/unicorn/riscv_const.go
+++ b/bindings/go/unicorn/riscv_const.go
@@ -1,6 +1,7 @@
 package unicorn
 // For Unicorn Engine. AUTO-GENERATED FILE, DO NOT EDIT [riscv_const.go]
 const (
+	RISCV_VLEN_MAX = 1024
 
 // RISCV32 CPU
 
@@ -218,7 +219,50 @@ const (
 	RISCV_REG_F31 = 189
 	RISCV_REG_PC = 190
 	RISCV_REG_PRIV = 191
-	RISCV_REG_ENDING = 192
+
+// Vector CSRs
+	RISCV_REG_VSTART = 192
+	RISCV_REG_VXSAT = 193
+	RISCV_REG_VXRM = 194
+	RISCV_REG_VCSR = 195
+	RISCV_REG_VL = 196
+	RISCV_REG_VTYPE = 197
+	RISCV_REG_VLENB = 198
+
+// Vector registers
+	RISCV_REG_V0 = 199
+	RISCV_REG_V1 = 200
+	RISCV_REG_V2 = 201
+	RISCV_REG_V3 = 202
+	RISCV_REG_V4 = 203
+	RISCV_REG_V5 = 204
+	RISCV_REG_V6 = 205
+	RISCV_REG_V7 = 206
+	RISCV_REG_V8 = 207
+	RISCV_REG_V9 = 208
+	RISCV_REG_V10 = 209
+	RISCV_REG_V11 = 210
+	RISCV_REG_V12 = 211
+	RISCV_REG_V13 = 212
+	RISCV_REG_V14 = 213
+	RISCV_REG_V15 = 214
+	RISCV_REG_V16 = 215
+	RISCV_REG_V17 = 216
+	RISCV_REG_V18 = 217
+	RISCV_REG_V19 = 218
+	RISCV_REG_V20 = 219
+	RISCV_REG_V21 = 220
+	RISCV_REG_V22 = 221
+	RISCV_REG_V23 = 222
+	RISCV_REG_V24 = 223
+	RISCV_REG_V25 = 224
+	RISCV_REG_V26 = 225
+	RISCV_REG_V27 = 226
+	RISCV_REG_V28 = 227
+	RISCV_REG_V29 = 228
+	RISCV_REG_V30 = 229
+	RISCV_REG_V31 = 230
+	RISCV_REG_ENDING = 231
 
 // Alias registers
 	RISCV_REG_ZERO = 1
diff --git a/bindings/go/unicorn/unicorn_const.go b/bindings/go/unicorn/unicorn_const.go
index 03a92f720b..68dfcbd39c 100644
--- a/bindings/go/unicorn/unicorn_const.go
+++ b/bindings/go/unicorn/unicorn_const.go
@@ -74,6 +74,9 @@ const (
 	ERR_RESOURCE = 20
 	ERR_EXCEPTION = 21
 	ERR_OVERFLOW = 22
+	ERR_MMU_READ = 23
+	ERR_MMU_WRITE = 24
+	ERR_MMU_FETCH = 25
 
 	PROT_NONE = 0
 	PROT_READ = 1
@@ -150,6 +153,7 @@ const (
 	CTL_PAUTH_SIGN = 15
 	CTL_PAUTH_STRIP = 16
 	CTL_PAUTH_AUTH = 17
+	CTL_INVALID_ADDR = 18
 	CTL_CONTEXT_CPU = 1
 	CTL_CONTEXT_MEMORY = 2
 )
\ No newline at end of file
diff --git a/bindings/java/src/main/java/unicorn/ArmConst.java b/bindings/java/src/main/java/unicorn/ArmConst.java
index a5fef8f1b3..9fecc32b3d 100644
--- a/bindings/java/src/main/java/unicorn/ArmConst.java
+++ b/bindings/java/src/main/java/unicorn/ArmConst.java
@@ -39,8 +39,9 @@ public interface ArmConst {
     public static final int UC_CPU_ARM_PXA270B1 = 30;
     public static final int UC_CPU_ARM_PXA270C0 = 31;
     public static final int UC_CPU_ARM_PXA270C5 = 32;
-    public static final int UC_CPU_ARM_MAX = 33;
-    public static final int UC_CPU_ARM_ENDING = 34;
+    public static final int UC_CPU_ARM_CORTEX_M55 = 33;
+    public static final int UC_CPU_ARM_MAX = 34;
+    public static final int UC_CPU_ARM_ENDING = 35;
 
     // ARM registers
 
@@ -185,7 +186,8 @@ public interface ArmConst {
     public static final int UC_ARM_REG_XPSR_NZCVQG = 138;
     public static final int UC_ARM_REG_CP_REG = 139;
     public static final int UC_ARM_REG_ESR = 140;
-    public static final int UC_ARM_REG_ENDING = 141;
+    public static final int UC_ARM_REG_VPR = 141;
+    public static final int UC_ARM_REG_ENDING = 142;
 
     // alias registers
     public static final int UC_ARM_REG_R13 = 12;
diff --git a/bindings/java/src/main/java/unicorn/M68kConst.java b/bindings/java/src/main/java/unicorn/M68kConst.java
index db62630b15..2087bea74d 100644
--- a/bindings/java/src/main/java/unicorn/M68kConst.java
+++ b/bindings/java/src/main/java/unicorn/M68kConst.java
@@ -15,7 +15,8 @@ public interface M68kConst {
     public static final int UC_CPU_M68K_M5208 = 6;
     public static final int UC_CPU_M68K_CFV4E = 7;
     public static final int UC_CPU_M68K_ANY = 8;
-    public static final int UC_CPU_M68K_ENDING = 9;
+    public static final int UC_CPU_M68K_M68010 = 9;
+    public static final int UC_CPU_M68K_ENDING = 10;
 
     // M68K registers
 
diff --git a/bindings/java/src/main/java/unicorn/MipsConst.java b/bindings/java/src/main/java/unicorn/MipsConst.java
index bf6d8cf2fe..503bd27ba6 100644
--- a/bindings/java/src/main/java/unicorn/MipsConst.java
+++ b/bindings/java/src/main/java/unicorn/MipsConst.java
@@ -39,7 +39,10 @@ public interface MipsConst {
     public static final int UC_CPU_MIPS64_LOONGSON_2E = 10;
     public static final int UC_CPU_MIPS64_LOONGSON_2F = 11;
     public static final int UC_CPU_MIPS64_MIPS64DSPR2 = 12;
-    public static final int UC_CPU_MIPS64_ENDING = 13;
+    public static final int UC_CPU_MIPS64_OCTEON68XX = 13;
+    public static final int UC_CPU_MIPS64_LOONGSON_3A1000 = 14;
+    public static final int UC_CPU_MIPS64_LOONGSON_3A4000 = 15;
+    public static final int UC_CPU_MIPS64_ENDING = 16;
 
     // MIPS registers
 
diff --git a/bindings/java/src/main/java/unicorn/RiscvConst.java b/bindings/java/src/main/java/unicorn/RiscvConst.java
index 5814180974..e1dcfca327 100644
--- a/bindings/java/src/main/java/unicorn/RiscvConst.java
+++ b/bindings/java/src/main/java/unicorn/RiscvConst.java
@@ -3,6 +3,7 @@
 package unicorn;
 
 public interface RiscvConst {
+    public static final int UC_RISCV_VLEN_MAX = 1024;
 
     // RISCV32 CPU
 
@@ -220,7 +221,50 @@ public interface RiscvConst {
     public static final int UC_RISCV_REG_F31 = 189;
     public static final int UC_RISCV_REG_PC = 190;
     public static final int UC_RISCV_REG_PRIV = 191;
-    public static final int UC_RISCV_REG_ENDING = 192;
+
+    // Vector CSRs
+    public static final int UC_RISCV_REG_VSTART = 192;
+    public static final int UC_RISCV_REG_VXSAT = 193;
+    public static final int UC_RISCV_REG_VXRM = 194;
+    public static final int UC_RISCV_REG_VCSR = 195;
+    public static final int UC_RISCV_REG_VL = 196;
+    public static final int UC_RISCV_REG_VTYPE = 197;
+    public static final int UC_RISCV_REG_VLENB = 198;
+
+    // Vector registers
+    public static final int UC_RISCV_REG_V0 = 199;
+    public static final int UC_RISCV_REG_V1 = 200;
+    public static final int UC_RISCV_REG_V2 = 201;
+    public static final int UC_RISCV_REG_V3 = 202;
+    public static final int UC_RISCV_REG_V4 = 203;
+    public static final int UC_RISCV_REG_V5 = 204;
+    public static final int UC_RISCV_REG_V6 = 205;
+    public static final int UC_RISCV_REG_V7 = 206;
+    public static final int UC_RISCV_REG_V8 = 207;
+    public static final int UC_RISCV_REG_V9 = 208;
+    public static final int UC_RISCV_REG_V10 = 209;
+    public static final int UC_RISCV_REG_V11 = 210;
+    public static final int UC_RISCV_REG_V12 = 211;
+    public static final int UC_RISCV_REG_V13 = 212;
+    public static final int UC_RISCV_REG_V14 = 213;
+    public static final int UC_RISCV_REG_V15 = 214;
+    public static final int UC_RISCV_REG_V16 = 215;
+    public static final int UC_RISCV_REG_V17 = 216;
+    public static final int UC_RISCV_REG_V18 = 217;
+    public static final int UC_RISCV_REG_V19 = 218;
+    public static final int UC_RISCV_REG_V20 = 219;
+    public static final int UC_RISCV_REG_V21 = 220;
+    public static final int UC_RISCV_REG_V22 = 221;
+    public static final int UC_RISCV_REG_V23 = 222;
+    public static final int UC_RISCV_REG_V24 = 223;
+    public static final int UC_RISCV_REG_V25 = 224;
+    public static final int UC_RISCV_REG_V26 = 225;
+    public static final int UC_RISCV_REG_V27 = 226;
+    public static final int UC_RISCV_REG_V28 = 227;
+    public static final int UC_RISCV_REG_V29 = 228;
+    public static final int UC_RISCV_REG_V30 = 229;
+    public static final int UC_RISCV_REG_V31 = 230;
+    public static final int UC_RISCV_REG_ENDING = 231;
 
     // Alias registers
     public static final int UC_RISCV_REG_ZERO = 1;
diff --git a/bindings/java/src/main/java/unicorn/UnicornConst.java b/bindings/java/src/main/java/unicorn/UnicornConst.java
index c46675e17f..57e109469a 100644
--- a/bindings/java/src/main/java/unicorn/UnicornConst.java
+++ b/bindings/java/src/main/java/unicorn/UnicornConst.java
@@ -76,6 +76,9 @@ public interface UnicornConst {
     public static final int UC_ERR_RESOURCE = 20;
     public static final int UC_ERR_EXCEPTION = 21;
     public static final int UC_ERR_OVERFLOW = 22;
+    public static final int UC_ERR_MMU_READ = 23;
+    public static final int UC_ERR_MMU_WRITE = 24;
+    public static final int UC_ERR_MMU_FETCH = 25;
 
     public static final int UC_PROT_NONE = 0;
     public static final int UC_PROT_READ = 1;
@@ -152,6 +155,7 @@ public interface UnicornConst {
     public static final int UC_CTL_PAUTH_SIGN = 15;
     public static final int UC_CTL_PAUTH_STRIP = 16;
     public static final int UC_CTL_PAUTH_AUTH = 17;
+    public static final int UC_CTL_INVALID_ADDR = 18;
     public static final int UC_CTL_CONTEXT_CPU = 1;
     public static final int UC_CTL_CONTEXT_MEMORY = 2;
 
diff --git a/bindings/pascal/unicorn/ArmConst.pas b/bindings/pascal/unicorn/ArmConst.pas
index bb923b002b..35e82bc0a4 100644
--- a/bindings/pascal/unicorn/ArmConst.pas
+++ b/bindings/pascal/unicorn/ArmConst.pas
@@ -40,8 +40,9 @@ interface
   UC_CPU_ARM_PXA270B1 = 30;
   UC_CPU_ARM_PXA270C0 = 31;
   UC_CPU_ARM_PXA270C5 = 32;
-  UC_CPU_ARM_MAX = 33;
-  UC_CPU_ARM_ENDING = 34;
+  UC_CPU_ARM_CORTEX_M55 = 33;
+  UC_CPU_ARM_MAX = 34;
+  UC_CPU_ARM_ENDING = 35;
 
 // ARM registers
 
@@ -186,7 +187,8 @@ interface
   UC_ARM_REG_XPSR_NZCVQG = 138;
   UC_ARM_REG_CP_REG = 139;
   UC_ARM_REG_ESR = 140;
-  UC_ARM_REG_ENDING = 141;
+  UC_ARM_REG_VPR = 141;
+  UC_ARM_REG_ENDING = 142;
 
 // alias registers
   UC_ARM_REG_R13 = 12;
diff --git a/bindings/pascal/unicorn/M68kConst.pas b/bindings/pascal/unicorn/M68kConst.pas
index 89b837f338..42f6082f2d 100644
--- a/bindings/pascal/unicorn/M68kConst.pas
+++ b/bindings/pascal/unicorn/M68kConst.pas
@@ -16,7 +16,8 @@ interface
   UC_CPU_M68K_M5208 = 6;
   UC_CPU_M68K_CFV4E = 7;
   UC_CPU_M68K_ANY = 8;
-  UC_CPU_M68K_ENDING = 9;
+  UC_CPU_M68K_M68010 = 9;
+  UC_CPU_M68K_ENDING = 10;
 
 // M68K registers
 
diff --git a/bindings/pascal/unicorn/MipsConst.pas b/bindings/pascal/unicorn/MipsConst.pas
index 3cbce39ac0..107969bda7 100644
--- a/bindings/pascal/unicorn/MipsConst.pas
+++ b/bindings/pascal/unicorn/MipsConst.pas
@@ -40,7 +40,10 @@ interface
   UC_CPU_MIPS64_LOONGSON_2E = 10;
   UC_CPU_MIPS64_LOONGSON_2F = 11;
   UC_CPU_MIPS64_MIPS64DSPR2 = 12;
-  UC_CPU_MIPS64_ENDING = 13;
+  UC_CPU_MIPS64_OCTEON68XX = 13;
+  UC_CPU_MIPS64_LOONGSON_3A1000 = 14;
+  UC_CPU_MIPS64_LOONGSON_3A4000 = 15;
+  UC_CPU_MIPS64_ENDING = 16;
 
 // MIPS registers
 
diff --git a/bindings/pascal/unicorn/RiscvConst.pas b/bindings/pascal/unicorn/RiscvConst.pas
index 075e271c65..03020083d7 100644
--- a/bindings/pascal/unicorn/RiscvConst.pas
+++ b/bindings/pascal/unicorn/RiscvConst.pas
@@ -4,7 +4,8 @@
 
 interface
 
-const
+const  UC_RISCV_VLEN_MAX = 1024;
+
 // RISCV32 CPU
 
   UC_CPU_RISCV32_ANY = 0;
@@ -221,7 +222,50 @@ interface
   UC_RISCV_REG_F31 = 189;
   UC_RISCV_REG_PC = 190;
   UC_RISCV_REG_PRIV = 191;
-  UC_RISCV_REG_ENDING = 192;
+
+// Vector CSRs
+  UC_RISCV_REG_VSTART = 192;
+  UC_RISCV_REG_VXSAT = 193;
+  UC_RISCV_REG_VXRM = 194;
+  UC_RISCV_REG_VCSR = 195;
+  UC_RISCV_REG_VL = 196;
+  UC_RISCV_REG_VTYPE = 197;
+  UC_RISCV_REG_VLENB = 198;
+
+// Vector registers
+  UC_RISCV_REG_V0 = 199;
+  UC_RISCV_REG_V1 = 200;
+  UC_RISCV_REG_V2 = 201;
+  UC_RISCV_REG_V3 = 202;
+  UC_RISCV_REG_V4 = 203;
+  UC_RISCV_REG_V5 = 204;
+  UC_RISCV_REG_V6 = 205;
+  UC_RISCV_REG_V7 = 206;
+  UC_RISCV_REG_V8 = 207;
+  UC_RISCV_REG_V9 = 208;
+  UC_RISCV_REG_V10 = 209;
+  UC_RISCV_REG_V11 = 210;
+  UC_RISCV_REG_V12 = 211;
+  UC_RISCV_REG_V13 = 212;
+  UC_RISCV_REG_V14 = 213;
+  UC_RISCV_REG_V15 = 214;
+  UC_RISCV_REG_V16 = 215;
+  UC_RISCV_REG_V17 = 216;
+  UC_RISCV_REG_V18 = 217;
+  UC_RISCV_REG_V19 = 218;
+  UC_RISCV_REG_V20 = 219;
+  UC_RISCV_REG_V21 = 220;
+  UC_RISCV_REG_V22 = 221;
+  UC_RISCV_REG_V23 = 222;
+  UC_RISCV_REG_V24 = 223;
+  UC_RISCV_REG_V25 = 224;
+  UC_RISCV_REG_V26 = 225;
+  UC_RISCV_REG_V27 = 226;
+  UC_RISCV_REG_V28 = 227;
+  UC_RISCV_REG_V29 = 228;
+  UC_RISCV_REG_V30 = 229;
+  UC_RISCV_REG_V31 = 230;
+  UC_RISCV_REG_ENDING = 231;
 
 // Alias registers
   UC_RISCV_REG_ZERO = 1;
diff --git a/bindings/pascal/unicorn/UnicornConst.pas b/bindings/pascal/unicorn/UnicornConst.pas
index 1ac1a92f5d..09635f51d6 100644
--- a/bindings/pascal/unicorn/UnicornConst.pas
+++ b/bindings/pascal/unicorn/UnicornConst.pas
@@ -77,6 +77,9 @@ interface
   UC_ERR_RESOURCE = 20;
   UC_ERR_EXCEPTION = 21;
   UC_ERR_OVERFLOW = 22;
+  UC_ERR_MMU_READ = 23;
+  UC_ERR_MMU_WRITE = 24;
+  UC_ERR_MMU_FETCH = 25;
 
   UC_PROT_NONE = 0;
   UC_PROT_READ = 1;
@@ -153,6 +156,7 @@ interface
   UC_CTL_PAUTH_SIGN = 15;
   UC_CTL_PAUTH_STRIP = 16;
   UC_CTL_PAUTH_AUTH = 17;
+  UC_CTL_INVALID_ADDR = 18;
   UC_CTL_CONTEXT_CPU = 1;
   UC_CTL_CONTEXT_MEMORY = 2;
 
diff --git a/bindings/python/unicorn/arm_const.py b/bindings/python/unicorn/arm_const.py
index 867b4347e5..92be24f122 100644
--- a/bindings/python/unicorn/arm_const.py
+++ b/bindings/python/unicorn/arm_const.py
@@ -35,8 +35,9 @@
 UC_CPU_ARM_PXA270B1 = 30
 UC_CPU_ARM_PXA270C0 = 31
 UC_CPU_ARM_PXA270C5 = 32
-UC_CPU_ARM_MAX = 33
-UC_CPU_ARM_ENDING = 34
+UC_CPU_ARM_CORTEX_M55 = 33
+UC_CPU_ARM_MAX = 34
+UC_CPU_ARM_ENDING = 35
 
 # ARM registers
 
@@ -181,7 +182,8 @@
 UC_ARM_REG_XPSR_NZCVQG = 138
 UC_ARM_REG_CP_REG = 139
 UC_ARM_REG_ESR = 140
-UC_ARM_REG_ENDING = 141
+UC_ARM_REG_VPR = 141
+UC_ARM_REG_ENDING = 142
 
 # alias registers
 UC_ARM_REG_R13 = 12
diff --git a/bindings/python/unicorn/m68k_const.py b/bindings/python/unicorn/m68k_const.py
index 824d300d84..70c4794f82 100644
--- a/bindings/python/unicorn/m68k_const.py
+++ b/bindings/python/unicorn/m68k_const.py
@@ -11,7 +11,8 @@
 UC_CPU_M68K_M5208 = 6
 UC_CPU_M68K_CFV4E = 7
 UC_CPU_M68K_ANY = 8
-UC_CPU_M68K_ENDING = 9
+UC_CPU_M68K_M68010 = 9
+UC_CPU_M68K_ENDING = 10
 
 # M68K registers
 
diff --git a/bindings/python/unicorn/mips_const.py b/bindings/python/unicorn/mips_const.py
index c60b2d0f77..c28f54eaa2 100644
--- a/bindings/python/unicorn/mips_const.py
+++ b/bindings/python/unicorn/mips_const.py
@@ -35,7 +35,10 @@
 UC_CPU_MIPS64_LOONGSON_2E = 10
 UC_CPU_MIPS64_LOONGSON_2F = 11
 UC_CPU_MIPS64_MIPS64DSPR2 = 12
-UC_CPU_MIPS64_ENDING = 13
+UC_CPU_MIPS64_OCTEON68XX = 13
+UC_CPU_MIPS64_LOONGSON_3A1000 = 14
+UC_CPU_MIPS64_LOONGSON_3A4000 = 15
+UC_CPU_MIPS64_ENDING = 16
 
 # MIPS registers
 
diff --git a/bindings/python/unicorn/riscv_const.py b/bindings/python/unicorn/riscv_const.py
index 3e63376fd5..8201f5a503 100644
--- a/bindings/python/unicorn/riscv_const.py
+++ b/bindings/python/unicorn/riscv_const.py
@@ -1,4 +1,5 @@
 # For Unicorn Engine. AUTO-GENERATED FILE, DO NOT EDIT [riscv_const.py]
+UC_RISCV_VLEN_MAX = 1024
 
 # RISCV32 CPU
 
@@ -216,7 +217,50 @@
 UC_RISCV_REG_F31 = 189
 UC_RISCV_REG_PC = 190
 UC_RISCV_REG_PRIV = 191
-UC_RISCV_REG_ENDING = 192
+
+# Vector CSRs
+UC_RISCV_REG_VSTART = 192
+UC_RISCV_REG_VXSAT = 193
+UC_RISCV_REG_VXRM = 194
+UC_RISCV_REG_VCSR = 195
+UC_RISCV_REG_VL = 196
+UC_RISCV_REG_VTYPE = 197
+UC_RISCV_REG_VLENB = 198
+
+# Vector registers
+UC_RISCV_REG_V0 = 199
+UC_RISCV_REG_V1 = 200
+UC_RISCV_REG_V2 = 201
+UC_RISCV_REG_V3 = 202
+UC_RISCV_REG_V4 = 203
+UC_RISCV_REG_V5 = 204
+UC_RISCV_REG_V6 = 205
+UC_RISCV_REG_V7 = 206
+UC_RISCV_REG_V8 = 207
+UC_RISCV_REG_V9 = 208
+UC_RISCV_REG_V10 = 209
+UC_RISCV_REG_V11 = 210
+UC_RISCV_REG_V12 = 211
+UC_RISCV_REG_V13 = 212
+UC_RISCV_REG_V14 = 213
+UC_RISCV_REG_V15 = 214
+UC_RISCV_REG_V16 = 215
+UC_RISCV_REG_V17 = 216
+UC_RISCV_REG_V18 = 217
+UC_RISCV_REG_V19 = 218
+UC_RISCV_REG_V20 = 219
+UC_RISCV_REG_V21 = 220
+UC_RISCV_REG_V22 = 221
+UC_RISCV_REG_V23 = 222
+UC_RISCV_REG_V24 = 223
+UC_RISCV_REG_V25 = 224
+UC_RISCV_REG_V26 = 225
+UC_RISCV_REG_V27 = 226
+UC_RISCV_REG_V28 = 227
+UC_RISCV_REG_V29 = 228
+UC_RISCV_REG_V30 = 229
+UC_RISCV_REG_V31 = 230
+UC_RISCV_REG_ENDING = 231
 
 # Alias registers
 UC_RISCV_REG_ZERO = 1
diff --git a/bindings/python/unicorn/unicorn_const.py b/bindings/python/unicorn/unicorn_const.py
index 3f280c77d4..37351676b4 100644
--- a/bindings/python/unicorn/unicorn_const.py
+++ b/bindings/python/unicorn/unicorn_const.py
@@ -72,6 +72,9 @@
 UC_ERR_RESOURCE = 20
 UC_ERR_EXCEPTION = 21
 UC_ERR_OVERFLOW = 22
+UC_ERR_MMU_READ = 23
+UC_ERR_MMU_WRITE = 24
+UC_ERR_MMU_FETCH = 25
 
 UC_PROT_NONE = 0
 UC_PROT_READ = 1
@@ -148,5 +151,6 @@
 UC_CTL_PAUTH_SIGN = 15
 UC_CTL_PAUTH_STRIP = 16
 UC_CTL_PAUTH_AUTH = 17
+UC_CTL_INVALID_ADDR = 18
 UC_CTL_CONTEXT_CPU = 1
 UC_CTL_CONTEXT_MEMORY = 2
diff --git a/bindings/ruby/unicorn_gem/lib/unicorn_engine/arm_const.rb b/bindings/ruby/unicorn_gem/lib/unicorn_engine/arm_const.rb
index 3b1a625ec9..326e047f9d 100644
--- a/bindings/ruby/unicorn_gem/lib/unicorn_engine/arm_const.rb
+++ b/bindings/ruby/unicorn_gem/lib/unicorn_engine/arm_const.rb
@@ -37,8 +37,9 @@ module UnicornEngine
 	UC_CPU_ARM_PXA270B1 = 30
 	UC_CPU_ARM_PXA270C0 = 31
 	UC_CPU_ARM_PXA270C5 = 32
-	UC_CPU_ARM_MAX = 33
-	UC_CPU_ARM_ENDING = 34
+	UC_CPU_ARM_CORTEX_M55 = 33
+	UC_CPU_ARM_MAX = 34
+	UC_CPU_ARM_ENDING = 35
 
 # ARM registers
 
@@ -183,7 +184,8 @@ module UnicornEngine
 	UC_ARM_REG_XPSR_NZCVQG = 138
 	UC_ARM_REG_CP_REG = 139
 	UC_ARM_REG_ESR = 140
-	UC_ARM_REG_ENDING = 141
+	UC_ARM_REG_VPR = 141
+	UC_ARM_REG_ENDING = 142
 
 # alias registers
 	UC_ARM_REG_R13 = 12
diff --git a/bindings/ruby/unicorn_gem/lib/unicorn_engine/m68k_const.rb b/bindings/ruby/unicorn_gem/lib/unicorn_engine/m68k_const.rb
index 7665ef3af5..49ed902265 100644
--- a/bindings/ruby/unicorn_gem/lib/unicorn_engine/m68k_const.rb
+++ b/bindings/ruby/unicorn_gem/lib/unicorn_engine/m68k_const.rb
@@ -13,7 +13,8 @@ module UnicornEngine
 	UC_CPU_M68K_M5208 = 6
 	UC_CPU_M68K_CFV4E = 7
 	UC_CPU_M68K_ANY = 8
-	UC_CPU_M68K_ENDING = 9
+	UC_CPU_M68K_M68010 = 9
+	UC_CPU_M68K_ENDING = 10
 
 # M68K registers
 
diff --git a/bindings/ruby/unicorn_gem/lib/unicorn_engine/mips_const.rb b/bindings/ruby/unicorn_gem/lib/unicorn_engine/mips_const.rb
index 374912a870..f60f6047a6 100644
--- a/bindings/ruby/unicorn_gem/lib/unicorn_engine/mips_const.rb
+++ b/bindings/ruby/unicorn_gem/lib/unicorn_engine/mips_const.rb
@@ -37,7 +37,10 @@ module UnicornEngine
 	UC_CPU_MIPS64_LOONGSON_2E = 10
 	UC_CPU_MIPS64_LOONGSON_2F = 11
 	UC_CPU_MIPS64_MIPS64DSPR2 = 12
-	UC_CPU_MIPS64_ENDING = 13
+	UC_CPU_MIPS64_OCTEON68XX = 13
+	UC_CPU_MIPS64_LOONGSON_3A1000 = 14
+	UC_CPU_MIPS64_LOONGSON_3A4000 = 15
+	UC_CPU_MIPS64_ENDING = 16
 
 # MIPS registers
 
diff --git a/bindings/ruby/unicorn_gem/lib/unicorn_engine/riscv_const.rb b/bindings/ruby/unicorn_gem/lib/unicorn_engine/riscv_const.rb
index 33203d0a4d..59c5fb811c 100644
--- a/bindings/ruby/unicorn_gem/lib/unicorn_engine/riscv_const.rb
+++ b/bindings/ruby/unicorn_gem/lib/unicorn_engine/riscv_const.rb
@@ -1,6 +1,7 @@
 # For Unicorn Engine. AUTO-GENERATED FILE, DO NOT EDIT [riscv_const.rb]
 
 module UnicornEngine
+	UC_RISCV_VLEN_MAX = 1024
 
 # RISCV32 CPU
 
@@ -218,7 +219,50 @@ module UnicornEngine
 	UC_RISCV_REG_F31 = 189
 	UC_RISCV_REG_PC = 190
 	UC_RISCV_REG_PRIV = 191
-	UC_RISCV_REG_ENDING = 192
+
+# Vector CSRs
+	UC_RISCV_REG_VSTART = 192
+	UC_RISCV_REG_VXSAT = 193
+	UC_RISCV_REG_VXRM = 194
+	UC_RISCV_REG_VCSR = 195
+	UC_RISCV_REG_VL = 196
+	UC_RISCV_REG_VTYPE = 197
+	UC_RISCV_REG_VLENB = 198
+
+# Vector registers
+	UC_RISCV_REG_V0 = 199
+	UC_RISCV_REG_V1 = 200
+	UC_RISCV_REG_V2 = 201
+	UC_RISCV_REG_V3 = 202
+	UC_RISCV_REG_V4 = 203
+	UC_RISCV_REG_V5 = 204
+	UC_RISCV_REG_V6 = 205
+	UC_RISCV_REG_V7 = 206
+	UC_RISCV_REG_V8 = 207
+	UC_RISCV_REG_V9 = 208
+	UC_RISCV_REG_V10 = 209
+	UC_RISCV_REG_V11 = 210
+	UC_RISCV_REG_V12 = 211
+	UC_RISCV_REG_V13 = 212
+	UC_RISCV_REG_V14 = 213
+	UC_RISCV_REG_V15 = 214
+	UC_RISCV_REG_V16 = 215
+	UC_RISCV_REG_V17 = 216
+	UC_RISCV_REG_V18 = 217
+	UC_RISCV_REG_V19 = 218
+	UC_RISCV_REG_V20 = 219
+	UC_RISCV_REG_V21 = 220
+	UC_RISCV_REG_V22 = 221
+	UC_RISCV_REG_V23 = 222
+	UC_RISCV_REG_V24 = 223
+	UC_RISCV_REG_V25 = 224
+	UC_RISCV_REG_V26 = 225
+	UC_RISCV_REG_V27 = 226
+	UC_RISCV_REG_V28 = 227
+	UC_RISCV_REG_V29 = 228
+	UC_RISCV_REG_V30 = 229
+	UC_RISCV_REG_V31 = 230
+	UC_RISCV_REG_ENDING = 231
 
 # Alias registers
 	UC_RISCV_REG_ZERO = 1
diff --git a/bindings/ruby/unicorn_gem/lib/unicorn_engine/unicorn_const.rb b/bindings/ruby/unicorn_gem/lib/unicorn_engine/unicorn_const.rb
index f39650f081..dfde2d4c94 100644
--- a/bindings/ruby/unicorn_gem/lib/unicorn_engine/unicorn_const.rb
+++ b/bindings/ruby/unicorn_gem/lib/unicorn_engine/unicorn_const.rb
@@ -74,6 +74,9 @@ module UnicornEngine
 	UC_ERR_RESOURCE = 20
 	UC_ERR_EXCEPTION = 21
 	UC_ERR_OVERFLOW = 22
+	UC_ERR_MMU_READ = 23
+	UC_ERR_MMU_WRITE = 24
+	UC_ERR_MMU_FETCH = 25
 
 	UC_PROT_NONE = 0
 	UC_PROT_READ = 1
@@ -150,6 +153,7 @@ module UnicornEngine
 	UC_CTL_PAUTH_SIGN = 15
 	UC_CTL_PAUTH_STRIP = 16
 	UC_CTL_PAUTH_AUTH = 17
+	UC_CTL_INVALID_ADDR = 18
 	UC_CTL_CONTEXT_CPU = 1
 	UC_CTL_CONTEXT_MEMORY = 2
 end
\ No newline at end of file
diff --git a/bindings/vb6/uc_def.bas b/bindings/vb6/uc_def.bas
index 6baa23615c..83407a3504 100644
--- a/bindings/vb6/uc_def.bas
+++ b/bindings/vb6/uc_def.bas
@@ -117,9 +117,9 @@ Public Enum uc_mode                   'from /bindings/dotnet/common.fs
 '     UC_MODE_THUMB = 16               'THUMB mode (including Thumb-2)
 '     UC_MODE_MCLASS = 32              'ARM's Cortex-M series (currently unsupported)
 '     UC_MODE_V8 = 64                  'ARMv8 A32 encodings for ARM (currently unsupported)
-'     UC_MODE_MICRO = 16               'MicroMips mode (currently unsupported)
-'     UC_MODE_MIPS3 = 32               'Mips III ISA (currently unsupported)
-'     UC_MODE_MIPS32R6 = 64            'Mips32r6 ISA (currently unsupported)
+'     UC_MODE_MICRO = 16               'MicroMips mode
+'     UC_MODE_MIPS3 = 32               'Mips III ISA
+'     UC_MODE_MIPS32R6 = 64            'Mips32r6 ISA
 '     UC_MODE_MIPS32 = 4               'Mips32 ISA
 '     UC_MODE_MIPS64 = 8               'Mips64 ISA
      UC_MODE_16 = 2                   '16-bit mode
diff --git a/bindings/zig/unicorn/arm_const.zig b/bindings/zig/unicorn/arm_const.zig
index d1e0f2d606..0a17215feb 100644
--- a/bindings/zig/unicorn/arm_const.zig
+++ b/bindings/zig/unicorn/arm_const.zig
@@ -37,8 +37,9 @@ pub const armConst = enum(c_int) {
 	CPU_ARM_PXA270B1 = 30,
 	CPU_ARM_PXA270C0 = 31,
 	CPU_ARM_PXA270C5 = 32,
-	CPU_ARM_MAX = 33,
-	CPU_ARM_ENDING = 34,
+	CPU_ARM_CORTEX_M55 = 33,
+	CPU_ARM_MAX = 34,
+	CPU_ARM_ENDING = 35,
 
 // ARM registers
 
@@ -183,7 +184,8 @@ pub const armConst = enum(c_int) {
 	ARM_REG_XPSR_NZCVQG = 138,
 	ARM_REG_CP_REG = 139,
 	ARM_REG_ESR = 140,
-	ARM_REG_ENDING = 141,
+	ARM_REG_VPR = 141,
+	ARM_REG_ENDING = 142,
 
 // alias registers
 	ARM_REG_R13 = 12,
diff --git a/bindings/zig/unicorn/m68k_const.zig b/bindings/zig/unicorn/m68k_const.zig
index 3124eeed9a..aaa95f6b81 100644
--- a/bindings/zig/unicorn/m68k_const.zig
+++ b/bindings/zig/unicorn/m68k_const.zig
@@ -13,7 +13,8 @@ pub const m68kConst = enum(c_int) {
 	CPU_M68K_M5208 = 6,
 	CPU_M68K_CFV4E = 7,
 	CPU_M68K_ANY = 8,
-	CPU_M68K_ENDING = 9,
+	CPU_M68K_M68010 = 9,
+	CPU_M68K_ENDING = 10,
 
 // M68K registers
 
diff --git a/bindings/zig/unicorn/mips_const.zig b/bindings/zig/unicorn/mips_const.zig
index 0987cb2f7b..ad357fe473 100644
--- a/bindings/zig/unicorn/mips_const.zig
+++ b/bindings/zig/unicorn/mips_const.zig
@@ -37,7 +37,10 @@ pub const mipsConst = enum(c_int) {
 	CPU_MIPS64_LOONGSON_2E = 10,
 	CPU_MIPS64_LOONGSON_2F = 11,
 	CPU_MIPS64_MIPS64DSPR2 = 12,
-	CPU_MIPS64_ENDING = 13,
+	CPU_MIPS64_OCTEON68XX = 13,
+	CPU_MIPS64_LOONGSON_3A1000 = 14,
+	CPU_MIPS64_LOONGSON_3A4000 = 15,
+	CPU_MIPS64_ENDING = 16,
 
 // MIPS registers
 
diff --git a/bindings/zig/unicorn/riscv_const.zig b/bindings/zig/unicorn/riscv_const.zig
index 00a34001f7..59a85d154c 100644
--- a/bindings/zig/unicorn/riscv_const.zig
+++ b/bindings/zig/unicorn/riscv_const.zig
@@ -1,6 +1,7 @@
 // For Unicorn Engine. AUTO-GENERATED FILE, DO NOT EDIT
 
 pub const riscvConst = enum(c_int) {
+	RISCV_VLEN_MAX = 1024,
 
 // RISCV32 CPU
 
@@ -218,7 +219,50 @@ pub const riscvConst = enum(c_int) {
 	RISCV_REG_F31 = 189,
 	RISCV_REG_PC = 190,
 	RISCV_REG_PRIV = 191,
-	RISCV_REG_ENDING = 192,
+
+// Vector CSRs
+	RISCV_REG_VSTART = 192,
+	RISCV_REG_VXSAT = 193,
+	RISCV_REG_VXRM = 194,
+	RISCV_REG_VCSR = 195,
+	RISCV_REG_VL = 196,
+	RISCV_REG_VTYPE = 197,
+	RISCV_REG_VLENB = 198,
+
+// Vector registers
+	RISCV_REG_V0 = 199,
+	RISCV_REG_V1 = 200,
+	RISCV_REG_V2 = 201,
+	RISCV_REG_V3 = 202,
+	RISCV_REG_V4 = 203,
+	RISCV_REG_V5 = 204,
+	RISCV_REG_V6 = 205,
+	RISCV_REG_V7 = 206,
+	RISCV_REG_V8 = 207,
+	RISCV_REG_V9 = 208,
+	RISCV_REG_V10 = 209,
+	RISCV_REG_V11 = 210,
+	RISCV_REG_V12 = 211,
+	RISCV_REG_V13 = 212,
+	RISCV_REG_V14 = 213,
+	RISCV_REG_V15 = 214,
+	RISCV_REG_V16 = 215,
+	RISCV_REG_V17 = 216,
+	RISCV_REG_V18 = 217,
+	RISCV_REG_V19 = 218,
+	RISCV_REG_V20 = 219,
+	RISCV_REG_V21 = 220,
+	RISCV_REG_V22 = 221,
+	RISCV_REG_V23 = 222,
+	RISCV_REG_V24 = 223,
+	RISCV_REG_V25 = 224,
+	RISCV_REG_V26 = 225,
+	RISCV_REG_V27 = 226,
+	RISCV_REG_V28 = 227,
+	RISCV_REG_V29 = 228,
+	RISCV_REG_V30 = 229,
+	RISCV_REG_V31 = 230,
+	RISCV_REG_ENDING = 231,
 
 // Alias registers
 	RISCV_REG_ZERO = 1,
diff --git a/bindings/zig/unicorn/unicorn_const.zig b/bindings/zig/unicorn/unicorn_const.zig
index fc94d2e452..c48893a2fe 100644
--- a/bindings/zig/unicorn/unicorn_const.zig
+++ b/bindings/zig/unicorn/unicorn_const.zig
@@ -74,6 +74,9 @@ pub const unicornConst = enum(c_int) {
 	ERR_RESOURCE = 20,
 	ERR_EXCEPTION = 21,
 	ERR_OVERFLOW = 22,
+	ERR_MMU_READ = 23,
+	ERR_MMU_WRITE = 24,
+	ERR_MMU_FETCH = 25,
 
 	PROT_NONE = 0,
 	PROT_READ = 1,
@@ -150,6 +153,7 @@ pub const unicornConst = enum(c_int) {
 	CTL_PAUTH_SIGN = 15,
 	CTL_PAUTH_STRIP = 16,
 	CTL_PAUTH_AUTH = 17,
+	CTL_INVALID_ADDR = 18,
 	CTL_CONTEXT_CPU = 1,
 	CTL_CONTEXT_MEMORY = 2,
 
diff --git a/glib_compat/gmacros.h b/glib_compat/gmacros.h
index 1fa0b149a3..25068eddfc 100644
--- a/glib_compat/gmacros.h
+++ b/glib_compat/gmacros.h
@@ -37,12 +37,22 @@
  * where this is valid. This allows for warningless compilation of
  * "long long" types even in the presence of '-ansi -pedantic'. 
  */
-#if     __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 8)
+#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 8))
 #define G_GNUC_EXTENSION __extension__
 #else
 #define G_GNUC_EXTENSION
 #endif
 
+#if defined(__GNUC__)
+#define G_GNUC_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#define G_NORETURN __attribute__((noreturn))
+#elif defined(_MSC_VER)
+#define G_NORETURN __declspec(noreturn)
+#else
+#define G_GNUC_WARN_UNUSED_RESULT
+#define G_NORETURN
+#endif
+
 #if !(defined (G_STMT_START) && defined (G_STMT_END))
 #define G_STMT_START  do
 #if defined (_MSC_VER) && (_MSC_VER >= 1500)
diff --git a/include/qemu.h b/include/qemu.h
index b6fedfacfb..ff491e0381 100644
--- a/include/qemu.h
+++ b/include/qemu.h
@@ -27,6 +27,8 @@ struct RAMBlock {
     ram_addr_t used_length;
     ram_addr_t max_length;
     uint32_t flags;
+    uint8_t *mte_tags;
+    ram_addr_t mte_tags_size;
     /* RCU-enabled, writes protected by the ramlist lock */
     QLIST_ENTRY(RAMBlock) next;
     size_t page_size;
diff --git a/include/uc_priv.h b/include/uc_priv.h
index 6a7cda8c58..4044057d67 100644
--- a/include/uc_priv.h
+++ b/include/uc_priv.h
@@ -23,8 +23,8 @@
      UC_MODE_ARM926 | UC_MODE_ARM946 | UC_MODE_ARM1176 | UC_MODE_BIG_ENDIAN |  \
      UC_MODE_ARMBE8)
 #define UC_MODE_MIPS_MASK                                                      \
-    (UC_MODE_MIPS32 | UC_MODE_MIPS64 | UC_MODE_LITTLE_ENDIAN |                 \
-     UC_MODE_BIG_ENDIAN)
+    (UC_MODE_MIPS32 | UC_MODE_MIPS64 | UC_MODE_MICRO | UC_MODE_MIPS3 |         \
+     UC_MODE_MIPS32R6 | UC_MODE_LITTLE_ENDIAN | UC_MODE_BIG_ENDIAN)
 #define UC_MODE_X86_MASK                                                       \
     (UC_MODE_16 | UC_MODE_32 | UC_MODE_64 | UC_MODE_LITTLE_ENDIAN)
 #define UC_MODE_PPC_MASK (UC_MODE_PPC32 | UC_MODE_PPC64 | UC_MODE_BIG_ENDIAN)
diff --git a/include/unicorn/arm.h b/include/unicorn/arm.h
index 8f8da827c3..8c6c8f5376 100644
--- a/include/unicorn/arm.h
+++ b/include/unicorn/arm.h
@@ -50,6 +50,7 @@ typedef enum uc_cpu_arm {
     UC_CPU_ARM_PXA270B1,
     UC_CPU_ARM_PXA270C0,
     UC_CPU_ARM_PXA270C5,
+    UC_CPU_ARM_CORTEX_M55,
     UC_CPU_ARM_MAX,
 
     UC_CPU_ARM_ENDING
@@ -222,6 +223,7 @@ typedef enum uc_arm_reg {
 	// A pseudo-register for fetching the exception syndrome
 	// from the CPU state. This is not a real register.
     UC_ARM_REG_ESR,
+    UC_ARM_REG_VPR,
     UC_ARM_REG_ENDING, // <-- mark the end of the list or registers
 
     //> alias registers
diff --git a/include/unicorn/m68k.h b/include/unicorn/m68k.h
index 3e2c52a4f0..b7b9c3b3ed 100644
--- a/include/unicorn/m68k.h
+++ b/include/unicorn/m68k.h
@@ -26,6 +26,7 @@ typedef enum uc_cpu_m68k {
     UC_CPU_M68K_M5208,
     UC_CPU_M68K_CFV4E,
     UC_CPU_M68K_ANY,
+    UC_CPU_M68K_M68010,
 
     UC_CPU_M68K_ENDING
 } uc_cpu_m68k;
diff --git a/include/unicorn/mips.h b/include/unicorn/mips.h
index 5b8ac26be5..537839bdc6 100644
--- a/include/unicorn/mips.h
+++ b/include/unicorn/mips.h
@@ -56,6 +56,9 @@ typedef enum uc_cpu_mips64 {
     UC_CPU_MIPS64_LOONGSON_2E,
     UC_CPU_MIPS64_LOONGSON_2F,
     UC_CPU_MIPS64_MIPS64DSPR2,
+    UC_CPU_MIPS64_OCTEON68XX,
+    UC_CPU_MIPS64_LOONGSON_3A1000,
+    UC_CPU_MIPS64_LOONGSON_3A4000,
 
     UC_CPU_MIPS64_ENDING
 } uc_cpu_mips64;
diff --git a/include/unicorn/riscv.h b/include/unicorn/riscv.h
index cf1595ae4f..8af4befaef 100644
--- a/include/unicorn/riscv.h
+++ b/include/unicorn/riscv.h
@@ -15,6 +15,12 @@ extern "C" {
 #pragma warning(disable : 4201)
 #endif
 
+#define UC_RISCV_VLEN_MAX 1024
+
+typedef struct uc_riscv_vreg {
+    unsigned char bytes[UC_RISCV_VLEN_MAX / 8];
+} uc_riscv_vreg;
+
 //> RISCV32 CPU
 typedef enum uc_cpu_riscv32 {
     UC_CPU_RISCV32_ANY = 0,
@@ -237,6 +243,49 @@ typedef enum uc_riscv_reg {
 
     UC_RISCV_REG_PRIV, // Virtual register for the current privilege level
 
+    //> Vector CSRs
+    UC_RISCV_REG_VSTART,
+    UC_RISCV_REG_VXSAT,
+    UC_RISCV_REG_VXRM,
+    UC_RISCV_REG_VCSR,
+    UC_RISCV_REG_VL,
+    UC_RISCV_REG_VTYPE,
+    UC_RISCV_REG_VLENB,
+
+    //> Vector registers
+    UC_RISCV_REG_V0,
+    UC_RISCV_REG_V1,
+    UC_RISCV_REG_V2,
+    UC_RISCV_REG_V3,
+    UC_RISCV_REG_V4,
+    UC_RISCV_REG_V5,
+    UC_RISCV_REG_V6,
+    UC_RISCV_REG_V7,
+    UC_RISCV_REG_V8,
+    UC_RISCV_REG_V9,
+    UC_RISCV_REG_V10,
+    UC_RISCV_REG_V11,
+    UC_RISCV_REG_V12,
+    UC_RISCV_REG_V13,
+    UC_RISCV_REG_V14,
+    UC_RISCV_REG_V15,
+    UC_RISCV_REG_V16,
+    UC_RISCV_REG_V17,
+    UC_RISCV_REG_V18,
+    UC_RISCV_REG_V19,
+    UC_RISCV_REG_V20,
+    UC_RISCV_REG_V21,
+    UC_RISCV_REG_V22,
+    UC_RISCV_REG_V23,
+    UC_RISCV_REG_V24,
+    UC_RISCV_REG_V25,
+    UC_RISCV_REG_V26,
+    UC_RISCV_REG_V27,
+    UC_RISCV_REG_V28,
+    UC_RISCV_REG_V29,
+    UC_RISCV_REG_V30,
+    UC_RISCV_REG_V31,
+
     UC_RISCV_REG_ENDING, // <-- mark the end of the list or registers
 
     //> Alias registers
diff --git a/include/unicorn/unicorn.h b/include/unicorn/unicorn.h
index cbf044d161..05dbc6bc88 100644
--- a/include/unicorn/unicorn.h
+++ b/include/unicorn/unicorn.h
@@ -130,9 +130,9 @@ typedef enum uc_mode {
     UC_MODE_ARM1176 = 1 << 9, // ARM1176 CPU type
 
     // mips
-    UC_MODE_MICRO = 1 << 4,    // MicroMips mode (currently unsupported)
-    UC_MODE_MIPS3 = 1 << 5,    // Mips III ISA (currently unsupported)
-    UC_MODE_MIPS32R6 = 1 << 6, // Mips32r6 ISA (currently unsupported)
+    UC_MODE_MICRO = 1 << 4,    // MicroMips mode
+    UC_MODE_MIPS3 = 1 << 5,    // Mips III ISA
+    UC_MODE_MIPS32R6 = 1 << 6, // Mips32r6 ISA
     UC_MODE_MIPS32 = 1 << 2,   // Mips32 ISA
     UC_MODE_MIPS64 = 1 << 3,   // Mips64 ISA
 
diff --git a/qemu/VERSION b/qemu/VERSION
index 6b244dcd69..28a73c73a7 100644
--- a/qemu/VERSION
+++ b/qemu/VERSION
@@ -1 +1 @@
-5.0.1
+7.2.22
diff --git a/qemu/aarch64.h b/qemu/aarch64.h
index ad096574a4..222dbe1156 100644
--- a/qemu/aarch64.h
+++ b/qemu/aarch64.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_aarch64
 #define float32_squash_input_denormal float32_squash_input_denormal_aarch64
 #define float64_squash_input_denormal float64_squash_input_denormal_aarch64
+#define bfloat16_add bfloat16_add_aarch64
+#define bfloat16_compare bfloat16_compare_aarch64
+#define bfloat16_compare_quiet bfloat16_compare_quiet_aarch64
+#define bfloat16_default_nan bfloat16_default_nan_aarch64
+#define bfloat16_div bfloat16_div_aarch64
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_aarch64
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_aarch64
+#define bfloat16_max bfloat16_max_aarch64
+#define bfloat16_maximum_number bfloat16_maximum_number_aarch64
+#define bfloat16_maxnum bfloat16_maxnum_aarch64
+#define bfloat16_maxnummag bfloat16_maxnummag_aarch64
+#define bfloat16_min bfloat16_min_aarch64
+#define bfloat16_minimum_number bfloat16_minimum_number_aarch64
+#define bfloat16_minnum bfloat16_minnum_aarch64
+#define bfloat16_minnummag bfloat16_minnummag_aarch64
+#define bfloat16_mul bfloat16_mul_aarch64
+#define bfloat16_muladd bfloat16_muladd_aarch64
+#define bfloat16_round_to_int bfloat16_round_to_int_aarch64
+#define bfloat16_scalbn bfloat16_scalbn_aarch64
+#define bfloat16_silence_nan bfloat16_silence_nan_aarch64
+#define bfloat16_sqrt bfloat16_sqrt_aarch64
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_aarch64
+#define bfloat16_sub bfloat16_sub_aarch64
+#define bfloat16_to_float32 bfloat16_to_float32_aarch64
+#define bfloat16_to_float64 bfloat16_to_float64_aarch64
+#define bfloat16_to_int16 bfloat16_to_int16_aarch64
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_aarch64
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_aarch64
+#define bfloat16_to_int32 bfloat16_to_int32_aarch64
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_aarch64
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_aarch64
+#define bfloat16_to_int64 bfloat16_to_int64_aarch64
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_aarch64
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_aarch64
+#define bfloat16_to_uint16 bfloat16_to_uint16_aarch64
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_aarch64
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_aarch64
+#define bfloat16_to_uint32 bfloat16_to_uint32_aarch64
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_aarch64
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_aarch64
+#define bfloat16_to_uint64 bfloat16_to_uint64_aarch64
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_aarch64
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_aarch64
+#define float128_maximum_number float128_maximum_number_aarch64
+#define float128_max float128_max_aarch64
+#define float128_maxnum float128_maxnum_aarch64
+#define float128_maxnummag float128_maxnummag_aarch64
+#define float128_min float128_min_aarch64
+#define float128_minimum_number float128_minimum_number_aarch64
+#define float128_minnum float128_minnum_aarch64
+#define float128_minnummag float128_minnummag_aarch64
+#define float128_muladd float128_muladd_aarch64
+#define float128_to_int128 float128_to_int128_aarch64
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_aarch64
+#define float128_to_uint128 float128_to_uint128_aarch64
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_aarch64
+#define float16_maximum_number float16_maximum_number_aarch64
+#define float16_minimum_number float16_minimum_number_aarch64
+#define float16_to_int8 float16_to_int8_aarch64
+#define float16_to_int8_scalbn float16_to_int8_scalbn_aarch64
+#define float16_to_uint8 float16_to_uint8_aarch64
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_aarch64
+#define float32_maximum_number float32_maximum_number_aarch64
+#define float32_minimum_number float32_minimum_number_aarch64
+#define float32_to_bfloat16 float32_to_bfloat16_aarch64
+#define float64_maximum_number float64_maximum_number_aarch64
+#define float64_minimum_number float64_minimum_number_aarch64
+#define float64_to_bfloat16 float64_to_bfloat16_aarch64
+#define float64r32_add float64r32_add_aarch64
+#define float64r32_div float64r32_div_aarch64
+#define float64r32_mul float64r32_mul_aarch64
+#define float64r32_muladd float64r32_muladd_aarch64
+#define float64r32_sqrt float64r32_sqrt_aarch64
+#define float64r32_sub float64r32_sub_aarch64
+#define floatx80_mod floatx80_mod_aarch64
+#define floatx80_modrem floatx80_modrem_aarch64
+#define int128_to_float128 int128_to_float128_aarch64
+#define int16_to_bfloat16 int16_to_bfloat16_aarch64
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_aarch64
+#define int32_to_bfloat16 int32_to_bfloat16_aarch64
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_aarch64
+#define int64_to_bfloat16 int64_to_bfloat16_aarch64
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_aarch64
+#define int8_to_float16 int8_to_float16_aarch64
+#define uint128_to_float128 uint128_to_float128_aarch64
+#define uint16_to_bfloat16 uint16_to_bfloat16_aarch64
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_aarch64
+#define uint32_to_bfloat16 uint32_to_bfloat16_aarch64
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_aarch64
+#define uint64_to_bfloat16 uint64_to_bfloat16_aarch64
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_aarch64
+#define uint8_to_float16 uint8_to_float16_aarch64
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_aarch64
 #define roundAndPackFloatx80 roundAndPackFloatx80_aarch64
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_aarch64
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_aarch64
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_aarch64
 #define helper_exit_atomic helper_exit_atomic_aarch64
+#define helper_memset helper_memset_aarch64
+#define helper_emu_stop helper_emu_stop_aarch64
+#define tcg_remove_ops_after tcg_remove_ops_after_aarch64
+#define tcg_constant_vec_matching tcg_constant_vec_matching_aarch64
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_aarch64
 #define helper_gvec_add8 helper_gvec_add8_aarch64
 #define helper_gvec_add16 helper_gvec_add16_aarch64
 #define helper_gvec_add32 helper_gvec_add32_aarch64
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_aarch64
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_aarch64
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_aarch64
+#define gen_helper_mve_vctp gen_helper_mve_vctp_aarch64
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_aarch64
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_aarch64
+#define gen_helper_mve_vdup gen_helper_mve_vdup_aarch64
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_aarch64
+#define gen_helper_mve_vandi gen_helper_mve_vandi_aarch64
+#define gen_helper_mve_vorri gen_helper_mve_vorri_aarch64
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_aarch64
+#define gen_helper_mve_viduph gen_helper_mve_viduph_aarch64
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_aarch64
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_aarch64
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_aarch64
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_aarch64
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_aarch64
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_aarch64
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_aarch64
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_aarch64
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_aarch64
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_aarch64
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_aarch64
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_aarch64
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_aarch64
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_aarch64
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_aarch64
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_aarch64
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_aarch64
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_aarch64
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_aarch64
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_aarch64
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_aarch64
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_aarch64
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_aarch64
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_aarch64
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_aarch64
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_aarch64
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_aarch64
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_aarch64
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_aarch64
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_aarch64
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_aarch64
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_aarch64
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_aarch64
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_aarch64
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_aarch64
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_aarch64
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_aarch64
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_aarch64
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_aarch64
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_aarch64
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_aarch64
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_aarch64
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_aarch64
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_aarch64
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_aarch64
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_aarch64
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_aarch64
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_aarch64
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_aarch64
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_aarch64
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_aarch64
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_aarch64
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_aarch64
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_aarch64
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_aarch64
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_aarch64
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_aarch64
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_aarch64
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_aarch64
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_aarch64
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_aarch64
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_aarch64
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_aarch64
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_aarch64
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_aarch64
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_aarch64
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_aarch64
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_aarch64
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_aarch64
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_aarch64
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_aarch64
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_aarch64
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_aarch64
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_aarch64
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_aarch64
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_aarch64
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_aarch64
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_aarch64
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_aarch64
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_aarch64
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_aarch64
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_aarch64
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_aarch64
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_aarch64
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_aarch64
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_aarch64
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_aarch64
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_aarch64
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_aarch64
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_aarch64
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_aarch64
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_aarch64
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_aarch64
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_aarch64
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_aarch64
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_aarch64
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_aarch64
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_aarch64
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_aarch64
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_aarch64
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_aarch64
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_aarch64
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_aarch64
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_aarch64
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_aarch64
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_aarch64
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_aarch64
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_aarch64
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_aarch64
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_aarch64
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_aarch64
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_aarch64
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_aarch64
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_aarch64
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_aarch64
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_aarch64
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_aarch64
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_aarch64
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_aarch64
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_aarch64
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_aarch64
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_aarch64
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_aarch64
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_aarch64
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_aarch64
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_aarch64
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_aarch64
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_aarch64
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_aarch64
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_aarch64
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_aarch64
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_aarch64
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_aarch64
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_aarch64
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_aarch64
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_aarch64
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_aarch64
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_aarch64
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_aarch64
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_aarch64
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_aarch64
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_aarch64
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_aarch64
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_aarch64
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_aarch64
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_aarch64
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_aarch64
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_aarch64
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_aarch64
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_aarch64
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_aarch64
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_aarch64
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_aarch64
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_aarch64
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_aarch64
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_aarch64
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_aarch64
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_aarch64
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_aarch64
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_aarch64
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_aarch64
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_aarch64
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_aarch64
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_aarch64
+#define gen_helper_mve_vand gen_helper_mve_vand_aarch64
+#define gen_helper_mve_vbic gen_helper_mve_vbic_aarch64
+#define gen_helper_mve_vorr gen_helper_mve_vorr_aarch64
+#define gen_helper_mve_vorn gen_helper_mve_vorn_aarch64
+#define gen_helper_mve_veor gen_helper_mve_veor_aarch64
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_aarch64
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_aarch64
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_aarch64
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_aarch64
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_aarch64
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_aarch64
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_aarch64
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_aarch64
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_aarch64
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_aarch64
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_aarch64
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_aarch64
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_aarch64
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_aarch64
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_aarch64
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_aarch64
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_aarch64
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_aarch64
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_aarch64
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_aarch64
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_aarch64
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_aarch64
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_aarch64
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_aarch64
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_aarch64
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_aarch64
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_aarch64
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_aarch64
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_aarch64
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_aarch64
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_aarch64
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_aarch64
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_aarch64
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_aarch64
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_aarch64
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_aarch64
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_aarch64
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_aarch64
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_aarch64
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_aarch64
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_aarch64
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_aarch64
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_aarch64
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_aarch64
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_aarch64
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_aarch64
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_aarch64
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_aarch64
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_aarch64
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_aarch64
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_aarch64
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_aarch64
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_aarch64
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_aarch64
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_aarch64
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_aarch64
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_aarch64
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_aarch64
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_aarch64
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_aarch64
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_aarch64
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_aarch64
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_aarch64
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_aarch64
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_aarch64
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_aarch64
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_aarch64
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_aarch64
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_aarch64
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_aarch64
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_aarch64
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_aarch64
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_aarch64
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_aarch64
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_aarch64
+#define gen_helper_mve_vminub gen_helper_mve_vminub_aarch64
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_aarch64
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_aarch64
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_aarch64
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_aarch64
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_aarch64
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_aarch64
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_aarch64
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_aarch64
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_aarch64
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_aarch64
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_aarch64
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_aarch64
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_aarch64
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_aarch64
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_aarch64
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_aarch64
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_aarch64
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_aarch64
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_aarch64
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_aarch64
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_aarch64
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_aarch64
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_aarch64
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_aarch64
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_aarch64
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_aarch64
+#define gen_helper_mve_vadc gen_helper_mve_vadc_aarch64
+#define gen_helper_mve_vadci gen_helper_mve_vadci_aarch64
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_aarch64
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_aarch64
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_aarch64
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_aarch64
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_aarch64
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_aarch64
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_aarch64
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_aarch64
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_aarch64
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_aarch64
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_aarch64
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_aarch64
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_aarch64
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_aarch64
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_aarch64
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_aarch64
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_aarch64
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_aarch64
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_aarch64
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_aarch64
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_aarch64
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_aarch64
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_aarch64
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_aarch64
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_aarch64
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_aarch64
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_aarch64
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_aarch64
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_aarch64
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_aarch64
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_aarch64
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_aarch64
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_aarch64
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_aarch64
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_aarch64
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_aarch64
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_aarch64
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_aarch64
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_aarch64
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_aarch64
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_aarch64
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_aarch64
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_aarch64
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_aarch64
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_aarch64
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_aarch64
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_aarch64
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_aarch64
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_aarch64
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_aarch64
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_aarch64
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_aarch64
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_aarch64
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_aarch64
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_aarch64
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_aarch64
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_aarch64
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_aarch64
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_aarch64
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_aarch64
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_aarch64
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_aarch64
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_aarch64
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_aarch64
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_aarch64
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_aarch64
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_aarch64
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_aarch64
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_aarch64
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_aarch64
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_aarch64
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_aarch64
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_aarch64
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_aarch64
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_aarch64
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_aarch64
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_aarch64
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_aarch64
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_aarch64
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_aarch64
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_aarch64
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_aarch64
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_aarch64
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_aarch64
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_aarch64
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_aarch64
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_aarch64
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_aarch64
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_aarch64
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_aarch64
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_aarch64
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_aarch64
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_aarch64
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_aarch64
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_aarch64
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_aarch64
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_aarch64
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_aarch64
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_aarch64
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_aarch64
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_aarch64
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_aarch64
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_aarch64
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_aarch64
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_aarch64
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_aarch64
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_aarch64
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_aarch64
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_aarch64
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_aarch64
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_aarch64
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_aarch64
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_aarch64
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_aarch64
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_aarch64
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_aarch64
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_aarch64
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_aarch64
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_aarch64
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_aarch64
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_aarch64
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_aarch64
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_aarch64
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_aarch64
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_aarch64
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_aarch64
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_aarch64
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_aarch64
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_aarch64
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_aarch64
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_aarch64
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_aarch64
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_aarch64
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_aarch64
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_aarch64
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_aarch64
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_aarch64
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_aarch64
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_aarch64
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_aarch64
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_aarch64
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_aarch64
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_aarch64
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_aarch64
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_aarch64
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_aarch64
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_aarch64
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_aarch64
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_aarch64
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_aarch64
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_aarch64
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_aarch64
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_aarch64
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_aarch64
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_aarch64
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_aarch64
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_aarch64
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_aarch64
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_aarch64
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_aarch64
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_aarch64
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_aarch64
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_aarch64
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_aarch64
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_aarch64
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_aarch64
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_aarch64
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_aarch64
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_aarch64
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_aarch64
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_aarch64
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_aarch64
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_aarch64
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_aarch64
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_aarch64
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_aarch64
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_aarch64
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_aarch64
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_aarch64
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_aarch64
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_aarch64
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_aarch64
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_aarch64
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_aarch64
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_aarch64
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_aarch64
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_aarch64
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_aarch64
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_aarch64
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_aarch64
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_aarch64
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_aarch64
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_aarch64
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_aarch64
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_aarch64
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_aarch64
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_aarch64
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_aarch64
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_aarch64
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_aarch64
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_aarch64
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_aarch64
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_aarch64
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_aarch64
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_aarch64
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_aarch64
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_aarch64
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_aarch64
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_aarch64
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_aarch64
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_aarch64
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_aarch64
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_aarch64
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_aarch64
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_aarch64
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_aarch64
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_aarch64
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_aarch64
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_aarch64
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_aarch64
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_aarch64
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_aarch64
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_aarch64
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_aarch64
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_aarch64
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_aarch64
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_aarch64
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_aarch64
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_aarch64
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_aarch64
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_aarch64
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_aarch64
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_aarch64
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_aarch64
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_aarch64
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_aarch64
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_aarch64
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_aarch64
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_aarch64
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_aarch64
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_aarch64
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_aarch64
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_aarch64
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_aarch64
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_aarch64
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_aarch64
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_aarch64
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_aarch64
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_aarch64
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_aarch64
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_aarch64
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_aarch64
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_aarch64
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_aarch64
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_aarch64
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_aarch64
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_aarch64
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_aarch64
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_aarch64
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_aarch64
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_aarch64
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_aarch64
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_aarch64
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_aarch64
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_aarch64
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_aarch64
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_aarch64
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_aarch64
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_aarch64
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_aarch64
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_aarch64
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_aarch64
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_aarch64
+#define gen_helper_mve_ushll gen_helper_mve_ushll_aarch64
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_aarch64
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_aarch64
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_aarch64
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_aarch64
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_aarch64
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_aarch64
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_aarch64
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_aarch64
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_aarch64
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_aarch64
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_aarch64
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_aarch64
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_aarch64
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_aarch64
+#define gen_helper_mve_vslib gen_helper_mve_vslib_aarch64
+#define gen_helper_mve_vslih gen_helper_mve_vslih_aarch64
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_aarch64
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_aarch64
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_aarch64
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_aarch64
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_aarch64
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_aarch64
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_aarch64
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_aarch64
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_aarch64
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_aarch64
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_aarch64
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_aarch64
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_aarch64
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_aarch64
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_aarch64
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_aarch64
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_aarch64
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_aarch64
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_aarch64
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_aarch64
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_aarch64
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_aarch64
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_aarch64
+#define gen_helper_mve_vminab gen_helper_mve_vminab_aarch64
+#define gen_helper_mve_vminah gen_helper_mve_vminah_aarch64
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_aarch64
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_aarch64
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_aarch64
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_aarch64
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_aarch64
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_aarch64
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_aarch64
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_aarch64
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_aarch64
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_aarch64
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_aarch64
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_aarch64
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_aarch64
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_aarch64
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_aarch64
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_aarch64
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_aarch64
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_aarch64
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_aarch64
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_aarch64
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_aarch64
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_aarch64
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_aarch64
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_aarch64
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_aarch64
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_aarch64
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_aarch64
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_aarch64
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_aarch64
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_aarch64
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_aarch64
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_aarch64
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_aarch64
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_aarch64
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_aarch64
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_aarch64
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_aarch64
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_aarch64
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_aarch64
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_aarch64
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_aarch64
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_aarch64
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_aarch64
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_aarch64
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_aarch64
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_aarch64
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_aarch64
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_aarch64
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_aarch64
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_aarch64
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_aarch64
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_aarch64
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_aarch64
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_aarch64
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_aarch64
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_aarch64
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_aarch64
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_aarch64
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_aarch64
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_aarch64
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_aarch64
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_aarch64
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_aarch64
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_aarch64
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_aarch64
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_aarch64
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_aarch64
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_aarch64
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_aarch64
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_aarch64
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_aarch64
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_aarch64
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_aarch64
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_aarch64
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_aarch64
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_aarch64
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_aarch64
 #define gen_helper_cpsr_read gen_helper_cpsr_read_aarch64
 #define gen_helper_cpsr_write gen_helper_cpsr_write_aarch64
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_aarch64
@@ -1306,6 +2077,8 @@
 #define cpu_arm_init cpu_arm_init_aarch64
 #define helper_crypto_aese helper_crypto_aese_aarch64
 #define helper_crypto_aesmc helper_crypto_aesmc_aarch64
+#define helper_crypto_sve_aese helper_crypto_sve_aese_aarch64
+#define helper_crypto_sve_aesmc helper_crypto_sve_aesmc_aarch64
 #define helper_crypto_sha1_3reg helper_crypto_sha1_3reg_aarch64
 #define helper_crypto_sha1h helper_crypto_sha1h_aarch64
 #define helper_crypto_sha1su1 helper_crypto_sha1su1_aarch64
@@ -1322,6 +2095,9 @@
 #define helper_crypto_sm3tt helper_crypto_sm3tt_aarch64
 #define helper_crypto_sm4e helper_crypto_sm4e_aarch64
 #define helper_crypto_sm4ekey helper_crypto_sm4ekey_aarch64
+#define helper_crypto_sve_sm4e helper_crypto_sve_sm4e_aarch64
+#define helper_crypto_sve_sm4ekey helper_crypto_sve_sm4ekey_aarch64
+#define helper_crypto_rax1 helper_crypto_rax1_aarch64
 #define helper_check_breakpoints helper_check_breakpoints_aarch64
 #define arm_debug_check_watchpoint arm_debug_check_watchpoint_aarch64
 #define arm_debug_excp_handler arm_debug_excp_handler_aarch64
@@ -1332,6 +2108,7 @@
 #define helper_msr_i_spsel helper_msr_i_spsel_aarch64
 #define helper_msr_i_daifset helper_msr_i_daifset_aarch64
 #define helper_msr_i_daifclear helper_msr_i_daifclear_aarch64
+#define helper_set_svcr helper_set_svcr_aarch64
 #define helper_vfp_cmph_a64 helper_vfp_cmph_a64_aarch64
 #define helper_vfp_cmpeh_a64 helper_vfp_cmpeh_a64_aarch64
 #define helper_vfp_cmps_a64 helper_vfp_cmps_a64_aarch64
@@ -1850,6 +2627,18 @@
 #define helper_pacda helper_pacda_aarch64
 #define helper_pacdb helper_pacdb_aarch64
 #define helper_pacga helper_pacga_aarch64
+#define helper_irg helper_irg_aarch64
+#define helper_addsubg helper_addsubg_aarch64
+#define helper_ldg helper_ldg_aarch64
+#define helper_stg helper_stg_aarch64
+#define helper_stg_stub helper_stg_stub_aarch64
+#define helper_st2g helper_st2g_aarch64
+#define helper_st2g_stub helper_st2g_stub_aarch64
+#define helper_ldgm helper_ldgm_aarch64
+#define helper_stgm helper_stgm_aarch64
+#define helper_stzgm_tags helper_stzgm_tags_aarch64
+#define helper_mte_check helper_mte_check_aarch64
+#define helper_mte_check_zva helper_mte_check_zva_aarch64
 #define helper_autia helper_autia_aarch64
 #define helper_autib helper_autib_aarch64
 #define helper_autda helper_autda_aarch64
@@ -2151,6 +2940,14 @@
 #define helper_sve_tbl_h helper_sve_tbl_h_aarch64
 #define helper_sve_tbl_s helper_sve_tbl_s_aarch64
 #define helper_sve_tbl_d helper_sve_tbl_d_aarch64
+#define helper_sve2_tbl_b helper_sve2_tbl_b_aarch64
+#define helper_sve2_tbl_h helper_sve2_tbl_h_aarch64
+#define helper_sve2_tbl_s helper_sve2_tbl_s_aarch64
+#define helper_sve2_tbl_d helper_sve2_tbl_d_aarch64
+#define helper_sve2_tbx_b helper_sve2_tbx_b_aarch64
+#define helper_sve2_tbx_h helper_sve2_tbx_h_aarch64
+#define helper_sve2_tbx_s helper_sve2_tbx_s_aarch64
+#define helper_sve2_tbx_d helper_sve2_tbx_d_aarch64
 #define helper_sve_sunpk_h helper_sve_sunpk_h_aarch64
 #define helper_sve_sunpk_s helper_sve_sunpk_s_aarch64
 #define helper_sve_sunpk_d helper_sve_sunpk_d_aarch64
@@ -2174,6 +2971,9 @@
 #define helper_sve_trn_h helper_sve_trn_h_aarch64
 #define helper_sve_trn_s helper_sve_trn_s_aarch64
 #define helper_sve_trn_d helper_sve_trn_d_aarch64
+#define helper_sve2_zip_q helper_sve2_zip_q_aarch64
+#define helper_sve2_uzp_q helper_sve2_uzp_q_aarch64
+#define helper_sve2_trn_q helper_sve2_trn_q_aarch64
 #define helper_sve_compact_s helper_sve_compact_s_aarch64
 #define helper_sve_compact_d helper_sve_compact_d_aarch64
 #define helper_sve_last_active_element helper_sve_last_active_element_aarch64
@@ -2182,6 +2982,106 @@
 #define helper_sve_sel_zpzz_h helper_sve_sel_zpzz_h_aarch64
 #define helper_sve_sel_zpzz_s helper_sve_sel_zpzz_s_aarch64
 #define helper_sve_sel_zpzz_d helper_sve_sel_zpzz_d_aarch64
+#define helper_sve_sel_zpzz_q helper_sve_sel_zpzz_q_aarch64
+#define helper_sme_zero helper_sme_zero_aarch64
+#define helper_sme_mova_cz_b helper_sme_mova_cz_b_aarch64
+#define helper_sme_mova_cz_h helper_sme_mova_cz_h_aarch64
+#define helper_sme_mova_cz_s helper_sme_mova_cz_s_aarch64
+#define helper_sme_mova_cz_d helper_sme_mova_cz_d_aarch64
+#define helper_sme_mova_cz_q helper_sme_mova_cz_q_aarch64
+#define helper_sme_mova_zc_b helper_sme_mova_zc_b_aarch64
+#define helper_sme_mova_zc_h helper_sme_mova_zc_h_aarch64
+#define helper_sme_mova_zc_s helper_sme_mova_zc_s_aarch64
+#define helper_sme_mova_zc_d helper_sme_mova_zc_d_aarch64
+#define helper_sme_mova_zc_q helper_sme_mova_zc_q_aarch64
+#define helper_sme_ld1b_h helper_sme_ld1b_h_aarch64
+#define helper_sme_ld1b_v helper_sme_ld1b_v_aarch64
+#define helper_sme_ld1h_le_h helper_sme_ld1h_le_h_aarch64
+#define helper_sme_ld1h_le_v helper_sme_ld1h_le_v_aarch64
+#define helper_sme_ld1h_be_h helper_sme_ld1h_be_h_aarch64
+#define helper_sme_ld1h_be_v helper_sme_ld1h_be_v_aarch64
+#define helper_sme_ld1s_le_h helper_sme_ld1s_le_h_aarch64
+#define helper_sme_ld1s_le_v helper_sme_ld1s_le_v_aarch64
+#define helper_sme_ld1s_be_h helper_sme_ld1s_be_h_aarch64
+#define helper_sme_ld1s_be_v helper_sme_ld1s_be_v_aarch64
+#define helper_sme_ld1d_le_h helper_sme_ld1d_le_h_aarch64
+#define helper_sme_ld1d_le_v helper_sme_ld1d_le_v_aarch64
+#define helper_sme_ld1d_be_h helper_sme_ld1d_be_h_aarch64
+#define helper_sme_ld1d_be_v helper_sme_ld1d_be_v_aarch64
+#define helper_sme_ld1q_le_h helper_sme_ld1q_le_h_aarch64
+#define helper_sme_ld1q_le_v helper_sme_ld1q_le_v_aarch64
+#define helper_sme_ld1q_be_h helper_sme_ld1q_be_h_aarch64
+#define helper_sme_ld1q_be_v helper_sme_ld1q_be_v_aarch64
+#define helper_sme_ld1b_h_mte helper_sme_ld1b_h_mte_aarch64
+#define helper_sme_ld1b_v_mte helper_sme_ld1b_v_mte_aarch64
+#define helper_sme_ld1h_le_h_mte helper_sme_ld1h_le_h_mte_aarch64
+#define helper_sme_ld1h_le_v_mte helper_sme_ld1h_le_v_mte_aarch64
+#define helper_sme_ld1h_be_h_mte helper_sme_ld1h_be_h_mte_aarch64
+#define helper_sme_ld1h_be_v_mte helper_sme_ld1h_be_v_mte_aarch64
+#define helper_sme_ld1s_le_h_mte helper_sme_ld1s_le_h_mte_aarch64
+#define helper_sme_ld1s_le_v_mte helper_sme_ld1s_le_v_mte_aarch64
+#define helper_sme_ld1s_be_h_mte helper_sme_ld1s_be_h_mte_aarch64
+#define helper_sme_ld1s_be_v_mte helper_sme_ld1s_be_v_mte_aarch64
+#define helper_sme_ld1d_le_h_mte helper_sme_ld1d_le_h_mte_aarch64
+#define helper_sme_ld1d_le_v_mte helper_sme_ld1d_le_v_mte_aarch64
+#define helper_sme_ld1d_be_h_mte helper_sme_ld1d_be_h_mte_aarch64
+#define helper_sme_ld1d_be_v_mte helper_sme_ld1d_be_v_mte_aarch64
+#define helper_sme_ld1q_le_h_mte helper_sme_ld1q_le_h_mte_aarch64
+#define helper_sme_ld1q_le_v_mte helper_sme_ld1q_le_v_mte_aarch64
+#define helper_sme_ld1q_be_h_mte helper_sme_ld1q_be_h_mte_aarch64
+#define helper_sme_ld1q_be_v_mte helper_sme_ld1q_be_v_mte_aarch64
+#define helper_sme_st1b_h helper_sme_st1b_h_aarch64
+#define helper_sme_st1b_v helper_sme_st1b_v_aarch64
+#define helper_sme_st1h_le_h helper_sme_st1h_le_h_aarch64
+#define helper_sme_st1h_le_v helper_sme_st1h_le_v_aarch64
+#define helper_sme_st1h_be_h helper_sme_st1h_be_h_aarch64
+#define helper_sme_st1h_be_v helper_sme_st1h_be_v_aarch64
+#define helper_sme_st1s_le_h helper_sme_st1s_le_h_aarch64
+#define helper_sme_st1s_le_v helper_sme_st1s_le_v_aarch64
+#define helper_sme_st1s_be_h helper_sme_st1s_be_h_aarch64
+#define helper_sme_st1s_be_v helper_sme_st1s_be_v_aarch64
+#define helper_sme_st1d_le_h helper_sme_st1d_le_h_aarch64
+#define helper_sme_st1d_le_v helper_sme_st1d_le_v_aarch64
+#define helper_sme_st1d_be_h helper_sme_st1d_be_h_aarch64
+#define helper_sme_st1d_be_v helper_sme_st1d_be_v_aarch64
+#define helper_sme_st1q_le_h helper_sme_st1q_le_h_aarch64
+#define helper_sme_st1q_le_v helper_sme_st1q_le_v_aarch64
+#define helper_sme_st1q_be_h helper_sme_st1q_be_h_aarch64
+#define helper_sme_st1q_be_v helper_sme_st1q_be_v_aarch64
+#define helper_sme_st1b_h_mte helper_sme_st1b_h_mte_aarch64
+#define helper_sme_st1b_v_mte helper_sme_st1b_v_mte_aarch64
+#define helper_sme_st1h_le_h_mte helper_sme_st1h_le_h_mte_aarch64
+#define helper_sme_st1h_le_v_mte helper_sme_st1h_le_v_mte_aarch64
+#define helper_sme_st1h_be_h_mte helper_sme_st1h_be_h_mte_aarch64
+#define helper_sme_st1h_be_v_mte helper_sme_st1h_be_v_mte_aarch64
+#define helper_sme_st1s_le_h_mte helper_sme_st1s_le_h_mte_aarch64
+#define helper_sme_st1s_le_v_mte helper_sme_st1s_le_v_mte_aarch64
+#define helper_sme_st1s_be_h_mte helper_sme_st1s_be_h_mte_aarch64
+#define helper_sme_st1s_be_v_mte helper_sme_st1s_be_v_mte_aarch64
+#define helper_sme_st1d_le_h_mte helper_sme_st1d_le_h_mte_aarch64
+#define helper_sme_st1d_le_v_mte helper_sme_st1d_le_v_mte_aarch64
+#define helper_sme_st1d_be_h_mte helper_sme_st1d_be_h_mte_aarch64
+#define helper_sme_st1d_be_v_mte helper_sme_st1d_be_v_mte_aarch64
+#define helper_sme_st1q_le_h_mte helper_sme_st1q_le_h_mte_aarch64
+#define helper_sme_st1q_le_v_mte helper_sme_st1q_le_v_mte_aarch64
+#define helper_sme_st1q_be_h_mte helper_sme_st1q_be_h_mte_aarch64
+#define helper_sme_st1q_be_v_mte helper_sme_st1q_be_v_mte_aarch64
+#define helper_sme_addha_s helper_sme_addha_s_aarch64
+#define helper_sme_addva_s helper_sme_addva_s_aarch64
+#define helper_sme_addha_d helper_sme_addha_d_aarch64
+#define helper_sme_addva_d helper_sme_addva_d_aarch64
+#define helper_sme_fmopa_h helper_sme_fmopa_h_aarch64
+#define helper_sme_fmopa_s helper_sme_fmopa_s_aarch64
+#define helper_sme_fmopa_d helper_sme_fmopa_d_aarch64
+#define helper_sme_bfmopa helper_sme_bfmopa_aarch64
+#define helper_sme_smopa_s helper_sme_smopa_s_aarch64
+#define helper_sme_umopa_s helper_sme_umopa_s_aarch64
+#define helper_sme_sumopa_s helper_sme_sumopa_s_aarch64
+#define helper_sme_usmopa_s helper_sme_usmopa_s_aarch64
+#define helper_sme_smopa_d helper_sme_smopa_d_aarch64
+#define helper_sme_umopa_d helper_sme_umopa_d_aarch64
+#define helper_sme_sumopa_d helper_sme_sumopa_d_aarch64
+#define helper_sme_usmopa_d helper_sme_usmopa_d_aarch64
 #define helper_sve_cmpeq_ppzz_b helper_sve_cmpeq_ppzz_b_aarch64
 #define helper_sve_cmpeq_ppzz_h helper_sve_cmpeq_ppzz_h_aarch64
 #define helper_sve_cmpeq_ppzz_s helper_sve_cmpeq_ppzz_s_aarch64
@@ -2373,6 +3273,13 @@
 #define helper_sve_fcvt_hd helper_sve_fcvt_hd_aarch64
 #define helper_sve_fcvt_ds helper_sve_fcvt_ds_aarch64
 #define helper_sve_fcvt_sd helper_sve_fcvt_sd_aarch64
+#define helper_sve2_fcvtnt_sh helper_sve2_fcvtnt_sh_aarch64
+#define helper_sve2_fcvtnt_ds helper_sve2_fcvtnt_ds_aarch64
+#define helper_sve2_fcvtlt_hs helper_sve2_fcvtlt_hs_aarch64
+#define helper_sve2_fcvtlt_sd helper_sve2_fcvtlt_sd_aarch64
+#define helper_flogb_h helper_flogb_h_aarch64
+#define helper_flogb_s helper_flogb_s_aarch64
+#define helper_flogb_d helper_flogb_d_aarch64
 #define helper_sve_fcvtzs_hh helper_sve_fcvtzs_hh_aarch64
 #define helper_sve_fcvtzs_hs helper_sve_fcvtzs_hs_aarch64
 #define helper_sve_fcvtzs_ss helper_sve_fcvtzs_ss_aarch64
@@ -2519,6 +3426,52 @@
 #define helper_sve_ld3dd_be_r helper_sve_ld3dd_be_r_aarch64
 #define helper_sve_ld4dd_le_r helper_sve_ld4dd_le_r_aarch64
 #define helper_sve_ld4dd_be_r helper_sve_ld4dd_be_r_aarch64
+#define helper_sve_ld1bb_r_mte helper_sve_ld1bb_r_mte_aarch64
+#define helper_sve_ld1bhu_r_mte helper_sve_ld1bhu_r_mte_aarch64
+#define helper_sve_ld1bhs_r_mte helper_sve_ld1bhs_r_mte_aarch64
+#define helper_sve_ld1bsu_r_mte helper_sve_ld1bsu_r_mte_aarch64
+#define helper_sve_ld1bss_r_mte helper_sve_ld1bss_r_mte_aarch64
+#define helper_sve_ld1bdu_r_mte helper_sve_ld1bdu_r_mte_aarch64
+#define helper_sve_ld1bds_r_mte helper_sve_ld1bds_r_mte_aarch64
+#define helper_sve_ld1hh_le_r_mte helper_sve_ld1hh_le_r_mte_aarch64
+#define helper_sve_ld1hh_be_r_mte helper_sve_ld1hh_be_r_mte_aarch64
+#define helper_sve_ld1hsu_le_r_mte helper_sve_ld1hsu_le_r_mte_aarch64
+#define helper_sve_ld1hsu_be_r_mte helper_sve_ld1hsu_be_r_mte_aarch64
+#define helper_sve_ld1hss_le_r_mte helper_sve_ld1hss_le_r_mte_aarch64
+#define helper_sve_ld1hss_be_r_mte helper_sve_ld1hss_be_r_mte_aarch64
+#define helper_sve_ld1hdu_le_r_mte helper_sve_ld1hdu_le_r_mte_aarch64
+#define helper_sve_ld1hdu_be_r_mte helper_sve_ld1hdu_be_r_mte_aarch64
+#define helper_sve_ld1hds_le_r_mte helper_sve_ld1hds_le_r_mte_aarch64
+#define helper_sve_ld1hds_be_r_mte helper_sve_ld1hds_be_r_mte_aarch64
+#define helper_sve_ld1ss_le_r_mte helper_sve_ld1ss_le_r_mte_aarch64
+#define helper_sve_ld1ss_be_r_mte helper_sve_ld1ss_be_r_mte_aarch64
+#define helper_sve_ld1sdu_le_r_mte helper_sve_ld1sdu_le_r_mte_aarch64
+#define helper_sve_ld1sdu_be_r_mte helper_sve_ld1sdu_be_r_mte_aarch64
+#define helper_sve_ld1sds_le_r_mte helper_sve_ld1sds_le_r_mte_aarch64
+#define helper_sve_ld1sds_be_r_mte helper_sve_ld1sds_be_r_mte_aarch64
+#define helper_sve_ld1dd_le_r_mte helper_sve_ld1dd_le_r_mte_aarch64
+#define helper_sve_ld1dd_be_r_mte helper_sve_ld1dd_be_r_mte_aarch64
+#define helper_sve_ld2bb_r_mte helper_sve_ld2bb_r_mte_aarch64
+#define helper_sve_ld3bb_r_mte helper_sve_ld3bb_r_mte_aarch64
+#define helper_sve_ld4bb_r_mte helper_sve_ld4bb_r_mte_aarch64
+#define helper_sve_ld2hh_le_r_mte helper_sve_ld2hh_le_r_mte_aarch64
+#define helper_sve_ld2hh_be_r_mte helper_sve_ld2hh_be_r_mte_aarch64
+#define helper_sve_ld3hh_le_r_mte helper_sve_ld3hh_le_r_mte_aarch64
+#define helper_sve_ld3hh_be_r_mte helper_sve_ld3hh_be_r_mte_aarch64
+#define helper_sve_ld4hh_le_r_mte helper_sve_ld4hh_le_r_mte_aarch64
+#define helper_sve_ld4hh_be_r_mte helper_sve_ld4hh_be_r_mte_aarch64
+#define helper_sve_ld2ss_le_r_mte helper_sve_ld2ss_le_r_mte_aarch64
+#define helper_sve_ld2ss_be_r_mte helper_sve_ld2ss_be_r_mte_aarch64
+#define helper_sve_ld3ss_le_r_mte helper_sve_ld3ss_le_r_mte_aarch64
+#define helper_sve_ld3ss_be_r_mte helper_sve_ld3ss_be_r_mte_aarch64
+#define helper_sve_ld4ss_le_r_mte helper_sve_ld4ss_le_r_mte_aarch64
+#define helper_sve_ld4ss_be_r_mte helper_sve_ld4ss_be_r_mte_aarch64
+#define helper_sve_ld2dd_le_r_mte helper_sve_ld2dd_le_r_mte_aarch64
+#define helper_sve_ld2dd_be_r_mte helper_sve_ld2dd_be_r_mte_aarch64
+#define helper_sve_ld3dd_le_r_mte helper_sve_ld3dd_le_r_mte_aarch64
+#define helper_sve_ld3dd_be_r_mte helper_sve_ld3dd_be_r_mte_aarch64
+#define helper_sve_ld4dd_le_r_mte helper_sve_ld4dd_le_r_mte_aarch64
+#define helper_sve_ld4dd_be_r_mte helper_sve_ld4dd_be_r_mte_aarch64
 #define helper_sve_ldff1bb_r helper_sve_ldff1bb_r_aarch64
 #define helper_sve_ldnf1bb_r helper_sve_ldnf1bb_r_aarch64
 #define helper_sve_ldff1bhu_r helper_sve_ldff1bhu_r_aarch64
@@ -2569,6 +3522,56 @@
 #define helper_sve_ldnf1dd_le_r helper_sve_ldnf1dd_le_r_aarch64
 #define helper_sve_ldff1dd_be_r helper_sve_ldff1dd_be_r_aarch64
 #define helper_sve_ldnf1dd_be_r helper_sve_ldnf1dd_be_r_aarch64
+#define helper_sve_ldff1bb_r_mte helper_sve_ldff1bb_r_mte_aarch64
+#define helper_sve_ldnf1bb_r_mte helper_sve_ldnf1bb_r_mte_aarch64
+#define helper_sve_ldff1bhu_r_mte helper_sve_ldff1bhu_r_mte_aarch64
+#define helper_sve_ldnf1bhu_r_mte helper_sve_ldnf1bhu_r_mte_aarch64
+#define helper_sve_ldff1bhs_r_mte helper_sve_ldff1bhs_r_mte_aarch64
+#define helper_sve_ldnf1bhs_r_mte helper_sve_ldnf1bhs_r_mte_aarch64
+#define helper_sve_ldff1bsu_r_mte helper_sve_ldff1bsu_r_mte_aarch64
+#define helper_sve_ldnf1bsu_r_mte helper_sve_ldnf1bsu_r_mte_aarch64
+#define helper_sve_ldff1bss_r_mte helper_sve_ldff1bss_r_mte_aarch64
+#define helper_sve_ldnf1bss_r_mte helper_sve_ldnf1bss_r_mte_aarch64
+#define helper_sve_ldff1bdu_r_mte helper_sve_ldff1bdu_r_mte_aarch64
+#define helper_sve_ldnf1bdu_r_mte helper_sve_ldnf1bdu_r_mte_aarch64
+#define helper_sve_ldff1bds_r_mte helper_sve_ldff1bds_r_mte_aarch64
+#define helper_sve_ldnf1bds_r_mte helper_sve_ldnf1bds_r_mte_aarch64
+#define helper_sve_ldff1hh_le_r_mte helper_sve_ldff1hh_le_r_mte_aarch64
+#define helper_sve_ldnf1hh_le_r_mte helper_sve_ldnf1hh_le_r_mte_aarch64
+#define helper_sve_ldff1hh_be_r_mte helper_sve_ldff1hh_be_r_mte_aarch64
+#define helper_sve_ldnf1hh_be_r_mte helper_sve_ldnf1hh_be_r_mte_aarch64
+#define helper_sve_ldff1hsu_le_r_mte helper_sve_ldff1hsu_le_r_mte_aarch64
+#define helper_sve_ldnf1hsu_le_r_mte helper_sve_ldnf1hsu_le_r_mte_aarch64
+#define helper_sve_ldff1hsu_be_r_mte helper_sve_ldff1hsu_be_r_mte_aarch64
+#define helper_sve_ldnf1hsu_be_r_mte helper_sve_ldnf1hsu_be_r_mte_aarch64
+#define helper_sve_ldff1hss_le_r_mte helper_sve_ldff1hss_le_r_mte_aarch64
+#define helper_sve_ldnf1hss_le_r_mte helper_sve_ldnf1hss_le_r_mte_aarch64
+#define helper_sve_ldff1hss_be_r_mte helper_sve_ldff1hss_be_r_mte_aarch64
+#define helper_sve_ldnf1hss_be_r_mte helper_sve_ldnf1hss_be_r_mte_aarch64
+#define helper_sve_ldff1hdu_le_r_mte helper_sve_ldff1hdu_le_r_mte_aarch64
+#define helper_sve_ldnf1hdu_le_r_mte helper_sve_ldnf1hdu_le_r_mte_aarch64
+#define helper_sve_ldff1hdu_be_r_mte helper_sve_ldff1hdu_be_r_mte_aarch64
+#define helper_sve_ldnf1hdu_be_r_mte helper_sve_ldnf1hdu_be_r_mte_aarch64
+#define helper_sve_ldff1hds_le_r_mte helper_sve_ldff1hds_le_r_mte_aarch64
+#define helper_sve_ldnf1hds_le_r_mte helper_sve_ldnf1hds_le_r_mte_aarch64
+#define helper_sve_ldff1hds_be_r_mte helper_sve_ldff1hds_be_r_mte_aarch64
+#define helper_sve_ldnf1hds_be_r_mte helper_sve_ldnf1hds_be_r_mte_aarch64
+#define helper_sve_ldff1ss_le_r_mte helper_sve_ldff1ss_le_r_mte_aarch64
+#define helper_sve_ldnf1ss_le_r_mte helper_sve_ldnf1ss_le_r_mte_aarch64
+#define helper_sve_ldff1ss_be_r_mte helper_sve_ldff1ss_be_r_mte_aarch64
+#define helper_sve_ldnf1ss_be_r_mte helper_sve_ldnf1ss_be_r_mte_aarch64
+#define helper_sve_ldff1sdu_le_r_mte helper_sve_ldff1sdu_le_r_mte_aarch64
+#define helper_sve_ldnf1sdu_le_r_mte helper_sve_ldnf1sdu_le_r_mte_aarch64
+#define helper_sve_ldff1sdu_be_r_mte helper_sve_ldff1sdu_be_r_mte_aarch64
+#define helper_sve_ldnf1sdu_be_r_mte helper_sve_ldnf1sdu_be_r_mte_aarch64
+#define helper_sve_ldff1sds_le_r_mte helper_sve_ldff1sds_le_r_mte_aarch64
+#define helper_sve_ldnf1sds_le_r_mte helper_sve_ldnf1sds_le_r_mte_aarch64
+#define helper_sve_ldff1sds_be_r_mte helper_sve_ldff1sds_be_r_mte_aarch64
+#define helper_sve_ldnf1sds_be_r_mte helper_sve_ldnf1sds_be_r_mte_aarch64
+#define helper_sve_ldff1dd_le_r_mte helper_sve_ldff1dd_le_r_mte_aarch64
+#define helper_sve_ldnf1dd_le_r_mte helper_sve_ldnf1dd_le_r_mte_aarch64
+#define helper_sve_ldff1dd_be_r_mte helper_sve_ldff1dd_be_r_mte_aarch64
+#define helper_sve_ldnf1dd_be_r_mte helper_sve_ldnf1dd_be_r_mte_aarch64
 #define helper_sve_st1bb_r helper_sve_st1bb_r_aarch64
 #define helper_sve_st1bh_r helper_sve_st1bh_r_aarch64
 #define helper_sve_st1bs_r helper_sve_st1bs_r_aarch64
@@ -2606,6 +3609,43 @@
 #define helper_sve_st3dd_be_r helper_sve_st3dd_be_r_aarch64
 #define helper_sve_st4dd_le_r helper_sve_st4dd_le_r_aarch64
 #define helper_sve_st4dd_be_r helper_sve_st4dd_be_r_aarch64
+#define helper_sve_st1bb_r_mte helper_sve_st1bb_r_mte_aarch64
+#define helper_sve_st1bh_r_mte helper_sve_st1bh_r_mte_aarch64
+#define helper_sve_st1bs_r_mte helper_sve_st1bs_r_mte_aarch64
+#define helper_sve_st1bd_r_mte helper_sve_st1bd_r_mte_aarch64
+#define helper_sve_st2bb_r_mte helper_sve_st2bb_r_mte_aarch64
+#define helper_sve_st3bb_r_mte helper_sve_st3bb_r_mte_aarch64
+#define helper_sve_st4bb_r_mte helper_sve_st4bb_r_mte_aarch64
+#define helper_sve_st1hh_le_r_mte helper_sve_st1hh_le_r_mte_aarch64
+#define helper_sve_st1hh_be_r_mte helper_sve_st1hh_be_r_mte_aarch64
+#define helper_sve_st1hs_le_r_mte helper_sve_st1hs_le_r_mte_aarch64
+#define helper_sve_st1hs_be_r_mte helper_sve_st1hs_be_r_mte_aarch64
+#define helper_sve_st1hd_le_r_mte helper_sve_st1hd_le_r_mte_aarch64
+#define helper_sve_st1hd_be_r_mte helper_sve_st1hd_be_r_mte_aarch64
+#define helper_sve_st2hh_le_r_mte helper_sve_st2hh_le_r_mte_aarch64
+#define helper_sve_st2hh_be_r_mte helper_sve_st2hh_be_r_mte_aarch64
+#define helper_sve_st3hh_le_r_mte helper_sve_st3hh_le_r_mte_aarch64
+#define helper_sve_st3hh_be_r_mte helper_sve_st3hh_be_r_mte_aarch64
+#define helper_sve_st4hh_le_r_mte helper_sve_st4hh_le_r_mte_aarch64
+#define helper_sve_st4hh_be_r_mte helper_sve_st4hh_be_r_mte_aarch64
+#define helper_sve_st1ss_le_r_mte helper_sve_st1ss_le_r_mte_aarch64
+#define helper_sve_st1ss_be_r_mte helper_sve_st1ss_be_r_mte_aarch64
+#define helper_sve_st1sd_le_r_mte helper_sve_st1sd_le_r_mte_aarch64
+#define helper_sve_st1sd_be_r_mte helper_sve_st1sd_be_r_mte_aarch64
+#define helper_sve_st2ss_le_r_mte helper_sve_st2ss_le_r_mte_aarch64
+#define helper_sve_st2ss_be_r_mte helper_sve_st2ss_be_r_mte_aarch64
+#define helper_sve_st3ss_le_r_mte helper_sve_st3ss_le_r_mte_aarch64
+#define helper_sve_st3ss_be_r_mte helper_sve_st3ss_be_r_mte_aarch64
+#define helper_sve_st4ss_le_r_mte helper_sve_st4ss_le_r_mte_aarch64
+#define helper_sve_st4ss_be_r_mte helper_sve_st4ss_be_r_mte_aarch64
+#define helper_sve_st1dd_le_r_mte helper_sve_st1dd_le_r_mte_aarch64
+#define helper_sve_st1dd_be_r_mte helper_sve_st1dd_be_r_mte_aarch64
+#define helper_sve_st2dd_le_r_mte helper_sve_st2dd_le_r_mte_aarch64
+#define helper_sve_st2dd_be_r_mte helper_sve_st2dd_be_r_mte_aarch64
+#define helper_sve_st3dd_le_r_mte helper_sve_st3dd_le_r_mte_aarch64
+#define helper_sve_st3dd_be_r_mte helper_sve_st3dd_be_r_mte_aarch64
+#define helper_sve_st4dd_le_r_mte helper_sve_st4dd_le_r_mte_aarch64
+#define helper_sve_st4dd_be_r_mte helper_sve_st4dd_be_r_mte_aarch64
 #define helper_sve_ldbsu_zsu helper_sve_ldbsu_zsu_aarch64
 #define helper_sve_ldbsu_zss helper_sve_ldbsu_zss_aarch64
 #define helper_sve_ldbdu_zsu helper_sve_ldbdu_zsu_aarch64
@@ -2658,6 +3698,58 @@
 #define helper_sve_lddd_be_zsu helper_sve_lddd_be_zsu_aarch64
 #define helper_sve_lddd_be_zss helper_sve_lddd_be_zss_aarch64
 #define helper_sve_lddd_be_zd helper_sve_lddd_be_zd_aarch64
+#define helper_sve_ldbsu_zsu_mte helper_sve_ldbsu_zsu_mte_aarch64
+#define helper_sve_ldbsu_zss_mte helper_sve_ldbsu_zss_mte_aarch64
+#define helper_sve_ldbdu_zsu_mte helper_sve_ldbdu_zsu_mte_aarch64
+#define helper_sve_ldbdu_zss_mte helper_sve_ldbdu_zss_mte_aarch64
+#define helper_sve_ldbdu_zd_mte helper_sve_ldbdu_zd_mte_aarch64
+#define helper_sve_ldbss_zsu_mte helper_sve_ldbss_zsu_mte_aarch64
+#define helper_sve_ldbss_zss_mte helper_sve_ldbss_zss_mte_aarch64
+#define helper_sve_ldbds_zsu_mte helper_sve_ldbds_zsu_mte_aarch64
+#define helper_sve_ldbds_zss_mte helper_sve_ldbds_zss_mte_aarch64
+#define helper_sve_ldbds_zd_mte helper_sve_ldbds_zd_mte_aarch64
+#define helper_sve_ldhsu_le_zsu_mte helper_sve_ldhsu_le_zsu_mte_aarch64
+#define helper_sve_ldhsu_le_zss_mte helper_sve_ldhsu_le_zss_mte_aarch64
+#define helper_sve_ldhdu_le_zsu_mte helper_sve_ldhdu_le_zsu_mte_aarch64
+#define helper_sve_ldhdu_le_zss_mte helper_sve_ldhdu_le_zss_mte_aarch64
+#define helper_sve_ldhdu_le_zd_mte helper_sve_ldhdu_le_zd_mte_aarch64
+#define helper_sve_ldhsu_be_zsu_mte helper_sve_ldhsu_be_zsu_mte_aarch64
+#define helper_sve_ldhsu_be_zss_mte helper_sve_ldhsu_be_zss_mte_aarch64
+#define helper_sve_ldhdu_be_zsu_mte helper_sve_ldhdu_be_zsu_mte_aarch64
+#define helper_sve_ldhdu_be_zss_mte helper_sve_ldhdu_be_zss_mte_aarch64
+#define helper_sve_ldhdu_be_zd_mte helper_sve_ldhdu_be_zd_mte_aarch64
+#define helper_sve_ldhss_le_zsu_mte helper_sve_ldhss_le_zsu_mte_aarch64
+#define helper_sve_ldhss_le_zss_mte helper_sve_ldhss_le_zss_mte_aarch64
+#define helper_sve_ldhds_le_zsu_mte helper_sve_ldhds_le_zsu_mte_aarch64
+#define helper_sve_ldhds_le_zss_mte helper_sve_ldhds_le_zss_mte_aarch64
+#define helper_sve_ldhds_le_zd_mte helper_sve_ldhds_le_zd_mte_aarch64
+#define helper_sve_ldhss_be_zsu_mte helper_sve_ldhss_be_zsu_mte_aarch64
+#define helper_sve_ldhss_be_zss_mte helper_sve_ldhss_be_zss_mte_aarch64
+#define helper_sve_ldhds_be_zsu_mte helper_sve_ldhds_be_zsu_mte_aarch64
+#define helper_sve_ldhds_be_zss_mte helper_sve_ldhds_be_zss_mte_aarch64
+#define helper_sve_ldhds_be_zd_mte helper_sve_ldhds_be_zd_mte_aarch64
+#define helper_sve_ldss_le_zsu_mte helper_sve_ldss_le_zsu_mte_aarch64
+#define helper_sve_ldss_le_zss_mte helper_sve_ldss_le_zss_mte_aarch64
+#define helper_sve_ldsdu_le_zsu_mte helper_sve_ldsdu_le_zsu_mte_aarch64
+#define helper_sve_ldsdu_le_zss_mte helper_sve_ldsdu_le_zss_mte_aarch64
+#define helper_sve_ldsdu_le_zd_mte helper_sve_ldsdu_le_zd_mte_aarch64
+#define helper_sve_ldss_be_zsu_mte helper_sve_ldss_be_zsu_mte_aarch64
+#define helper_sve_ldss_be_zss_mte helper_sve_ldss_be_zss_mte_aarch64
+#define helper_sve_ldsdu_be_zsu_mte helper_sve_ldsdu_be_zsu_mte_aarch64
+#define helper_sve_ldsdu_be_zss_mte helper_sve_ldsdu_be_zss_mte_aarch64
+#define helper_sve_ldsdu_be_zd_mte helper_sve_ldsdu_be_zd_mte_aarch64
+#define helper_sve_ldsds_le_zsu_mte helper_sve_ldsds_le_zsu_mte_aarch64
+#define helper_sve_ldsds_le_zss_mte helper_sve_ldsds_le_zss_mte_aarch64
+#define helper_sve_ldsds_le_zd_mte helper_sve_ldsds_le_zd_mte_aarch64
+#define helper_sve_ldsds_be_zsu_mte helper_sve_ldsds_be_zsu_mte_aarch64
+#define helper_sve_ldsds_be_zss_mte helper_sve_ldsds_be_zss_mte_aarch64
+#define helper_sve_ldsds_be_zd_mte helper_sve_ldsds_be_zd_mte_aarch64
+#define helper_sve_lddd_le_zsu_mte helper_sve_lddd_le_zsu_mte_aarch64
+#define helper_sve_lddd_le_zss_mte helper_sve_lddd_le_zss_mte_aarch64
+#define helper_sve_lddd_le_zd_mte helper_sve_lddd_le_zd_mte_aarch64
+#define helper_sve_lddd_be_zsu_mte helper_sve_lddd_be_zsu_mte_aarch64
+#define helper_sve_lddd_be_zss_mte helper_sve_lddd_be_zss_mte_aarch64
+#define helper_sve_lddd_be_zd_mte helper_sve_lddd_be_zd_mte_aarch64
 #define helper_sve_ldffbsu_zsu helper_sve_ldffbsu_zsu_aarch64
 #define helper_sve_ldffbsu_zss helper_sve_ldffbsu_zss_aarch64
 #define helper_sve_ldffbdu_zsu helper_sve_ldffbdu_zsu_aarch64
@@ -2710,6 +3802,58 @@
 #define helper_sve_ldffdd_be_zsu helper_sve_ldffdd_be_zsu_aarch64
 #define helper_sve_ldffdd_be_zss helper_sve_ldffdd_be_zss_aarch64
 #define helper_sve_ldffdd_be_zd helper_sve_ldffdd_be_zd_aarch64
+#define helper_sve_ldffbsu_zsu_mte helper_sve_ldffbsu_zsu_mte_aarch64
+#define helper_sve_ldffbsu_zss_mte helper_sve_ldffbsu_zss_mte_aarch64
+#define helper_sve_ldffbdu_zsu_mte helper_sve_ldffbdu_zsu_mte_aarch64
+#define helper_sve_ldffbdu_zss_mte helper_sve_ldffbdu_zss_mte_aarch64
+#define helper_sve_ldffbdu_zd_mte helper_sve_ldffbdu_zd_mte_aarch64
+#define helper_sve_ldffbss_zsu_mte helper_sve_ldffbss_zsu_mte_aarch64
+#define helper_sve_ldffbss_zss_mte helper_sve_ldffbss_zss_mte_aarch64
+#define helper_sve_ldffbds_zsu_mte helper_sve_ldffbds_zsu_mte_aarch64
+#define helper_sve_ldffbds_zss_mte helper_sve_ldffbds_zss_mte_aarch64
+#define helper_sve_ldffbds_zd_mte helper_sve_ldffbds_zd_mte_aarch64
+#define helper_sve_ldffhsu_le_zsu_mte helper_sve_ldffhsu_le_zsu_mte_aarch64
+#define helper_sve_ldffhsu_le_zss_mte helper_sve_ldffhsu_le_zss_mte_aarch64
+#define helper_sve_ldffhdu_le_zsu_mte helper_sve_ldffhdu_le_zsu_mte_aarch64
+#define helper_sve_ldffhdu_le_zss_mte helper_sve_ldffhdu_le_zss_mte_aarch64
+#define helper_sve_ldffhdu_le_zd_mte helper_sve_ldffhdu_le_zd_mte_aarch64
+#define helper_sve_ldffhsu_be_zsu_mte helper_sve_ldffhsu_be_zsu_mte_aarch64
+#define helper_sve_ldffhsu_be_zss_mte helper_sve_ldffhsu_be_zss_mte_aarch64
+#define helper_sve_ldffhdu_be_zsu_mte helper_sve_ldffhdu_be_zsu_mte_aarch64
+#define helper_sve_ldffhdu_be_zss_mte helper_sve_ldffhdu_be_zss_mte_aarch64
+#define helper_sve_ldffhdu_be_zd_mte helper_sve_ldffhdu_be_zd_mte_aarch64
+#define helper_sve_ldffhss_le_zsu_mte helper_sve_ldffhss_le_zsu_mte_aarch64
+#define helper_sve_ldffhss_le_zss_mte helper_sve_ldffhss_le_zss_mte_aarch64
+#define helper_sve_ldffhds_le_zsu_mte helper_sve_ldffhds_le_zsu_mte_aarch64
+#define helper_sve_ldffhds_le_zss_mte helper_sve_ldffhds_le_zss_mte_aarch64
+#define helper_sve_ldffhds_le_zd_mte helper_sve_ldffhds_le_zd_mte_aarch64
+#define helper_sve_ldffhss_be_zsu_mte helper_sve_ldffhss_be_zsu_mte_aarch64
+#define helper_sve_ldffhss_be_zss_mte helper_sve_ldffhss_be_zss_mte_aarch64
+#define helper_sve_ldffhds_be_zsu_mte helper_sve_ldffhds_be_zsu_mte_aarch64
+#define helper_sve_ldffhds_be_zss_mte helper_sve_ldffhds_be_zss_mte_aarch64
+#define helper_sve_ldffhds_be_zd_mte helper_sve_ldffhds_be_zd_mte_aarch64
+#define helper_sve_ldffss_le_zsu_mte helper_sve_ldffss_le_zsu_mte_aarch64
+#define helper_sve_ldffss_le_zss_mte helper_sve_ldffss_le_zss_mte_aarch64
+#define helper_sve_ldffsdu_le_zsu_mte helper_sve_ldffsdu_le_zsu_mte_aarch64
+#define helper_sve_ldffsdu_le_zss_mte helper_sve_ldffsdu_le_zss_mte_aarch64
+#define helper_sve_ldffsdu_le_zd_mte helper_sve_ldffsdu_le_zd_mte_aarch64
+#define helper_sve_ldffss_be_zsu_mte helper_sve_ldffss_be_zsu_mte_aarch64
+#define helper_sve_ldffss_be_zss_mte helper_sve_ldffss_be_zss_mte_aarch64
+#define helper_sve_ldffsdu_be_zsu_mte helper_sve_ldffsdu_be_zsu_mte_aarch64
+#define helper_sve_ldffsdu_be_zss_mte helper_sve_ldffsdu_be_zss_mte_aarch64
+#define helper_sve_ldffsdu_be_zd_mte helper_sve_ldffsdu_be_zd_mte_aarch64
+#define helper_sve_ldffsds_le_zsu_mte helper_sve_ldffsds_le_zsu_mte_aarch64
+#define helper_sve_ldffsds_le_zss_mte helper_sve_ldffsds_le_zss_mte_aarch64
+#define helper_sve_ldffsds_le_zd_mte helper_sve_ldffsds_le_zd_mte_aarch64
+#define helper_sve_ldffsds_be_zsu_mte helper_sve_ldffsds_be_zsu_mte_aarch64
+#define helper_sve_ldffsds_be_zss_mte helper_sve_ldffsds_be_zss_mte_aarch64
+#define helper_sve_ldffsds_be_zd_mte helper_sve_ldffsds_be_zd_mte_aarch64
+#define helper_sve_ldffdd_le_zsu_mte helper_sve_ldffdd_le_zsu_mte_aarch64
+#define helper_sve_ldffdd_le_zss_mte helper_sve_ldffdd_le_zss_mte_aarch64
+#define helper_sve_ldffdd_le_zd_mte helper_sve_ldffdd_le_zd_mte_aarch64
+#define helper_sve_ldffdd_be_zsu_mte helper_sve_ldffdd_be_zsu_mte_aarch64
+#define helper_sve_ldffdd_be_zss_mte helper_sve_ldffdd_be_zss_mte_aarch64
+#define helper_sve_ldffdd_be_zd_mte helper_sve_ldffdd_be_zd_mte_aarch64
 #define helper_sve_stbs_zsu helper_sve_stbs_zsu_aarch64
 #define helper_sve_sths_le_zsu helper_sve_sths_le_zsu_aarch64
 #define helper_sve_sths_be_zsu helper_sve_sths_be_zsu_aarch64
@@ -2741,6 +3885,37 @@
 #define helper_sve_stsd_be_zd helper_sve_stsd_be_zd_aarch64
 #define helper_sve_stdd_le_zd helper_sve_stdd_le_zd_aarch64
 #define helper_sve_stdd_be_zd helper_sve_stdd_be_zd_aarch64
+#define helper_sve_stbs_zsu_mte helper_sve_stbs_zsu_mte_aarch64
+#define helper_sve_sths_le_zsu_mte helper_sve_sths_le_zsu_mte_aarch64
+#define helper_sve_sths_be_zsu_mte helper_sve_sths_be_zsu_mte_aarch64
+#define helper_sve_stss_le_zsu_mte helper_sve_stss_le_zsu_mte_aarch64
+#define helper_sve_stss_be_zsu_mte helper_sve_stss_be_zsu_mte_aarch64
+#define helper_sve_stbs_zss_mte helper_sve_stbs_zss_mte_aarch64
+#define helper_sve_sths_le_zss_mte helper_sve_sths_le_zss_mte_aarch64
+#define helper_sve_sths_be_zss_mte helper_sve_sths_be_zss_mte_aarch64
+#define helper_sve_stss_le_zss_mte helper_sve_stss_le_zss_mte_aarch64
+#define helper_sve_stss_be_zss_mte helper_sve_stss_be_zss_mte_aarch64
+#define helper_sve_stbd_zsu_mte helper_sve_stbd_zsu_mte_aarch64
+#define helper_sve_sthd_le_zsu_mte helper_sve_sthd_le_zsu_mte_aarch64
+#define helper_sve_sthd_be_zsu_mte helper_sve_sthd_be_zsu_mte_aarch64
+#define helper_sve_stsd_le_zsu_mte helper_sve_stsd_le_zsu_mte_aarch64
+#define helper_sve_stsd_be_zsu_mte helper_sve_stsd_be_zsu_mte_aarch64
+#define helper_sve_stdd_le_zsu_mte helper_sve_stdd_le_zsu_mte_aarch64
+#define helper_sve_stdd_be_zsu_mte helper_sve_stdd_be_zsu_mte_aarch64
+#define helper_sve_stbd_zss_mte helper_sve_stbd_zss_mte_aarch64
+#define helper_sve_sthd_le_zss_mte helper_sve_sthd_le_zss_mte_aarch64
+#define helper_sve_sthd_be_zss_mte helper_sve_sthd_be_zss_mte_aarch64
+#define helper_sve_stsd_le_zss_mte helper_sve_stsd_le_zss_mte_aarch64
+#define helper_sve_stsd_be_zss_mte helper_sve_stsd_be_zss_mte_aarch64
+#define helper_sve_stdd_le_zss_mte helper_sve_stdd_le_zss_mte_aarch64
+#define helper_sve_stdd_be_zss_mte helper_sve_stdd_be_zss_mte_aarch64
+#define helper_sve_stbd_zd_mte helper_sve_stbd_zd_mte_aarch64
+#define helper_sve_sthd_le_zd_mte helper_sve_sthd_le_zd_mte_aarch64
+#define helper_sve_sthd_be_zd_mte helper_sve_sthd_be_zd_mte_aarch64
+#define helper_sve_stsd_le_zd_mte helper_sve_stsd_le_zd_mte_aarch64
+#define helper_sve_stsd_be_zd_mte helper_sve_stsd_be_zd_mte_aarch64
+#define helper_sve_stdd_le_zd_mte helper_sve_stdd_le_zd_mte_aarch64
+#define helper_sve_stdd_be_zd_mte helper_sve_stdd_be_zd_mte_aarch64
 #define arm_cpu_do_unaligned_access arm_cpu_do_unaligned_access_aarch64
 #define arm_cpu_do_transaction_failed arm_cpu_do_transaction_failed_aarch64
 #define arm_cpu_tlb_fill arm_cpu_tlb_fill_aarch64
@@ -2781,12 +3956,23 @@
 #define helper_gvec_qrdmlsh_s32 helper_gvec_qrdmlsh_s32_aarch64
 #define helper_gvec_sdot_b helper_gvec_sdot_b_aarch64
 #define helper_gvec_udot_b helper_gvec_udot_b_aarch64
+#define helper_gvec_usdot_b helper_gvec_usdot_b_aarch64
 #define helper_gvec_sdot_h helper_gvec_sdot_h_aarch64
 #define helper_gvec_udot_h helper_gvec_udot_h_aarch64
 #define helper_gvec_sdot_idx_b helper_gvec_sdot_idx_b_aarch64
 #define helper_gvec_udot_idx_b helper_gvec_udot_idx_b_aarch64
+#define helper_gvec_sudot_idx_b helper_gvec_sudot_idx_b_aarch64
+#define helper_gvec_usdot_idx_b helper_gvec_usdot_idx_b_aarch64
 #define helper_gvec_sdot_idx_h helper_gvec_sdot_idx_h_aarch64
 #define helper_gvec_udot_idx_h helper_gvec_udot_idx_h_aarch64
+#define helper_gvec_smmla_b helper_gvec_smmla_b_aarch64
+#define helper_gvec_ummla_b helper_gvec_ummla_b_aarch64
+#define helper_gvec_usmmla_b helper_gvec_usmmla_b_aarch64
+#define helper_gvec_bfdot helper_gvec_bfdot_aarch64
+#define helper_gvec_bfdot_idx helper_gvec_bfdot_idx_aarch64
+#define helper_gvec_bfmmla helper_gvec_bfmmla_aarch64
+#define helper_gvec_bfmlal helper_gvec_bfmlal_aarch64
+#define helper_gvec_bfmlal_idx helper_gvec_bfmlal_idx_aarch64
 #define helper_gvec_fcaddh helper_gvec_fcaddh_aarch64
 #define helper_gvec_fcadds helper_gvec_fcadds_aarch64
 #define helper_gvec_fcaddd helper_gvec_fcaddd_aarch64
@@ -2841,10 +4027,20 @@
 #define helper_gvec_uqsub_d helper_gvec_uqsub_d_aarch64
 #define helper_gvec_sqadd_d helper_gvec_sqadd_d_aarch64
 #define helper_gvec_sqsub_d helper_gvec_sqsub_d_aarch64
+#define helper_gvec_saba_b helper_gvec_saba_b_aarch64
+#define helper_gvec_saba_h helper_gvec_saba_h_aarch64
+#define helper_gvec_saba_s helper_gvec_saba_s_aarch64
+#define helper_gvec_saba_d helper_gvec_saba_d_aarch64
+#define helper_gvec_uaba_b helper_gvec_uaba_b_aarch64
+#define helper_gvec_uaba_h helper_gvec_uaba_h_aarch64
+#define helper_gvec_uaba_s helper_gvec_uaba_s_aarch64
+#define helper_gvec_uaba_d helper_gvec_uaba_d_aarch64
 #define helper_gvec_fmlal_a32 helper_gvec_fmlal_a32_aarch64
 #define helper_gvec_fmlal_a64 helper_gvec_fmlal_a64_aarch64
 #define helper_gvec_fmlal_idx_a32 helper_gvec_fmlal_idx_a32_aarch64
 #define helper_gvec_fmlal_idx_a64 helper_gvec_fmlal_idx_a64_aarch64
+#define helper_sve2_fmlal_zzzw_s helper_sve2_fmlal_zzzw_s_aarch64
+#define helper_sve2_fmlal_zzxw_s helper_sve2_fmlal_zzxw_s_aarch64
 #define helper_gvec_sshl_b helper_gvec_sshl_b_aarch64
 #define helper_gvec_sshl_h helper_gvec_sshl_h_aarch64
 #define helper_gvec_ushl_b helper_gvec_ushl_b_aarch64
@@ -2852,11 +4048,1103 @@
 #define helper_gvec_pmul_b helper_gvec_pmul_b_aarch64
 #define helper_gvec_pmull_q helper_gvec_pmull_q_aarch64
 #define helper_neon_pmull_h helper_neon_pmull_h_aarch64
+#define helper_sve2_sqabs_b helper_sve2_sqabs_b_aarch64
+#define helper_sve2_sqabs_h helper_sve2_sqabs_h_aarch64
+#define helper_sve2_sqabs_s helper_sve2_sqabs_s_aarch64
+#define helper_sve2_sqabs_d helper_sve2_sqabs_d_aarch64
+#define helper_sve2_sqneg_b helper_sve2_sqneg_b_aarch64
+#define helper_sve2_sqneg_h helper_sve2_sqneg_h_aarch64
+#define helper_sve2_sqneg_s helper_sve2_sqneg_s_aarch64
+#define helper_sve2_sqneg_d helper_sve2_sqneg_d_aarch64
+#define helper_sve2_urecpe_s helper_sve2_urecpe_s_aarch64
+#define helper_sve2_ursqrte_s helper_sve2_ursqrte_s_aarch64
+#define helper_sve2_sadalp_zpzz_h helper_sve2_sadalp_zpzz_h_aarch64
+#define helper_sve2_sadalp_zpzz_s helper_sve2_sadalp_zpzz_s_aarch64
+#define helper_sve2_sadalp_zpzz_d helper_sve2_sadalp_zpzz_d_aarch64
+#define helper_sve2_uadalp_zpzz_h helper_sve2_uadalp_zpzz_h_aarch64
+#define helper_sve2_uadalp_zpzz_s helper_sve2_uadalp_zpzz_s_aarch64
+#define helper_sve2_uadalp_zpzz_d helper_sve2_uadalp_zpzz_d_aarch64
+#define helper_sve2_shadd_zpzz_b helper_sve2_shadd_zpzz_b_aarch64
+#define helper_sve2_shadd_zpzz_h helper_sve2_shadd_zpzz_h_aarch64
+#define helper_sve2_shadd_zpzz_s helper_sve2_shadd_zpzz_s_aarch64
+#define helper_sve2_shadd_zpzz_d helper_sve2_shadd_zpzz_d_aarch64
+#define helper_sve2_uhadd_zpzz_b helper_sve2_uhadd_zpzz_b_aarch64
+#define helper_sve2_uhadd_zpzz_h helper_sve2_uhadd_zpzz_h_aarch64
+#define helper_sve2_uhadd_zpzz_s helper_sve2_uhadd_zpzz_s_aarch64
+#define helper_sve2_uhadd_zpzz_d helper_sve2_uhadd_zpzz_d_aarch64
+#define helper_sve2_srhadd_zpzz_b helper_sve2_srhadd_zpzz_b_aarch64
+#define helper_sve2_srhadd_zpzz_h helper_sve2_srhadd_zpzz_h_aarch64
+#define helper_sve2_srhadd_zpzz_s helper_sve2_srhadd_zpzz_s_aarch64
+#define helper_sve2_srhadd_zpzz_d helper_sve2_srhadd_zpzz_d_aarch64
+#define helper_sve2_urhadd_zpzz_b helper_sve2_urhadd_zpzz_b_aarch64
+#define helper_sve2_urhadd_zpzz_h helper_sve2_urhadd_zpzz_h_aarch64
+#define helper_sve2_urhadd_zpzz_s helper_sve2_urhadd_zpzz_s_aarch64
+#define helper_sve2_urhadd_zpzz_d helper_sve2_urhadd_zpzz_d_aarch64
+#define helper_sve2_shsub_zpzz_b helper_sve2_shsub_zpzz_b_aarch64
+#define helper_sve2_shsub_zpzz_h helper_sve2_shsub_zpzz_h_aarch64
+#define helper_sve2_shsub_zpzz_s helper_sve2_shsub_zpzz_s_aarch64
+#define helper_sve2_shsub_zpzz_d helper_sve2_shsub_zpzz_d_aarch64
+#define helper_sve2_uhsub_zpzz_b helper_sve2_uhsub_zpzz_b_aarch64
+#define helper_sve2_uhsub_zpzz_h helper_sve2_uhsub_zpzz_h_aarch64
+#define helper_sve2_uhsub_zpzz_s helper_sve2_uhsub_zpzz_s_aarch64
+#define helper_sve2_uhsub_zpzz_d helper_sve2_uhsub_zpzz_d_aarch64
+#define helper_sve2_addp_zpzz_b helper_sve2_addp_zpzz_b_aarch64
+#define helper_sve2_addp_zpzz_h helper_sve2_addp_zpzz_h_aarch64
+#define helper_sve2_addp_zpzz_s helper_sve2_addp_zpzz_s_aarch64
+#define helper_sve2_addp_zpzz_d helper_sve2_addp_zpzz_d_aarch64
+#define helper_sve2_smaxp_zpzz_b helper_sve2_smaxp_zpzz_b_aarch64
+#define helper_sve2_smaxp_zpzz_h helper_sve2_smaxp_zpzz_h_aarch64
+#define helper_sve2_smaxp_zpzz_s helper_sve2_smaxp_zpzz_s_aarch64
+#define helper_sve2_smaxp_zpzz_d helper_sve2_smaxp_zpzz_d_aarch64
+#define helper_sve2_umaxp_zpzz_b helper_sve2_umaxp_zpzz_b_aarch64
+#define helper_sve2_umaxp_zpzz_h helper_sve2_umaxp_zpzz_h_aarch64
+#define helper_sve2_umaxp_zpzz_s helper_sve2_umaxp_zpzz_s_aarch64
+#define helper_sve2_umaxp_zpzz_d helper_sve2_umaxp_zpzz_d_aarch64
+#define helper_sve2_sminp_zpzz_b helper_sve2_sminp_zpzz_b_aarch64
+#define helper_sve2_sminp_zpzz_h helper_sve2_sminp_zpzz_h_aarch64
+#define helper_sve2_sminp_zpzz_s helper_sve2_sminp_zpzz_s_aarch64
+#define helper_sve2_sminp_zpzz_d helper_sve2_sminp_zpzz_d_aarch64
+#define helper_sve2_uminp_zpzz_b helper_sve2_uminp_zpzz_b_aarch64
+#define helper_sve2_uminp_zpzz_h helper_sve2_uminp_zpzz_h_aarch64
+#define helper_sve2_uminp_zpzz_s helper_sve2_uminp_zpzz_s_aarch64
+#define helper_sve2_uminp_zpzz_d helper_sve2_uminp_zpzz_d_aarch64
+#define helper_sve2_faddp_zpzz_h helper_sve2_faddp_zpzz_h_aarch64
+#define helper_sve2_faddp_zpzz_s helper_sve2_faddp_zpzz_s_aarch64
+#define helper_sve2_faddp_zpzz_d helper_sve2_faddp_zpzz_d_aarch64
+#define helper_sve2_fmaxnmp_zpzz_h helper_sve2_fmaxnmp_zpzz_h_aarch64
+#define helper_sve2_fmaxnmp_zpzz_s helper_sve2_fmaxnmp_zpzz_s_aarch64
+#define helper_sve2_fmaxnmp_zpzz_d helper_sve2_fmaxnmp_zpzz_d_aarch64
+#define helper_sve2_fminnmp_zpzz_h helper_sve2_fminnmp_zpzz_h_aarch64
+#define helper_sve2_fminnmp_zpzz_s helper_sve2_fminnmp_zpzz_s_aarch64
+#define helper_sve2_fminnmp_zpzz_d helper_sve2_fminnmp_zpzz_d_aarch64
+#define helper_sve2_fmaxp_zpzz_h helper_sve2_fmaxp_zpzz_h_aarch64
+#define helper_sve2_fmaxp_zpzz_s helper_sve2_fmaxp_zpzz_s_aarch64
+#define helper_sve2_fmaxp_zpzz_d helper_sve2_fmaxp_zpzz_d_aarch64
+#define helper_sve2_fminp_zpzz_h helper_sve2_fminp_zpzz_h_aarch64
+#define helper_sve2_fminp_zpzz_s helper_sve2_fminp_zpzz_s_aarch64
+#define helper_sve2_fminp_zpzz_d helper_sve2_fminp_zpzz_d_aarch64
+#define helper_sve2_srshl_zpzz_b helper_sve2_srshl_zpzz_b_aarch64
+#define helper_sve2_srshl_zpzz_h helper_sve2_srshl_zpzz_h_aarch64
+#define helper_sve2_srshl_zpzz_s helper_sve2_srshl_zpzz_s_aarch64
+#define helper_sve2_srshl_zpzz_d helper_sve2_srshl_zpzz_d_aarch64
+#define helper_sve2_urshl_zpzz_b helper_sve2_urshl_zpzz_b_aarch64
+#define helper_sve2_urshl_zpzz_h helper_sve2_urshl_zpzz_h_aarch64
+#define helper_sve2_urshl_zpzz_s helper_sve2_urshl_zpzz_s_aarch64
+#define helper_sve2_urshl_zpzz_d helper_sve2_urshl_zpzz_d_aarch64
+#define helper_sve2_sqshl_zpzz_b helper_sve2_sqshl_zpzz_b_aarch64
+#define helper_sve2_sqshl_zpzz_h helper_sve2_sqshl_zpzz_h_aarch64
+#define helper_sve2_sqshl_zpzz_s helper_sve2_sqshl_zpzz_s_aarch64
+#define helper_sve2_sqshl_zpzz_d helper_sve2_sqshl_zpzz_d_aarch64
+#define helper_sve2_uqshl_zpzz_b helper_sve2_uqshl_zpzz_b_aarch64
+#define helper_sve2_uqshl_zpzz_h helper_sve2_uqshl_zpzz_h_aarch64
+#define helper_sve2_uqshl_zpzz_s helper_sve2_uqshl_zpzz_s_aarch64
+#define helper_sve2_uqshl_zpzz_d helper_sve2_uqshl_zpzz_d_aarch64
+#define helper_sve2_sqrshl_zpzz_b helper_sve2_sqrshl_zpzz_b_aarch64
+#define helper_sve2_sqrshl_zpzz_h helper_sve2_sqrshl_zpzz_h_aarch64
+#define helper_sve2_sqrshl_zpzz_s helper_sve2_sqrshl_zpzz_s_aarch64
+#define helper_sve2_sqrshl_zpzz_d helper_sve2_sqrshl_zpzz_d_aarch64
+#define helper_sve2_uqrshl_zpzz_b helper_sve2_uqrshl_zpzz_b_aarch64
+#define helper_sve2_uqrshl_zpzz_h helper_sve2_uqrshl_zpzz_h_aarch64
+#define helper_sve2_uqrshl_zpzz_s helper_sve2_uqrshl_zpzz_s_aarch64
+#define helper_sve2_uqrshl_zpzz_d helper_sve2_uqrshl_zpzz_d_aarch64
+#define helper_sve2_sqadd_zpzz_b helper_sve2_sqadd_zpzz_b_aarch64
+#define helper_sve2_sqadd_zpzz_h helper_sve2_sqadd_zpzz_h_aarch64
+#define helper_sve2_sqadd_zpzz_s helper_sve2_sqadd_zpzz_s_aarch64
+#define helper_sve2_sqadd_zpzz_d helper_sve2_sqadd_zpzz_d_aarch64
+#define helper_sve2_uqadd_zpzz_b helper_sve2_uqadd_zpzz_b_aarch64
+#define helper_sve2_uqadd_zpzz_h helper_sve2_uqadd_zpzz_h_aarch64
+#define helper_sve2_uqadd_zpzz_s helper_sve2_uqadd_zpzz_s_aarch64
+#define helper_sve2_uqadd_zpzz_d helper_sve2_uqadd_zpzz_d_aarch64
+#define helper_sve2_sqsub_zpzz_b helper_sve2_sqsub_zpzz_b_aarch64
+#define helper_sve2_sqsub_zpzz_h helper_sve2_sqsub_zpzz_h_aarch64
+#define helper_sve2_sqsub_zpzz_s helper_sve2_sqsub_zpzz_s_aarch64
+#define helper_sve2_sqsub_zpzz_d helper_sve2_sqsub_zpzz_d_aarch64
+#define helper_sve2_uqsub_zpzz_b helper_sve2_uqsub_zpzz_b_aarch64
+#define helper_sve2_uqsub_zpzz_h helper_sve2_uqsub_zpzz_h_aarch64
+#define helper_sve2_uqsub_zpzz_s helper_sve2_uqsub_zpzz_s_aarch64
+#define helper_sve2_uqsub_zpzz_d helper_sve2_uqsub_zpzz_d_aarch64
+#define helper_sve2_suqadd_zpzz_b helper_sve2_suqadd_zpzz_b_aarch64
+#define helper_sve2_suqadd_zpzz_h helper_sve2_suqadd_zpzz_h_aarch64
+#define helper_sve2_suqadd_zpzz_s helper_sve2_suqadd_zpzz_s_aarch64
+#define helper_sve2_suqadd_zpzz_d helper_sve2_suqadd_zpzz_d_aarch64
+#define helper_sve2_usqadd_zpzz_b helper_sve2_usqadd_zpzz_b_aarch64
+#define helper_sve2_usqadd_zpzz_h helper_sve2_usqadd_zpzz_h_aarch64
+#define helper_sve2_usqadd_zpzz_s helper_sve2_usqadd_zpzz_s_aarch64
+#define helper_sve2_usqadd_zpzz_d helper_sve2_usqadd_zpzz_d_aarch64
+#define helper_sve2_eor3 helper_sve2_eor3_aarch64
+#define helper_sve2_bcax helper_sve2_bcax_aarch64
+#define helper_sve2_bsl1n helper_sve2_bsl1n_aarch64
+#define helper_sve2_bsl2n helper_sve2_bsl2n_aarch64
+#define helper_sve2_nbsl helper_sve2_nbsl_aarch64
+#define helper_sve2_xar_b helper_sve2_xar_b_aarch64
+#define helper_sve2_xar_h helper_sve2_xar_h_aarch64
+#define helper_sve2_xar_s helper_sve2_xar_s_aarch64
+#define helper_sve2_xar_d helper_sve2_xar_d_aarch64
+#define helper_fmmla_s helper_fmmla_s_aarch64
+#define helper_fmmla_d helper_fmmla_d_aarch64
 #define helper_sve2_pmull_h helper_sve2_pmull_h_aarch64
+#define helper_sve2_pmull_d helper_sve2_pmull_d_aarch64
+#define helper_sve2_eoril_b helper_sve2_eoril_b_aarch64
+#define helper_sve2_eoril_h helper_sve2_eoril_h_aarch64
+#define helper_sve2_eoril_s helper_sve2_eoril_s_aarch64
+#define helper_sve2_eoril_d helper_sve2_eoril_d_aarch64
+#define helper_sve2_bext_b helper_sve2_bext_b_aarch64
+#define helper_sve2_bext_h helper_sve2_bext_h_aarch64
+#define helper_sve2_bext_s helper_sve2_bext_s_aarch64
+#define helper_sve2_bext_d helper_sve2_bext_d_aarch64
+#define helper_sve2_bdep_b helper_sve2_bdep_b_aarch64
+#define helper_sve2_bdep_h helper_sve2_bdep_h_aarch64
+#define helper_sve2_bdep_s helper_sve2_bdep_s_aarch64
+#define helper_sve2_bdep_d helper_sve2_bdep_d_aarch64
+#define helper_sve2_bgrp_b helper_sve2_bgrp_b_aarch64
+#define helper_sve2_bgrp_h helper_sve2_bgrp_h_aarch64
+#define helper_sve2_bgrp_s helper_sve2_bgrp_s_aarch64
+#define helper_sve2_bgrp_d helper_sve2_bgrp_d_aarch64
+#define helper_sve2_cadd_b helper_sve2_cadd_b_aarch64
+#define helper_sve2_cadd_h helper_sve2_cadd_h_aarch64
+#define helper_sve2_cadd_s helper_sve2_cadd_s_aarch64
+#define helper_sve2_cadd_d helper_sve2_cadd_d_aarch64
+#define helper_sve2_sqcadd_b helper_sve2_sqcadd_b_aarch64
+#define helper_sve2_sqcadd_h helper_sve2_sqcadd_h_aarch64
+#define helper_sve2_sqcadd_s helper_sve2_sqcadd_s_aarch64
+#define helper_sve2_sqcadd_d helper_sve2_sqcadd_d_aarch64
+#define helper_sve2_smulh_zzz_b helper_sve2_smulh_zzz_b_aarch64
+#define helper_sve2_smulh_zzz_h helper_sve2_smulh_zzz_h_aarch64
+#define helper_sve2_smulh_zzz_s helper_sve2_smulh_zzz_s_aarch64
+#define helper_sve2_smulh_zzz_d helper_sve2_smulh_zzz_d_aarch64
+#define helper_sve2_umulh_zzz_b helper_sve2_umulh_zzz_b_aarch64
+#define helper_sve2_umulh_zzz_h helper_sve2_umulh_zzz_h_aarch64
+#define helper_sve2_umulh_zzz_s helper_sve2_umulh_zzz_s_aarch64
+#define helper_sve2_umulh_zzz_d helper_sve2_umulh_zzz_d_aarch64
+#define helper_sve2_sqdmulh_b helper_sve2_sqdmulh_b_aarch64
+#define helper_sve2_sqdmulh_h helper_sve2_sqdmulh_h_aarch64
+#define helper_sve2_sqdmulh_s helper_sve2_sqdmulh_s_aarch64
+#define helper_sve2_sqdmulh_d helper_sve2_sqdmulh_d_aarch64
+#define helper_sve2_sqrdmulh_b helper_sve2_sqrdmulh_b_aarch64
+#define helper_sve2_sqrdmulh_h helper_sve2_sqrdmulh_h_aarch64
+#define helper_sve2_sqrdmulh_s helper_sve2_sqrdmulh_s_aarch64
+#define helper_sve2_sqrdmulh_d helper_sve2_sqrdmulh_d_aarch64
+#define helper_sve2_sqrdmlah_b helper_sve2_sqrdmlah_b_aarch64
+#define helper_sve2_sqrdmlah_h helper_sve2_sqrdmlah_h_aarch64
+#define helper_sve2_sqrdmlah_s helper_sve2_sqrdmlah_s_aarch64
+#define helper_sve2_sqrdmlah_d helper_sve2_sqrdmlah_d_aarch64
+#define helper_sve2_sqrdmlsh_b helper_sve2_sqrdmlsh_b_aarch64
+#define helper_sve2_sqrdmlsh_h helper_sve2_sqrdmlsh_h_aarch64
+#define helper_sve2_sqrdmlsh_s helper_sve2_sqrdmlsh_s_aarch64
+#define helper_sve2_sqrdmlsh_d helper_sve2_sqrdmlsh_d_aarch64
+#define helper_sve2_cmla_zzzz_b helper_sve2_cmla_zzzz_b_aarch64
+#define helper_sve2_cmla_zzzz_h helper_sve2_cmla_zzzz_h_aarch64
+#define helper_sve2_cmla_zzzz_s helper_sve2_cmla_zzzz_s_aarch64
+#define helper_sve2_cmla_zzzz_d helper_sve2_cmla_zzzz_d_aarch64
+#define helper_sve2_sqrdcmlah_zzzz_b helper_sve2_sqrdcmlah_zzzz_b_aarch64
+#define helper_sve2_sqrdcmlah_zzzz_h helper_sve2_sqrdcmlah_zzzz_h_aarch64
+#define helper_sve2_sqrdcmlah_zzzz_s helper_sve2_sqrdcmlah_zzzz_s_aarch64
+#define helper_sve2_sqrdcmlah_zzzz_d helper_sve2_sqrdcmlah_zzzz_d_aarch64
+#define helper_sve2_cdot_zzzz_s helper_sve2_cdot_zzzz_s_aarch64
+#define helper_sve2_cdot_zzzz_d helper_sve2_cdot_zzzz_d_aarch64
+#define helper_sve2_mul_idx_h helper_sve2_mul_idx_h_aarch64
+#define helper_sve2_mul_idx_s helper_sve2_mul_idx_s_aarch64
+#define helper_sve2_mul_idx_d helper_sve2_mul_idx_d_aarch64
+#define helper_sve2_sqdmulh_idx_h helper_sve2_sqdmulh_idx_h_aarch64
+#define helper_sve2_sqdmulh_idx_s helper_sve2_sqdmulh_idx_s_aarch64
+#define helper_sve2_sqdmulh_idx_d helper_sve2_sqdmulh_idx_d_aarch64
+#define helper_sve2_sqrdmulh_idx_h helper_sve2_sqrdmulh_idx_h_aarch64
+#define helper_sve2_sqrdmulh_idx_s helper_sve2_sqrdmulh_idx_s_aarch64
+#define helper_sve2_sqrdmulh_idx_d helper_sve2_sqrdmulh_idx_d_aarch64
+#define helper_sve2_sqrdmlah_idx_h helper_sve2_sqrdmlah_idx_h_aarch64
+#define helper_sve2_sqrdmlah_idx_s helper_sve2_sqrdmlah_idx_s_aarch64
+#define helper_sve2_sqrdmlah_idx_d helper_sve2_sqrdmlah_idx_d_aarch64
+#define helper_sve2_sqrdmlsh_idx_h helper_sve2_sqrdmlsh_idx_h_aarch64
+#define helper_sve2_sqrdmlsh_idx_s helper_sve2_sqrdmlsh_idx_s_aarch64
+#define helper_sve2_sqrdmlsh_idx_d helper_sve2_sqrdmlsh_idx_d_aarch64
+#define helper_sve2_cmla_idx_h helper_sve2_cmla_idx_h_aarch64
+#define helper_sve2_cmla_idx_s helper_sve2_cmla_idx_s_aarch64
+#define helper_sve2_sqrdcmlah_idx_h helper_sve2_sqrdcmlah_idx_h_aarch64
+#define helper_sve2_sqrdcmlah_idx_s helper_sve2_sqrdcmlah_idx_s_aarch64
+#define helper_sve2_cdot_idx_s helper_sve2_cdot_idx_s_aarch64
+#define helper_sve2_cdot_idx_d helper_sve2_cdot_idx_d_aarch64
+#define helper_sve_bfcvt helper_sve_bfcvt_aarch64
+#define helper_sve_bfcvtnt helper_sve_bfcvtnt_aarch64
+#define helper_sve2_saddl_h helper_sve2_saddl_h_aarch64
+#define helper_sve2_saddl_s helper_sve2_saddl_s_aarch64
+#define helper_sve2_saddl_d helper_sve2_saddl_d_aarch64
+#define helper_sve2_uaddl_h helper_sve2_uaddl_h_aarch64
+#define helper_sve2_uaddl_s helper_sve2_uaddl_s_aarch64
+#define helper_sve2_uaddl_d helper_sve2_uaddl_d_aarch64
+#define helper_sve2_ssubl_h helper_sve2_ssubl_h_aarch64
+#define helper_sve2_ssubl_s helper_sve2_ssubl_s_aarch64
+#define helper_sve2_ssubl_d helper_sve2_ssubl_d_aarch64
+#define helper_sve2_usubl_h helper_sve2_usubl_h_aarch64
+#define helper_sve2_usubl_s helper_sve2_usubl_s_aarch64
+#define helper_sve2_usubl_d helper_sve2_usubl_d_aarch64
+#define helper_sve2_sabdl_h helper_sve2_sabdl_h_aarch64
+#define helper_sve2_sabdl_s helper_sve2_sabdl_s_aarch64
+#define helper_sve2_sabdl_d helper_sve2_sabdl_d_aarch64
+#define helper_sve2_uabdl_h helper_sve2_uabdl_h_aarch64
+#define helper_sve2_uabdl_s helper_sve2_uabdl_s_aarch64
+#define helper_sve2_uabdl_d helper_sve2_uabdl_d_aarch64
+#define helper_sve2_smull_zzz_h helper_sve2_smull_zzz_h_aarch64
+#define helper_sve2_smull_zzz_s helper_sve2_smull_zzz_s_aarch64
+#define helper_sve2_smull_zzz_d helper_sve2_smull_zzz_d_aarch64
+#define helper_sve2_umull_zzz_h helper_sve2_umull_zzz_h_aarch64
+#define helper_sve2_umull_zzz_s helper_sve2_umull_zzz_s_aarch64
+#define helper_sve2_umull_zzz_d helper_sve2_umull_zzz_d_aarch64
+#define helper_sve2_sqdmull_zzz_h helper_sve2_sqdmull_zzz_h_aarch64
+#define helper_sve2_sqdmull_zzz_s helper_sve2_sqdmull_zzz_s_aarch64
+#define helper_sve2_sqdmull_zzz_d helper_sve2_sqdmull_zzz_d_aarch64
+#define helper_sve2_smull_idx_s helper_sve2_smull_idx_s_aarch64
+#define helper_sve2_smull_idx_d helper_sve2_smull_idx_d_aarch64
+#define helper_sve2_umull_idx_s helper_sve2_umull_idx_s_aarch64
+#define helper_sve2_umull_idx_d helper_sve2_umull_idx_d_aarch64
+#define helper_sve2_sqdmull_idx_s helper_sve2_sqdmull_idx_s_aarch64
+#define helper_sve2_sqdmull_idx_d helper_sve2_sqdmull_idx_d_aarch64
+#define helper_sve2_sabal_h helper_sve2_sabal_h_aarch64
+#define helper_sve2_sabal_s helper_sve2_sabal_s_aarch64
+#define helper_sve2_sabal_d helper_sve2_sabal_d_aarch64
+#define helper_sve2_uabal_h helper_sve2_uabal_h_aarch64
+#define helper_sve2_uabal_s helper_sve2_uabal_s_aarch64
+#define helper_sve2_uabal_d helper_sve2_uabal_d_aarch64
+#define helper_sve2_smlal_zzzw_h helper_sve2_smlal_zzzw_h_aarch64
+#define helper_sve2_smlal_zzzw_s helper_sve2_smlal_zzzw_s_aarch64
+#define helper_sve2_smlal_zzzw_d helper_sve2_smlal_zzzw_d_aarch64
+#define helper_sve2_umlal_zzzw_h helper_sve2_umlal_zzzw_h_aarch64
+#define helper_sve2_umlal_zzzw_s helper_sve2_umlal_zzzw_s_aarch64
+#define helper_sve2_umlal_zzzw_d helper_sve2_umlal_zzzw_d_aarch64
+#define helper_sve2_smlsl_zzzw_h helper_sve2_smlsl_zzzw_h_aarch64
+#define helper_sve2_smlsl_zzzw_s helper_sve2_smlsl_zzzw_s_aarch64
+#define helper_sve2_smlsl_zzzw_d helper_sve2_smlsl_zzzw_d_aarch64
+#define helper_sve2_umlsl_zzzw_h helper_sve2_umlsl_zzzw_h_aarch64
+#define helper_sve2_umlsl_zzzw_s helper_sve2_umlsl_zzzw_s_aarch64
+#define helper_sve2_umlsl_zzzw_d helper_sve2_umlsl_zzzw_d_aarch64
+#define helper_sve2_sqdmlal_zzzw_h helper_sve2_sqdmlal_zzzw_h_aarch64
+#define helper_sve2_sqdmlal_zzzw_s helper_sve2_sqdmlal_zzzw_s_aarch64
+#define helper_sve2_sqdmlal_zzzw_d helper_sve2_sqdmlal_zzzw_d_aarch64
+#define helper_sve2_sqdmlsl_zzzw_h helper_sve2_sqdmlsl_zzzw_h_aarch64
+#define helper_sve2_sqdmlsl_zzzw_s helper_sve2_sqdmlsl_zzzw_s_aarch64
+#define helper_sve2_sqdmlsl_zzzw_d helper_sve2_sqdmlsl_zzzw_d_aarch64
+#define helper_sve2_smlal_idx_s helper_sve2_smlal_idx_s_aarch64
+#define helper_sve2_smlal_idx_d helper_sve2_smlal_idx_d_aarch64
+#define helper_sve2_umlal_idx_s helper_sve2_umlal_idx_s_aarch64
+#define helper_sve2_umlal_idx_d helper_sve2_umlal_idx_d_aarch64
+#define helper_sve2_smlsl_idx_s helper_sve2_smlsl_idx_s_aarch64
+#define helper_sve2_smlsl_idx_d helper_sve2_smlsl_idx_d_aarch64
+#define helper_sve2_umlsl_idx_s helper_sve2_umlsl_idx_s_aarch64
+#define helper_sve2_umlsl_idx_d helper_sve2_umlsl_idx_d_aarch64
+#define helper_sve2_sqdmlal_idx_s helper_sve2_sqdmlal_idx_s_aarch64
+#define helper_sve2_sqdmlal_idx_d helper_sve2_sqdmlal_idx_d_aarch64
+#define helper_sve2_sqdmlsl_idx_s helper_sve2_sqdmlsl_idx_s_aarch64
+#define helper_sve2_sqdmlsl_idx_d helper_sve2_sqdmlsl_idx_d_aarch64
+#define helper_sve2_saddw_h helper_sve2_saddw_h_aarch64
+#define helper_sve2_saddw_s helper_sve2_saddw_s_aarch64
+#define helper_sve2_saddw_d helper_sve2_saddw_d_aarch64
+#define helper_sve2_uaddw_h helper_sve2_uaddw_h_aarch64
+#define helper_sve2_uaddw_s helper_sve2_uaddw_s_aarch64
+#define helper_sve2_uaddw_d helper_sve2_uaddw_d_aarch64
+#define helper_sve2_ssubw_h helper_sve2_ssubw_h_aarch64
+#define helper_sve2_ssubw_s helper_sve2_ssubw_s_aarch64
+#define helper_sve2_ssubw_d helper_sve2_ssubw_d_aarch64
+#define helper_sve2_usubw_h helper_sve2_usubw_h_aarch64
+#define helper_sve2_usubw_s helper_sve2_usubw_s_aarch64
+#define helper_sve2_usubw_d helper_sve2_usubw_d_aarch64
+#define helper_sve2_sshll_h helper_sve2_sshll_h_aarch64
+#define helper_sve2_sshll_s helper_sve2_sshll_s_aarch64
+#define helper_sve2_sshll_d helper_sve2_sshll_d_aarch64
+#define helper_sve2_ushll_h helper_sve2_ushll_h_aarch64
+#define helper_sve2_ushll_s helper_sve2_ushll_s_aarch64
+#define helper_sve2_ushll_d helper_sve2_ushll_d_aarch64
+#define helper_sve2_ssra_b helper_sve2_ssra_b_aarch64
+#define helper_sve2_ssra_h helper_sve2_ssra_h_aarch64
+#define helper_sve2_ssra_s helper_sve2_ssra_s_aarch64
+#define helper_sve2_ssra_d helper_sve2_ssra_d_aarch64
+#define helper_sve2_usra_b helper_sve2_usra_b_aarch64
+#define helper_sve2_usra_h helper_sve2_usra_h_aarch64
+#define helper_sve2_usra_s helper_sve2_usra_s_aarch64
+#define helper_sve2_usra_d helper_sve2_usra_d_aarch64
+#define helper_sve2_srsra_b helper_sve2_srsra_b_aarch64
+#define helper_sve2_srsra_h helper_sve2_srsra_h_aarch64
+#define helper_sve2_srsra_s helper_sve2_srsra_s_aarch64
+#define helper_sve2_srsra_d helper_sve2_srsra_d_aarch64
+#define helper_sve2_ursra_b helper_sve2_ursra_b_aarch64
+#define helper_sve2_ursra_h helper_sve2_ursra_h_aarch64
+#define helper_sve2_ursra_s helper_sve2_ursra_s_aarch64
+#define helper_sve2_ursra_d helper_sve2_ursra_d_aarch64
+#define helper_sve2_sqshrunb_h helper_sve2_sqshrunb_h_aarch64
+#define helper_sve2_sqshrunb_s helper_sve2_sqshrunb_s_aarch64
+#define helper_sve2_sqshrunb_d helper_sve2_sqshrunb_d_aarch64
+#define helper_sve2_sqshrunt_h helper_sve2_sqshrunt_h_aarch64
+#define helper_sve2_sqshrunt_s helper_sve2_sqshrunt_s_aarch64
+#define helper_sve2_sqshrunt_d helper_sve2_sqshrunt_d_aarch64
+#define helper_sve2_sqrshrunb_h helper_sve2_sqrshrunb_h_aarch64
+#define helper_sve2_sqrshrunb_s helper_sve2_sqrshrunb_s_aarch64
+#define helper_sve2_sqrshrunb_d helper_sve2_sqrshrunb_d_aarch64
+#define helper_sve2_sqrshrunt_h helper_sve2_sqrshrunt_h_aarch64
+#define helper_sve2_sqrshrunt_s helper_sve2_sqrshrunt_s_aarch64
+#define helper_sve2_sqrshrunt_d helper_sve2_sqrshrunt_d_aarch64
+#define helper_sve2_shrnb_h helper_sve2_shrnb_h_aarch64
+#define helper_sve2_shrnb_s helper_sve2_shrnb_s_aarch64
+#define helper_sve2_shrnb_d helper_sve2_shrnb_d_aarch64
+#define helper_sve2_shrnt_h helper_sve2_shrnt_h_aarch64
+#define helper_sve2_shrnt_s helper_sve2_shrnt_s_aarch64
+#define helper_sve2_shrnt_d helper_sve2_shrnt_d_aarch64
+#define helper_sve2_rshrnb_h helper_sve2_rshrnb_h_aarch64
+#define helper_sve2_rshrnb_s helper_sve2_rshrnb_s_aarch64
+#define helper_sve2_rshrnb_d helper_sve2_rshrnb_d_aarch64
+#define helper_sve2_rshrnt_h helper_sve2_rshrnt_h_aarch64
+#define helper_sve2_rshrnt_s helper_sve2_rshrnt_s_aarch64
+#define helper_sve2_rshrnt_d helper_sve2_rshrnt_d_aarch64
+#define helper_sve2_sqshrnb_h helper_sve2_sqshrnb_h_aarch64
+#define helper_sve2_sqshrnb_s helper_sve2_sqshrnb_s_aarch64
+#define helper_sve2_sqshrnb_d helper_sve2_sqshrnb_d_aarch64
+#define helper_sve2_sqshrnt_h helper_sve2_sqshrnt_h_aarch64
+#define helper_sve2_sqshrnt_s helper_sve2_sqshrnt_s_aarch64
+#define helper_sve2_sqshrnt_d helper_sve2_sqshrnt_d_aarch64
+#define helper_sve2_sqrshrnb_h helper_sve2_sqrshrnb_h_aarch64
+#define helper_sve2_sqrshrnb_s helper_sve2_sqrshrnb_s_aarch64
+#define helper_sve2_sqrshrnb_d helper_sve2_sqrshrnb_d_aarch64
+#define helper_sve2_sqrshrnt_h helper_sve2_sqrshrnt_h_aarch64
+#define helper_sve2_sqrshrnt_s helper_sve2_sqrshrnt_s_aarch64
+#define helper_sve2_sqrshrnt_d helper_sve2_sqrshrnt_d_aarch64
+#define helper_sve2_uqshrnb_h helper_sve2_uqshrnb_h_aarch64
+#define helper_sve2_uqshrnb_s helper_sve2_uqshrnb_s_aarch64
+#define helper_sve2_uqshrnb_d helper_sve2_uqshrnb_d_aarch64
+#define helper_sve2_uqshrnt_h helper_sve2_uqshrnt_h_aarch64
+#define helper_sve2_uqshrnt_s helper_sve2_uqshrnt_s_aarch64
+#define helper_sve2_uqshrnt_d helper_sve2_uqshrnt_d_aarch64
+#define helper_sve2_uqrshrnb_h helper_sve2_uqrshrnb_h_aarch64
+#define helper_sve2_uqrshrnb_s helper_sve2_uqrshrnb_s_aarch64
+#define helper_sve2_uqrshrnb_d helper_sve2_uqrshrnb_d_aarch64
+#define helper_sve2_uqrshrnt_h helper_sve2_uqrshrnt_h_aarch64
+#define helper_sve2_uqrshrnt_s helper_sve2_uqrshrnt_s_aarch64
+#define helper_sve2_uqrshrnt_d helper_sve2_uqrshrnt_d_aarch64
+#define helper_sve2_addhnb_h helper_sve2_addhnb_h_aarch64
+#define helper_sve2_addhnb_s helper_sve2_addhnb_s_aarch64
+#define helper_sve2_addhnb_d helper_sve2_addhnb_d_aarch64
+#define helper_sve2_addhnt_h helper_sve2_addhnt_h_aarch64
+#define helper_sve2_addhnt_s helper_sve2_addhnt_s_aarch64
+#define helper_sve2_addhnt_d helper_sve2_addhnt_d_aarch64
+#define helper_sve2_raddhnb_h helper_sve2_raddhnb_h_aarch64
+#define helper_sve2_raddhnb_s helper_sve2_raddhnb_s_aarch64
+#define helper_sve2_raddhnb_d helper_sve2_raddhnb_d_aarch64
+#define helper_sve2_raddhnt_h helper_sve2_raddhnt_h_aarch64
+#define helper_sve2_raddhnt_s helper_sve2_raddhnt_s_aarch64
+#define helper_sve2_raddhnt_d helper_sve2_raddhnt_d_aarch64
+#define helper_sve2_subhnb_h helper_sve2_subhnb_h_aarch64
+#define helper_sve2_subhnb_s helper_sve2_subhnb_s_aarch64
+#define helper_sve2_subhnb_d helper_sve2_subhnb_d_aarch64
+#define helper_sve2_subhnt_h helper_sve2_subhnt_h_aarch64
+#define helper_sve2_subhnt_s helper_sve2_subhnt_s_aarch64
+#define helper_sve2_subhnt_d helper_sve2_subhnt_d_aarch64
+#define helper_sve2_rsubhnb_h helper_sve2_rsubhnb_h_aarch64
+#define helper_sve2_rsubhnb_s helper_sve2_rsubhnb_s_aarch64
+#define helper_sve2_rsubhnb_d helper_sve2_rsubhnb_d_aarch64
+#define helper_sve2_rsubhnt_h helper_sve2_rsubhnt_h_aarch64
+#define helper_sve2_rsubhnt_s helper_sve2_rsubhnt_s_aarch64
+#define helper_sve2_rsubhnt_d helper_sve2_rsubhnt_d_aarch64
+#define helper_sve2_sqxtnb_h helper_sve2_sqxtnb_h_aarch64
+#define helper_sve2_sqxtnb_s helper_sve2_sqxtnb_s_aarch64
+#define helper_sve2_sqxtnb_d helper_sve2_sqxtnb_d_aarch64
+#define helper_sve2_sqxtnt_h helper_sve2_sqxtnt_h_aarch64
+#define helper_sve2_sqxtnt_s helper_sve2_sqxtnt_s_aarch64
+#define helper_sve2_sqxtnt_d helper_sve2_sqxtnt_d_aarch64
+#define helper_sve2_uqxtnb_h helper_sve2_uqxtnb_h_aarch64
+#define helper_sve2_uqxtnb_s helper_sve2_uqxtnb_s_aarch64
+#define helper_sve2_uqxtnb_d helper_sve2_uqxtnb_d_aarch64
+#define helper_sve2_uqxtnt_h helper_sve2_uqxtnt_h_aarch64
+#define helper_sve2_uqxtnt_s helper_sve2_uqxtnt_s_aarch64
+#define helper_sve2_uqxtnt_d helper_sve2_uqxtnt_d_aarch64
+#define helper_sve2_sqxtunb_h helper_sve2_sqxtunb_h_aarch64
+#define helper_sve2_sqxtunb_s helper_sve2_sqxtunb_s_aarch64
+#define helper_sve2_sqxtunb_d helper_sve2_sqxtunb_d_aarch64
+#define helper_sve2_sqxtunt_h helper_sve2_sqxtunt_h_aarch64
+#define helper_sve2_sqxtunt_s helper_sve2_sqxtunt_s_aarch64
+#define helper_sve2_sqxtunt_d helper_sve2_sqxtunt_d_aarch64
+#define helper_sve2_match_ppzz_b helper_sve2_match_ppzz_b_aarch64
+#define helper_sve2_match_ppzz_h helper_sve2_match_ppzz_h_aarch64
+#define helper_sve2_nmatch_ppzz_b helper_sve2_nmatch_ppzz_b_aarch64
+#define helper_sve2_nmatch_ppzz_h helper_sve2_nmatch_ppzz_h_aarch64
+#define helper_sve2_histcnt_s helper_sve2_histcnt_s_aarch64
+#define helper_sve2_histcnt_d helper_sve2_histcnt_d_aarch64
+#define helper_sve2_histseg helper_sve2_histseg_aarch64
+#define helper_sve2_adcl_s helper_sve2_adcl_s_aarch64
+#define helper_sve2_adcl_d helper_sve2_adcl_d_aarch64
 #define helper_vfp_get_fpscr helper_vfp_get_fpscr_aarch64
 #define vfp_get_fpscr vfp_get_fpscr_aarch64
 #define helper_vfp_set_fpscr helper_vfp_set_fpscr_aarch64
 #define vfp_set_fpscr vfp_set_fpscr_aarch64
+#define helper_mve_vctp helper_mve_vctp_aarch64
+#define helper_mve_vpnot helper_mve_vpnot_aarch64
+#define helper_mve_vpsel helper_mve_vpsel_aarch64
+#define helper_mve_vdup helper_mve_vdup_aarch64
+#define helper_mve_vmovi helper_mve_vmovi_aarch64
+#define helper_mve_vandi helper_mve_vandi_aarch64
+#define helper_mve_vorri helper_mve_vorri_aarch64
+#define helper_mve_vidupb helper_mve_vidupb_aarch64
+#define helper_mve_viduph helper_mve_viduph_aarch64
+#define helper_mve_vidupw helper_mve_vidupw_aarch64
+#define helper_mve_viwdupb helper_mve_viwdupb_aarch64
+#define helper_mve_viwduph helper_mve_viwduph_aarch64
+#define helper_mve_viwdupw helper_mve_viwdupw_aarch64
+#define helper_mve_vdwdupb helper_mve_vdwdupb_aarch64
+#define helper_mve_vdwduph helper_mve_vdwduph_aarch64
+#define helper_mve_vdwdupw helper_mve_vdwdupw_aarch64
+#define helper_mve_vcmpeqb helper_mve_vcmpeqb_aarch64
+#define helper_mve_vcmpeqh helper_mve_vcmpeqh_aarch64
+#define helper_mve_vcmpeqw helper_mve_vcmpeqw_aarch64
+#define helper_mve_vcmpeq_scalarb helper_mve_vcmpeq_scalarb_aarch64
+#define helper_mve_vcmpeq_scalarh helper_mve_vcmpeq_scalarh_aarch64
+#define helper_mve_vcmpeq_scalarw helper_mve_vcmpeq_scalarw_aarch64
+#define helper_mve_vcmpneb helper_mve_vcmpneb_aarch64
+#define helper_mve_vcmpneh helper_mve_vcmpneh_aarch64
+#define helper_mve_vcmpnew helper_mve_vcmpnew_aarch64
+#define helper_mve_vcmpne_scalarb helper_mve_vcmpne_scalarb_aarch64
+#define helper_mve_vcmpne_scalarh helper_mve_vcmpne_scalarh_aarch64
+#define helper_mve_vcmpne_scalarw helper_mve_vcmpne_scalarw_aarch64
+#define helper_mve_vcmpcsb helper_mve_vcmpcsb_aarch64
+#define helper_mve_vcmpcsh helper_mve_vcmpcsh_aarch64
+#define helper_mve_vcmpcsw helper_mve_vcmpcsw_aarch64
+#define helper_mve_vcmpcs_scalarb helper_mve_vcmpcs_scalarb_aarch64
+#define helper_mve_vcmpcs_scalarh helper_mve_vcmpcs_scalarh_aarch64
+#define helper_mve_vcmpcs_scalarw helper_mve_vcmpcs_scalarw_aarch64
+#define helper_mve_vcmphib helper_mve_vcmphib_aarch64
+#define helper_mve_vcmphih helper_mve_vcmphih_aarch64
+#define helper_mve_vcmphiw helper_mve_vcmphiw_aarch64
+#define helper_mve_vcmphi_scalarb helper_mve_vcmphi_scalarb_aarch64
+#define helper_mve_vcmphi_scalarh helper_mve_vcmphi_scalarh_aarch64
+#define helper_mve_vcmphi_scalarw helper_mve_vcmphi_scalarw_aarch64
+#define helper_mve_vcmpgeb helper_mve_vcmpgeb_aarch64
+#define helper_mve_vcmpgeh helper_mve_vcmpgeh_aarch64
+#define helper_mve_vcmpgew helper_mve_vcmpgew_aarch64
+#define helper_mve_vcmpge_scalarb helper_mve_vcmpge_scalarb_aarch64
+#define helper_mve_vcmpge_scalarh helper_mve_vcmpge_scalarh_aarch64
+#define helper_mve_vcmpge_scalarw helper_mve_vcmpge_scalarw_aarch64
+#define helper_mve_vcmpltb helper_mve_vcmpltb_aarch64
+#define helper_mve_vcmplth helper_mve_vcmplth_aarch64
+#define helper_mve_vcmpltw helper_mve_vcmpltw_aarch64
+#define helper_mve_vcmplt_scalarb helper_mve_vcmplt_scalarb_aarch64
+#define helper_mve_vcmplt_scalarh helper_mve_vcmplt_scalarh_aarch64
+#define helper_mve_vcmplt_scalarw helper_mve_vcmplt_scalarw_aarch64
+#define helper_mve_vcmpgtb helper_mve_vcmpgtb_aarch64
+#define helper_mve_vcmpgth helper_mve_vcmpgth_aarch64
+#define helper_mve_vcmpgtw helper_mve_vcmpgtw_aarch64
+#define helper_mve_vcmpgt_scalarb helper_mve_vcmpgt_scalarb_aarch64
+#define helper_mve_vcmpgt_scalarh helper_mve_vcmpgt_scalarh_aarch64
+#define helper_mve_vcmpgt_scalarw helper_mve_vcmpgt_scalarw_aarch64
+#define helper_mve_vcmpleb helper_mve_vcmpleb_aarch64
+#define helper_mve_vcmpleh helper_mve_vcmpleh_aarch64
+#define helper_mve_vcmplew helper_mve_vcmplew_aarch64
+#define helper_mve_vcmple_scalarb helper_mve_vcmple_scalarb_aarch64
+#define helper_mve_vcmple_scalarh helper_mve_vcmple_scalarh_aarch64
+#define helper_mve_vcmple_scalarw helper_mve_vcmple_scalarw_aarch64
+#define helper_mve_vfcmpeqh helper_mve_vfcmpeqh_aarch64
+#define helper_mve_vfcmpeqs helper_mve_vfcmpeqs_aarch64
+#define helper_mve_vfcmpneh helper_mve_vfcmpneh_aarch64
+#define helper_mve_vfcmpnes helper_mve_vfcmpnes_aarch64
+#define helper_mve_vfcmpgeh helper_mve_vfcmpgeh_aarch64
+#define helper_mve_vfcmpges helper_mve_vfcmpges_aarch64
+#define helper_mve_vfcmplth helper_mve_vfcmplth_aarch64
+#define helper_mve_vfcmplts helper_mve_vfcmplts_aarch64
+#define helper_mve_vfcmpgth helper_mve_vfcmpgth_aarch64
+#define helper_mve_vfcmpgts helper_mve_vfcmpgts_aarch64
+#define helper_mve_vfcmpleh helper_mve_vfcmpleh_aarch64
+#define helper_mve_vfcmples helper_mve_vfcmples_aarch64
+#define helper_mve_vfcmpeq_scalarh helper_mve_vfcmpeq_scalarh_aarch64
+#define helper_mve_vfcmpeq_scalars helper_mve_vfcmpeq_scalars_aarch64
+#define helper_mve_vfcmpne_scalarh helper_mve_vfcmpne_scalarh_aarch64
+#define helper_mve_vfcmpne_scalars helper_mve_vfcmpne_scalars_aarch64
+#define helper_mve_vfcmpge_scalarh helper_mve_vfcmpge_scalarh_aarch64
+#define helper_mve_vfcmpge_scalars helper_mve_vfcmpge_scalars_aarch64
+#define helper_mve_vfcmplt_scalarh helper_mve_vfcmplt_scalarh_aarch64
+#define helper_mve_vfcmplt_scalars helper_mve_vfcmplt_scalars_aarch64
+#define helper_mve_vfcmpgt_scalarh helper_mve_vfcmpgt_scalarh_aarch64
+#define helper_mve_vfcmpgt_scalars helper_mve_vfcmpgt_scalars_aarch64
+#define helper_mve_vfcmple_scalarh helper_mve_vfcmple_scalarh_aarch64
+#define helper_mve_vfcmple_scalars helper_mve_vfcmple_scalars_aarch64
+#define helper_mve_vfabsh helper_mve_vfabsh_aarch64
+#define helper_mve_vfabss helper_mve_vfabss_aarch64
+#define helper_mve_vfnegh helper_mve_vfnegh_aarch64
+#define helper_mve_vfnegs helper_mve_vfnegs_aarch64
+#define helper_mve_vldrb helper_mve_vldrb_aarch64
+#define helper_mve_vldrh helper_mve_vldrh_aarch64
+#define helper_mve_vldrw helper_mve_vldrw_aarch64
+#define helper_mve_vldrb_sh helper_mve_vldrb_sh_aarch64
+#define helper_mve_vldrb_uh helper_mve_vldrb_uh_aarch64
+#define helper_mve_vldrb_sw helper_mve_vldrb_sw_aarch64
+#define helper_mve_vldrb_uw helper_mve_vldrb_uw_aarch64
+#define helper_mve_vldrh_sw helper_mve_vldrh_sw_aarch64
+#define helper_mve_vldrh_uw helper_mve_vldrh_uw_aarch64
+#define helper_mve_vstrb helper_mve_vstrb_aarch64
+#define helper_mve_vstrh helper_mve_vstrh_aarch64
+#define helper_mve_vstrw helper_mve_vstrw_aarch64
+#define helper_mve_vstrb_h helper_mve_vstrb_h_aarch64
+#define helper_mve_vstrb_w helper_mve_vstrb_w_aarch64
+#define helper_mve_vstrh_w helper_mve_vstrh_w_aarch64
+#define helper_mve_vldrb_sg_sh helper_mve_vldrb_sg_sh_aarch64
+#define helper_mve_vldrb_sg_sw helper_mve_vldrb_sg_sw_aarch64
+#define helper_mve_vldrh_sg_sw helper_mve_vldrh_sg_sw_aarch64
+#define helper_mve_vldrb_sg_ub helper_mve_vldrb_sg_ub_aarch64
+#define helper_mve_vldrb_sg_uh helper_mve_vldrb_sg_uh_aarch64
+#define helper_mve_vldrb_sg_uw helper_mve_vldrb_sg_uw_aarch64
+#define helper_mve_vldrh_sg_uh helper_mve_vldrh_sg_uh_aarch64
+#define helper_mve_vldrh_sg_uw helper_mve_vldrh_sg_uw_aarch64
+#define helper_mve_vldrw_sg_uw helper_mve_vldrw_sg_uw_aarch64
+#define helper_mve_vldrd_sg_ud helper_mve_vldrd_sg_ud_aarch64
+#define helper_mve_vldrh_sg_os_sw helper_mve_vldrh_sg_os_sw_aarch64
+#define helper_mve_vldrh_sg_os_uh helper_mve_vldrh_sg_os_uh_aarch64
+#define helper_mve_vldrh_sg_os_uw helper_mve_vldrh_sg_os_uw_aarch64
+#define helper_mve_vldrw_sg_os_uw helper_mve_vldrw_sg_os_uw_aarch64
+#define helper_mve_vldrd_sg_os_ud helper_mve_vldrd_sg_os_ud_aarch64
+#define helper_mve_vstrb_sg_ub helper_mve_vstrb_sg_ub_aarch64
+#define helper_mve_vstrb_sg_uh helper_mve_vstrb_sg_uh_aarch64
+#define helper_mve_vstrb_sg_uw helper_mve_vstrb_sg_uw_aarch64
+#define helper_mve_vstrh_sg_uh helper_mve_vstrh_sg_uh_aarch64
+#define helper_mve_vstrh_sg_uw helper_mve_vstrh_sg_uw_aarch64
+#define helper_mve_vstrw_sg_uw helper_mve_vstrw_sg_uw_aarch64
+#define helper_mve_vstrd_sg_ud helper_mve_vstrd_sg_ud_aarch64
+#define helper_mve_vstrh_sg_os_uh helper_mve_vstrh_sg_os_uh_aarch64
+#define helper_mve_vstrh_sg_os_uw helper_mve_vstrh_sg_os_uw_aarch64
+#define helper_mve_vstrw_sg_os_uw helper_mve_vstrw_sg_os_uw_aarch64
+#define helper_mve_vstrd_sg_os_ud helper_mve_vstrd_sg_os_ud_aarch64
+#define helper_mve_vldrw_sg_wb_uw helper_mve_vldrw_sg_wb_uw_aarch64
+#define helper_mve_vldrd_sg_wb_ud helper_mve_vldrd_sg_wb_ud_aarch64
+#define helper_mve_vstrw_sg_wb_uw helper_mve_vstrw_sg_wb_uw_aarch64
+#define helper_mve_vstrd_sg_wb_ud helper_mve_vstrd_sg_wb_ud_aarch64
+#define helper_mve_vld20b helper_mve_vld20b_aarch64
+#define helper_mve_vld20h helper_mve_vld20h_aarch64
+#define helper_mve_vld20w helper_mve_vld20w_aarch64
+#define helper_mve_vld21b helper_mve_vld21b_aarch64
+#define helper_mve_vld21h helper_mve_vld21h_aarch64
+#define helper_mve_vld21w helper_mve_vld21w_aarch64
+#define helper_mve_vld40b helper_mve_vld40b_aarch64
+#define helper_mve_vld40h helper_mve_vld40h_aarch64
+#define helper_mve_vld40w helper_mve_vld40w_aarch64
+#define helper_mve_vld41b helper_mve_vld41b_aarch64
+#define helper_mve_vld41h helper_mve_vld41h_aarch64
+#define helper_mve_vld41w helper_mve_vld41w_aarch64
+#define helper_mve_vld42b helper_mve_vld42b_aarch64
+#define helper_mve_vld42h helper_mve_vld42h_aarch64
+#define helper_mve_vld42w helper_mve_vld42w_aarch64
+#define helper_mve_vld43b helper_mve_vld43b_aarch64
+#define helper_mve_vld43h helper_mve_vld43h_aarch64
+#define helper_mve_vld43w helper_mve_vld43w_aarch64
+#define helper_mve_vst20b helper_mve_vst20b_aarch64
+#define helper_mve_vst20h helper_mve_vst20h_aarch64
+#define helper_mve_vst20w helper_mve_vst20w_aarch64
+#define helper_mve_vst21b helper_mve_vst21b_aarch64
+#define helper_mve_vst21h helper_mve_vst21h_aarch64
+#define helper_mve_vst21w helper_mve_vst21w_aarch64
+#define helper_mve_vst40b helper_mve_vst40b_aarch64
+#define helper_mve_vst40h helper_mve_vst40h_aarch64
+#define helper_mve_vst40w helper_mve_vst40w_aarch64
+#define helper_mve_vst41b helper_mve_vst41b_aarch64
+#define helper_mve_vst41h helper_mve_vst41h_aarch64
+#define helper_mve_vst41w helper_mve_vst41w_aarch64
+#define helper_mve_vst42b helper_mve_vst42b_aarch64
+#define helper_mve_vst42h helper_mve_vst42h_aarch64
+#define helper_mve_vst42w helper_mve_vst42w_aarch64
+#define helper_mve_vst43b helper_mve_vst43b_aarch64
+#define helper_mve_vst43h helper_mve_vst43h_aarch64
+#define helper_mve_vst43w helper_mve_vst43w_aarch64
+#define helper_mve_vand helper_mve_vand_aarch64
+#define helper_mve_vbic helper_mve_vbic_aarch64
+#define helper_mve_vorr helper_mve_vorr_aarch64
+#define helper_mve_vorn helper_mve_vorn_aarch64
+#define helper_mve_veor helper_mve_veor_aarch64
+#define helper_mve_vaddb helper_mve_vaddb_aarch64
+#define helper_mve_vaddh helper_mve_vaddh_aarch64
+#define helper_mve_vaddw helper_mve_vaddw_aarch64
+#define helper_mve_vadd_scalarb helper_mve_vadd_scalarb_aarch64
+#define helper_mve_vadd_scalarh helper_mve_vadd_scalarh_aarch64
+#define helper_mve_vadd_scalarw helper_mve_vadd_scalarw_aarch64
+#define helper_mve_vsubb helper_mve_vsubb_aarch64
+#define helper_mve_vsubh helper_mve_vsubh_aarch64
+#define helper_mve_vsubw helper_mve_vsubw_aarch64
+#define helper_mve_vsub_scalarb helper_mve_vsub_scalarb_aarch64
+#define helper_mve_vsub_scalarh helper_mve_vsub_scalarh_aarch64
+#define helper_mve_vsub_scalarw helper_mve_vsub_scalarw_aarch64
+#define helper_mve_vmulb helper_mve_vmulb_aarch64
+#define helper_mve_vmulh helper_mve_vmulh_aarch64
+#define helper_mve_vmulw helper_mve_vmulw_aarch64
+#define helper_mve_vmul_scalarb helper_mve_vmul_scalarb_aarch64
+#define helper_mve_vmul_scalarh helper_mve_vmul_scalarh_aarch64
+#define helper_mve_vmul_scalarw helper_mve_vmul_scalarw_aarch64
+#define helper_mve_vmulhsb helper_mve_vmulhsb_aarch64
+#define helper_mve_vmulhsh helper_mve_vmulhsh_aarch64
+#define helper_mve_vmulhsw helper_mve_vmulhsw_aarch64
+#define helper_mve_vmulhub helper_mve_vmulhub_aarch64
+#define helper_mve_vmulhuh helper_mve_vmulhuh_aarch64
+#define helper_mve_vmulhuw helper_mve_vmulhuw_aarch64
+#define helper_mve_vrmulhsb helper_mve_vrmulhsb_aarch64
+#define helper_mve_vrmulhsh helper_mve_vrmulhsh_aarch64
+#define helper_mve_vrmulhsw helper_mve_vrmulhsw_aarch64
+#define helper_mve_vrmulhub helper_mve_vrmulhub_aarch64
+#define helper_mve_vrmulhuh helper_mve_vrmulhuh_aarch64
+#define helper_mve_vrmulhuw helper_mve_vrmulhuw_aarch64
+#define helper_mve_vmullbsb helper_mve_vmullbsb_aarch64
+#define helper_mve_vmullbsh helper_mve_vmullbsh_aarch64
+#define helper_mve_vmullbsw helper_mve_vmullbsw_aarch64
+#define helper_mve_vmullbub helper_mve_vmullbub_aarch64
+#define helper_mve_vmullbuh helper_mve_vmullbuh_aarch64
+#define helper_mve_vmullbuw helper_mve_vmullbuw_aarch64
+#define helper_mve_vmulltsb helper_mve_vmulltsb_aarch64
+#define helper_mve_vmulltsh helper_mve_vmulltsh_aarch64
+#define helper_mve_vmulltsw helper_mve_vmulltsw_aarch64
+#define helper_mve_vmulltub helper_mve_vmulltub_aarch64
+#define helper_mve_vmulltuh helper_mve_vmulltuh_aarch64
+#define helper_mve_vmulltuw helper_mve_vmulltuw_aarch64
+#define helper_mve_vmullpbh helper_mve_vmullpbh_aarch64
+#define helper_mve_vmullpth helper_mve_vmullpth_aarch64
+#define helper_mve_vmullpbw helper_mve_vmullpbw_aarch64
+#define helper_mve_vmullptw helper_mve_vmullptw_aarch64
+#define helper_mve_vqdmullbh helper_mve_vqdmullbh_aarch64
+#define helper_mve_vqdmullbw helper_mve_vqdmullbw_aarch64
+#define helper_mve_vqdmullth helper_mve_vqdmullth_aarch64
+#define helper_mve_vqdmulltw helper_mve_vqdmulltw_aarch64
+#define helper_mve_vqdmullb_scalarh helper_mve_vqdmullb_scalarh_aarch64
+#define helper_mve_vqdmullb_scalarw helper_mve_vqdmullb_scalarw_aarch64
+#define helper_mve_vqdmullt_scalarh helper_mve_vqdmullt_scalarh_aarch64
+#define helper_mve_vqdmullt_scalarw helper_mve_vqdmullt_scalarw_aarch64
+#define helper_mve_vcadd90b helper_mve_vcadd90b_aarch64
+#define helper_mve_vcadd90h helper_mve_vcadd90h_aarch64
+#define helper_mve_vcadd90w helper_mve_vcadd90w_aarch64
+#define helper_mve_vcadd270b helper_mve_vcadd270b_aarch64
+#define helper_mve_vcadd270h helper_mve_vcadd270h_aarch64
+#define helper_mve_vcadd270w helper_mve_vcadd270w_aarch64
+#define helper_mve_vhcadd90b helper_mve_vhcadd90b_aarch64
+#define helper_mve_vhcadd90h helper_mve_vhcadd90h_aarch64
+#define helper_mve_vhcadd90w helper_mve_vhcadd90w_aarch64
+#define helper_mve_vhcadd270b helper_mve_vhcadd270b_aarch64
+#define helper_mve_vhcadd270h helper_mve_vhcadd270h_aarch64
+#define helper_mve_vhcadd270w helper_mve_vhcadd270w_aarch64
+#define helper_mve_vmaxsb helper_mve_vmaxsb_aarch64
+#define helper_mve_vmaxsh helper_mve_vmaxsh_aarch64
+#define helper_mve_vmaxsw helper_mve_vmaxsw_aarch64
+#define helper_mve_vmaxub helper_mve_vmaxub_aarch64
+#define helper_mve_vmaxuh helper_mve_vmaxuh_aarch64
+#define helper_mve_vmaxuw helper_mve_vmaxuw_aarch64
+#define helper_mve_vminsb helper_mve_vminsb_aarch64
+#define helper_mve_vminsh helper_mve_vminsh_aarch64
+#define helper_mve_vminsw helper_mve_vminsw_aarch64
+#define helper_mve_vminub helper_mve_vminub_aarch64
+#define helper_mve_vminuh helper_mve_vminuh_aarch64
+#define helper_mve_vminuw helper_mve_vminuw_aarch64
+#define helper_mve_vabdsb helper_mve_vabdsb_aarch64
+#define helper_mve_vabdsh helper_mve_vabdsh_aarch64
+#define helper_mve_vabdsw helper_mve_vabdsw_aarch64
+#define helper_mve_vabdub helper_mve_vabdub_aarch64
+#define helper_mve_vabduh helper_mve_vabduh_aarch64
+#define helper_mve_vabduw helper_mve_vabduw_aarch64
+#define helper_mve_vhaddsb helper_mve_vhaddsb_aarch64
+#define helper_mve_vhaddsh helper_mve_vhaddsh_aarch64
+#define helper_mve_vhaddsw helper_mve_vhaddsw_aarch64
+#define helper_mve_vhaddub helper_mve_vhaddub_aarch64
+#define helper_mve_vhadduh helper_mve_vhadduh_aarch64
+#define helper_mve_vhadduw helper_mve_vhadduw_aarch64
+#define helper_mve_vhadds_scalarb helper_mve_vhadds_scalarb_aarch64
+#define helper_mve_vhadds_scalarh helper_mve_vhadds_scalarh_aarch64
+#define helper_mve_vhadds_scalarw helper_mve_vhadds_scalarw_aarch64
+#define helper_mve_vhaddu_scalarb helper_mve_vhaddu_scalarb_aarch64
+#define helper_mve_vhaddu_scalarh helper_mve_vhaddu_scalarh_aarch64
+#define helper_mve_vhaddu_scalarw helper_mve_vhaddu_scalarw_aarch64
+#define helper_mve_vrhaddsb helper_mve_vrhaddsb_aarch64
+#define helper_mve_vrhaddsh helper_mve_vrhaddsh_aarch64
+#define helper_mve_vrhaddsw helper_mve_vrhaddsw_aarch64
+#define helper_mve_vrhaddub helper_mve_vrhaddub_aarch64
+#define helper_mve_vrhadduh helper_mve_vrhadduh_aarch64
+#define helper_mve_vrhadduw helper_mve_vrhadduw_aarch64
+#define helper_mve_vadc helper_mve_vadc_aarch64
+#define helper_mve_vadci helper_mve_vadci_aarch64
+#define helper_mve_vsbc helper_mve_vsbc_aarch64
+#define helper_mve_vsbci helper_mve_vsbci_aarch64
+#define helper_mve_vhsubsb helper_mve_vhsubsb_aarch64
+#define helper_mve_vhsubsh helper_mve_vhsubsh_aarch64
+#define helper_mve_vhsubsw helper_mve_vhsubsw_aarch64
+#define helper_mve_vhsubub helper_mve_vhsubub_aarch64
+#define helper_mve_vhsubuh helper_mve_vhsubuh_aarch64
+#define helper_mve_vhsubuw helper_mve_vhsubuw_aarch64
+#define helper_mve_vhsubs_scalarb helper_mve_vhsubs_scalarb_aarch64
+#define helper_mve_vhsubs_scalarh helper_mve_vhsubs_scalarh_aarch64
+#define helper_mve_vhsubs_scalarw helper_mve_vhsubs_scalarw_aarch64
+#define helper_mve_vhsubu_scalarb helper_mve_vhsubu_scalarb_aarch64
+#define helper_mve_vhsubu_scalarh helper_mve_vhsubu_scalarh_aarch64
+#define helper_mve_vhsubu_scalarw helper_mve_vhsubu_scalarw_aarch64
+#define helper_mve_vqaddsb helper_mve_vqaddsb_aarch64
+#define helper_mve_vqaddsh helper_mve_vqaddsh_aarch64
+#define helper_mve_vqaddsw helper_mve_vqaddsw_aarch64
+#define helper_mve_vqaddub helper_mve_vqaddub_aarch64
+#define helper_mve_vqadduh helper_mve_vqadduh_aarch64
+#define helper_mve_vqadduw helper_mve_vqadduw_aarch64
+#define helper_mve_vqsubsb helper_mve_vqsubsb_aarch64
+#define helper_mve_vqsubsh helper_mve_vqsubsh_aarch64
+#define helper_mve_vqsubsw helper_mve_vqsubsw_aarch64
+#define helper_mve_vqsubub helper_mve_vqsubub_aarch64
+#define helper_mve_vqsubuh helper_mve_vqsubuh_aarch64
+#define helper_mve_vqsubuw helper_mve_vqsubuw_aarch64
+#define helper_mve_vqdmulhb helper_mve_vqdmulhb_aarch64
+#define helper_mve_vqdmulhh helper_mve_vqdmulhh_aarch64
+#define helper_mve_vqdmulhw helper_mve_vqdmulhw_aarch64
+#define helper_mve_vqrdmulhb helper_mve_vqrdmulhb_aarch64
+#define helper_mve_vqrdmulhh helper_mve_vqrdmulhh_aarch64
+#define helper_mve_vqrdmulhw helper_mve_vqrdmulhw_aarch64
+#define helper_mve_vqadds_scalarb helper_mve_vqadds_scalarb_aarch64
+#define helper_mve_vqadds_scalarh helper_mve_vqadds_scalarh_aarch64
+#define helper_mve_vqadds_scalarw helper_mve_vqadds_scalarw_aarch64
+#define helper_mve_vqaddu_scalarb helper_mve_vqaddu_scalarb_aarch64
+#define helper_mve_vqaddu_scalarh helper_mve_vqaddu_scalarh_aarch64
+#define helper_mve_vqaddu_scalarw helper_mve_vqaddu_scalarw_aarch64
+#define helper_mve_vqsubs_scalarb helper_mve_vqsubs_scalarb_aarch64
+#define helper_mve_vqsubs_scalarh helper_mve_vqsubs_scalarh_aarch64
+#define helper_mve_vqsubs_scalarw helper_mve_vqsubs_scalarw_aarch64
+#define helper_mve_vqsubu_scalarb helper_mve_vqsubu_scalarb_aarch64
+#define helper_mve_vqsubu_scalarh helper_mve_vqsubu_scalarh_aarch64
+#define helper_mve_vqsubu_scalarw helper_mve_vqsubu_scalarw_aarch64
+#define helper_mve_vqdmulh_scalarb helper_mve_vqdmulh_scalarb_aarch64
+#define helper_mve_vqdmulh_scalarh helper_mve_vqdmulh_scalarh_aarch64
+#define helper_mve_vqdmulh_scalarw helper_mve_vqdmulh_scalarw_aarch64
+#define helper_mve_vqrdmulh_scalarb helper_mve_vqrdmulh_scalarb_aarch64
+#define helper_mve_vqrdmulh_scalarh helper_mve_vqrdmulh_scalarh_aarch64
+#define helper_mve_vqrdmulh_scalarw helper_mve_vqrdmulh_scalarw_aarch64
+#define helper_mve_vmlab helper_mve_vmlab_aarch64
+#define helper_mve_vmlah helper_mve_vmlah_aarch64
+#define helper_mve_vmlaw helper_mve_vmlaw_aarch64
+#define helper_mve_vmlasb helper_mve_vmlasb_aarch64
+#define helper_mve_vmlash helper_mve_vmlash_aarch64
+#define helper_mve_vmlasw helper_mve_vmlasw_aarch64
+#define helper_mve_vqdmlahb helper_mve_vqdmlahb_aarch64
+#define helper_mve_vqdmlahh helper_mve_vqdmlahh_aarch64
+#define helper_mve_vqdmlahw helper_mve_vqdmlahw_aarch64
+#define helper_mve_vqrdmlahb helper_mve_vqrdmlahb_aarch64
+#define helper_mve_vqrdmlahh helper_mve_vqrdmlahh_aarch64
+#define helper_mve_vqrdmlahw helper_mve_vqrdmlahw_aarch64
+#define helper_mve_vqdmlashb helper_mve_vqdmlashb_aarch64
+#define helper_mve_vqdmlashh helper_mve_vqdmlashh_aarch64
+#define helper_mve_vqdmlashw helper_mve_vqdmlashw_aarch64
+#define helper_mve_vqrdmlashb helper_mve_vqrdmlashb_aarch64
+#define helper_mve_vqrdmlashh helper_mve_vqrdmlashh_aarch64
+#define helper_mve_vqrdmlashw helper_mve_vqrdmlashw_aarch64
+#define helper_mve_vfaddh helper_mve_vfaddh_aarch64
+#define helper_mve_vfadds helper_mve_vfadds_aarch64
+#define helper_mve_vfsubh helper_mve_vfsubh_aarch64
+#define helper_mve_vfsubs helper_mve_vfsubs_aarch64
+#define helper_mve_vfmulh helper_mve_vfmulh_aarch64
+#define helper_mve_vfmuls helper_mve_vfmuls_aarch64
+#define helper_mve_vfabdh helper_mve_vfabdh_aarch64
+#define helper_mve_vfabds helper_mve_vfabds_aarch64
+#define helper_mve_vmaxnmh helper_mve_vmaxnmh_aarch64
+#define helper_mve_vmaxnms helper_mve_vmaxnms_aarch64
+#define helper_mve_vminnmh helper_mve_vminnmh_aarch64
+#define helper_mve_vminnms helper_mve_vminnms_aarch64
+#define helper_mve_vmaxnmah helper_mve_vmaxnmah_aarch64
+#define helper_mve_vmaxnmas helper_mve_vmaxnmas_aarch64
+#define helper_mve_vminnmah helper_mve_vminnmah_aarch64
+#define helper_mve_vminnmas helper_mve_vminnmas_aarch64
+#define helper_mve_vfcadd90h helper_mve_vfcadd90h_aarch64
+#define helper_mve_vfcadd90s helper_mve_vfcadd90s_aarch64
+#define helper_mve_vfcadd270h helper_mve_vfcadd270h_aarch64
+#define helper_mve_vfcadd270s helper_mve_vfcadd270s_aarch64
+#define helper_mve_vcmul0h helper_mve_vcmul0h_aarch64
+#define helper_mve_vcmul0s helper_mve_vcmul0s_aarch64
+#define helper_mve_vcmul90h helper_mve_vcmul90h_aarch64
+#define helper_mve_vcmul90s helper_mve_vcmul90s_aarch64
+#define helper_mve_vcmul180h helper_mve_vcmul180h_aarch64
+#define helper_mve_vcmul180s helper_mve_vcmul180s_aarch64
+#define helper_mve_vcmul270h helper_mve_vcmul270h_aarch64
+#define helper_mve_vcmul270s helper_mve_vcmul270s_aarch64
+#define helper_mve_vcmla0h helper_mve_vcmla0h_aarch64
+#define helper_mve_vcmla0s helper_mve_vcmla0s_aarch64
+#define helper_mve_vcmla90h helper_mve_vcmla90h_aarch64
+#define helper_mve_vcmla90s helper_mve_vcmla90s_aarch64
+#define helper_mve_vcmla180h helper_mve_vcmla180h_aarch64
+#define helper_mve_vcmla180s helper_mve_vcmla180s_aarch64
+#define helper_mve_vcmla270h helper_mve_vcmla270h_aarch64
+#define helper_mve_vcmla270s helper_mve_vcmla270s_aarch64
+#define helper_mve_vfmah helper_mve_vfmah_aarch64
+#define helper_mve_vfmas helper_mve_vfmas_aarch64
+#define helper_mve_vfmsh helper_mve_vfmsh_aarch64
+#define helper_mve_vfmss helper_mve_vfmss_aarch64
+#define helper_mve_vfadd_scalarh helper_mve_vfadd_scalarh_aarch64
+#define helper_mve_vfadd_scalars helper_mve_vfadd_scalars_aarch64
+#define helper_mve_vfsub_scalarh helper_mve_vfsub_scalarh_aarch64
+#define helper_mve_vfsub_scalars helper_mve_vfsub_scalars_aarch64
+#define helper_mve_vfmul_scalarh helper_mve_vfmul_scalarh_aarch64
+#define helper_mve_vfmul_scalars helper_mve_vfmul_scalars_aarch64
+#define helper_mve_vfma_scalarh helper_mve_vfma_scalarh_aarch64
+#define helper_mve_vfma_scalars helper_mve_vfma_scalars_aarch64
+#define helper_mve_vfmas_scalarh helper_mve_vfmas_scalarh_aarch64
+#define helper_mve_vfmas_scalars helper_mve_vfmas_scalars_aarch64
+#define helper_mve_vcvt_sh helper_mve_vcvt_sh_aarch64
+#define helper_mve_vcvt_uh helper_mve_vcvt_uh_aarch64
+#define helper_mve_vcvt_hs helper_mve_vcvt_hs_aarch64
+#define helper_mve_vcvt_hu helper_mve_vcvt_hu_aarch64
+#define helper_mve_vcvt_sf helper_mve_vcvt_sf_aarch64
+#define helper_mve_vcvt_uf helper_mve_vcvt_uf_aarch64
+#define helper_mve_vcvt_fs helper_mve_vcvt_fs_aarch64
+#define helper_mve_vcvt_fu helper_mve_vcvt_fu_aarch64
+#define helper_mve_vcvtb_sh helper_mve_vcvtb_sh_aarch64
+#define helper_mve_vcvtt_sh helper_mve_vcvtt_sh_aarch64
+#define helper_mve_vcvtb_hs helper_mve_vcvtb_hs_aarch64
+#define helper_mve_vcvtt_hs helper_mve_vcvtt_hs_aarch64
+#define helper_mve_vcvt_rm_sh helper_mve_vcvt_rm_sh_aarch64
+#define helper_mve_vcvt_rm_uh helper_mve_vcvt_rm_uh_aarch64
+#define helper_mve_vcvt_rm_ss helper_mve_vcvt_rm_ss_aarch64
+#define helper_mve_vcvt_rm_us helper_mve_vcvt_rm_us_aarch64
+#define helper_mve_vrint_rm_h helper_mve_vrint_rm_h_aarch64
+#define helper_mve_vrint_rm_s helper_mve_vrint_rm_s_aarch64
+#define helper_mve_vrintx_h helper_mve_vrintx_h_aarch64
+#define helper_mve_vrintx_s helper_mve_vrintx_s_aarch64
+#define helper_mve_vshlsb helper_mve_vshlsb_aarch64
+#define helper_mve_vshlsh helper_mve_vshlsh_aarch64
+#define helper_mve_vshlsw helper_mve_vshlsw_aarch64
+#define helper_mve_vshlub helper_mve_vshlub_aarch64
+#define helper_mve_vshluh helper_mve_vshluh_aarch64
+#define helper_mve_vshluw helper_mve_vshluw_aarch64
+#define helper_mve_vrshlsb helper_mve_vrshlsb_aarch64
+#define helper_mve_vrshlsh helper_mve_vrshlsh_aarch64
+#define helper_mve_vrshlsw helper_mve_vrshlsw_aarch64
+#define helper_mve_vrshlub helper_mve_vrshlub_aarch64
+#define helper_mve_vrshluh helper_mve_vrshluh_aarch64
+#define helper_mve_vrshluw helper_mve_vrshluw_aarch64
+#define helper_mve_vqshlsb helper_mve_vqshlsb_aarch64
+#define helper_mve_vqshlsh helper_mve_vqshlsh_aarch64
+#define helper_mve_vqshlsw helper_mve_vqshlsw_aarch64
+#define helper_mve_vqshlub helper_mve_vqshlub_aarch64
+#define helper_mve_vqshluh helper_mve_vqshluh_aarch64
+#define helper_mve_vqshluw helper_mve_vqshluw_aarch64
+#define helper_mve_vqrshlsb helper_mve_vqrshlsb_aarch64
+#define helper_mve_vqrshlsh helper_mve_vqrshlsh_aarch64
+#define helper_mve_vqrshlsw helper_mve_vqrshlsw_aarch64
+#define helper_mve_vqrshlub helper_mve_vqrshlub_aarch64
+#define helper_mve_vqrshluh helper_mve_vqrshluh_aarch64
+#define helper_mve_vqrshluw helper_mve_vqrshluw_aarch64
+#define helper_mve_vqdmladhb helper_mve_vqdmladhb_aarch64
+#define helper_mve_vqdmladhh helper_mve_vqdmladhh_aarch64
+#define helper_mve_vqdmladhw helper_mve_vqdmladhw_aarch64
+#define helper_mve_vqdmladhxb helper_mve_vqdmladhxb_aarch64
+#define helper_mve_vqdmladhxh helper_mve_vqdmladhxh_aarch64
+#define helper_mve_vqdmladhxw helper_mve_vqdmladhxw_aarch64
+#define helper_mve_vqrdmladhb helper_mve_vqrdmladhb_aarch64
+#define helper_mve_vqrdmladhh helper_mve_vqrdmladhh_aarch64
+#define helper_mve_vqrdmladhw helper_mve_vqrdmladhw_aarch64
+#define helper_mve_vqrdmladhxb helper_mve_vqrdmladhxb_aarch64
+#define helper_mve_vqrdmladhxh helper_mve_vqrdmladhxh_aarch64
+#define helper_mve_vqrdmladhxw helper_mve_vqrdmladhxw_aarch64
+#define helper_mve_vqdmlsdhb helper_mve_vqdmlsdhb_aarch64
+#define helper_mve_vqdmlsdhh helper_mve_vqdmlsdhh_aarch64
+#define helper_mve_vqdmlsdhw helper_mve_vqdmlsdhw_aarch64
+#define helper_mve_vqdmlsdhxb helper_mve_vqdmlsdhxb_aarch64
+#define helper_mve_vqdmlsdhxh helper_mve_vqdmlsdhxh_aarch64
+#define helper_mve_vqdmlsdhxw helper_mve_vqdmlsdhxw_aarch64
+#define helper_mve_vqrdmlsdhb helper_mve_vqrdmlsdhb_aarch64
+#define helper_mve_vqrdmlsdhh helper_mve_vqrdmlsdhh_aarch64
+#define helper_mve_vqrdmlsdhw helper_mve_vqrdmlsdhw_aarch64
+#define helper_mve_vqrdmlsdhxb helper_mve_vqrdmlsdhxb_aarch64
+#define helper_mve_vqrdmlsdhxh helper_mve_vqrdmlsdhxh_aarch64
+#define helper_mve_vqrdmlsdhxw helper_mve_vqrdmlsdhxw_aarch64
+#define helper_mve_vbrsrb helper_mve_vbrsrb_aarch64
+#define helper_mve_vbrsrh helper_mve_vbrsrh_aarch64
+#define helper_mve_vbrsrw helper_mve_vbrsrw_aarch64
+#define helper_mve_vshli_sb helper_mve_vshli_sb_aarch64
+#define helper_mve_vshli_sh helper_mve_vshli_sh_aarch64
+#define helper_mve_vshli_sw helper_mve_vshli_sw_aarch64
+#define helper_mve_vshli_ub helper_mve_vshli_ub_aarch64
+#define helper_mve_vshli_uh helper_mve_vshli_uh_aarch64
+#define helper_mve_vshli_uw helper_mve_vshli_uw_aarch64
+#define helper_mve_vrshli_sb helper_mve_vrshli_sb_aarch64
+#define helper_mve_vrshli_sh helper_mve_vrshli_sh_aarch64
+#define helper_mve_vrshli_sw helper_mve_vrshli_sw_aarch64
+#define helper_mve_vrshli_ub helper_mve_vrshli_ub_aarch64
+#define helper_mve_vrshli_uh helper_mve_vrshli_uh_aarch64
+#define helper_mve_vrshli_uw helper_mve_vrshli_uw_aarch64
+#define helper_mve_vqshli_sb helper_mve_vqshli_sb_aarch64
+#define helper_mve_vqshli_sh helper_mve_vqshli_sh_aarch64
+#define helper_mve_vqshli_sw helper_mve_vqshli_sw_aarch64
+#define helper_mve_vqshli_ub helper_mve_vqshli_ub_aarch64
+#define helper_mve_vqshli_uh helper_mve_vqshli_uh_aarch64
+#define helper_mve_vqshli_uw helper_mve_vqshli_uw_aarch64
+#define helper_mve_vqrshli_sb helper_mve_vqrshli_sb_aarch64
+#define helper_mve_vqrshli_sh helper_mve_vqrshli_sh_aarch64
+#define helper_mve_vqrshli_sw helper_mve_vqrshli_sw_aarch64
+#define helper_mve_vqrshli_ub helper_mve_vqrshli_ub_aarch64
+#define helper_mve_vqrshli_uh helper_mve_vqrshli_uh_aarch64
+#define helper_mve_vqrshli_uw helper_mve_vqrshli_uw_aarch64
+#define helper_mve_vqshlui_sb helper_mve_vqshlui_sb_aarch64
+#define helper_mve_vqshlui_sh helper_mve_vqshlui_sh_aarch64
+#define helper_mve_vqshlui_sw helper_mve_vqshlui_sw_aarch64
+#define helper_mve_vshllbsb helper_mve_vshllbsb_aarch64
+#define helper_mve_vshllbsh helper_mve_vshllbsh_aarch64
+#define helper_mve_vshllbub helper_mve_vshllbub_aarch64
+#define helper_mve_vshllbuh helper_mve_vshllbuh_aarch64
+#define helper_mve_vshlltsb helper_mve_vshlltsb_aarch64
+#define helper_mve_vshlltsh helper_mve_vshlltsh_aarch64
+#define helper_mve_vshlltub helper_mve_vshlltub_aarch64
+#define helper_mve_vshlltuh helper_mve_vshlltuh_aarch64
+#define helper_mve_vshrnbb helper_mve_vshrnbb_aarch64
+#define helper_mve_vshrnbh helper_mve_vshrnbh_aarch64
+#define helper_mve_vshrntb helper_mve_vshrntb_aarch64
+#define helper_mve_vshrnth helper_mve_vshrnth_aarch64
+#define helper_mve_vrshrnbb helper_mve_vrshrnbb_aarch64
+#define helper_mve_vrshrnbh helper_mve_vrshrnbh_aarch64
+#define helper_mve_vrshrntb helper_mve_vrshrntb_aarch64
+#define helper_mve_vrshrnth helper_mve_vrshrnth_aarch64
+#define helper_mve_vqshrnb_sb helper_mve_vqshrnb_sb_aarch64
+#define helper_mve_vqshrnb_sh helper_mve_vqshrnb_sh_aarch64
+#define helper_mve_vqshrnt_sb helper_mve_vqshrnt_sb_aarch64
+#define helper_mve_vqshrnt_sh helper_mve_vqshrnt_sh_aarch64
+#define helper_mve_vqshrnb_ub helper_mve_vqshrnb_ub_aarch64
+#define helper_mve_vqshrnb_uh helper_mve_vqshrnb_uh_aarch64
+#define helper_mve_vqshrnt_ub helper_mve_vqshrnt_ub_aarch64
+#define helper_mve_vqshrnt_uh helper_mve_vqshrnt_uh_aarch64
+#define helper_mve_vqshrunbb helper_mve_vqshrunbb_aarch64
+#define helper_mve_vqshrunbh helper_mve_vqshrunbh_aarch64
+#define helper_mve_vqshruntb helper_mve_vqshruntb_aarch64
+#define helper_mve_vqshrunth helper_mve_vqshrunth_aarch64
+#define helper_mve_vqrshrnb_sb helper_mve_vqrshrnb_sb_aarch64
+#define helper_mve_vqrshrnb_sh helper_mve_vqrshrnb_sh_aarch64
+#define helper_mve_vqrshrnt_sb helper_mve_vqrshrnt_sb_aarch64
+#define helper_mve_vqrshrnt_sh helper_mve_vqrshrnt_sh_aarch64
+#define helper_mve_vqrshrnb_ub helper_mve_vqrshrnb_ub_aarch64
+#define helper_mve_vqrshrnb_uh helper_mve_vqrshrnb_uh_aarch64
+#define helper_mve_vqrshrnt_ub helper_mve_vqrshrnt_ub_aarch64
+#define helper_mve_vqrshrnt_uh helper_mve_vqrshrnt_uh_aarch64
+#define helper_mve_vqrshrunbb helper_mve_vqrshrunbb_aarch64
+#define helper_mve_vqrshrunbh helper_mve_vqrshrunbh_aarch64
+#define helper_mve_vqrshruntb helper_mve_vqrshruntb_aarch64
+#define helper_mve_vqrshrunth helper_mve_vqrshrunth_aarch64
+#define helper_mve_vmovnbb helper_mve_vmovnbb_aarch64
+#define helper_mve_vmovnbh helper_mve_vmovnbh_aarch64
+#define helper_mve_vmovntb helper_mve_vmovntb_aarch64
+#define helper_mve_vmovnth helper_mve_vmovnth_aarch64
+#define helper_mve_vqmovnbsb helper_mve_vqmovnbsb_aarch64
+#define helper_mve_vqmovnbsh helper_mve_vqmovnbsh_aarch64
+#define helper_mve_vqmovntsb helper_mve_vqmovntsb_aarch64
+#define helper_mve_vqmovntsh helper_mve_vqmovntsh_aarch64
+#define helper_mve_vqmovnbub helper_mve_vqmovnbub_aarch64
+#define helper_mve_vqmovnbuh helper_mve_vqmovnbuh_aarch64
+#define helper_mve_vqmovntub helper_mve_vqmovntub_aarch64
+#define helper_mve_vqmovntuh helper_mve_vqmovntuh_aarch64
+#define helper_mve_vqmovunbb helper_mve_vqmovunbb_aarch64
+#define helper_mve_vqmovunbh helper_mve_vqmovunbh_aarch64
+#define helper_mve_vqmovuntb helper_mve_vqmovuntb_aarch64
+#define helper_mve_vqmovunth helper_mve_vqmovunth_aarch64
+#define helper_mve_sshrl helper_mve_sshrl_aarch64
+#define helper_mve_ushll helper_mve_ushll_aarch64
+#define helper_mve_sqshll helper_mve_sqshll_aarch64
+#define helper_mve_uqshll helper_mve_uqshll_aarch64
+#define helper_mve_sqrshrl helper_mve_sqrshrl_aarch64
+#define helper_mve_uqrshll helper_mve_uqrshll_aarch64
+#define helper_mve_sqrshrl48 helper_mve_sqrshrl48_aarch64
+#define helper_mve_uqrshll48 helper_mve_uqrshll48_aarch64
+#define helper_mve_uqshl helper_mve_uqshl_aarch64
+#define helper_mve_sqshl helper_mve_sqshl_aarch64
+#define helper_mve_uqrshl helper_mve_uqrshl_aarch64
+#define helper_mve_sqrshr helper_mve_sqrshr_aarch64
+#define helper_mve_vshlc helper_mve_vshlc_aarch64
+#define helper_mve_vsrib helper_mve_vsrib_aarch64
+#define helper_mve_vsrih helper_mve_vsrih_aarch64
+#define helper_mve_vsriw helper_mve_vsriw_aarch64
+#define helper_mve_vslib helper_mve_vslib_aarch64
+#define helper_mve_vslih helper_mve_vslih_aarch64
+#define helper_mve_vsliw helper_mve_vsliw_aarch64
+#define helper_mve_vclsb helper_mve_vclsb_aarch64
+#define helper_mve_vclsh helper_mve_vclsh_aarch64
+#define helper_mve_vclsw helper_mve_vclsw_aarch64
+#define helper_mve_vclzb helper_mve_vclzb_aarch64
+#define helper_mve_vclzh helper_mve_vclzh_aarch64
+#define helper_mve_vclzw helper_mve_vclzw_aarch64
+#define helper_mve_vrev16b helper_mve_vrev16b_aarch64
+#define helper_mve_vrev32b helper_mve_vrev32b_aarch64
+#define helper_mve_vrev32h helper_mve_vrev32h_aarch64
+#define helper_mve_vrev64b helper_mve_vrev64b_aarch64
+#define helper_mve_vrev64h helper_mve_vrev64h_aarch64
+#define helper_mve_vrev64w helper_mve_vrev64w_aarch64
+#define helper_mve_vmvn helper_mve_vmvn_aarch64
+#define helper_mve_vabsb helper_mve_vabsb_aarch64
+#define helper_mve_vabsh helper_mve_vabsh_aarch64
+#define helper_mve_vabsw helper_mve_vabsw_aarch64
+#define helper_mve_vnegb helper_mve_vnegb_aarch64
+#define helper_mve_vnegh helper_mve_vnegh_aarch64
+#define helper_mve_vnegw helper_mve_vnegw_aarch64
+#define helper_mve_vmaxab helper_mve_vmaxab_aarch64
+#define helper_mve_vmaxah helper_mve_vmaxah_aarch64
+#define helper_mve_vmaxaw helper_mve_vmaxaw_aarch64
+#define helper_mve_vminab helper_mve_vminab_aarch64
+#define helper_mve_vminah helper_mve_vminah_aarch64
+#define helper_mve_vminaw helper_mve_vminaw_aarch64
+#define helper_mve_vqabsb helper_mve_vqabsb_aarch64
+#define helper_mve_vqabsh helper_mve_vqabsh_aarch64
+#define helper_mve_vqabsw helper_mve_vqabsw_aarch64
+#define helper_mve_vqnegb helper_mve_vqnegb_aarch64
+#define helper_mve_vqnegh helper_mve_vqnegh_aarch64
+#define helper_mve_vqnegw helper_mve_vqnegw_aarch64
+#define helper_mve_vmlaldavsh helper_mve_vmlaldavsh_aarch64
+#define helper_mve_vmlaldavsw helper_mve_vmlaldavsw_aarch64
+#define helper_mve_vmlaldavxsh helper_mve_vmlaldavxsh_aarch64
+#define helper_mve_vmlaldavxsw helper_mve_vmlaldavxsw_aarch64
+#define helper_mve_vmlaldavuh helper_mve_vmlaldavuh_aarch64
+#define helper_mve_vmlaldavuw helper_mve_vmlaldavuw_aarch64
+#define helper_mve_vmlsldavsh helper_mve_vmlsldavsh_aarch64
+#define helper_mve_vmlsldavsw helper_mve_vmlsldavsw_aarch64
+#define helper_mve_vmlsldavxsh helper_mve_vmlsldavxsh_aarch64
+#define helper_mve_vmlsldavxsw helper_mve_vmlsldavxsw_aarch64
+#define helper_mve_vrmlaldavhsw helper_mve_vrmlaldavhsw_aarch64
+#define helper_mve_vrmlaldavhxsw helper_mve_vrmlaldavhxsw_aarch64
+#define helper_mve_vrmlaldavhuw helper_mve_vrmlaldavhuw_aarch64
+#define helper_mve_vrmlsldavhsw helper_mve_vrmlsldavhsw_aarch64
+#define helper_mve_vrmlsldavhxsw helper_mve_vrmlsldavhxsw_aarch64
+#define helper_mve_vmladavsb helper_mve_vmladavsb_aarch64
+#define helper_mve_vmladavsh helper_mve_vmladavsh_aarch64
+#define helper_mve_vmladavsw helper_mve_vmladavsw_aarch64
+#define helper_mve_vmladavub helper_mve_vmladavub_aarch64
+#define helper_mve_vmladavuh helper_mve_vmladavuh_aarch64
+#define helper_mve_vmladavuw helper_mve_vmladavuw_aarch64
+#define helper_mve_vmlsdavb helper_mve_vmlsdavb_aarch64
+#define helper_mve_vmlsdavh helper_mve_vmlsdavh_aarch64
+#define helper_mve_vmlsdavw helper_mve_vmlsdavw_aarch64
+#define helper_mve_vmladavsxb helper_mve_vmladavsxb_aarch64
+#define helper_mve_vmladavsxh helper_mve_vmladavsxh_aarch64
+#define helper_mve_vmladavsxw helper_mve_vmladavsxw_aarch64
+#define helper_mve_vmlsdavxb helper_mve_vmlsdavxb_aarch64
+#define helper_mve_vmlsdavxh helper_mve_vmlsdavxh_aarch64
+#define helper_mve_vmlsdavxw helper_mve_vmlsdavxw_aarch64
+#define helper_mve_vaddvsb helper_mve_vaddvsb_aarch64
+#define helper_mve_vaddvsh helper_mve_vaddvsh_aarch64
+#define helper_mve_vaddvsw helper_mve_vaddvsw_aarch64
+#define helper_mve_vaddvub helper_mve_vaddvub_aarch64
+#define helper_mve_vaddvuh helper_mve_vaddvuh_aarch64
+#define helper_mve_vaddvuw helper_mve_vaddvuw_aarch64
+#define helper_mve_vmaxvsb helper_mve_vmaxvsb_aarch64
+#define helper_mve_vmaxvsh helper_mve_vmaxvsh_aarch64
+#define helper_mve_vmaxvsw helper_mve_vmaxvsw_aarch64
+#define helper_mve_vmaxvub helper_mve_vmaxvub_aarch64
+#define helper_mve_vmaxvuh helper_mve_vmaxvuh_aarch64
+#define helper_mve_vmaxvuw helper_mve_vmaxvuw_aarch64
+#define helper_mve_vmaxavb helper_mve_vmaxavb_aarch64
+#define helper_mve_vmaxavh helper_mve_vmaxavh_aarch64
+#define helper_mve_vmaxavw helper_mve_vmaxavw_aarch64
+#define helper_mve_vminvsb helper_mve_vminvsb_aarch64
+#define helper_mve_vminvsh helper_mve_vminvsh_aarch64
+#define helper_mve_vminvsw helper_mve_vminvsw_aarch64
+#define helper_mve_vminvub helper_mve_vminvub_aarch64
+#define helper_mve_vminvuh helper_mve_vminvuh_aarch64
+#define helper_mve_vminvuw helper_mve_vminvuw_aarch64
+#define helper_mve_vminavb helper_mve_vminavb_aarch64
+#define helper_mve_vminavh helper_mve_vminavh_aarch64
+#define helper_mve_vminavw helper_mve_vminavw_aarch64
+#define helper_mve_vmaxnmvh helper_mve_vmaxnmvh_aarch64
+#define helper_mve_vmaxnmvs helper_mve_vmaxnmvs_aarch64
+#define helper_mve_vminnmvh helper_mve_vminnmvh_aarch64
+#define helper_mve_vminnmvs helper_mve_vminnmvs_aarch64
+#define helper_mve_vmaxnmavh helper_mve_vmaxnmavh_aarch64
+#define helper_mve_vmaxnmavs helper_mve_vmaxnmavs_aarch64
+#define helper_mve_vminnmavh helper_mve_vminnmavh_aarch64
+#define helper_mve_vminnmavs helper_mve_vminnmavs_aarch64
+#define helper_mve_vaddlv_s helper_mve_vaddlv_s_aarch64
+#define helper_mve_vaddlv_u helper_mve_vaddlv_u_aarch64
+#define helper_mve_vabavsb helper_mve_vabavsb_aarch64
+#define helper_mve_vabavsh helper_mve_vabavsh_aarch64
+#define helper_mve_vabavsw helper_mve_vabavsw_aarch64
+#define helper_mve_vabavub helper_mve_vabavub_aarch64
+#define helper_mve_vabavuh helper_mve_vabavuh_aarch64
+#define helper_mve_vabavuw helper_mve_vabavuw_aarch64
 #define helper_vfp_adds helper_vfp_adds_aarch64
 #define helper_vfp_addd helper_vfp_addd_aarch64
 #define helper_vfp_subs helper_vfp_subs_aarch64
@@ -2935,10 +5223,14 @@
 #define helper_vfp_touls helper_vfp_touls_aarch64
 #define helper_vfp_uqtos helper_vfp_uqtos_aarch64
 #define helper_vfp_touqs helper_vfp_touqs_aarch64
+#define helper_vfp_shtoh helper_vfp_shtoh_aarch64
+#define helper_vfp_uhtoh helper_vfp_uhtoh_aarch64
 #define helper_vfp_sltoh helper_vfp_sltoh_aarch64
 #define helper_vfp_ultoh helper_vfp_ultoh_aarch64
 #define helper_vfp_sqtoh helper_vfp_sqtoh_aarch64
 #define helper_vfp_uqtoh helper_vfp_uqtoh_aarch64
+#define helper_vfp_toshh_round_to_zero helper_vfp_toshh_round_to_zero_aarch64
+#define helper_vfp_touhh_round_to_zero helper_vfp_touhh_round_to_zero_aarch64
 #define helper_vfp_toshh helper_vfp_toshh_aarch64
 #define helper_vfp_touhh helper_vfp_touhh_aarch64
 #define helper_vfp_toslh helper_vfp_toslh_aarch64
@@ -2951,6 +5243,8 @@
 #define helper_vfp_fcvt_f32_to_f16 helper_vfp_fcvt_f32_to_f16_aarch64
 #define helper_vfp_fcvt_f16_to_f64 helper_vfp_fcvt_f16_to_f64_aarch64
 #define helper_vfp_fcvt_f64_to_f16 helper_vfp_fcvt_f64_to_f16_aarch64
+#define helper_bfcvt helper_bfcvt_aarch64
+#define helper_bfcvt_pair helper_bfcvt_pair_aarch64
 #define helper_recps_f32 helper_recps_f32_aarch64
 #define helper_rsqrts_f32 helper_rsqrts_f32_aarch64
 #define helper_recpe_f16 helper_recpe_f16_aarch64
@@ -2963,8 +5257,10 @@
 #define helper_rsqrte_u32 helper_rsqrte_u32_aarch64
 #define helper_vfp_muladds helper_vfp_muladds_aarch64
 #define helper_vfp_muladdd helper_vfp_muladdd_aarch64
+#define helper_rinth_exact helper_rinth_exact_aarch64
 #define helper_rints_exact helper_rints_exact_aarch64
 #define helper_rintd_exact helper_rintd_exact_aarch64
+#define helper_rinth helper_rinth_aarch64
 #define helper_rints helper_rints_aarch64
 #define helper_rintd helper_rintd_aarch64
 #define arm_rmode_to_sf arm_rmode_to_sf_aarch64
diff --git a/qemu/accel/tcg/cpu-exec-common.c b/qemu/accel/tcg/cpu-exec-common.c
index f1110b434a..f9f50dd144 100644
--- a/qemu/accel/tcg/cpu-exec-common.c
+++ b/qemu/accel/tcg/cpu-exec-common.c
@@ -18,7 +18,6 @@
  */
 
 #include "qemu/osdep.h"
-#include "cpu.h"
 #include "sysemu/cpus.h"
 #include "sysemu/tcg.h"
 #include "exec/exec-all.h"
diff --git a/qemu/accel/tcg/cpu-exec.c b/qemu/accel/tcg/cpu-exec.c
index 77168f35a7..1343b81571 100644
--- a/qemu/accel/tcg/cpu-exec.c
+++ b/qemu/accel/tcg/cpu-exec.c
@@ -26,6 +26,7 @@
 #include "qemu/timer.h"
 #include "exec/tb-hash.h"
 #include "exec/tb-lookup.h"
+#include "hw/core/tcg-cpu-ops.h"
 #include "sysemu/cpus.h"
 #include "uc_priv.h"
 
@@ -74,7 +75,6 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
          * counter hit zero); we must restore the guest PC to the address
          * of the start of the TB.
          */
-        CPUClass *cc = CPU_GET_CLASS(cpu);
         if (!HOOK_EXISTS(env->uc, UC_HOOK_CODE)) {
             // We should sync pc for R/W error.
             switch (env->uc->invalid_error) {
@@ -91,12 +91,7 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
                 default:
                     // If we receive a quit request, users has sync-ed PC themselves.
                     if (!cpu->uc->quit_request) {
-                        if (cc->synchronize_from_tb) {
-                            cc->synchronize_from_tb(cpu, last_tb);
-                        } else {
-                            assert(cc->set_pc);
-                            cc->set_pc(cpu, last_tb->pc);
-                        }
+                        cpu_tcg_synchronize_from_tb(cpu, last_tb);
                     }
             }
         }
@@ -321,7 +316,6 @@ static inline bool cpu_handle_halt(CPUState *cpu)
 
 static inline void cpu_handle_debug_exception(CPUState *cpu)
 {
-    CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUWatchpoint *wp;
 
     if (!cpu->watchpoint_hit) {
@@ -330,7 +324,7 @@ static inline void cpu_handle_debug_exception(CPUState *cpu)
         }
     }
 
-    cc->debug_excp_handler(cpu);
+    cpu_tcg_debug_excp_handler(cpu);
 }
 
 static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
@@ -355,7 +349,9 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
             }
         }
         if (!catched) {
-            uc->invalid_error = UC_ERR_INSN_INVALID;
+            if (uc->invalid_error == UC_ERR_OK) {
+                uc->invalid_error = UC_ERR_INSN_INVALID;
+            }
             // we want to stop emulation
             *ret = EXCP_HLT;
             return true;
@@ -433,8 +429,6 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
 static inline bool cpu_handle_interrupt(CPUState *cpu,
                                         TranslationBlock **last_tb)
 {
-    CPUClass *cc = CPU_GET_CLASS(cpu);
-
     /* Clear the interrupt flag now since we're processing
      * cpu->interrupt_request and cpu->exit_request.
      * Ensure zeroing happens before reading cpu->exit_request or
@@ -476,7 +470,7 @@ static inline bool cpu_handle_interrupt(CPUState *cpu,
            True when it is, and we should restart on a new TB,
            and via longjmp via cpu_loop_exit.  */
         else {
-            if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
+            if (cpu_tcg_exec_interrupt(cpu, interrupt_request)) {
                 //replay_interrupt();
                 cpu->exception_index = -1;
                 *last_tb = NULL;
@@ -562,7 +556,7 @@ int cpu_exec(struct uc_struct *uc, CPUState *cpu)
 
     // rcu_read_lock();
 
-    cc->cpu_exec_enter(cpu);
+    cpu_tcg_exec_enter(cpu);
 
     /* Calculate difference between guest clock and host clock.
      * This delay includes the delay of the last cycle, so
@@ -624,7 +618,7 @@ int cpu_exec(struct uc_struct *uc, CPUState *cpu)
     // Unicorn: Clear any TCG exit flag that might have been left set by exit requests
     uc->cpu->tcg_exit_req = 0;
 
-    cc->cpu_exec_exit(cpu);
+    cpu_tcg_exec_exit(cpu);
     // rcu_read_unlock();
 
     return ret;
diff --git a/qemu/accel/tcg/cputlb.c b/qemu/accel/tcg/cputlb.c
index 3ab0206084..14267b4f6c 100644
--- a/qemu/accel/tcg/cputlb.c
+++ b/qemu/accel/tcg/cputlb.c
@@ -29,6 +29,7 @@
 #include "exec/helper-proto.h"
 #include "qemu/atomic.h"
 #include "qemu/atomic128.h"
+#include "hw/core/tcg-cpu-ops.h"
 #include "translate-all.h"
 #include "exec/cpu-common.h"
 #include "trace/mem.h"
@@ -998,7 +999,6 @@ static inline ram_addr_t qemu_ram_addr_from_host_nofail(struct uc_struct *uc, vo
 static void tlb_fill(CPUState *cpu, target_ulong addr, int size,
                      MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
 {
-    CPUClass *cc = CPU_GET_CLASS(cpu);
 #ifndef NDEBUG
     bool ok;
 
@@ -1006,10 +1006,11 @@ static void tlb_fill(CPUState *cpu, target_ulong addr, int size,
      * This is not a probe, so only valid return is success; failure
      * should result in exception + longjmp to the cpu loop.
      */
-    ok = cc->tlb_fill(cpu, addr, size, access_type, mmu_idx, false, retaddr);
+    ok = cpu_tcg_tlb_fill(cpu, addr, size, access_type, mmu_idx,
+                          false, retaddr);
     assert(ok);
 #else
-    cc->tlb_fill(cpu, addr, size, access_type, mmu_idx, false, retaddr);
+    cpu_tcg_tlb_fill(cpu, addr, size, access_type, mmu_idx, false, retaddr);
 #endif
 }
 
@@ -1326,9 +1327,8 @@ bool tlb_vaddr_to_paddr(CPUArchState *env, abi_ptr addr,
 
         if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, page)) {
             CPUState *cs = env_cpu(env);
-            CPUClass *cc = CPU_GET_CLASS(cs);
-
-            if (!cc->tlb_fill(cs, addr, 0, access_type, mmu_idx, true, 0)) {
+            if (!cpu_tcg_tlb_fill(cs, addr, 0, access_type, mmu_idx, true,
+                                  0)) {
                 /* Non-faulting page table read failed.  */
                 return false;
             }
@@ -1373,9 +1373,8 @@ void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
 
         if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, page)) {
             CPUState *cs = env_cpu(env);
-            CPUClass *cc = CPU_GET_CLASS(cs);
-
-            if (!cc->tlb_fill(cs, addr, 0, access_type, mmu_idx, true, 0)) {
+            if (!cpu_tcg_tlb_fill(cs, addr, 0, access_type, mmu_idx, true,
+                                  0)) {
                 /* Non-faulting page table read failed.  */
                 return NULL;
             }
@@ -1417,8 +1416,8 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
     /* Enforce guest required alignment.  */
     if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
         /* ??? Maybe indicate atomic op to cpu_unaligned_access */
-        cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
-                             mmu_idx, retaddr);
+        cpu_tcg_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
+                                 mmu_idx, retaddr);
     }
 
     /* Enforce qemu required alignment.  */
@@ -1584,8 +1583,8 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
 
     /* Handle CPU specific unaligned behaviour */
     if (addr & ((1 << a_bits) - 1)) {
-        cpu_unaligned_access(env_cpu(env), addr, access_type,
-                             mmu_idx, retaddr);
+        cpu_tcg_unaligned_access(env_cpu(env), addr, access_type,
+                                 mmu_idx, retaddr);
     }
 
     /* If the TLB entry is for a different page, reload and try again.  */
@@ -2212,8 +2211,8 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
 
     /* Handle CPU specific unaligned behaviour */
     if (addr & ((1 << a_bits) - 1)) {
-        cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
-                             mmu_idx, retaddr);
+        cpu_tcg_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
+                                 mmu_idx, retaddr);
     }
 
     /* If the TLB entry is for a different page, reload and try again.  */
diff --git a/qemu/accel/tcg/tcg-runtime.c b/qemu/accel/tcg/tcg-runtime.c
index 777287abb9..216ea89cc5 100644
--- a/qemu/accel/tcg/tcg-runtime.c
+++ b/qemu/accel/tcg/tcg-runtime.c
@@ -160,6 +160,19 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env)
     return tb->tc.ptr;
 }
 
+void *HELPER(memset)(void *ptr, int val, void *size)
+{
+    return memset(ptr, val, (uintptr_t)size);
+}
+
+void HELPER(emu_stop)(void *p)
+{
+    uc_engine *uc = p;
+
+    uc->stop_request = true;
+    break_translation_loop(uc);
+}
+
 void HELPER(exit_atomic)(CPUArchState *env)
 {
     cpu_loop_exit_atomic(env_cpu(env), GETPC());
@@ -185,4 +198,4 @@ void HELPER(check_exit_request)(void *p, uint32_t in_delay_slot) {
             cpu_loop_exit_restore(uc->cpu, GETPC());
         }
     }
-}
\ No newline at end of file
+}
diff --git a/qemu/accel/tcg/tcg-runtime.h b/qemu/accel/tcg/tcg-runtime.h
index 8ba10e1835..42df0fca09 100644
--- a/qemu/accel/tcg/tcg-runtime.h
+++ b/qemu/accel/tcg/tcg-runtime.h
@@ -28,6 +28,9 @@ DEF_HELPER_FLAGS_1(lookup_tb_ptr, TCG_CALL_NO_WG_SE, ptr, env)
 
 DEF_HELPER_FLAGS_1(exit_atomic, TCG_CALL_NO_WG, noreturn, env)
 
+DEF_HELPER_FLAGS_3(memset, TCG_CALL_NO_RWG, ptr, ptr, int, ptr)
+DEF_HELPER_1(emu_stop, void, ptr)
+
 DEF_HELPER_FLAGS_5(atomic_cmpxchgb, TCG_CALL_NO_WG,
                    i32, env, tl, i32, i32, i32)
 DEF_HELPER_FLAGS_5(atomic_cmpxchgw_be, TCG_CALL_NO_WG,
@@ -260,4 +263,4 @@ DEF_HELPER_FLAGS_4(gvec_leu64, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_5(gvec_bitsel, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
 
-DEF_HELPER_2(check_exit_request, void, ptr, i32)
\ No newline at end of file
+DEF_HELPER_2(check_exit_request, void, ptr, i32)
diff --git a/qemu/accel/tcg/translator.c b/qemu/accel/tcg/translator.c
index 72f21c410c..bae828a388 100644
--- a/qemu/accel/tcg/translator.c
+++ b/qemu/accel/tcg/translator.c
@@ -45,6 +45,11 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
     db->tb = tb;
     db->pc_first = tb->pc;
     db->pc_next = db->pc_first;
+#ifdef TARGET_PAGE_BITS_VARY
+    db->target_page_mask = uc->init_target_page->mask;
+#else
+    db->target_page_mask = (target_ulong)-1 << TARGET_PAGE_BITS;
+#endif
     db->is_jmp = DISAS_NEXT;
     db->num_insns = 0;
     db->max_insns = max_insns;
@@ -59,13 +64,16 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
     /* Unicorn: early check to see if the address of this block is
      * the "run until" address. */
     if (uc_addr_is_exit(uc, tb->pc)) {
-        // This should catch that instruction is at the end
-        // and generate appropriate halting code.
+        TCGv_ptr puc = tcg_const_ptr(tcg_ctx, uc);
+
         gen_tb_start(tcg_ctx, db->tb);
         ops->tb_start(db, cpu);
         db->num_insns++;
         ops->insn_start(db, cpu);
-        ops->translate_insn(db, cpu);
+        gen_helper_emu_stop(tcg_ctx, puc);
+        tcg_temp_free_ptr(tcg_ctx, puc);
+        check_exit_request(tcg_ctx);
+        db->is_jmp = DISAS_NORETURN;
         goto _end_loop;
     }
 
diff --git a/qemu/arm.h b/qemu/arm.h
index cd81f540f0..63466c452e 100644
--- a/qemu/arm.h
+++ b/qemu/arm.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_arm
 #define float32_squash_input_denormal float32_squash_input_denormal_arm
 #define float64_squash_input_denormal float64_squash_input_denormal_arm
+#define bfloat16_add bfloat16_add_arm
+#define bfloat16_compare bfloat16_compare_arm
+#define bfloat16_compare_quiet bfloat16_compare_quiet_arm
+#define bfloat16_default_nan bfloat16_default_nan_arm
+#define bfloat16_div bfloat16_div_arm
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_arm
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_arm
+#define bfloat16_max bfloat16_max_arm
+#define bfloat16_maximum_number bfloat16_maximum_number_arm
+#define bfloat16_maxnum bfloat16_maxnum_arm
+#define bfloat16_maxnummag bfloat16_maxnummag_arm
+#define bfloat16_min bfloat16_min_arm
+#define bfloat16_minimum_number bfloat16_minimum_number_arm
+#define bfloat16_minnum bfloat16_minnum_arm
+#define bfloat16_minnummag bfloat16_minnummag_arm
+#define bfloat16_mul bfloat16_mul_arm
+#define bfloat16_muladd bfloat16_muladd_arm
+#define bfloat16_round_to_int bfloat16_round_to_int_arm
+#define bfloat16_scalbn bfloat16_scalbn_arm
+#define bfloat16_silence_nan bfloat16_silence_nan_arm
+#define bfloat16_sqrt bfloat16_sqrt_arm
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_arm
+#define bfloat16_sub bfloat16_sub_arm
+#define bfloat16_to_float32 bfloat16_to_float32_arm
+#define bfloat16_to_float64 bfloat16_to_float64_arm
+#define bfloat16_to_int16 bfloat16_to_int16_arm
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_arm
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_arm
+#define bfloat16_to_int32 bfloat16_to_int32_arm
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_arm
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_arm
+#define bfloat16_to_int64 bfloat16_to_int64_arm
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_arm
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_arm
+#define bfloat16_to_uint16 bfloat16_to_uint16_arm
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_arm
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_arm
+#define bfloat16_to_uint32 bfloat16_to_uint32_arm
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_arm
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_arm
+#define bfloat16_to_uint64 bfloat16_to_uint64_arm
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_arm
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_arm
+#define float128_maximum_number float128_maximum_number_arm
+#define float128_max float128_max_arm
+#define float128_maxnum float128_maxnum_arm
+#define float128_maxnummag float128_maxnummag_arm
+#define float128_min float128_min_arm
+#define float128_minimum_number float128_minimum_number_arm
+#define float128_minnum float128_minnum_arm
+#define float128_minnummag float128_minnummag_arm
+#define float128_muladd float128_muladd_arm
+#define float128_to_int128 float128_to_int128_arm
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_arm
+#define float128_to_uint128 float128_to_uint128_arm
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_arm
+#define float16_maximum_number float16_maximum_number_arm
+#define float16_minimum_number float16_minimum_number_arm
+#define float16_to_int8 float16_to_int8_arm
+#define float16_to_int8_scalbn float16_to_int8_scalbn_arm
+#define float16_to_uint8 float16_to_uint8_arm
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_arm
+#define float32_maximum_number float32_maximum_number_arm
+#define float32_minimum_number float32_minimum_number_arm
+#define float32_to_bfloat16 float32_to_bfloat16_arm
+#define float64_maximum_number float64_maximum_number_arm
+#define float64_minimum_number float64_minimum_number_arm
+#define float64_to_bfloat16 float64_to_bfloat16_arm
+#define float64r32_add float64r32_add_arm
+#define float64r32_div float64r32_div_arm
+#define float64r32_mul float64r32_mul_arm
+#define float64r32_muladd float64r32_muladd_arm
+#define float64r32_sqrt float64r32_sqrt_arm
+#define float64r32_sub float64r32_sub_arm
+#define floatx80_mod floatx80_mod_arm
+#define floatx80_modrem floatx80_modrem_arm
+#define int128_to_float128 int128_to_float128_arm
+#define int16_to_bfloat16 int16_to_bfloat16_arm
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_arm
+#define int32_to_bfloat16 int32_to_bfloat16_arm
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_arm
+#define int64_to_bfloat16 int64_to_bfloat16_arm
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_arm
+#define int8_to_float16 int8_to_float16_arm
+#define uint128_to_float128 uint128_to_float128_arm
+#define uint16_to_bfloat16 uint16_to_bfloat16_arm
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_arm
+#define uint32_to_bfloat16 uint32_to_bfloat16_arm
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_arm
+#define uint64_to_bfloat16 uint64_to_bfloat16_arm
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_arm
+#define uint8_to_float16 uint8_to_float16_arm
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_arm
 #define roundAndPackFloatx80 roundAndPackFloatx80_arm
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_arm
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_arm
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_arm
 #define helper_exit_atomic helper_exit_atomic_arm
+#define helper_memset helper_memset_arm
+#define helper_emu_stop helper_emu_stop_arm
+#define tcg_remove_ops_after tcg_remove_ops_after_arm
+#define tcg_constant_vec_matching tcg_constant_vec_matching_arm
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_arm
 #define helper_gvec_add8 helper_gvec_add8_arm
 #define helper_gvec_add16 helper_gvec_add16_arm
 #define helper_gvec_add32 helper_gvec_add32_arm
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_arm
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_arm
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_arm
+#define gen_helper_mve_vctp gen_helper_mve_vctp_arm
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_arm
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_arm
+#define gen_helper_mve_vdup gen_helper_mve_vdup_arm
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_arm
+#define gen_helper_mve_vandi gen_helper_mve_vandi_arm
+#define gen_helper_mve_vorri gen_helper_mve_vorri_arm
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_arm
+#define gen_helper_mve_viduph gen_helper_mve_viduph_arm
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_arm
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_arm
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_arm
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_arm
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_arm
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_arm
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_arm
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_arm
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_arm
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_arm
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_arm
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_arm
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_arm
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_arm
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_arm
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_arm
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_arm
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_arm
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_arm
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_arm
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_arm
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_arm
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_arm
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_arm
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_arm
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_arm
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_arm
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_arm
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_arm
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_arm
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_arm
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_arm
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_arm
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_arm
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_arm
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_arm
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_arm
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_arm
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_arm
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_arm
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_arm
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_arm
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_arm
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_arm
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_arm
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_arm
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_arm
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_arm
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_arm
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_arm
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_arm
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_arm
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_arm
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_arm
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_arm
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_arm
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_arm
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_arm
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_arm
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_arm
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_arm
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_arm
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_arm
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_arm
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_arm
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_arm
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_arm
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_arm
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_arm
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_arm
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_arm
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_arm
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_arm
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_arm
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_arm
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_arm
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_arm
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_arm
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_arm
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_arm
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_arm
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_arm
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_arm
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_arm
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_arm
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_arm
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_arm
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_arm
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_arm
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_arm
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_arm
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_arm
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_arm
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_arm
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_arm
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_arm
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_arm
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_arm
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_arm
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_arm
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_arm
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_arm
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_arm
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_arm
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_arm
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_arm
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_arm
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_arm
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_arm
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_arm
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_arm
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_arm
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_arm
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_arm
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_arm
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_arm
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_arm
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_arm
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_arm
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_arm
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_arm
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_arm
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_arm
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_arm
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_arm
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_arm
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_arm
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_arm
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_arm
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_arm
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_arm
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_arm
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_arm
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_arm
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_arm
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_arm
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_arm
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_arm
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_arm
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_arm
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_arm
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_arm
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_arm
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_arm
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_arm
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_arm
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_arm
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_arm
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_arm
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_arm
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_arm
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_arm
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_arm
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_arm
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_arm
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_arm
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_arm
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_arm
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_arm
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_arm
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_arm
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_arm
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_arm
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_arm
+#define gen_helper_mve_vand gen_helper_mve_vand_arm
+#define gen_helper_mve_vbic gen_helper_mve_vbic_arm
+#define gen_helper_mve_vorr gen_helper_mve_vorr_arm
+#define gen_helper_mve_vorn gen_helper_mve_vorn_arm
+#define gen_helper_mve_veor gen_helper_mve_veor_arm
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_arm
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_arm
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_arm
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_arm
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_arm
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_arm
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_arm
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_arm
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_arm
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_arm
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_arm
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_arm
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_arm
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_arm
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_arm
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_arm
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_arm
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_arm
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_arm
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_arm
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_arm
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_arm
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_arm
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_arm
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_arm
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_arm
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_arm
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_arm
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_arm
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_arm
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_arm
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_arm
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_arm
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_arm
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_arm
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_arm
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_arm
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_arm
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_arm
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_arm
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_arm
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_arm
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_arm
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_arm
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_arm
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_arm
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_arm
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_arm
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_arm
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_arm
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_arm
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_arm
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_arm
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_arm
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_arm
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_arm
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_arm
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_arm
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_arm
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_arm
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_arm
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_arm
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_arm
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_arm
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_arm
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_arm
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_arm
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_arm
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_arm
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_arm
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_arm
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_arm
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_arm
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_arm
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_arm
+#define gen_helper_mve_vminub gen_helper_mve_vminub_arm
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_arm
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_arm
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_arm
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_arm
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_arm
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_arm
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_arm
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_arm
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_arm
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_arm
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_arm
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_arm
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_arm
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_arm
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_arm
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_arm
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_arm
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_arm
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_arm
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_arm
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_arm
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_arm
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_arm
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_arm
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_arm
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_arm
+#define gen_helper_mve_vadc gen_helper_mve_vadc_arm
+#define gen_helper_mve_vadci gen_helper_mve_vadci_arm
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_arm
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_arm
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_arm
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_arm
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_arm
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_arm
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_arm
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_arm
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_arm
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_arm
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_arm
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_arm
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_arm
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_arm
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_arm
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_arm
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_arm
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_arm
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_arm
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_arm
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_arm
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_arm
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_arm
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_arm
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_arm
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_arm
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_arm
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_arm
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_arm
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_arm
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_arm
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_arm
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_arm
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_arm
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_arm
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_arm
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_arm
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_arm
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_arm
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_arm
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_arm
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_arm
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_arm
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_arm
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_arm
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_arm
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_arm
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_arm
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_arm
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_arm
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_arm
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_arm
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_arm
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_arm
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_arm
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_arm
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_arm
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_arm
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_arm
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_arm
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_arm
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_arm
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_arm
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_arm
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_arm
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_arm
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_arm
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_arm
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_arm
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_arm
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_arm
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_arm
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_arm
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_arm
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_arm
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_arm
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_arm
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_arm
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_arm
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_arm
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_arm
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_arm
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_arm
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_arm
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_arm
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_arm
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_arm
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_arm
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_arm
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_arm
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_arm
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_arm
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_arm
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_arm
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_arm
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_arm
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_arm
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_arm
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_arm
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_arm
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_arm
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_arm
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_arm
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_arm
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_arm
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_arm
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_arm
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_arm
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_arm
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_arm
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_arm
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_arm
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_arm
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_arm
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_arm
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_arm
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_arm
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_arm
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_arm
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_arm
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_arm
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_arm
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_arm
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_arm
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_arm
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_arm
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_arm
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_arm
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_arm
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_arm
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_arm
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_arm
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_arm
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_arm
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_arm
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_arm
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_arm
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_arm
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_arm
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_arm
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_arm
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_arm
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_arm
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_arm
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_arm
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_arm
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_arm
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_arm
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_arm
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_arm
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_arm
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_arm
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_arm
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_arm
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_arm
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_arm
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_arm
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_arm
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_arm
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_arm
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_arm
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_arm
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_arm
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_arm
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_arm
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_arm
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_arm
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_arm
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_arm
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_arm
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_arm
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_arm
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_arm
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_arm
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_arm
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_arm
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_arm
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_arm
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_arm
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_arm
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_arm
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_arm
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_arm
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_arm
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_arm
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_arm
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_arm
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_arm
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_arm
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_arm
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_arm
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_arm
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_arm
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_arm
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_arm
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_arm
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_arm
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_arm
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_arm
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_arm
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_arm
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_arm
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_arm
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_arm
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_arm
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_arm
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_arm
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_arm
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_arm
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_arm
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_arm
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_arm
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_arm
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_arm
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_arm
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_arm
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_arm
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_arm
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_arm
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_arm
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_arm
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_arm
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_arm
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_arm
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_arm
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_arm
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_arm
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_arm
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_arm
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_arm
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_arm
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_arm
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_arm
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_arm
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_arm
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_arm
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_arm
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_arm
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_arm
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_arm
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_arm
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_arm
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_arm
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_arm
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_arm
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_arm
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_arm
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_arm
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_arm
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_arm
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_arm
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_arm
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_arm
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_arm
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_arm
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_arm
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_arm
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_arm
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_arm
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_arm
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_arm
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_arm
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_arm
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_arm
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_arm
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_arm
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_arm
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_arm
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_arm
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_arm
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_arm
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_arm
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_arm
+#define gen_helper_mve_ushll gen_helper_mve_ushll_arm
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_arm
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_arm
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_arm
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_arm
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_arm
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_arm
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_arm
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_arm
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_arm
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_arm
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_arm
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_arm
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_arm
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_arm
+#define gen_helper_mve_vslib gen_helper_mve_vslib_arm
+#define gen_helper_mve_vslih gen_helper_mve_vslih_arm
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_arm
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_arm
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_arm
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_arm
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_arm
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_arm
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_arm
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_arm
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_arm
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_arm
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_arm
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_arm
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_arm
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_arm
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_arm
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_arm
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_arm
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_arm
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_arm
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_arm
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_arm
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_arm
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_arm
+#define gen_helper_mve_vminab gen_helper_mve_vminab_arm
+#define gen_helper_mve_vminah gen_helper_mve_vminah_arm
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_arm
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_arm
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_arm
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_arm
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_arm
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_arm
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_arm
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_arm
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_arm
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_arm
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_arm
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_arm
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_arm
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_arm
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_arm
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_arm
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_arm
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_arm
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_arm
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_arm
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_arm
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_arm
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_arm
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_arm
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_arm
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_arm
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_arm
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_arm
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_arm
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_arm
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_arm
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_arm
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_arm
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_arm
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_arm
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_arm
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_arm
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_arm
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_arm
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_arm
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_arm
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_arm
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_arm
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_arm
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_arm
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_arm
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_arm
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_arm
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_arm
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_arm
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_arm
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_arm
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_arm
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_arm
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_arm
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_arm
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_arm
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_arm
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_arm
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_arm
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_arm
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_arm
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_arm
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_arm
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_arm
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_arm
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_arm
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_arm
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_arm
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_arm
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_arm
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_arm
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_arm
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_arm
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_arm
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_arm
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_arm
 #define gen_helper_cpsr_read gen_helper_cpsr_read_arm
 #define gen_helper_cpsr_write gen_helper_cpsr_write_arm
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_arm
@@ -1306,6 +2077,8 @@
 #define cpu_arm_init cpu_arm_init_arm
 #define helper_crypto_aese helper_crypto_aese_arm
 #define helper_crypto_aesmc helper_crypto_aesmc_arm
+#define helper_crypto_sve_aese helper_crypto_sve_aese_arm
+#define helper_crypto_sve_aesmc helper_crypto_sve_aesmc_arm
 #define helper_crypto_sha1_3reg helper_crypto_sha1_3reg_arm
 #define helper_crypto_sha1h helper_crypto_sha1h_arm
 #define helper_crypto_sha1su1 helper_crypto_sha1su1_arm
@@ -1322,6 +2095,9 @@
 #define helper_crypto_sm3tt helper_crypto_sm3tt_arm
 #define helper_crypto_sm4e helper_crypto_sm4e_arm
 #define helper_crypto_sm4ekey helper_crypto_sm4ekey_arm
+#define helper_crypto_sve_sm4e helper_crypto_sve_sm4e_arm
+#define helper_crypto_sve_sm4ekey helper_crypto_sve_sm4ekey_arm
+#define helper_crypto_rax1 helper_crypto_rax1_arm
 #define helper_check_breakpoints helper_check_breakpoints_arm
 #define arm_debug_check_watchpoint arm_debug_check_watchpoint_arm
 #define arm_debug_excp_handler arm_debug_excp_handler_arm
@@ -1799,12 +2575,23 @@
 #define helper_gvec_qrdmlsh_s32 helper_gvec_qrdmlsh_s32_arm
 #define helper_gvec_sdot_b helper_gvec_sdot_b_arm
 #define helper_gvec_udot_b helper_gvec_udot_b_arm
+#define helper_gvec_usdot_b helper_gvec_usdot_b_arm
 #define helper_gvec_sdot_h helper_gvec_sdot_h_arm
 #define helper_gvec_udot_h helper_gvec_udot_h_arm
 #define helper_gvec_sdot_idx_b helper_gvec_sdot_idx_b_arm
 #define helper_gvec_udot_idx_b helper_gvec_udot_idx_b_arm
+#define helper_gvec_sudot_idx_b helper_gvec_sudot_idx_b_arm
+#define helper_gvec_usdot_idx_b helper_gvec_usdot_idx_b_arm
 #define helper_gvec_sdot_idx_h helper_gvec_sdot_idx_h_arm
 #define helper_gvec_udot_idx_h helper_gvec_udot_idx_h_arm
+#define helper_gvec_smmla_b helper_gvec_smmla_b_arm
+#define helper_gvec_ummla_b helper_gvec_ummla_b_arm
+#define helper_gvec_usmmla_b helper_gvec_usmmla_b_arm
+#define helper_gvec_bfdot helper_gvec_bfdot_arm
+#define helper_gvec_bfdot_idx helper_gvec_bfdot_idx_arm
+#define helper_gvec_bfmmla helper_gvec_bfmmla_arm
+#define helper_gvec_bfmlal helper_gvec_bfmlal_arm
+#define helper_gvec_bfmlal_idx helper_gvec_bfmlal_idx_arm
 #define helper_gvec_fcaddh helper_gvec_fcaddh_arm
 #define helper_gvec_fcadds helper_gvec_fcadds_arm
 #define helper_gvec_fcaddd helper_gvec_fcaddd_arm
@@ -1853,10 +2640,20 @@
 #define helper_gvec_uqsub_d helper_gvec_uqsub_d_arm
 #define helper_gvec_sqadd_d helper_gvec_sqadd_d_arm
 #define helper_gvec_sqsub_d helper_gvec_sqsub_d_arm
+#define helper_gvec_saba_b helper_gvec_saba_b_arm
+#define helper_gvec_saba_h helper_gvec_saba_h_arm
+#define helper_gvec_saba_s helper_gvec_saba_s_arm
+#define helper_gvec_saba_d helper_gvec_saba_d_arm
+#define helper_gvec_uaba_b helper_gvec_uaba_b_arm
+#define helper_gvec_uaba_h helper_gvec_uaba_h_arm
+#define helper_gvec_uaba_s helper_gvec_uaba_s_arm
+#define helper_gvec_uaba_d helper_gvec_uaba_d_arm
 #define helper_gvec_fmlal_a32 helper_gvec_fmlal_a32_arm
 #define helper_gvec_fmlal_a64 helper_gvec_fmlal_a64_arm
 #define helper_gvec_fmlal_idx_a32 helper_gvec_fmlal_idx_a32_arm
 #define helper_gvec_fmlal_idx_a64 helper_gvec_fmlal_idx_a64_arm
+#define helper_sve2_fmlal_zzzw_s helper_sve2_fmlal_zzzw_s_arm
+#define helper_sve2_fmlal_zzxw_s helper_sve2_fmlal_zzxw_s_arm
 #define helper_gvec_sshl_b helper_gvec_sshl_b_arm
 #define helper_gvec_sshl_h helper_gvec_sshl_h_arm
 #define helper_gvec_ushl_b helper_gvec_ushl_b_arm
@@ -1868,6 +2665,680 @@
 #define vfp_get_fpscr vfp_get_fpscr_arm
 #define helper_vfp_set_fpscr helper_vfp_set_fpscr_arm
 #define vfp_set_fpscr vfp_set_fpscr_arm
+#define helper_mve_vctp helper_mve_vctp_arm
+#define helper_mve_vpnot helper_mve_vpnot_arm
+#define helper_mve_vpsel helper_mve_vpsel_arm
+#define helper_mve_vdup helper_mve_vdup_arm
+#define helper_mve_vmovi helper_mve_vmovi_arm
+#define helper_mve_vandi helper_mve_vandi_arm
+#define helper_mve_vorri helper_mve_vorri_arm
+#define helper_mve_vidupb helper_mve_vidupb_arm
+#define helper_mve_viduph helper_mve_viduph_arm
+#define helper_mve_vidupw helper_mve_vidupw_arm
+#define helper_mve_viwdupb helper_mve_viwdupb_arm
+#define helper_mve_viwduph helper_mve_viwduph_arm
+#define helper_mve_viwdupw helper_mve_viwdupw_arm
+#define helper_mve_vdwdupb helper_mve_vdwdupb_arm
+#define helper_mve_vdwduph helper_mve_vdwduph_arm
+#define helper_mve_vdwdupw helper_mve_vdwdupw_arm
+#define helper_mve_vcmpeqb helper_mve_vcmpeqb_arm
+#define helper_mve_vcmpeqh helper_mve_vcmpeqh_arm
+#define helper_mve_vcmpeqw helper_mve_vcmpeqw_arm
+#define helper_mve_vcmpeq_scalarb helper_mve_vcmpeq_scalarb_arm
+#define helper_mve_vcmpeq_scalarh helper_mve_vcmpeq_scalarh_arm
+#define helper_mve_vcmpeq_scalarw helper_mve_vcmpeq_scalarw_arm
+#define helper_mve_vcmpneb helper_mve_vcmpneb_arm
+#define helper_mve_vcmpneh helper_mve_vcmpneh_arm
+#define helper_mve_vcmpnew helper_mve_vcmpnew_arm
+#define helper_mve_vcmpne_scalarb helper_mve_vcmpne_scalarb_arm
+#define helper_mve_vcmpne_scalarh helper_mve_vcmpne_scalarh_arm
+#define helper_mve_vcmpne_scalarw helper_mve_vcmpne_scalarw_arm
+#define helper_mve_vcmpcsb helper_mve_vcmpcsb_arm
+#define helper_mve_vcmpcsh helper_mve_vcmpcsh_arm
+#define helper_mve_vcmpcsw helper_mve_vcmpcsw_arm
+#define helper_mve_vcmpcs_scalarb helper_mve_vcmpcs_scalarb_arm
+#define helper_mve_vcmpcs_scalarh helper_mve_vcmpcs_scalarh_arm
+#define helper_mve_vcmpcs_scalarw helper_mve_vcmpcs_scalarw_arm
+#define helper_mve_vcmphib helper_mve_vcmphib_arm
+#define helper_mve_vcmphih helper_mve_vcmphih_arm
+#define helper_mve_vcmphiw helper_mve_vcmphiw_arm
+#define helper_mve_vcmphi_scalarb helper_mve_vcmphi_scalarb_arm
+#define helper_mve_vcmphi_scalarh helper_mve_vcmphi_scalarh_arm
+#define helper_mve_vcmphi_scalarw helper_mve_vcmphi_scalarw_arm
+#define helper_mve_vcmpgeb helper_mve_vcmpgeb_arm
+#define helper_mve_vcmpgeh helper_mve_vcmpgeh_arm
+#define helper_mve_vcmpgew helper_mve_vcmpgew_arm
+#define helper_mve_vcmpge_scalarb helper_mve_vcmpge_scalarb_arm
+#define helper_mve_vcmpge_scalarh helper_mve_vcmpge_scalarh_arm
+#define helper_mve_vcmpge_scalarw helper_mve_vcmpge_scalarw_arm
+#define helper_mve_vcmpltb helper_mve_vcmpltb_arm
+#define helper_mve_vcmplth helper_mve_vcmplth_arm
+#define helper_mve_vcmpltw helper_mve_vcmpltw_arm
+#define helper_mve_vcmplt_scalarb helper_mve_vcmplt_scalarb_arm
+#define helper_mve_vcmplt_scalarh helper_mve_vcmplt_scalarh_arm
+#define helper_mve_vcmplt_scalarw helper_mve_vcmplt_scalarw_arm
+#define helper_mve_vcmpgtb helper_mve_vcmpgtb_arm
+#define helper_mve_vcmpgth helper_mve_vcmpgth_arm
+#define helper_mve_vcmpgtw helper_mve_vcmpgtw_arm
+#define helper_mve_vcmpgt_scalarb helper_mve_vcmpgt_scalarb_arm
+#define helper_mve_vcmpgt_scalarh helper_mve_vcmpgt_scalarh_arm
+#define helper_mve_vcmpgt_scalarw helper_mve_vcmpgt_scalarw_arm
+#define helper_mve_vcmpleb helper_mve_vcmpleb_arm
+#define helper_mve_vcmpleh helper_mve_vcmpleh_arm
+#define helper_mve_vcmplew helper_mve_vcmplew_arm
+#define helper_mve_vcmple_scalarb helper_mve_vcmple_scalarb_arm
+#define helper_mve_vcmple_scalarh helper_mve_vcmple_scalarh_arm
+#define helper_mve_vcmple_scalarw helper_mve_vcmple_scalarw_arm
+#define helper_mve_vfcmpeqh helper_mve_vfcmpeqh_arm
+#define helper_mve_vfcmpeqs helper_mve_vfcmpeqs_arm
+#define helper_mve_vfcmpneh helper_mve_vfcmpneh_arm
+#define helper_mve_vfcmpnes helper_mve_vfcmpnes_arm
+#define helper_mve_vfcmpgeh helper_mve_vfcmpgeh_arm
+#define helper_mve_vfcmpges helper_mve_vfcmpges_arm
+#define helper_mve_vfcmplth helper_mve_vfcmplth_arm
+#define helper_mve_vfcmplts helper_mve_vfcmplts_arm
+#define helper_mve_vfcmpgth helper_mve_vfcmpgth_arm
+#define helper_mve_vfcmpgts helper_mve_vfcmpgts_arm
+#define helper_mve_vfcmpleh helper_mve_vfcmpleh_arm
+#define helper_mve_vfcmples helper_mve_vfcmples_arm
+#define helper_mve_vfcmpeq_scalarh helper_mve_vfcmpeq_scalarh_arm
+#define helper_mve_vfcmpeq_scalars helper_mve_vfcmpeq_scalars_arm
+#define helper_mve_vfcmpne_scalarh helper_mve_vfcmpne_scalarh_arm
+#define helper_mve_vfcmpne_scalars helper_mve_vfcmpne_scalars_arm
+#define helper_mve_vfcmpge_scalarh helper_mve_vfcmpge_scalarh_arm
+#define helper_mve_vfcmpge_scalars helper_mve_vfcmpge_scalars_arm
+#define helper_mve_vfcmplt_scalarh helper_mve_vfcmplt_scalarh_arm
+#define helper_mve_vfcmplt_scalars helper_mve_vfcmplt_scalars_arm
+#define helper_mve_vfcmpgt_scalarh helper_mve_vfcmpgt_scalarh_arm
+#define helper_mve_vfcmpgt_scalars helper_mve_vfcmpgt_scalars_arm
+#define helper_mve_vfcmple_scalarh helper_mve_vfcmple_scalarh_arm
+#define helper_mve_vfcmple_scalars helper_mve_vfcmple_scalars_arm
+#define helper_mve_vfabsh helper_mve_vfabsh_arm
+#define helper_mve_vfabss helper_mve_vfabss_arm
+#define helper_mve_vfnegh helper_mve_vfnegh_arm
+#define helper_mve_vfnegs helper_mve_vfnegs_arm
+#define helper_mve_vldrb helper_mve_vldrb_arm
+#define helper_mve_vldrh helper_mve_vldrh_arm
+#define helper_mve_vldrw helper_mve_vldrw_arm
+#define helper_mve_vldrb_sh helper_mve_vldrb_sh_arm
+#define helper_mve_vldrb_uh helper_mve_vldrb_uh_arm
+#define helper_mve_vldrb_sw helper_mve_vldrb_sw_arm
+#define helper_mve_vldrb_uw helper_mve_vldrb_uw_arm
+#define helper_mve_vldrh_sw helper_mve_vldrh_sw_arm
+#define helper_mve_vldrh_uw helper_mve_vldrh_uw_arm
+#define helper_mve_vstrb helper_mve_vstrb_arm
+#define helper_mve_vstrh helper_mve_vstrh_arm
+#define helper_mve_vstrw helper_mve_vstrw_arm
+#define helper_mve_vstrb_h helper_mve_vstrb_h_arm
+#define helper_mve_vstrb_w helper_mve_vstrb_w_arm
+#define helper_mve_vstrh_w helper_mve_vstrh_w_arm
+#define helper_mve_vldrb_sg_sh helper_mve_vldrb_sg_sh_arm
+#define helper_mve_vldrb_sg_sw helper_mve_vldrb_sg_sw_arm
+#define helper_mve_vldrh_sg_sw helper_mve_vldrh_sg_sw_arm
+#define helper_mve_vldrb_sg_ub helper_mve_vldrb_sg_ub_arm
+#define helper_mve_vldrb_sg_uh helper_mve_vldrb_sg_uh_arm
+#define helper_mve_vldrb_sg_uw helper_mve_vldrb_sg_uw_arm
+#define helper_mve_vldrh_sg_uh helper_mve_vldrh_sg_uh_arm
+#define helper_mve_vldrh_sg_uw helper_mve_vldrh_sg_uw_arm
+#define helper_mve_vldrw_sg_uw helper_mve_vldrw_sg_uw_arm
+#define helper_mve_vldrd_sg_ud helper_mve_vldrd_sg_ud_arm
+#define helper_mve_vldrh_sg_os_sw helper_mve_vldrh_sg_os_sw_arm
+#define helper_mve_vldrh_sg_os_uh helper_mve_vldrh_sg_os_uh_arm
+#define helper_mve_vldrh_sg_os_uw helper_mve_vldrh_sg_os_uw_arm
+#define helper_mve_vldrw_sg_os_uw helper_mve_vldrw_sg_os_uw_arm
+#define helper_mve_vldrd_sg_os_ud helper_mve_vldrd_sg_os_ud_arm
+#define helper_mve_vstrb_sg_ub helper_mve_vstrb_sg_ub_arm
+#define helper_mve_vstrb_sg_uh helper_mve_vstrb_sg_uh_arm
+#define helper_mve_vstrb_sg_uw helper_mve_vstrb_sg_uw_arm
+#define helper_mve_vstrh_sg_uh helper_mve_vstrh_sg_uh_arm
+#define helper_mve_vstrh_sg_uw helper_mve_vstrh_sg_uw_arm
+#define helper_mve_vstrw_sg_uw helper_mve_vstrw_sg_uw_arm
+#define helper_mve_vstrd_sg_ud helper_mve_vstrd_sg_ud_arm
+#define helper_mve_vstrh_sg_os_uh helper_mve_vstrh_sg_os_uh_arm
+#define helper_mve_vstrh_sg_os_uw helper_mve_vstrh_sg_os_uw_arm
+#define helper_mve_vstrw_sg_os_uw helper_mve_vstrw_sg_os_uw_arm
+#define helper_mve_vstrd_sg_os_ud helper_mve_vstrd_sg_os_ud_arm
+#define helper_mve_vldrw_sg_wb_uw helper_mve_vldrw_sg_wb_uw_arm
+#define helper_mve_vldrd_sg_wb_ud helper_mve_vldrd_sg_wb_ud_arm
+#define helper_mve_vstrw_sg_wb_uw helper_mve_vstrw_sg_wb_uw_arm
+#define helper_mve_vstrd_sg_wb_ud helper_mve_vstrd_sg_wb_ud_arm
+#define helper_mve_vld20b helper_mve_vld20b_arm
+#define helper_mve_vld20h helper_mve_vld20h_arm
+#define helper_mve_vld20w helper_mve_vld20w_arm
+#define helper_mve_vld21b helper_mve_vld21b_arm
+#define helper_mve_vld21h helper_mve_vld21h_arm
+#define helper_mve_vld21w helper_mve_vld21w_arm
+#define helper_mve_vld40b helper_mve_vld40b_arm
+#define helper_mve_vld40h helper_mve_vld40h_arm
+#define helper_mve_vld40w helper_mve_vld40w_arm
+#define helper_mve_vld41b helper_mve_vld41b_arm
+#define helper_mve_vld41h helper_mve_vld41h_arm
+#define helper_mve_vld41w helper_mve_vld41w_arm
+#define helper_mve_vld42b helper_mve_vld42b_arm
+#define helper_mve_vld42h helper_mve_vld42h_arm
+#define helper_mve_vld42w helper_mve_vld42w_arm
+#define helper_mve_vld43b helper_mve_vld43b_arm
+#define helper_mve_vld43h helper_mve_vld43h_arm
+#define helper_mve_vld43w helper_mve_vld43w_arm
+#define helper_mve_vst20b helper_mve_vst20b_arm
+#define helper_mve_vst20h helper_mve_vst20h_arm
+#define helper_mve_vst20w helper_mve_vst20w_arm
+#define helper_mve_vst21b helper_mve_vst21b_arm
+#define helper_mve_vst21h helper_mve_vst21h_arm
+#define helper_mve_vst21w helper_mve_vst21w_arm
+#define helper_mve_vst40b helper_mve_vst40b_arm
+#define helper_mve_vst40h helper_mve_vst40h_arm
+#define helper_mve_vst40w helper_mve_vst40w_arm
+#define helper_mve_vst41b helper_mve_vst41b_arm
+#define helper_mve_vst41h helper_mve_vst41h_arm
+#define helper_mve_vst41w helper_mve_vst41w_arm
+#define helper_mve_vst42b helper_mve_vst42b_arm
+#define helper_mve_vst42h helper_mve_vst42h_arm
+#define helper_mve_vst42w helper_mve_vst42w_arm
+#define helper_mve_vst43b helper_mve_vst43b_arm
+#define helper_mve_vst43h helper_mve_vst43h_arm
+#define helper_mve_vst43w helper_mve_vst43w_arm
+#define helper_mve_vand helper_mve_vand_arm
+#define helper_mve_vbic helper_mve_vbic_arm
+#define helper_mve_vorr helper_mve_vorr_arm
+#define helper_mve_vorn helper_mve_vorn_arm
+#define helper_mve_veor helper_mve_veor_arm
+#define helper_mve_vaddb helper_mve_vaddb_arm
+#define helper_mve_vaddh helper_mve_vaddh_arm
+#define helper_mve_vaddw helper_mve_vaddw_arm
+#define helper_mve_vadd_scalarb helper_mve_vadd_scalarb_arm
+#define helper_mve_vadd_scalarh helper_mve_vadd_scalarh_arm
+#define helper_mve_vadd_scalarw helper_mve_vadd_scalarw_arm
+#define helper_mve_vsubb helper_mve_vsubb_arm
+#define helper_mve_vsubh helper_mve_vsubh_arm
+#define helper_mve_vsubw helper_mve_vsubw_arm
+#define helper_mve_vsub_scalarb helper_mve_vsub_scalarb_arm
+#define helper_mve_vsub_scalarh helper_mve_vsub_scalarh_arm
+#define helper_mve_vsub_scalarw helper_mve_vsub_scalarw_arm
+#define helper_mve_vmulb helper_mve_vmulb_arm
+#define helper_mve_vmulh helper_mve_vmulh_arm
+#define helper_mve_vmulw helper_mve_vmulw_arm
+#define helper_mve_vmul_scalarb helper_mve_vmul_scalarb_arm
+#define helper_mve_vmul_scalarh helper_mve_vmul_scalarh_arm
+#define helper_mve_vmul_scalarw helper_mve_vmul_scalarw_arm
+#define helper_mve_vmulhsb helper_mve_vmulhsb_arm
+#define helper_mve_vmulhsh helper_mve_vmulhsh_arm
+#define helper_mve_vmulhsw helper_mve_vmulhsw_arm
+#define helper_mve_vmulhub helper_mve_vmulhub_arm
+#define helper_mve_vmulhuh helper_mve_vmulhuh_arm
+#define helper_mve_vmulhuw helper_mve_vmulhuw_arm
+#define helper_mve_vrmulhsb helper_mve_vrmulhsb_arm
+#define helper_mve_vrmulhsh helper_mve_vrmulhsh_arm
+#define helper_mve_vrmulhsw helper_mve_vrmulhsw_arm
+#define helper_mve_vrmulhub helper_mve_vrmulhub_arm
+#define helper_mve_vrmulhuh helper_mve_vrmulhuh_arm
+#define helper_mve_vrmulhuw helper_mve_vrmulhuw_arm
+#define helper_mve_vmullbsb helper_mve_vmullbsb_arm
+#define helper_mve_vmullbsh helper_mve_vmullbsh_arm
+#define helper_mve_vmullbsw helper_mve_vmullbsw_arm
+#define helper_mve_vmullbub helper_mve_vmullbub_arm
+#define helper_mve_vmullbuh helper_mve_vmullbuh_arm
+#define helper_mve_vmullbuw helper_mve_vmullbuw_arm
+#define helper_mve_vmulltsb helper_mve_vmulltsb_arm
+#define helper_mve_vmulltsh helper_mve_vmulltsh_arm
+#define helper_mve_vmulltsw helper_mve_vmulltsw_arm
+#define helper_mve_vmulltub helper_mve_vmulltub_arm
+#define helper_mve_vmulltuh helper_mve_vmulltuh_arm
+#define helper_mve_vmulltuw helper_mve_vmulltuw_arm
+#define helper_mve_vmullpbh helper_mve_vmullpbh_arm
+#define helper_mve_vmullpth helper_mve_vmullpth_arm
+#define helper_mve_vmullpbw helper_mve_vmullpbw_arm
+#define helper_mve_vmullptw helper_mve_vmullptw_arm
+#define helper_mve_vqdmullbh helper_mve_vqdmullbh_arm
+#define helper_mve_vqdmullbw helper_mve_vqdmullbw_arm
+#define helper_mve_vqdmullth helper_mve_vqdmullth_arm
+#define helper_mve_vqdmulltw helper_mve_vqdmulltw_arm
+#define helper_mve_vqdmullb_scalarh helper_mve_vqdmullb_scalarh_arm
+#define helper_mve_vqdmullb_scalarw helper_mve_vqdmullb_scalarw_arm
+#define helper_mve_vqdmullt_scalarh helper_mve_vqdmullt_scalarh_arm
+#define helper_mve_vqdmullt_scalarw helper_mve_vqdmullt_scalarw_arm
+#define helper_mve_vcadd90b helper_mve_vcadd90b_arm
+#define helper_mve_vcadd90h helper_mve_vcadd90h_arm
+#define helper_mve_vcadd90w helper_mve_vcadd90w_arm
+#define helper_mve_vcadd270b helper_mve_vcadd270b_arm
+#define helper_mve_vcadd270h helper_mve_vcadd270h_arm
+#define helper_mve_vcadd270w helper_mve_vcadd270w_arm
+#define helper_mve_vhcadd90b helper_mve_vhcadd90b_arm
+#define helper_mve_vhcadd90h helper_mve_vhcadd90h_arm
+#define helper_mve_vhcadd90w helper_mve_vhcadd90w_arm
+#define helper_mve_vhcadd270b helper_mve_vhcadd270b_arm
+#define helper_mve_vhcadd270h helper_mve_vhcadd270h_arm
+#define helper_mve_vhcadd270w helper_mve_vhcadd270w_arm
+#define helper_mve_vmaxsb helper_mve_vmaxsb_arm
+#define helper_mve_vmaxsh helper_mve_vmaxsh_arm
+#define helper_mve_vmaxsw helper_mve_vmaxsw_arm
+#define helper_mve_vmaxub helper_mve_vmaxub_arm
+#define helper_mve_vmaxuh helper_mve_vmaxuh_arm
+#define helper_mve_vmaxuw helper_mve_vmaxuw_arm
+#define helper_mve_vminsb helper_mve_vminsb_arm
+#define helper_mve_vminsh helper_mve_vminsh_arm
+#define helper_mve_vminsw helper_mve_vminsw_arm
+#define helper_mve_vminub helper_mve_vminub_arm
+#define helper_mve_vminuh helper_mve_vminuh_arm
+#define helper_mve_vminuw helper_mve_vminuw_arm
+#define helper_mve_vabdsb helper_mve_vabdsb_arm
+#define helper_mve_vabdsh helper_mve_vabdsh_arm
+#define helper_mve_vabdsw helper_mve_vabdsw_arm
+#define helper_mve_vabdub helper_mve_vabdub_arm
+#define helper_mve_vabduh helper_mve_vabduh_arm
+#define helper_mve_vabduw helper_mve_vabduw_arm
+#define helper_mve_vhaddsb helper_mve_vhaddsb_arm
+#define helper_mve_vhaddsh helper_mve_vhaddsh_arm
+#define helper_mve_vhaddsw helper_mve_vhaddsw_arm
+#define helper_mve_vhaddub helper_mve_vhaddub_arm
+#define helper_mve_vhadduh helper_mve_vhadduh_arm
+#define helper_mve_vhadduw helper_mve_vhadduw_arm
+#define helper_mve_vhadds_scalarb helper_mve_vhadds_scalarb_arm
+#define helper_mve_vhadds_scalarh helper_mve_vhadds_scalarh_arm
+#define helper_mve_vhadds_scalarw helper_mve_vhadds_scalarw_arm
+#define helper_mve_vhaddu_scalarb helper_mve_vhaddu_scalarb_arm
+#define helper_mve_vhaddu_scalarh helper_mve_vhaddu_scalarh_arm
+#define helper_mve_vhaddu_scalarw helper_mve_vhaddu_scalarw_arm
+#define helper_mve_vrhaddsb helper_mve_vrhaddsb_arm
+#define helper_mve_vrhaddsh helper_mve_vrhaddsh_arm
+#define helper_mve_vrhaddsw helper_mve_vrhaddsw_arm
+#define helper_mve_vrhaddub helper_mve_vrhaddub_arm
+#define helper_mve_vrhadduh helper_mve_vrhadduh_arm
+#define helper_mve_vrhadduw helper_mve_vrhadduw_arm
+#define helper_mve_vadc helper_mve_vadc_arm
+#define helper_mve_vadci helper_mve_vadci_arm
+#define helper_mve_vsbc helper_mve_vsbc_arm
+#define helper_mve_vsbci helper_mve_vsbci_arm
+#define helper_mve_vhsubsb helper_mve_vhsubsb_arm
+#define helper_mve_vhsubsh helper_mve_vhsubsh_arm
+#define helper_mve_vhsubsw helper_mve_vhsubsw_arm
+#define helper_mve_vhsubub helper_mve_vhsubub_arm
+#define helper_mve_vhsubuh helper_mve_vhsubuh_arm
+#define helper_mve_vhsubuw helper_mve_vhsubuw_arm
+#define helper_mve_vhsubs_scalarb helper_mve_vhsubs_scalarb_arm
+#define helper_mve_vhsubs_scalarh helper_mve_vhsubs_scalarh_arm
+#define helper_mve_vhsubs_scalarw helper_mve_vhsubs_scalarw_arm
+#define helper_mve_vhsubu_scalarb helper_mve_vhsubu_scalarb_arm
+#define helper_mve_vhsubu_scalarh helper_mve_vhsubu_scalarh_arm
+#define helper_mve_vhsubu_scalarw helper_mve_vhsubu_scalarw_arm
+#define helper_mve_vqaddsb helper_mve_vqaddsb_arm
+#define helper_mve_vqaddsh helper_mve_vqaddsh_arm
+#define helper_mve_vqaddsw helper_mve_vqaddsw_arm
+#define helper_mve_vqaddub helper_mve_vqaddub_arm
+#define helper_mve_vqadduh helper_mve_vqadduh_arm
+#define helper_mve_vqadduw helper_mve_vqadduw_arm
+#define helper_mve_vqsubsb helper_mve_vqsubsb_arm
+#define helper_mve_vqsubsh helper_mve_vqsubsh_arm
+#define helper_mve_vqsubsw helper_mve_vqsubsw_arm
+#define helper_mve_vqsubub helper_mve_vqsubub_arm
+#define helper_mve_vqsubuh helper_mve_vqsubuh_arm
+#define helper_mve_vqsubuw helper_mve_vqsubuw_arm
+#define helper_mve_vqdmulhb helper_mve_vqdmulhb_arm
+#define helper_mve_vqdmulhh helper_mve_vqdmulhh_arm
+#define helper_mve_vqdmulhw helper_mve_vqdmulhw_arm
+#define helper_mve_vqrdmulhb helper_mve_vqrdmulhb_arm
+#define helper_mve_vqrdmulhh helper_mve_vqrdmulhh_arm
+#define helper_mve_vqrdmulhw helper_mve_vqrdmulhw_arm
+#define helper_mve_vqadds_scalarb helper_mve_vqadds_scalarb_arm
+#define helper_mve_vqadds_scalarh helper_mve_vqadds_scalarh_arm
+#define helper_mve_vqadds_scalarw helper_mve_vqadds_scalarw_arm
+#define helper_mve_vqaddu_scalarb helper_mve_vqaddu_scalarb_arm
+#define helper_mve_vqaddu_scalarh helper_mve_vqaddu_scalarh_arm
+#define helper_mve_vqaddu_scalarw helper_mve_vqaddu_scalarw_arm
+#define helper_mve_vqsubs_scalarb helper_mve_vqsubs_scalarb_arm
+#define helper_mve_vqsubs_scalarh helper_mve_vqsubs_scalarh_arm
+#define helper_mve_vqsubs_scalarw helper_mve_vqsubs_scalarw_arm
+#define helper_mve_vqsubu_scalarb helper_mve_vqsubu_scalarb_arm
+#define helper_mve_vqsubu_scalarh helper_mve_vqsubu_scalarh_arm
+#define helper_mve_vqsubu_scalarw helper_mve_vqsubu_scalarw_arm
+#define helper_mve_vqdmulh_scalarb helper_mve_vqdmulh_scalarb_arm
+#define helper_mve_vqdmulh_scalarh helper_mve_vqdmulh_scalarh_arm
+#define helper_mve_vqdmulh_scalarw helper_mve_vqdmulh_scalarw_arm
+#define helper_mve_vqrdmulh_scalarb helper_mve_vqrdmulh_scalarb_arm
+#define helper_mve_vqrdmulh_scalarh helper_mve_vqrdmulh_scalarh_arm
+#define helper_mve_vqrdmulh_scalarw helper_mve_vqrdmulh_scalarw_arm
+#define helper_mve_vmlab helper_mve_vmlab_arm
+#define helper_mve_vmlah helper_mve_vmlah_arm
+#define helper_mve_vmlaw helper_mve_vmlaw_arm
+#define helper_mve_vmlasb helper_mve_vmlasb_arm
+#define helper_mve_vmlash helper_mve_vmlash_arm
+#define helper_mve_vmlasw helper_mve_vmlasw_arm
+#define helper_mve_vqdmlahb helper_mve_vqdmlahb_arm
+#define helper_mve_vqdmlahh helper_mve_vqdmlahh_arm
+#define helper_mve_vqdmlahw helper_mve_vqdmlahw_arm
+#define helper_mve_vqrdmlahb helper_mve_vqrdmlahb_arm
+#define helper_mve_vqrdmlahh helper_mve_vqrdmlahh_arm
+#define helper_mve_vqrdmlahw helper_mve_vqrdmlahw_arm
+#define helper_mve_vqdmlashb helper_mve_vqdmlashb_arm
+#define helper_mve_vqdmlashh helper_mve_vqdmlashh_arm
+#define helper_mve_vqdmlashw helper_mve_vqdmlashw_arm
+#define helper_mve_vqrdmlashb helper_mve_vqrdmlashb_arm
+#define helper_mve_vqrdmlashh helper_mve_vqrdmlashh_arm
+#define helper_mve_vqrdmlashw helper_mve_vqrdmlashw_arm
+#define helper_mve_vfaddh helper_mve_vfaddh_arm
+#define helper_mve_vfadds helper_mve_vfadds_arm
+#define helper_mve_vfsubh helper_mve_vfsubh_arm
+#define helper_mve_vfsubs helper_mve_vfsubs_arm
+#define helper_mve_vfmulh helper_mve_vfmulh_arm
+#define helper_mve_vfmuls helper_mve_vfmuls_arm
+#define helper_mve_vfabdh helper_mve_vfabdh_arm
+#define helper_mve_vfabds helper_mve_vfabds_arm
+#define helper_mve_vmaxnmh helper_mve_vmaxnmh_arm
+#define helper_mve_vmaxnms helper_mve_vmaxnms_arm
+#define helper_mve_vminnmh helper_mve_vminnmh_arm
+#define helper_mve_vminnms helper_mve_vminnms_arm
+#define helper_mve_vmaxnmah helper_mve_vmaxnmah_arm
+#define helper_mve_vmaxnmas helper_mve_vmaxnmas_arm
+#define helper_mve_vminnmah helper_mve_vminnmah_arm
+#define helper_mve_vminnmas helper_mve_vminnmas_arm
+#define helper_mve_vfcadd90h helper_mve_vfcadd90h_arm
+#define helper_mve_vfcadd90s helper_mve_vfcadd90s_arm
+#define helper_mve_vfcadd270h helper_mve_vfcadd270h_arm
+#define helper_mve_vfcadd270s helper_mve_vfcadd270s_arm
+#define helper_mve_vcmul0h helper_mve_vcmul0h_arm
+#define helper_mve_vcmul0s helper_mve_vcmul0s_arm
+#define helper_mve_vcmul90h helper_mve_vcmul90h_arm
+#define helper_mve_vcmul90s helper_mve_vcmul90s_arm
+#define helper_mve_vcmul180h helper_mve_vcmul180h_arm
+#define helper_mve_vcmul180s helper_mve_vcmul180s_arm
+#define helper_mve_vcmul270h helper_mve_vcmul270h_arm
+#define helper_mve_vcmul270s helper_mve_vcmul270s_arm
+#define helper_mve_vcmla0h helper_mve_vcmla0h_arm
+#define helper_mve_vcmla0s helper_mve_vcmla0s_arm
+#define helper_mve_vcmla90h helper_mve_vcmla90h_arm
+#define helper_mve_vcmla90s helper_mve_vcmla90s_arm
+#define helper_mve_vcmla180h helper_mve_vcmla180h_arm
+#define helper_mve_vcmla180s helper_mve_vcmla180s_arm
+#define helper_mve_vcmla270h helper_mve_vcmla270h_arm
+#define helper_mve_vcmla270s helper_mve_vcmla270s_arm
+#define helper_mve_vfmah helper_mve_vfmah_arm
+#define helper_mve_vfmas helper_mve_vfmas_arm
+#define helper_mve_vfmsh helper_mve_vfmsh_arm
+#define helper_mve_vfmss helper_mve_vfmss_arm
+#define helper_mve_vfadd_scalarh helper_mve_vfadd_scalarh_arm
+#define helper_mve_vfadd_scalars helper_mve_vfadd_scalars_arm
+#define helper_mve_vfsub_scalarh helper_mve_vfsub_scalarh_arm
+#define helper_mve_vfsub_scalars helper_mve_vfsub_scalars_arm
+#define helper_mve_vfmul_scalarh helper_mve_vfmul_scalarh_arm
+#define helper_mve_vfmul_scalars helper_mve_vfmul_scalars_arm
+#define helper_mve_vfma_scalarh helper_mve_vfma_scalarh_arm
+#define helper_mve_vfma_scalars helper_mve_vfma_scalars_arm
+#define helper_mve_vfmas_scalarh helper_mve_vfmas_scalarh_arm
+#define helper_mve_vfmas_scalars helper_mve_vfmas_scalars_arm
+#define helper_mve_vcvt_sh helper_mve_vcvt_sh_arm
+#define helper_mve_vcvt_uh helper_mve_vcvt_uh_arm
+#define helper_mve_vcvt_hs helper_mve_vcvt_hs_arm
+#define helper_mve_vcvt_hu helper_mve_vcvt_hu_arm
+#define helper_mve_vcvt_sf helper_mve_vcvt_sf_arm
+#define helper_mve_vcvt_uf helper_mve_vcvt_uf_arm
+#define helper_mve_vcvt_fs helper_mve_vcvt_fs_arm
+#define helper_mve_vcvt_fu helper_mve_vcvt_fu_arm
+#define helper_mve_vcvtb_sh helper_mve_vcvtb_sh_arm
+#define helper_mve_vcvtt_sh helper_mve_vcvtt_sh_arm
+#define helper_mve_vcvtb_hs helper_mve_vcvtb_hs_arm
+#define helper_mve_vcvtt_hs helper_mve_vcvtt_hs_arm
+#define helper_mve_vcvt_rm_sh helper_mve_vcvt_rm_sh_arm
+#define helper_mve_vcvt_rm_uh helper_mve_vcvt_rm_uh_arm
+#define helper_mve_vcvt_rm_ss helper_mve_vcvt_rm_ss_arm
+#define helper_mve_vcvt_rm_us helper_mve_vcvt_rm_us_arm
+#define helper_mve_vrint_rm_h helper_mve_vrint_rm_h_arm
+#define helper_mve_vrint_rm_s helper_mve_vrint_rm_s_arm
+#define helper_mve_vrintx_h helper_mve_vrintx_h_arm
+#define helper_mve_vrintx_s helper_mve_vrintx_s_arm
+#define helper_mve_vshlsb helper_mve_vshlsb_arm
+#define helper_mve_vshlsh helper_mve_vshlsh_arm
+#define helper_mve_vshlsw helper_mve_vshlsw_arm
+#define helper_mve_vshlub helper_mve_vshlub_arm
+#define helper_mve_vshluh helper_mve_vshluh_arm
+#define helper_mve_vshluw helper_mve_vshluw_arm
+#define helper_mve_vrshlsb helper_mve_vrshlsb_arm
+#define helper_mve_vrshlsh helper_mve_vrshlsh_arm
+#define helper_mve_vrshlsw helper_mve_vrshlsw_arm
+#define helper_mve_vrshlub helper_mve_vrshlub_arm
+#define helper_mve_vrshluh helper_mve_vrshluh_arm
+#define helper_mve_vrshluw helper_mve_vrshluw_arm
+#define helper_mve_vqshlsb helper_mve_vqshlsb_arm
+#define helper_mve_vqshlsh helper_mve_vqshlsh_arm
+#define helper_mve_vqshlsw helper_mve_vqshlsw_arm
+#define helper_mve_vqshlub helper_mve_vqshlub_arm
+#define helper_mve_vqshluh helper_mve_vqshluh_arm
+#define helper_mve_vqshluw helper_mve_vqshluw_arm
+#define helper_mve_vqrshlsb helper_mve_vqrshlsb_arm
+#define helper_mve_vqrshlsh helper_mve_vqrshlsh_arm
+#define helper_mve_vqrshlsw helper_mve_vqrshlsw_arm
+#define helper_mve_vqrshlub helper_mve_vqrshlub_arm
+#define helper_mve_vqrshluh helper_mve_vqrshluh_arm
+#define helper_mve_vqrshluw helper_mve_vqrshluw_arm
+#define helper_mve_vqdmladhb helper_mve_vqdmladhb_arm
+#define helper_mve_vqdmladhh helper_mve_vqdmladhh_arm
+#define helper_mve_vqdmladhw helper_mve_vqdmladhw_arm
+#define helper_mve_vqdmladhxb helper_mve_vqdmladhxb_arm
+#define helper_mve_vqdmladhxh helper_mve_vqdmladhxh_arm
+#define helper_mve_vqdmladhxw helper_mve_vqdmladhxw_arm
+#define helper_mve_vqrdmladhb helper_mve_vqrdmladhb_arm
+#define helper_mve_vqrdmladhh helper_mve_vqrdmladhh_arm
+#define helper_mve_vqrdmladhw helper_mve_vqrdmladhw_arm
+#define helper_mve_vqrdmladhxb helper_mve_vqrdmladhxb_arm
+#define helper_mve_vqrdmladhxh helper_mve_vqrdmladhxh_arm
+#define helper_mve_vqrdmladhxw helper_mve_vqrdmladhxw_arm
+#define helper_mve_vqdmlsdhb helper_mve_vqdmlsdhb_arm
+#define helper_mve_vqdmlsdhh helper_mve_vqdmlsdhh_arm
+#define helper_mve_vqdmlsdhw helper_mve_vqdmlsdhw_arm
+#define helper_mve_vqdmlsdhxb helper_mve_vqdmlsdhxb_arm
+#define helper_mve_vqdmlsdhxh helper_mve_vqdmlsdhxh_arm
+#define helper_mve_vqdmlsdhxw helper_mve_vqdmlsdhxw_arm
+#define helper_mve_vqrdmlsdhb helper_mve_vqrdmlsdhb_arm
+#define helper_mve_vqrdmlsdhh helper_mve_vqrdmlsdhh_arm
+#define helper_mve_vqrdmlsdhw helper_mve_vqrdmlsdhw_arm
+#define helper_mve_vqrdmlsdhxb helper_mve_vqrdmlsdhxb_arm
+#define helper_mve_vqrdmlsdhxh helper_mve_vqrdmlsdhxh_arm
+#define helper_mve_vqrdmlsdhxw helper_mve_vqrdmlsdhxw_arm
+#define helper_mve_vbrsrb helper_mve_vbrsrb_arm
+#define helper_mve_vbrsrh helper_mve_vbrsrh_arm
+#define helper_mve_vbrsrw helper_mve_vbrsrw_arm
+#define helper_mve_vshli_sb helper_mve_vshli_sb_arm
+#define helper_mve_vshli_sh helper_mve_vshli_sh_arm
+#define helper_mve_vshli_sw helper_mve_vshli_sw_arm
+#define helper_mve_vshli_ub helper_mve_vshli_ub_arm
+#define helper_mve_vshli_uh helper_mve_vshli_uh_arm
+#define helper_mve_vshli_uw helper_mve_vshli_uw_arm
+#define helper_mve_vrshli_sb helper_mve_vrshli_sb_arm
+#define helper_mve_vrshli_sh helper_mve_vrshli_sh_arm
+#define helper_mve_vrshli_sw helper_mve_vrshli_sw_arm
+#define helper_mve_vrshli_ub helper_mve_vrshli_ub_arm
+#define helper_mve_vrshli_uh helper_mve_vrshli_uh_arm
+#define helper_mve_vrshli_uw helper_mve_vrshli_uw_arm
+#define helper_mve_vqshli_sb helper_mve_vqshli_sb_arm
+#define helper_mve_vqshli_sh helper_mve_vqshli_sh_arm
+#define helper_mve_vqshli_sw helper_mve_vqshli_sw_arm
+#define helper_mve_vqshli_ub helper_mve_vqshli_ub_arm
+#define helper_mve_vqshli_uh helper_mve_vqshli_uh_arm
+#define helper_mve_vqshli_uw helper_mve_vqshli_uw_arm
+#define helper_mve_vqrshli_sb helper_mve_vqrshli_sb_arm
+#define helper_mve_vqrshli_sh helper_mve_vqrshli_sh_arm
+#define helper_mve_vqrshli_sw helper_mve_vqrshli_sw_arm
+#define helper_mve_vqrshli_ub helper_mve_vqrshli_ub_arm
+#define helper_mve_vqrshli_uh helper_mve_vqrshli_uh_arm
+#define helper_mve_vqrshli_uw helper_mve_vqrshli_uw_arm
+#define helper_mve_vqshlui_sb helper_mve_vqshlui_sb_arm
+#define helper_mve_vqshlui_sh helper_mve_vqshlui_sh_arm
+#define helper_mve_vqshlui_sw helper_mve_vqshlui_sw_arm
+#define helper_mve_vshllbsb helper_mve_vshllbsb_arm
+#define helper_mve_vshllbsh helper_mve_vshllbsh_arm
+#define helper_mve_vshllbub helper_mve_vshllbub_arm
+#define helper_mve_vshllbuh helper_mve_vshllbuh_arm
+#define helper_mve_vshlltsb helper_mve_vshlltsb_arm
+#define helper_mve_vshlltsh helper_mve_vshlltsh_arm
+#define helper_mve_vshlltub helper_mve_vshlltub_arm
+#define helper_mve_vshlltuh helper_mve_vshlltuh_arm
+#define helper_mve_vshrnbb helper_mve_vshrnbb_arm
+#define helper_mve_vshrnbh helper_mve_vshrnbh_arm
+#define helper_mve_vshrntb helper_mve_vshrntb_arm
+#define helper_mve_vshrnth helper_mve_vshrnth_arm
+#define helper_mve_vrshrnbb helper_mve_vrshrnbb_arm
+#define helper_mve_vrshrnbh helper_mve_vrshrnbh_arm
+#define helper_mve_vrshrntb helper_mve_vrshrntb_arm
+#define helper_mve_vrshrnth helper_mve_vrshrnth_arm
+#define helper_mve_vqshrnb_sb helper_mve_vqshrnb_sb_arm
+#define helper_mve_vqshrnb_sh helper_mve_vqshrnb_sh_arm
+#define helper_mve_vqshrnt_sb helper_mve_vqshrnt_sb_arm
+#define helper_mve_vqshrnt_sh helper_mve_vqshrnt_sh_arm
+#define helper_mve_vqshrnb_ub helper_mve_vqshrnb_ub_arm
+#define helper_mve_vqshrnb_uh helper_mve_vqshrnb_uh_arm
+#define helper_mve_vqshrnt_ub helper_mve_vqshrnt_ub_arm
+#define helper_mve_vqshrnt_uh helper_mve_vqshrnt_uh_arm
+#define helper_mve_vqshrunbb helper_mve_vqshrunbb_arm
+#define helper_mve_vqshrunbh helper_mve_vqshrunbh_arm
+#define helper_mve_vqshruntb helper_mve_vqshruntb_arm
+#define helper_mve_vqshrunth helper_mve_vqshrunth_arm
+#define helper_mve_vqrshrnb_sb helper_mve_vqrshrnb_sb_arm
+#define helper_mve_vqrshrnb_sh helper_mve_vqrshrnb_sh_arm
+#define helper_mve_vqrshrnt_sb helper_mve_vqrshrnt_sb_arm
+#define helper_mve_vqrshrnt_sh helper_mve_vqrshrnt_sh_arm
+#define helper_mve_vqrshrnb_ub helper_mve_vqrshrnb_ub_arm
+#define helper_mve_vqrshrnb_uh helper_mve_vqrshrnb_uh_arm
+#define helper_mve_vqrshrnt_ub helper_mve_vqrshrnt_ub_arm
+#define helper_mve_vqrshrnt_uh helper_mve_vqrshrnt_uh_arm
+#define helper_mve_vqrshrunbb helper_mve_vqrshrunbb_arm
+#define helper_mve_vqrshrunbh helper_mve_vqrshrunbh_arm
+#define helper_mve_vqrshruntb helper_mve_vqrshruntb_arm
+#define helper_mve_vqrshrunth helper_mve_vqrshrunth_arm
+#define helper_mve_vmovnbb helper_mve_vmovnbb_arm
+#define helper_mve_vmovnbh helper_mve_vmovnbh_arm
+#define helper_mve_vmovntb helper_mve_vmovntb_arm
+#define helper_mve_vmovnth helper_mve_vmovnth_arm
+#define helper_mve_vqmovnbsb helper_mve_vqmovnbsb_arm
+#define helper_mve_vqmovnbsh helper_mve_vqmovnbsh_arm
+#define helper_mve_vqmovntsb helper_mve_vqmovntsb_arm
+#define helper_mve_vqmovntsh helper_mve_vqmovntsh_arm
+#define helper_mve_vqmovnbub helper_mve_vqmovnbub_arm
+#define helper_mve_vqmovnbuh helper_mve_vqmovnbuh_arm
+#define helper_mve_vqmovntub helper_mve_vqmovntub_arm
+#define helper_mve_vqmovntuh helper_mve_vqmovntuh_arm
+#define helper_mve_vqmovunbb helper_mve_vqmovunbb_arm
+#define helper_mve_vqmovunbh helper_mve_vqmovunbh_arm
+#define helper_mve_vqmovuntb helper_mve_vqmovuntb_arm
+#define helper_mve_vqmovunth helper_mve_vqmovunth_arm
+#define helper_mve_sshrl helper_mve_sshrl_arm
+#define helper_mve_ushll helper_mve_ushll_arm
+#define helper_mve_sqshll helper_mve_sqshll_arm
+#define helper_mve_uqshll helper_mve_uqshll_arm
+#define helper_mve_sqrshrl helper_mve_sqrshrl_arm
+#define helper_mve_uqrshll helper_mve_uqrshll_arm
+#define helper_mve_sqrshrl48 helper_mve_sqrshrl48_arm
+#define helper_mve_uqrshll48 helper_mve_uqrshll48_arm
+#define helper_mve_uqshl helper_mve_uqshl_arm
+#define helper_mve_sqshl helper_mve_sqshl_arm
+#define helper_mve_uqrshl helper_mve_uqrshl_arm
+#define helper_mve_sqrshr helper_mve_sqrshr_arm
+#define helper_mve_vshlc helper_mve_vshlc_arm
+#define helper_mve_vsrib helper_mve_vsrib_arm
+#define helper_mve_vsrih helper_mve_vsrih_arm
+#define helper_mve_vsriw helper_mve_vsriw_arm
+#define helper_mve_vslib helper_mve_vslib_arm
+#define helper_mve_vslih helper_mve_vslih_arm
+#define helper_mve_vsliw helper_mve_vsliw_arm
+#define helper_mve_vclsb helper_mve_vclsb_arm
+#define helper_mve_vclsh helper_mve_vclsh_arm
+#define helper_mve_vclsw helper_mve_vclsw_arm
+#define helper_mve_vclzb helper_mve_vclzb_arm
+#define helper_mve_vclzh helper_mve_vclzh_arm
+#define helper_mve_vclzw helper_mve_vclzw_arm
+#define helper_mve_vrev16b helper_mve_vrev16b_arm
+#define helper_mve_vrev32b helper_mve_vrev32b_arm
+#define helper_mve_vrev32h helper_mve_vrev32h_arm
+#define helper_mve_vrev64b helper_mve_vrev64b_arm
+#define helper_mve_vrev64h helper_mve_vrev64h_arm
+#define helper_mve_vrev64w helper_mve_vrev64w_arm
+#define helper_mve_vmvn helper_mve_vmvn_arm
+#define helper_mve_vabsb helper_mve_vabsb_arm
+#define helper_mve_vabsh helper_mve_vabsh_arm
+#define helper_mve_vabsw helper_mve_vabsw_arm
+#define helper_mve_vnegb helper_mve_vnegb_arm
+#define helper_mve_vnegh helper_mve_vnegh_arm
+#define helper_mve_vnegw helper_mve_vnegw_arm
+#define helper_mve_vmaxab helper_mve_vmaxab_arm
+#define helper_mve_vmaxah helper_mve_vmaxah_arm
+#define helper_mve_vmaxaw helper_mve_vmaxaw_arm
+#define helper_mve_vminab helper_mve_vminab_arm
+#define helper_mve_vminah helper_mve_vminah_arm
+#define helper_mve_vminaw helper_mve_vminaw_arm
+#define helper_mve_vqabsb helper_mve_vqabsb_arm
+#define helper_mve_vqabsh helper_mve_vqabsh_arm
+#define helper_mve_vqabsw helper_mve_vqabsw_arm
+#define helper_mve_vqnegb helper_mve_vqnegb_arm
+#define helper_mve_vqnegh helper_mve_vqnegh_arm
+#define helper_mve_vqnegw helper_mve_vqnegw_arm
+#define helper_mve_vmlaldavsh helper_mve_vmlaldavsh_arm
+#define helper_mve_vmlaldavsw helper_mve_vmlaldavsw_arm
+#define helper_mve_vmlaldavxsh helper_mve_vmlaldavxsh_arm
+#define helper_mve_vmlaldavxsw helper_mve_vmlaldavxsw_arm
+#define helper_mve_vmlaldavuh helper_mve_vmlaldavuh_arm
+#define helper_mve_vmlaldavuw helper_mve_vmlaldavuw_arm
+#define helper_mve_vmlsldavsh helper_mve_vmlsldavsh_arm
+#define helper_mve_vmlsldavsw helper_mve_vmlsldavsw_arm
+#define helper_mve_vmlsldavxsh helper_mve_vmlsldavxsh_arm
+#define helper_mve_vmlsldavxsw helper_mve_vmlsldavxsw_arm
+#define helper_mve_vrmlaldavhsw helper_mve_vrmlaldavhsw_arm
+#define helper_mve_vrmlaldavhxsw helper_mve_vrmlaldavhxsw_arm
+#define helper_mve_vrmlaldavhuw helper_mve_vrmlaldavhuw_arm
+#define helper_mve_vrmlsldavhsw helper_mve_vrmlsldavhsw_arm
+#define helper_mve_vrmlsldavhxsw helper_mve_vrmlsldavhxsw_arm
+#define helper_mve_vmladavsb helper_mve_vmladavsb_arm
+#define helper_mve_vmladavsh helper_mve_vmladavsh_arm
+#define helper_mve_vmladavsw helper_mve_vmladavsw_arm
+#define helper_mve_vmladavub helper_mve_vmladavub_arm
+#define helper_mve_vmladavuh helper_mve_vmladavuh_arm
+#define helper_mve_vmladavuw helper_mve_vmladavuw_arm
+#define helper_mve_vmlsdavb helper_mve_vmlsdavb_arm
+#define helper_mve_vmlsdavh helper_mve_vmlsdavh_arm
+#define helper_mve_vmlsdavw helper_mve_vmlsdavw_arm
+#define helper_mve_vmladavsxb helper_mve_vmladavsxb_arm
+#define helper_mve_vmladavsxh helper_mve_vmladavsxh_arm
+#define helper_mve_vmladavsxw helper_mve_vmladavsxw_arm
+#define helper_mve_vmlsdavxb helper_mve_vmlsdavxb_arm
+#define helper_mve_vmlsdavxh helper_mve_vmlsdavxh_arm
+#define helper_mve_vmlsdavxw helper_mve_vmlsdavxw_arm
+#define helper_mve_vaddvsb helper_mve_vaddvsb_arm
+#define helper_mve_vaddvsh helper_mve_vaddvsh_arm
+#define helper_mve_vaddvsw helper_mve_vaddvsw_arm
+#define helper_mve_vaddvub helper_mve_vaddvub_arm
+#define helper_mve_vaddvuh helper_mve_vaddvuh_arm
+#define helper_mve_vaddvuw helper_mve_vaddvuw_arm
+#define helper_mve_vmaxvsb helper_mve_vmaxvsb_arm
+#define helper_mve_vmaxvsh helper_mve_vmaxvsh_arm
+#define helper_mve_vmaxvsw helper_mve_vmaxvsw_arm
+#define helper_mve_vmaxvub helper_mve_vmaxvub_arm
+#define helper_mve_vmaxvuh helper_mve_vmaxvuh_arm
+#define helper_mve_vmaxvuw helper_mve_vmaxvuw_arm
+#define helper_mve_vmaxavb helper_mve_vmaxavb_arm
+#define helper_mve_vmaxavh helper_mve_vmaxavh_arm
+#define helper_mve_vmaxavw helper_mve_vmaxavw_arm
+#define helper_mve_vminvsb helper_mve_vminvsb_arm
+#define helper_mve_vminvsh helper_mve_vminvsh_arm
+#define helper_mve_vminvsw helper_mve_vminvsw_arm
+#define helper_mve_vminvub helper_mve_vminvub_arm
+#define helper_mve_vminvuh helper_mve_vminvuh_arm
+#define helper_mve_vminvuw helper_mve_vminvuw_arm
+#define helper_mve_vminavb helper_mve_vminavb_arm
+#define helper_mve_vminavh helper_mve_vminavh_arm
+#define helper_mve_vminavw helper_mve_vminavw_arm
+#define helper_mve_vmaxnmvh helper_mve_vmaxnmvh_arm
+#define helper_mve_vmaxnmvs helper_mve_vmaxnmvs_arm
+#define helper_mve_vminnmvh helper_mve_vminnmvh_arm
+#define helper_mve_vminnmvs helper_mve_vminnmvs_arm
+#define helper_mve_vmaxnmavh helper_mve_vmaxnmavh_arm
+#define helper_mve_vmaxnmavs helper_mve_vmaxnmavs_arm
+#define helper_mve_vminnmavh helper_mve_vminnmavh_arm
+#define helper_mve_vminnmavs helper_mve_vminnmavs_arm
+#define helper_mve_vaddlv_s helper_mve_vaddlv_s_arm
+#define helper_mve_vaddlv_u helper_mve_vaddlv_u_arm
+#define helper_mve_vabavsb helper_mve_vabavsb_arm
+#define helper_mve_vabavsh helper_mve_vabavsh_arm
+#define helper_mve_vabavsw helper_mve_vabavsw_arm
+#define helper_mve_vabavub helper_mve_vabavub_arm
+#define helper_mve_vabavuh helper_mve_vabavuh_arm
+#define helper_mve_vabavuw helper_mve_vabavuw_arm
 #define helper_vfp_adds helper_vfp_adds_arm
 #define helper_vfp_addd helper_vfp_addd_arm
 #define helper_vfp_subs helper_vfp_subs_arm
@@ -1946,10 +3417,14 @@
 #define helper_vfp_touls helper_vfp_touls_arm
 #define helper_vfp_uqtos helper_vfp_uqtos_arm
 #define helper_vfp_touqs helper_vfp_touqs_arm
+#define helper_vfp_shtoh helper_vfp_shtoh_arm
+#define helper_vfp_uhtoh helper_vfp_uhtoh_arm
 #define helper_vfp_sltoh helper_vfp_sltoh_arm
 #define helper_vfp_ultoh helper_vfp_ultoh_arm
 #define helper_vfp_sqtoh helper_vfp_sqtoh_arm
 #define helper_vfp_uqtoh helper_vfp_uqtoh_arm
+#define helper_vfp_toshh_round_to_zero helper_vfp_toshh_round_to_zero_arm
+#define helper_vfp_touhh_round_to_zero helper_vfp_touhh_round_to_zero_arm
 #define helper_vfp_toshh helper_vfp_toshh_arm
 #define helper_vfp_touhh helper_vfp_touhh_arm
 #define helper_vfp_toslh helper_vfp_toslh_arm
@@ -1962,6 +3437,8 @@
 #define helper_vfp_fcvt_f32_to_f16 helper_vfp_fcvt_f32_to_f16_arm
 #define helper_vfp_fcvt_f16_to_f64 helper_vfp_fcvt_f16_to_f64_arm
 #define helper_vfp_fcvt_f64_to_f16 helper_vfp_fcvt_f64_to_f16_arm
+#define helper_bfcvt helper_bfcvt_arm
+#define helper_bfcvt_pair helper_bfcvt_pair_arm
 #define helper_recps_f32 helper_recps_f32_arm
 #define helper_rsqrts_f32 helper_rsqrts_f32_arm
 #define helper_recpe_f16 helper_recpe_f16_arm
@@ -1974,8 +3451,10 @@
 #define helper_rsqrte_u32 helper_rsqrte_u32_arm
 #define helper_vfp_muladds helper_vfp_muladds_arm
 #define helper_vfp_muladdd helper_vfp_muladdd_arm
+#define helper_rinth_exact helper_rinth_exact_arm
 #define helper_rints_exact helper_rints_exact_arm
 #define helper_rintd_exact helper_rintd_exact_arm
+#define helper_rinth helper_rinth_arm
 #define helper_rints helper_rints_arm
 #define helper_rintd helper_rintd_arm
 #define arm_rmode_to_sf arm_rmode_to_sf_arm
diff --git a/qemu/crypto/aes.c b/qemu/crypto/aes.c
index 0f6a195af8..af72ff7779 100644
--- a/qemu/crypto/aes.c
+++ b/qemu/crypto/aes.c
@@ -1080,9 +1080,9 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
 
         rk = key->rd_key;
 
-        if (bits==128)
+        if (bits == 128)
                 key->rounds = 10;
-        else if (bits==192)
+        else if (bits == 192)
                 key->rounds = 12;
         else
                 key->rounds = 14;
@@ -1182,7 +1182,7 @@ int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
         rk = key->rd_key;
 
         /* invert the order of the round keys: */
-        for (i = 0, j = 4*(key->rounds); i < j; i += 4, j -= 4) {
+        for (i = 0, j = 4 * (key->rounds); i < j; i += 4, j -= 4) {
                 temp = rk[i    ]; rk[i    ] = rk[j    ]; rk[j    ] = temp;
                 temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
                 temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
@@ -1599,54 +1599,3 @@ void AES_decrypt(const unsigned char *in, unsigned char *out,
 }
 
 #endif /* AES_ASM */
-
-void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
-                     const unsigned long length, const AES_KEY *key,
-                     unsigned char *ivec, const int enc)
-{
-
-        unsigned long n;
-        unsigned long len = length;
-        unsigned char tmp[AES_BLOCK_SIZE];
-
-        assert(in && out && key && ivec);
-
-        if (enc) {
-                while (len >= AES_BLOCK_SIZE) {
-                        for(n=0; n < AES_BLOCK_SIZE; ++n)
-                                tmp[n] = in[n] ^ ivec[n];
-                        AES_encrypt(tmp, out, key);
-                        memcpy(ivec, out, AES_BLOCK_SIZE);
-                        len -= AES_BLOCK_SIZE;
-                        in += AES_BLOCK_SIZE;
-                        out += AES_BLOCK_SIZE;
-                }
-                if (len) {
-                        for(n=0; n < len; ++n)
-                                tmp[n] = in[n] ^ ivec[n];
-                        for(n=len; n < AES_BLOCK_SIZE; ++n)
-                                tmp[n] = ivec[n];
-                        AES_encrypt(tmp, tmp, key);
-                        memcpy(out, tmp, AES_BLOCK_SIZE);
-                        memcpy(ivec, tmp, AES_BLOCK_SIZE);
-                }
-        } else {
-                while (len >= AES_BLOCK_SIZE) {
-                        memcpy(tmp, in, AES_BLOCK_SIZE);
-                        AES_decrypt(in, out, key);
-                        for(n=0; n < AES_BLOCK_SIZE; ++n)
-                                out[n] ^= ivec[n];
-                        memcpy(ivec, tmp, AES_BLOCK_SIZE);
-                        len -= AES_BLOCK_SIZE;
-                        in += AES_BLOCK_SIZE;
-                        out += AES_BLOCK_SIZE;
-                }
-                if (len) {
-                        memcpy(tmp, in, AES_BLOCK_SIZE);
-                        AES_decrypt(tmp, tmp, key);
-                        for(n=0; n < len; ++n)
-                                out[n] = tmp[n] ^ ivec[n];
-                        memcpy(ivec, tmp, AES_BLOCK_SIZE);
-                }
-        }
-}
diff --git a/qemu/crypto/sm4.c b/qemu/crypto/sm4.c
new file mode 100644
index 0000000000..0f0034c63c
--- /dev/null
+++ b/qemu/crypto/sm4.c
@@ -0,0 +1,48 @@
+/*
+ * QEMU crypto sm4 support
+ *
+ * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/sm4.h"
+
+uint8_t const sm4_sbox[] = {
+    0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7,
+    0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05,
+    0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3,
+    0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99,
+    0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a,
+    0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62,
+    0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95,
+    0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6,
+    0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba,
+    0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8,
+    0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b,
+    0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35,
+    0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2,
+    0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87,
+    0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52,
+    0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e,
+    0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5,
+    0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1,
+    0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55,
+    0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3,
+    0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60,
+    0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f,
+    0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f,
+    0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51,
+    0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f,
+    0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8,
+    0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd,
+    0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0,
+    0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e,
+    0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84,
+    0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20,
+    0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48,
+};
diff --git a/qemu/exec.c b/qemu/exec.c
index 002c14dfc1..b2e38de005 100644
--- a/qemu/exec.c
+++ b/qemu/exec.c
@@ -1156,6 +1156,7 @@ static void reclaim_ramblock(struct uc_struct *uc, RAMBlock *block)
     } else {
         qemu_anon_ram_free(uc, block->host, block->max_length);
     }
+    g_free(block->mte_tags);
     g_free(block);
 }
 
diff --git a/qemu/fpu/softfloat-parts-addsub.c.inc b/qemu/fpu/softfloat-parts-addsub.c.inc
new file mode 100644
index 0000000000..ae5c1017c5
--- /dev/null
+++ b/qemu/fpu/softfloat-parts-addsub.c.inc
@@ -0,0 +1,62 @@
+/*
+ * Floating point arithmetic implementation
+ *
+ * The code in this source file is derived from release 2a of the SoftFloat
+ * IEC/IEEE Floating-point Arithmetic Package. Those parts of the code (and
+ * some later contributions) are provided under that license, as detailed below.
+ * It has subsequently been modified by contributors to the QEMU Project,
+ * so some portions are provided under:
+ *  the SoftFloat-2a license
+ *  the BSD license
+ *  GPL-v2-or-later
+ *
+ * Any future contributions to this file after December 1st 2014 will be
+ * taken to be licensed under the Softfloat-2a license unless specifically
+ * indicated otherwise.
+ */
+
+static void partsN(add_normal)(FloatPartsN *a, FloatPartsN *b)
+{
+    int exp_diff = a->exp - b->exp;
+
+    if (exp_diff > 0) {
+        frac_shrjam(b, exp_diff);
+    } else if (exp_diff < 0) {
+        frac_shrjam(a, -exp_diff);
+        a->exp = b->exp;
+    }
+
+    if (frac_add(a, a, b)) {
+        frac_shrjam(a, 1);
+        a->frac_hi |= DECOMPOSED_IMPLICIT_BIT;
+        a->exp += 1;
+    }
+}
+
+static bool partsN(sub_normal)(FloatPartsN *a, FloatPartsN *b)
+{
+    int exp_diff = a->exp - b->exp;
+    int shift;
+
+    if (exp_diff > 0) {
+        frac_shrjam(b, exp_diff);
+        frac_sub(a, a, b);
+    } else if (exp_diff < 0) {
+        a->exp = b->exp;
+        a->sign ^= 1;
+        frac_shrjam(a, -exp_diff);
+        frac_sub(a, b, a);
+    } else if (frac_sub(a, a, b)) {
+        /* Overflow means that A was less than B. */
+        frac_neg(a);
+        a->sign ^= 1;
+    }
+
+    shift = frac_normalize(a);
+    if (likely(shift < N)) {
+        a->exp -= shift;
+	return true;
+    }
+    a->cls = float_class_zero;
+    return false;
+}
diff --git a/qemu/fpu/softfloat-parts.c.inc b/qemu/fpu/softfloat-parts.c.inc
new file mode 100644
index 0000000000..247400031c
--- /dev/null
+++ b/qemu/fpu/softfloat-parts.c.inc
@@ -0,0 +1,1547 @@
+/*
+ * QEMU float support
+ *
+ * The code in this source file is derived from release 2a of the SoftFloat
+ * IEC/IEEE Floating-point Arithmetic Package. Those parts of the code (and
+ * some later contributions) are provided under that license, as detailed below.
+ * It has subsequently been modified by contributors to the QEMU Project,
+ * so some portions are provided under:
+ *  the SoftFloat-2a license
+ *  the BSD license
+ *  GPL-v2-or-later
+ *
+ * Any future contributions to this file after December 1st 2014 will be
+ * taken to be licensed under the Softfloat-2a license unless specifically
+ * indicated otherwise.
+ */
+
+static void partsN(return_nan)(FloatPartsN *a, float_status *s)
+{
+    switch (a->cls) {
+    case float_class_snan:
+        float_raise(float_flag_invalid | float_flag_invalid_snan, s);
+        if (s->default_nan_mode) {
+            parts_default_nan(a, s);
+        } else {
+            parts_silence_nan(a, s);
+        }
+        break;
+    case float_class_qnan:
+        if (s->default_nan_mode) {
+            parts_default_nan(a, s);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static FloatPartsN *partsN(pick_nan)(FloatPartsN *a, FloatPartsN *b,
+                                     float_status *s)
+{
+    if (is_snan(a->cls) || is_snan(b->cls)) {
+        float_raise(float_flag_invalid | float_flag_invalid_snan, s);
+    }
+
+    if (s->default_nan_mode) {
+        parts_default_nan(a, s);
+    } else {
+        int cmp = frac_cmp(a, b);
+        if (cmp == 0) {
+            cmp = a->sign < b->sign;
+        }
+
+        if (pickNaN(a->cls, b->cls, cmp > 0, s)) {
+            a = b;
+        }
+        if (is_snan(a->cls)) {
+            parts_silence_nan(a, s);
+        }
+    }
+    return a;
+}
+
+static FloatPartsN *partsN(pick_nan_muladd)(FloatPartsN *a, FloatPartsN *b,
+                                            FloatPartsN *c, float_status *s,
+                                            int ab_mask, int abc_mask)
+{
+    int which;
+
+    if (unlikely(abc_mask & float_cmask_snan)) {
+        float_raise(float_flag_invalid | float_flag_invalid_snan, s);
+    }
+
+    which = pickNaNMulAdd(a->cls, b->cls, c->cls,
+                          ab_mask == float_cmask_infzero, s);
+
+    if (s->default_nan_mode || which == 3) {
+        /*
+         * Note that this check is after pickNaNMulAdd so that function
+         * has an opportunity to set the Invalid flag for infzero.
+         */
+        parts_default_nan(a, s);
+        return a;
+    }
+
+    switch (which) {
+    case 0:
+        break;
+    case 1:
+        a = b;
+        break;
+    case 2:
+        a = c;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    if (is_snan(a->cls)) {
+        parts_silence_nan(a, s);
+    }
+    return a;
+}
+
+/*
+ * Canonicalize the FloatParts structure.  Determine the class,
+ * unbias the exponent, and normalize the fraction.
+ */
+static void partsN(canonicalize)(FloatPartsN *p, float_status *status,
+                                 const FloatFmt *fmt)
+{
+    if (unlikely(p->exp == 0)) {
+        if (likely(frac_eqz(p))) {
+            p->cls = float_class_zero;
+        } else if (status->flush_inputs_to_zero) {
+            float_raise(float_flag_input_denormal, status);
+            p->cls = float_class_zero;
+            frac_clear(p);
+        } else {
+            int shift = frac_normalize(p);
+            p->cls = float_class_normal;
+            p->exp = fmt->frac_shift - fmt->exp_bias - shift + 1;
+        }
+    } else if (likely(p->exp < fmt->exp_max) || fmt->arm_althp) {
+        p->cls = float_class_normal;
+        p->exp -= fmt->exp_bias;
+        frac_shl(p, fmt->frac_shift);
+        p->frac_hi |= DECOMPOSED_IMPLICIT_BIT;
+    } else if (likely(frac_eqz(p))) {
+        p->cls = float_class_inf;
+    } else {
+        frac_shl(p, fmt->frac_shift);
+        p->cls = (parts_is_snan_frac(p->frac_hi, status)
+                  ? float_class_snan : float_class_qnan);
+    }
+}
+
+/*
+ * Round and uncanonicalize a floating-point number by parts. There
+ * are FRAC_SHIFT bits that may require rounding at the bottom of the
+ * fraction; these bits will be removed. The exponent will be biased
+ * by EXP_BIAS and must be bounded by [EXP_MAX-1, 0].
+ */
+static void partsN(uncanon_normal)(FloatPartsN *p, float_status *s,
+                                   const FloatFmt *fmt)
+{
+    const int exp_max = fmt->exp_max;
+    const int frac_shift = fmt->frac_shift;
+    const uint64_t round_mask = fmt->round_mask;
+    const uint64_t frac_lsb = round_mask + 1;
+    const uint64_t frac_lsbm1 = round_mask ^ (round_mask >> 1);
+    const uint64_t roundeven_mask = round_mask | frac_lsb;
+    uint64_t inc;
+    bool overflow_norm = false;
+    int exp, flags = 0;
+
+    switch (s->float_rounding_mode) {
+    case float_round_nearest_even:
+        if (N > 64 && frac_lsb == 0) {
+            inc = ((p->frac_hi & 1) || (p->frac_lo & round_mask) != frac_lsbm1
+                   ? frac_lsbm1 : 0);
+        } else {
+            inc = ((p->frac_lo & roundeven_mask) != frac_lsbm1
+                   ? frac_lsbm1 : 0);
+        }
+        break;
+    case float_round_ties_away:
+        inc = frac_lsbm1;
+        break;
+    case float_round_to_zero:
+        overflow_norm = true;
+        inc = 0;
+        break;
+    case float_round_up:
+        inc = p->sign ? 0 : round_mask;
+        overflow_norm = p->sign;
+        break;
+    case float_round_down:
+        inc = p->sign ? round_mask : 0;
+        overflow_norm = !p->sign;
+        break;
+    case float_round_to_odd:
+        overflow_norm = true;
+        /* fall through */
+    case float_round_to_odd_inf:
+        if (N > 64 && frac_lsb == 0) {
+            inc = p->frac_hi & 1 ? 0 : round_mask;
+        } else {
+            inc = p->frac_lo & frac_lsb ? 0 : round_mask;
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    exp = p->exp + fmt->exp_bias;
+    if (likely(exp > 0)) {
+        if (p->frac_lo & round_mask) {
+            flags |= float_flag_inexact;
+            if (frac_addi(p, p, inc)) {
+                frac_shr(p, 1);
+                p->frac_hi |= DECOMPOSED_IMPLICIT_BIT;
+                exp++;
+            }
+            p->frac_lo &= ~round_mask;
+        }
+
+        if (fmt->arm_althp) {
+            /* ARM Alt HP eschews Inf and NaN for a wider exponent.  */
+            if (unlikely(exp > exp_max)) {
+                /* Overflow.  Return the maximum normal.  */
+                flags = float_flag_invalid;
+                exp = exp_max;
+                frac_allones(p);
+                p->frac_lo &= ~round_mask;
+            }
+        } else if (unlikely(exp >= exp_max)) {
+            flags |= float_flag_overflow;
+            if (s->rebias_overflow) {
+                exp -= fmt->exp_re_bias;
+            } else if (overflow_norm) {
+                flags |= float_flag_inexact;
+                exp = exp_max - 1;
+                frac_allones(p);
+                p->frac_lo &= ~round_mask;
+            } else {
+                flags |= float_flag_inexact;
+                p->cls = float_class_inf;
+                exp = exp_max;
+                frac_clear(p);
+            }
+        }
+        frac_shr(p, frac_shift);
+    } else if (unlikely(s->rebias_underflow)) {
+        flags |= float_flag_underflow;
+        exp += fmt->exp_re_bias;
+        if (p->frac_lo & round_mask) {
+            flags |= float_flag_inexact;
+            if (frac_addi(p, p, inc)) {
+                frac_shr(p, 1);
+                p->frac_hi |= DECOMPOSED_IMPLICIT_BIT;
+                exp++;
+            }
+            p->frac_lo &= ~round_mask;
+        }
+        frac_shr(p, frac_shift);
+    } else if (s->flush_to_zero) {
+        flags |= float_flag_output_denormal;
+        p->cls = float_class_zero;
+        exp = 0;
+        frac_clear(p);
+    } else {
+        bool is_tiny = s->tininess_before_rounding || exp < 0;
+
+        if (!is_tiny) {
+            FloatPartsN discard;
+            is_tiny = !frac_addi(&discard, p, inc);
+        }
+
+        frac_shrjam(p, 1 - exp);
+
+        if (p->frac_lo & round_mask) {
+            /* Need to recompute round-to-even/round-to-odd. */
+            switch (s->float_rounding_mode) {
+            case float_round_nearest_even:
+                if (N > 64 && frac_lsb == 0) {
+                    inc = ((p->frac_hi & 1) ||
+                           (p->frac_lo & round_mask) != frac_lsbm1
+                           ? frac_lsbm1 : 0);
+                } else {
+                    inc = ((p->frac_lo & roundeven_mask) != frac_lsbm1
+                           ? frac_lsbm1 : 0);
+                }
+                break;
+            case float_round_to_odd:
+            case float_round_to_odd_inf:
+                if (N > 64 && frac_lsb == 0) {
+                    inc = p->frac_hi & 1 ? 0 : round_mask;
+                } else {
+                    inc = p->frac_lo & frac_lsb ? 0 : round_mask;
+                }
+                break;
+            default:
+                break;
+            }
+            flags |= float_flag_inexact;
+            frac_addi(p, p, inc);
+            p->frac_lo &= ~round_mask;
+        }
+
+        exp = (p->frac_hi & DECOMPOSED_IMPLICIT_BIT) != 0;
+        frac_shr(p, frac_shift);
+
+        if (is_tiny && (flags & float_flag_inexact)) {
+            flags |= float_flag_underflow;
+        }
+        if (exp == 0 && frac_eqz(p)) {
+            p->cls = float_class_zero;
+        }
+    }
+    p->exp = exp;
+    float_raise(flags, s);
+}
+
+static void partsN(uncanon)(FloatPartsN *p, float_status *s,
+                            const FloatFmt *fmt)
+{
+    if (likely(p->cls == float_class_normal)) {
+        parts_uncanon_normal(p, s, fmt);
+    } else {
+        switch (p->cls) {
+        case float_class_zero:
+            p->exp = 0;
+            frac_clear(p);
+            return;
+        case float_class_inf:
+            g_assert(!fmt->arm_althp);
+            p->exp = fmt->exp_max;
+            frac_clear(p);
+            return;
+        case float_class_qnan:
+        case float_class_snan:
+            g_assert(!fmt->arm_althp);
+            p->exp = fmt->exp_max;
+            frac_shr(p, fmt->frac_shift);
+            return;
+        default:
+            break;
+        }
+        g_assert_not_reached();
+    }
+}
+
+/*
+ * Returns the result of adding or subtracting the values of the
+ * floating-point values `a' and `b'. The operation is performed
+ * according to the IEC/IEEE Standard for Binary Floating-Point
+ * Arithmetic.
+ */
+static FloatPartsN *partsN(addsub)(FloatPartsN *a, FloatPartsN *b,
+                                   float_status *s, bool subtract)
+{
+    bool b_sign = b->sign ^ subtract;
+    int ab_mask = float_cmask(a->cls) | float_cmask(b->cls);
+
+    if (a->sign != b_sign) {
+        /* Subtraction */
+        if (likely(ab_mask == float_cmask_normal)) {
+            if (parts_sub_normal(a, b)) {
+                return a;
+            }
+            /* Subtract was exact, fall through to set sign. */
+            ab_mask = float_cmask_zero;
+        }
+
+        if (ab_mask == float_cmask_zero) {
+            a->sign = s->float_rounding_mode == float_round_down;
+            return a;
+        }
+
+        if (unlikely(ab_mask & float_cmask_anynan)) {
+            goto p_nan;
+        }
+
+        if (ab_mask & float_cmask_inf) {
+            if (a->cls != float_class_inf) {
+                /* N - Inf */
+                goto return_b;
+            }
+            if (b->cls != float_class_inf) {
+                /* Inf - N */
+                return a;
+            }
+            /* Inf - Inf */
+            float_raise(float_flag_invalid | float_flag_invalid_isi, s);
+            parts_default_nan(a, s);
+            return a;
+        }
+    } else {
+        /* Addition */
+        if (likely(ab_mask == float_cmask_normal)) {
+            parts_add_normal(a, b);
+            return a;
+        }
+
+        if (ab_mask == float_cmask_zero) {
+            return a;
+        }
+
+        if (unlikely(ab_mask & float_cmask_anynan)) {
+            goto p_nan;
+        }
+
+        if (ab_mask & float_cmask_inf) {
+            a->cls = float_class_inf;
+            return a;
+        }
+    }
+
+    if (b->cls == float_class_zero) {
+        g_assert(a->cls == float_class_normal);
+        return a;
+    }
+
+    g_assert(a->cls == float_class_zero);
+    g_assert(b->cls == float_class_normal);
+ return_b:
+    b->sign = b_sign;
+    return b;
+
+ p_nan:
+    return parts_pick_nan(a, b, s);
+}
+
+/*
+ * Returns the result of multiplying the floating-point values `a' and
+ * `b'. The operation is performed according to the IEC/IEEE Standard
+ * for Binary Floating-Point Arithmetic.
+ */
+static FloatPartsN *partsN(mul)(FloatPartsN *a, FloatPartsN *b,
+                                float_status *s)
+{
+    int ab_mask = float_cmask(a->cls) | float_cmask(b->cls);
+    bool sign = a->sign ^ b->sign;
+
+    if (likely(ab_mask == float_cmask_normal)) {
+        FloatPartsW tmp;
+
+        frac_mulw(&tmp, a, b);
+        frac_truncjam(a, &tmp);
+
+        a->exp += b->exp + 1;
+        if (!(a->frac_hi & DECOMPOSED_IMPLICIT_BIT)) {
+            frac_add(a, a, a);
+            a->exp -= 1;
+        }
+
+        a->sign = sign;
+        return a;
+    }
+
+    /* Inf * Zero == NaN */
+    if (unlikely(ab_mask == float_cmask_infzero)) {
+        float_raise(float_flag_invalid | float_flag_invalid_imz, s);
+        parts_default_nan(a, s);
+        return a;
+    }
+
+    if (unlikely(ab_mask & float_cmask_anynan)) {
+        return parts_pick_nan(a, b, s);
+    }
+
+    /* Multiply by 0 or Inf */
+    if (ab_mask & float_cmask_inf) {
+        a->cls = float_class_inf;
+        a->sign = sign;
+        return a;
+    }
+
+    g_assert(ab_mask & float_cmask_zero);
+    a->cls = float_class_zero;
+    a->sign = sign;
+    return a;
+}
+
+/*
+ * Returns the result of multiplying the floating-point values `a' and
+ * `b' then adding 'c', with no intermediate rounding step after the
+ * multiplication. The operation is performed according to the
+ * IEC/IEEE Standard for Binary Floating-Point Arithmetic 754-2008.
+ * The flags argument allows the caller to select negation of the
+ * addend, the intermediate product, or the final result. (The
+ * difference between this and having the caller do a separate
+ * negation is that negating externally will flip the sign bit on NaNs.)
+ *
+ * Requires A and C extracted into a double-sized structure to provide the
+ * extra space for the widening multiply.
+ */
+static FloatPartsN *partsN(muladd)(FloatPartsN *a, FloatPartsN *b,
+                                   FloatPartsN *c, int flags, float_status *s)
+{
+    int ab_mask, abc_mask;
+    FloatPartsW p_widen, c_widen;
+
+    ab_mask = float_cmask(a->cls) | float_cmask(b->cls);
+    abc_mask = float_cmask(c->cls) | ab_mask;
+
+    /*
+     * It is implementation-defined whether the cases of (0,inf,qnan)
+     * and (inf,0,qnan) raise InvalidOperation or not (and what QNaN
+     * they return if they do), so we have to hand this information
+     * off to the target-specific pick-a-NaN routine.
+     */
+    if (unlikely(abc_mask & float_cmask_anynan)) {
+        return parts_pick_nan_muladd(a, b, c, s, ab_mask, abc_mask);
+    }
+
+    if (flags & float_muladd_negate_c) {
+        c->sign ^= 1;
+    }
+
+    /* Compute the sign of the product into A. */
+    a->sign ^= b->sign;
+    if (flags & float_muladd_negate_product) {
+        a->sign ^= 1;
+    }
+
+    if (unlikely(ab_mask != float_cmask_normal)) {
+        if (unlikely(ab_mask == float_cmask_infzero)) {
+            float_raise(float_flag_invalid | float_flag_invalid_imz, s);
+            goto d_nan;
+        }
+
+        if (ab_mask & float_cmask_inf) {
+            if (c->cls == float_class_inf && a->sign != c->sign) {
+                float_raise(float_flag_invalid | float_flag_invalid_isi, s);
+                goto d_nan;
+            }
+            goto return_inf;
+        }
+
+        g_assert(ab_mask & float_cmask_zero);
+        if (c->cls == float_class_normal) {
+            *a = *c;
+            goto return_normal;
+        }
+        if (c->cls == float_class_zero) {
+            if (a->sign != c->sign) {
+                goto return_sub_zero;
+            }
+            goto return_zero;
+        }
+        g_assert(c->cls == float_class_inf);
+    }
+
+    if (unlikely(c->cls == float_class_inf)) {
+        a->sign = c->sign;
+        goto return_inf;
+    }
+
+    /* Perform the multiplication step. */
+    p_widen.sign = a->sign;
+    p_widen.exp = a->exp + b->exp + 1;
+    frac_mulw(&p_widen, a, b);
+    if (!(p_widen.frac_hi & DECOMPOSED_IMPLICIT_BIT)) {
+        frac_add(&p_widen, &p_widen, &p_widen);
+        p_widen.exp -= 1;
+    }
+
+    /* Perform the addition step. */
+    if (c->cls != float_class_zero) {
+        /* Zero-extend C to less significant bits. */
+        frac_widen(&c_widen, c);
+        c_widen.exp = c->exp;
+
+        if (a->sign == c->sign) {
+            parts_add_normal(&p_widen, &c_widen);
+        } else if (!parts_sub_normal(&p_widen, &c_widen)) {
+            goto return_sub_zero;
+        }
+    }
+
+    /* Narrow with sticky bit, for proper rounding later. */
+    frac_truncjam(a, &p_widen);
+    a->sign = p_widen.sign;
+    a->exp = p_widen.exp;
+
+ return_normal:
+    if (flags & float_muladd_halve_result) {
+        a->exp -= 1;
+    }
+ finish_sign:
+    if (flags & float_muladd_negate_result) {
+        a->sign ^= 1;
+    }
+    return a;
+
+ return_sub_zero:
+    a->sign = s->float_rounding_mode == float_round_down;
+ return_zero:
+    a->cls = float_class_zero;
+    goto finish_sign;
+
+ return_inf:
+    a->cls = float_class_inf;
+    goto finish_sign;
+
+ d_nan:
+    parts_default_nan(a, s);
+    return a;
+}
+
+/*
+ * Returns the result of dividing the floating-point value `a' by the
+ * corresponding value `b'. The operation is performed according to
+ * the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+ */
+static FloatPartsN *partsN(div)(FloatPartsN *a, FloatPartsN *b,
+                                float_status *s)
+{
+    int ab_mask = float_cmask(a->cls) | float_cmask(b->cls);
+    bool sign = a->sign ^ b->sign;
+
+    if (likely(ab_mask == float_cmask_normal)) {
+        a->sign = sign;
+        a->exp -= b->exp + frac_div(a, b);
+        return a;
+    }
+
+    /* 0/0 or Inf/Inf => NaN */
+    if (unlikely(ab_mask == float_cmask_zero)) {
+        float_raise(float_flag_invalid | float_flag_invalid_zdz, s);
+        goto d_nan;
+    }
+    if (unlikely(ab_mask == float_cmask_inf)) {
+        float_raise(float_flag_invalid | float_flag_invalid_idi, s);
+        goto d_nan;
+    }
+
+    /* All the NaN cases */
+    if (unlikely(ab_mask & float_cmask_anynan)) {
+        return parts_pick_nan(a, b, s);
+    }
+
+    a->sign = sign;
+
+    /* Inf / X */
+    if (a->cls == float_class_inf) {
+        return a;
+    }
+
+    /* 0 / X */
+    if (a->cls == float_class_zero) {
+        return a;
+    }
+
+    /* X / Inf */
+    if (b->cls == float_class_inf) {
+        a->cls = float_class_zero;
+        return a;
+    }
+
+    /* X / 0 => Inf */
+    g_assert(b->cls == float_class_zero);
+    float_raise(float_flag_divbyzero, s);
+    a->cls = float_class_inf;
+    return a;
+
+ d_nan:
+    parts_default_nan(a, s);
+    return a;
+}
+
+/*
+ * Floating point remainder, per IEC/IEEE, or modulus.
+ */
+static FloatPartsN *partsN(modrem)(FloatPartsN *a, FloatPartsN *b,
+                                   uint64_t *mod_quot, float_status *s)
+{
+    int ab_mask = float_cmask(a->cls) | float_cmask(b->cls);
+
+    if (likely(ab_mask == float_cmask_normal)) {
+        frac_modrem(a, b, mod_quot);
+        return a;
+    }
+
+    if (mod_quot) {
+        *mod_quot = 0;
+    }
+
+    /* All the NaN cases */
+    if (unlikely(ab_mask & float_cmask_anynan)) {
+        return parts_pick_nan(a, b, s);
+    }
+
+    /* Inf % N; N % 0 */
+    if (a->cls == float_class_inf || b->cls == float_class_zero) {
+        float_raise(float_flag_invalid, s);
+        parts_default_nan(a, s);
+        return a;
+    }
+
+    /* N % Inf; 0 % N */
+    g_assert(b->cls == float_class_inf || a->cls == float_class_zero);
+    return a;
+}
+
+/*
+ * Square Root
+ *
+ * The base algorithm is lifted from
+ * https://git.musl-libc.org/cgit/musl/tree/src/math/sqrtf.c
+ * https://git.musl-libc.org/cgit/musl/tree/src/math/sqrt.c
+ * https://git.musl-libc.org/cgit/musl/tree/src/math/sqrtl.c
+ * and is thus MIT licenced.
+ */
+static void partsN(sqrt)(FloatPartsN *a, float_status *status,
+                         const FloatFmt *fmt)
+{
+    const uint32_t three32 = 3u << 30;
+    const uint64_t three64 = 3ull << 62;
+    uint32_t d32, m32, r32, s32, u32;            /* 32-bit computation */
+    uint64_t d64, m64, r64, s64, u64;            /* 64-bit computation */
+    uint64_t dh, dl, rh, rl, sh, sl, uh, ul;     /* 128-bit computation */
+    uint64_t d0h, d0l, d1h, d1l, d2h, d2l;
+    uint64_t discard;
+    bool exp_odd;
+    size_t index;
+
+    if (unlikely(a->cls != float_class_normal)) {
+        switch (a->cls) {
+        case float_class_snan:
+        case float_class_qnan:
+            parts_return_nan(a, status);
+            return;
+        case float_class_zero:
+            return;
+        case float_class_inf:
+            if (unlikely(a->sign)) {
+                goto d_nan;
+            }
+            return;
+        default:
+            g_assert_not_reached();
+        }
+    }
+
+    if (unlikely(a->sign)) {
+        goto d_nan;
+    }
+
+    /*
+     * Argument reduction.
+     * x = 4^e frac; with integer e, and frac in [1, 4)
+     * m = frac fixed point at bit 62, since we're in base 4.
+     * If base-2 exponent is odd, exchange that for multiply by 2,
+     * which results in no shift.
+     */
+    exp_odd = a->exp & 1;
+    index = extract64(a->frac_hi, 57, 6) | (!exp_odd << 6);
+    if (!exp_odd) {
+        frac_shr(a, 1);
+    }
+
+    /*
+     * Approximate r ~= 1/sqrt(m) and s ~= sqrt(m) when m in [1, 4).
+     *
+     * Initial estimate:
+     * 7-bit lookup table (1-bit exponent and 6-bit significand).
+     *
+     * The relative error (e = r0*sqrt(m)-1) of a linear estimate
+     * (r0 = a*m + b) is |e| < 0.085955 ~ 0x1.6p-4 at best;
+     * a table lookup is faster and needs one less iteration.
+     * The 7-bit table gives |e| < 0x1.fdp-9.
+     *
+     * A Newton-Raphson iteration for r is
+     *   s = m*r
+     *   d = s*r
+     *   u = 3 - d
+     *   r = r*u/2
+     *
+     * Fixed point representations:
+     *   m, s, d, u, three are all 2.30; r is 0.32
+     */
+    m64 = a->frac_hi;
+    m32 = m64 >> 32;
+
+    r32 = rsqrt_tab[index] << 16;
+    /* |r*sqrt(m) - 1| < 0x1.FDp-9 */
+
+    s32 = ((uint64_t)m32 * r32) >> 32;
+    d32 = ((uint64_t)s32 * r32) >> 32;
+    u32 = three32 - d32;
+
+    if (N == 64) {
+        /* float64 or smaller */
+
+        r32 = ((uint64_t)r32 * u32) >> 31;
+        /* |r*sqrt(m) - 1| < 0x1.7Bp-16 */
+
+        s32 = ((uint64_t)m32 * r32) >> 32;
+        d32 = ((uint64_t)s32 * r32) >> 32;
+        u32 = three32 - d32;
+
+        if (fmt->frac_size <= 23) {
+            /* float32 or smaller */
+
+            s32 = ((uint64_t)s32 * u32) >> 32;  /* 3.29 */
+            s32 = (s32 - 1) >> 6;               /* 9.23 */
+            /* s < sqrt(m) < s + 0x1.08p-23 */
+
+            /* compute nearest rounded result to 2.23 bits */
+            uint32_t d0 = (m32 << 16) - s32 * s32;
+            uint32_t d1 = s32 - d0;
+            uint32_t d2 = d1 + s32 + 1;
+            s32 += d1 >> 31;
+            a->frac_hi = (uint64_t)s32 << (64 - 25);
+
+            /* increment or decrement for inexact */
+            if (d2 != 0) {
+                a->frac_hi += ((int32_t)(d1 ^ d2) < 0 ? -1 : 1);
+            }
+            goto done;
+        }
+
+        /* float64 */
+
+        r64 = (uint64_t)r32 * u32 * 2;
+        /* |r*sqrt(m) - 1| < 0x1.37-p29; convert to 64-bit arithmetic */
+        mul64To128(m64, r64, &s64, &discard);
+        mul64To128(s64, r64, &d64, &discard);
+        u64 = three64 - d64;
+
+        mul64To128(s64, u64, &s64, &discard);  /* 3.61 */
+        s64 = (s64 - 2) >> 9;                  /* 12.52 */
+
+        /* Compute nearest rounded result */
+        uint64_t d0 = (m64 << 42) - s64 * s64;
+        uint64_t d1 = s64 - d0;
+        uint64_t d2 = d1 + s64 + 1;
+        s64 += d1 >> 63;
+        a->frac_hi = s64 << (64 - 54);
+
+        /* increment or decrement for inexact */
+        if (d2 != 0) {
+            a->frac_hi += ((int64_t)(d1 ^ d2) < 0 ? -1 : 1);
+        }
+        goto done;
+    }
+
+    r64 = (uint64_t)r32 * u32 * 2;
+    /* |r*sqrt(m) - 1| < 0x1.7Bp-16; convert to 64-bit arithmetic */
+
+    mul64To128(m64, r64, &s64, &discard);
+    mul64To128(s64, r64, &d64, &discard);
+    u64 = three64 - d64;
+    mul64To128(u64, r64, &r64, &discard);
+    r64 <<= 1;
+    /* |r*sqrt(m) - 1| < 0x1.a5p-31 */
+
+    mul64To128(m64, r64, &s64, &discard);
+    mul64To128(s64, r64, &d64, &discard);
+    u64 = three64 - d64;
+    mul64To128(u64, r64, &rh, &rl);
+    add128(rh, rl, rh, rl, &rh, &rl);
+    /* |r*sqrt(m) - 1| < 0x1.c001p-59; change to 128-bit arithmetic */
+
+    mul128To256(a->frac_hi, a->frac_lo, rh, rl, &sh, &sl, &discard, &discard);
+    mul128To256(sh, sl, rh, rl, &dh, &dl, &discard, &discard);
+    sub128(three64, 0, dh, dl, &uh, &ul);
+    mul128To256(uh, ul, sh, sl, &sh, &sl, &discard, &discard);  /* 3.125 */
+    /* -0x1p-116 < s - sqrt(m) < 0x3.8001p-125 */
+
+    sub128(sh, sl, 0, 4, &sh, &sl);
+    shift128Right(sh, sl, 13, &sh, &sl);  /* 16.112 */
+    /* s < sqrt(m) < s + 1ulp */
+
+    /* Compute nearest rounded result */
+    mul64To128(sl, sl, &d0h, &d0l);
+    d0h += 2 * sh * sl;
+    sub128(a->frac_lo << 34, 0, d0h, d0l, &d0h, &d0l);
+    sub128(sh, sl, d0h, d0l, &d1h, &d1l);
+    add128(sh, sl, 0, 1, &d2h, &d2l);
+    add128(d2h, d2l, d1h, d1l, &d2h, &d2l);
+    add128(sh, sl, 0, d1h >> 63, &sh, &sl);
+    shift128Left(sh, sl, 128 - 114, &sh, &sl);
+
+    /* increment or decrement for inexact */
+    if (d2h | d2l) {
+        if ((int64_t)(d1h ^ d2h) < 0) {
+            sub128(sh, sl, 0, 1, &sh, &sl);
+        } else {
+            add128(sh, sl, 0, 1, &sh, &sl);
+        }
+    }
+    a->frac_lo = sl;
+    a->frac_hi = sh;
+
+ done:
+    /* Convert back from base 4 to base 2. */
+    a->exp >>= 1;
+    if (!(a->frac_hi & DECOMPOSED_IMPLICIT_BIT)) {
+        frac_add(a, a, a);
+    } else {
+        a->exp += 1;
+    }
+    return;
+
+ d_nan:
+    float_raise(float_flag_invalid | float_flag_invalid_sqrt, status);
+    parts_default_nan(a, status);
+}
+
+/*
+ * Rounds the floating-point value `a' to an integer, and returns the
+ * result as a floating-point value. The operation is performed
+ * according to the IEC/IEEE Standard for Binary Floating-Point
+ * Arithmetic.
+ *
+ * parts_round_to_int_normal is an internal helper function for
+ * normal numbers only, returning true for inexact but not directly
+ * raising float_flag_inexact.
+ */
+static bool partsN(round_to_int_normal)(FloatPartsN *a, FloatRoundMode rmode,
+                                        int scale, int frac_size)
+{
+    uint64_t frac_lsb, frac_lsbm1, rnd_even_mask, rnd_mask, inc;
+    int shift_adj;
+
+    scale = MIN(MAX(scale, -0x10000), 0x10000);
+    a->exp += scale;
+
+    if (a->exp < 0) {
+        bool one;
+
+        /* All fractional */
+        switch (rmode) {
+        case float_round_nearest_even:
+            one = false;
+            if (a->exp == -1) {
+                FloatPartsN tmp;
+                /* Shift left one, discarding DECOMPOSED_IMPLICIT_BIT */
+                frac_add(&tmp, a, a);
+                /* Anything remaining means frac > 0.5. */
+                one = !frac_eqz(&tmp);
+            }
+            break;
+        case float_round_ties_away:
+            one = a->exp == -1;
+            break;
+        case float_round_to_zero:
+            one = false;
+            break;
+        case float_round_up:
+            one = !a->sign;
+            break;
+        case float_round_down:
+            one = a->sign;
+            break;
+        case float_round_to_odd:
+            one = true;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+
+        frac_clear(a);
+        a->exp = 0;
+        if (one) {
+            a->frac_hi = DECOMPOSED_IMPLICIT_BIT;
+        } else {
+            a->cls = float_class_zero;
+        }
+        return true;
+    }
+
+    if (a->exp >= frac_size) {
+        /* All integral */
+        return false;
+    }
+
+    if (N > 64 && a->exp < N - 64) {
+        /*
+         * Rounding is not in the low word -- shift lsb to bit 2,
+         * which leaves room for sticky and rounding bit.
+         */
+        shift_adj = (N - 1) - (a->exp + 2);
+        frac_shrjam(a, shift_adj);
+        frac_lsb = 1 << 2;
+    } else {
+        shift_adj = 0;
+        frac_lsb = DECOMPOSED_IMPLICIT_BIT >> (a->exp & 63);
+    }
+
+    frac_lsbm1 = frac_lsb >> 1;
+    rnd_mask = frac_lsb - 1;
+    rnd_even_mask = rnd_mask | frac_lsb;
+
+    if (!(a->frac_lo & rnd_mask)) {
+        /* Fractional bits already clear, undo the shift above. */
+        frac_shl(a, shift_adj);
+        return false;
+    }
+
+    switch (rmode) {
+    case float_round_nearest_even:
+        inc = ((a->frac_lo & rnd_even_mask) != frac_lsbm1 ? frac_lsbm1 : 0);
+        break;
+    case float_round_ties_away:
+        inc = frac_lsbm1;
+        break;
+    case float_round_to_zero:
+        inc = 0;
+        break;
+    case float_round_up:
+        inc = a->sign ? 0 : rnd_mask;
+        break;
+    case float_round_down:
+        inc = a->sign ? rnd_mask : 0;
+        break;
+    case float_round_to_odd:
+        inc = a->frac_lo & frac_lsb ? 0 : rnd_mask;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if (shift_adj == 0) {
+        if (frac_addi(a, a, inc)) {
+            frac_shr(a, 1);
+            a->frac_hi |= DECOMPOSED_IMPLICIT_BIT;
+            a->exp++;
+        }
+        a->frac_lo &= ~rnd_mask;
+    } else {
+        frac_addi(a, a, inc);
+        a->frac_lo &= ~rnd_mask;
+        /* Be careful shifting back, not to overflow */
+        frac_shl(a, shift_adj - 1);
+        if (a->frac_hi & DECOMPOSED_IMPLICIT_BIT) {
+            a->exp++;
+        } else {
+            frac_add(a, a, a);
+        }
+    }
+    return true;
+}
+
+static void partsN(round_to_int)(FloatPartsN *a, FloatRoundMode rmode,
+                                 int scale, float_status *s,
+                                 const FloatFmt *fmt)
+{
+    switch (a->cls) {
+    case float_class_qnan:
+    case float_class_snan:
+        parts_return_nan(a, s);
+        break;
+    case float_class_zero:
+    case float_class_inf:
+        break;
+    case float_class_normal:
+        if (parts_round_to_int_normal(a, rmode, scale, fmt->frac_size)) {
+            float_raise(float_flag_inexact, s);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+/*
+ * Returns the result of converting the floating-point value `a' to
+ * the two's complement integer format. The conversion is performed
+ * according to the IEC/IEEE Standard for Binary Floating-Point
+ * Arithmetic---which means in particular that the conversion is
+ * rounded according to the current rounding mode. If `a' is a NaN,
+ * the largest positive integer is returned. Otherwise, if the
+ * conversion overflows, the largest integer with the same sign as `a'
+ * is returned.
+ */
+static int64_t partsN(float_to_sint)(FloatPartsN *p, FloatRoundMode rmode,
+                                     int scale, int64_t min, int64_t max,
+                                     float_status *s)
+{
+    int flags = 0;
+    uint64_t r;
+
+    switch (p->cls) {
+    case float_class_snan:
+        flags |= float_flag_invalid_snan;
+        /* fall through */
+    case float_class_qnan:
+        flags |= float_flag_invalid;
+        r = max;
+        break;
+
+    case float_class_inf:
+        flags = float_flag_invalid | float_flag_invalid_cvti;
+        r = p->sign ? min : max;
+        break;
+
+    case float_class_zero:
+        return 0;
+
+    case float_class_normal:
+        /* TODO: N - 2 is frac_size for rounding; could use input fmt. */
+        if (parts_round_to_int_normal(p, rmode, scale, N - 2)) {
+            flags = float_flag_inexact;
+        }
+
+        if (p->exp <= DECOMPOSED_BINARY_POINT) {
+            r = p->frac_hi >> (DECOMPOSED_BINARY_POINT - p->exp);
+        } else {
+            r = UINT64_MAX;
+        }
+        if (p->sign) {
+            if (r <= -(uint64_t)min) {
+                r = -r;
+            } else {
+                flags = float_flag_invalid | float_flag_invalid_cvti;
+                r = min;
+            }
+        } else if (r > max) {
+            flags = float_flag_invalid | float_flag_invalid_cvti;
+            r = max;
+        }
+        break;
+
+    default:
+        g_assert_not_reached();
+    }
+
+    float_raise(flags, s);
+    return r;
+}
+
+/*
+ *  Returns the result of converting the floating-point value `a' to
+ *  the unsigned integer format. The conversion is performed according
+ *  to the IEC/IEEE Standard for Binary Floating-Point
+ *  Arithmetic---which means in particular that the conversion is
+ *  rounded according to the current rounding mode. If `a' is a NaN,
+ *  the largest unsigned integer is returned. Otherwise, if the
+ *  conversion overflows, the largest unsigned integer is returned. If
+ *  the 'a' is negative, the result is rounded and zero is returned;
+ *  values that do not round to zero will raise the inexact exception
+ *  flag.
+ */
+static uint64_t partsN(float_to_uint)(FloatPartsN *p, FloatRoundMode rmode,
+                                      int scale, uint64_t max, float_status *s)
+{
+    int flags = 0;
+    uint64_t r;
+
+    switch (p->cls) {
+    case float_class_snan:
+        flags |= float_flag_invalid_snan;
+        /* fall through */
+    case float_class_qnan:
+        flags |= float_flag_invalid;
+        r = max;
+        break;
+
+    case float_class_inf:
+        flags = float_flag_invalid | float_flag_invalid_cvti;
+        r = p->sign ? 0 : max;
+        break;
+
+    case float_class_zero:
+        return 0;
+
+    case float_class_normal:
+        /* TODO: N - 2 is frac_size for rounding; could use input fmt. */
+        if (parts_round_to_int_normal(p, rmode, scale, N - 2)) {
+            flags = float_flag_inexact;
+            if (p->cls == float_class_zero) {
+                r = 0;
+                break;
+            }
+        }
+
+        if (p->sign) {
+            flags = float_flag_invalid | float_flag_invalid_cvti;
+            r = 0;
+        } else if (p->exp > DECOMPOSED_BINARY_POINT) {
+            flags = float_flag_invalid | float_flag_invalid_cvti;
+            r = max;
+        } else {
+            r = p->frac_hi >> (DECOMPOSED_BINARY_POINT - p->exp);
+            if (r > max) {
+                flags = float_flag_invalid | float_flag_invalid_cvti;
+                r = max;
+            }
+        }
+        break;
+
+    default:
+        g_assert_not_reached();
+    }
+
+    float_raise(flags, s);
+    return r;
+}
+
+/*
+ * Integer to float conversions
+ *
+ * Returns the result of converting the two's complement integer `a'
+ * to the floating-point format. The conversion is performed according
+ * to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+ */
+static void partsN(sint_to_float)(FloatPartsN *p, int64_t a,
+                                  int scale, float_status *s)
+{
+    uint64_t f = a;
+    int shift;
+
+    memset(p, 0, sizeof(*p));
+
+    if (a == 0) {
+        p->cls = float_class_zero;
+        return;
+    }
+
+    p->cls = float_class_normal;
+    if (a < 0) {
+        f = -f;
+        p->sign = true;
+    }
+    shift = clz64(f);
+    scale = MIN(MAX(scale, -0x10000), 0x10000);
+
+    p->exp = DECOMPOSED_BINARY_POINT - shift + scale;
+    p->frac_hi = f << shift;
+}
+
+/*
+ * Unsigned Integer to float conversions
+ *
+ * Returns the result of converting the unsigned integer `a' to the
+ * floating-point format. The conversion is performed according to the
+ * IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+ */
+static void partsN(uint_to_float)(FloatPartsN *p, uint64_t a,
+                                  int scale, float_status *status)
+{
+    memset(p, 0, sizeof(*p));
+
+    if (a == 0) {
+        p->cls = float_class_zero;
+    } else {
+        int shift = clz64(a);
+        scale = MIN(MAX(scale, -0x10000), 0x10000);
+        p->cls = float_class_normal;
+        p->exp = DECOMPOSED_BINARY_POINT - shift + scale;
+        p->frac_hi = a << shift;
+    }
+}
+
+/*
+ * Float min/max.
+ */
+static FloatPartsN *partsN(minmax)(FloatPartsN *a, FloatPartsN *b,
+                                   float_status *s, int flags)
+{
+    int ab_mask = float_cmask(a->cls) | float_cmask(b->cls);
+    int a_exp, b_exp, cmp;
+
+    if (unlikely(ab_mask & float_cmask_anynan)) {
+        /*
+         * For minNum/maxNum (IEEE 754-2008)
+         * or minimumNumber/maximumNumber (IEEE 754-2019),
+         * if one operand is a QNaN, and the other
+         * operand is numerical, then return numerical argument.
+         */
+        if ((flags & (minmax_isnum | minmax_isnumber))
+            && !(ab_mask & float_cmask_snan)
+            && (ab_mask & ~float_cmask_qnan)) {
+            return is_nan(a->cls) ? b : a;
+        }
+
+        /*
+         * In IEEE 754-2019, minNum, maxNum, minNumMag and maxNumMag
+         * are removed and replaced with minimum, minimumNumber, maximum
+         * and maximumNumber.
+         * minimumNumber/maximumNumber behavior for SNaN is changed to:
+         *   If both operands are NaNs, a QNaN is returned.
+         *   If either operand is a SNaN,
+         *   an invalid operation exception is signaled,
+         *   but unless both operands are NaNs,
+         *   the SNaN is otherwise ignored and not converted to a QNaN.
+         */
+        if ((flags & minmax_isnumber)
+            && (ab_mask & float_cmask_snan)
+            && (ab_mask & ~float_cmask_anynan)) {
+            float_raise(float_flag_invalid, s);
+            return is_nan(a->cls) ? b : a;
+        }
+
+        return parts_pick_nan(a, b, s);
+    }
+
+    a_exp = a->exp;
+    b_exp = b->exp;
+
+    if (unlikely(ab_mask != float_cmask_normal)) {
+        switch (a->cls) {
+        case float_class_normal:
+            break;
+        case float_class_inf:
+            a_exp = INT16_MAX;
+            break;
+        case float_class_zero:
+            a_exp = INT16_MIN;
+            break;
+        default:
+            g_assert_not_reached();
+            break;
+        }
+        switch (b->cls) {
+        case float_class_normal:
+            break;
+        case float_class_inf:
+            b_exp = INT16_MAX;
+            break;
+        case float_class_zero:
+            b_exp = INT16_MIN;
+            break;
+        default:
+            g_assert_not_reached();
+            break;
+        }
+    }
+
+    /* Compare magnitudes. */
+    cmp = a_exp - b_exp;
+    if (cmp == 0) {
+        cmp = frac_cmp(a, b);
+    }
+
+    /*
+     * Take the sign into account.
+     * For ismag, only do this if the magnitudes are equal.
+     */
+    if (!(flags & minmax_ismag) || cmp == 0) {
+        if (a->sign != b->sign) {
+            /* For differing signs, the negative operand is less. */
+            cmp = a->sign ? -1 : 1;
+        } else if (a->sign) {
+            /* For two negative operands, invert the magnitude comparison. */
+            cmp = -cmp;
+        }
+    }
+
+    if (flags & minmax_ismin) {
+        cmp = -cmp;
+    }
+    return cmp < 0 ? b : a;
+}
+
+/*
+ * Floating point compare
+ */
+static FloatRelation partsN(compare)(FloatPartsN *a, FloatPartsN *b,
+                                     float_status *s, bool is_quiet)
+{
+    int ab_mask = float_cmask(a->cls) | float_cmask(b->cls);
+
+    if (likely(ab_mask == float_cmask_normal)) {
+        FloatRelation cmp;
+
+        if (a->sign != b->sign) {
+            goto a_sign;
+        }
+        if (a->exp == b->exp) {
+            cmp = frac_cmp(a, b);
+        } else if (a->exp < b->exp) {
+            cmp = float_relation_less;
+        } else {
+            cmp = float_relation_greater;
+        }
+        if (a->sign) {
+            cmp = -cmp;
+        }
+        return cmp;
+    }
+
+    if (unlikely(ab_mask & float_cmask_anynan)) {
+        if (ab_mask & float_cmask_snan) {
+            float_raise(float_flag_invalid | float_flag_invalid_snan, s);
+        } else if (!is_quiet) {
+            float_raise(float_flag_invalid, s);
+        }
+        return float_relation_unordered;
+    }
+
+    if (ab_mask & float_cmask_zero) {
+        if (ab_mask == float_cmask_zero) {
+            return float_relation_equal;
+        } else if (a->cls == float_class_zero) {
+            goto b_sign;
+        } else {
+            goto a_sign;
+        }
+    }
+
+    if (ab_mask == float_cmask_inf) {
+        if (a->sign == b->sign) {
+            return float_relation_equal;
+        }
+    } else if (b->cls == float_class_inf) {
+        goto b_sign;
+    } else {
+        g_assert(a->cls == float_class_inf);
+    }
+
+ a_sign:
+    return a->sign ? float_relation_less : float_relation_greater;
+ b_sign:
+    return b->sign ? float_relation_greater : float_relation_less;
+}
+
+/*
+ * Multiply A by 2 raised to the power N.
+ */
+static void partsN(scalbn)(FloatPartsN *a, int n, float_status *s)
+{
+    switch (a->cls) {
+    case float_class_snan:
+    case float_class_qnan:
+        parts_return_nan(a, s);
+        break;
+    case float_class_zero:
+    case float_class_inf:
+        break;
+    case float_class_normal:
+        a->exp += MIN(MAX(n, -0x10000), 0x10000);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+/*
+ * Return log2(A)
+ */
+static void partsN(log2)(FloatPartsN *a, float_status *s, const FloatFmt *fmt)
+{
+    uint64_t a0, a1, r, t, ign;
+    FloatPartsN f;
+    int i, n, a_exp, f_exp;
+
+    if (unlikely(a->cls != float_class_normal)) {
+        switch (a->cls) {
+        case float_class_snan:
+        case float_class_qnan:
+            parts_return_nan(a, s);
+            return;
+        case float_class_zero:
+            float_raise(float_flag_divbyzero, s);
+            /* log2(0) = -inf */
+            a->cls = float_class_inf;
+            a->sign = 1;
+            return;
+        case float_class_inf:
+            if (unlikely(a->sign)) {
+                goto d_nan;
+            }
+            return;
+        default:
+            break;
+        }
+        g_assert_not_reached();
+    }
+    if (unlikely(a->sign)) {
+        goto d_nan;
+    }
+
+    /* TODO: This algorithm looses bits too quickly for float128. */
+    g_assert(N == 64);
+
+    a_exp = a->exp;
+    f_exp = -1;
+
+    r = 0;
+    t = DECOMPOSED_IMPLICIT_BIT;
+    a0 = a->frac_hi;
+    a1 = 0;
+
+    n = fmt->frac_size + 2;
+    if (unlikely(a_exp == -1)) {
+        /*
+         * When a_exp == -1, we're computing the log2 of a value [0.5,1.0).
+         * When the value is very close to 1.0, there are lots of 1's in
+         * the msb parts of the fraction.  At the end, when we subtract
+         * this value from -1.0, we can see a catastrophic loss of precision,
+         * as 0x800..000 - 0x7ff..ffx becomes 0x000..00y, leaving only the
+         * bits of y in the final result.  To minimize this, compute as many
+         * digits as we can.
+         * ??? This case needs another algorithm to avoid this.
+         */
+        n = fmt->frac_size * 2 + 2;
+        /* Don't compute a value overlapping the sticky bit */
+        n = MIN(n, 62);
+    }
+
+    for (i = 0; i < n; i++) {
+        if (a1) {
+            mul128To256(a0, a1, a0, a1, &a0, &a1, &ign, &ign);
+        } else if (a0 & 0xffffffffull) {
+            mul64To128(a0, a0, &a0, &a1);
+        } else if (a0 & ~DECOMPOSED_IMPLICIT_BIT) {
+            a0 >>= 32;
+            a0 *= a0;
+        } else {
+            goto exact;
+        }
+
+        if (a0 & DECOMPOSED_IMPLICIT_BIT) {
+            if (unlikely(a_exp == 0 && r == 0)) {
+                /*
+                 * When a_exp == 0, we're computing the log2 of a value
+                 * [1.0,2.0).  When the value is very close to 1.0, there
+                 * are lots of 0's in the msb parts of the fraction.
+                 * We need to compute more digits to produce a correct
+                 * result -- restart at the top of the fraction.
+                 * ??? This is likely to lose precision quickly, as for
+                 * float128; we may need another method.
+                 */
+                f_exp -= i;
+                t = r = DECOMPOSED_IMPLICIT_BIT;
+                i = 0;
+            } else {
+                r |= t;
+            }
+        } else {
+            add128(a0, a1, a0, a1, &a0, &a1);
+        }
+        t >>= 1;
+    }
+
+    /* Set sticky for inexact. */
+    r |= (a1 || a0 & ~DECOMPOSED_IMPLICIT_BIT);
+
+ exact:
+    parts_sint_to_float(a, a_exp, 0, s);
+    if (r == 0) {
+        return;
+    }
+
+    memset(&f, 0, sizeof(f));
+    f.cls = float_class_normal;
+    f.frac_hi = r;
+    f.exp = f_exp - frac_normalize(&f);
+
+    if (a_exp < 0) {
+        parts_sub_normal(a, &f);
+    } else if (a_exp > 0) {
+        parts_add_normal(a, &f);
+    } else {
+        *a = f;
+    }
+    return;
+
+ d_nan:
+    float_raise(float_flag_invalid, s);
+    parts_default_nan(a, s);
+}
diff --git a/qemu/fpu/softfloat-specialize.inc.c b/qemu/fpu/softfloat-specialize.c.inc
similarity index 55%
rename from qemu/fpu/softfloat-specialize.inc.c
rename to qemu/fpu/softfloat-specialize.c.inc
index 5ab2fa1941..1610472cfc 100644
--- a/qemu/fpu/softfloat-specialize.inc.c
+++ b/qemu/fpu/softfloat-specialize.c.inc
@@ -79,12 +79,18 @@ this code that are retained.
  * version 2 or later. See the COPYING file in the top-level directory.
  */
 
-/* Define for architectures which deviate from IEEE in not supporting
+/*
+ * Define whether architecture deviates from IEEE in not supporting
  * signaling NaNs (so all NaNs are treated as quiet).
  */
+static inline bool no_signaling_nans(float_status *status)
+{
 #if defined(TARGET_XTENSA)
-#define NO_SIGNALING_NANS 1
+    return status->no_signaling_nans;
+#else
+    return false;
 #endif
+}
 
 /* Define how the architecture discriminates signaling NaNs.
  * This done with the most significant bit of the fraction.
@@ -93,11 +99,11 @@ this code that are retained.
  * 2008 revision and backward compatibility with their original choice.
  * Thus for MIPS we must make the choice at runtime.
  */
-static inline flag snan_bit_is_one(float_status *status)
+static inline bool snan_bit_is_one(float_status *status)
 {
 #if defined(TARGET_MIPS)
     return status->snan_bit_is_one;
-#elif defined(TARGET_HPPA) || defined(TARGET_UNICORE32) || defined(TARGET_SH4)
+#elif defined(TARGET_HPPA) || defined(TARGET_SH4)
     return 1;
 #else
     return 0;
@@ -111,19 +117,19 @@ static inline flag snan_bit_is_one(float_status *status)
 
 static bool parts_is_snan_frac(uint64_t frac, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return false;
-#else
-    flag msb = extract64(frac, DECOMPOSED_BINARY_POINT - 1, 1);
-    return msb == snan_bit_is_one(status);
-#endif
+    if (no_signaling_nans(status)) {
+        return false;
+    } else {
+        bool msb = extract64(frac, DECOMPOSED_BINARY_POINT - 1, 1);
+        return msb == snan_bit_is_one(status);
+    }
 }
 
 /*----------------------------------------------------------------------------
 | The pattern for a default generated deconstructed floating-point NaN.
 *----------------------------------------------------------------------------*/
 
-static FloatParts parts_default_nan(float_status *status)
+static void parts64_default_nan(FloatParts64 *p, float_status *status)
 {
     bool sign = 0;
     uint64_t frac;
@@ -139,12 +145,14 @@ static FloatParts parts_default_nan(float_status *status)
 #elif defined(TARGET_HPPA)
     /* snan_bit_is_one, set msb-1.  */
     frac = 1ULL << (DECOMPOSED_BINARY_POINT - 2);
+#elif defined(TARGET_HEXAGON)
+    sign = 1;
+    frac = ~0ULL;
 #else
-    /* This case is true for Alpha, ARM, MIPS, OpenRISC, PPC, RISC-V,
-     * S390, SH4, TriCore, and Xtensa.  I cannot find documentation
-     * for Unicore32; the choice from the original commit is unchanged.
-     * Our other supported targets, CRIS, LM32, Moxie, Nios2, and Tile,
-     * do not have floating-point.
+    /*
+     * This case is true for Alpha, ARM, MIPS, OpenRISC, PPC, RISC-V,
+     * S390, SH4, TriCore, and Xtensa.  Our other supported targets,
+     * CRIS, Nios2, and Tile, do not have floating-point.
      */
     if (snan_bit_is_one(status)) {
         /* set all bits other than msb */
@@ -155,7 +163,7 @@ static FloatParts parts_default_nan(float_status *status)
     }
 #endif
 
-    return (FloatParts) {
+    *p = (FloatParts64) {
         .cls = float_class_qnan,
         .sign = sign,
         .exp = INT_MAX,
@@ -163,27 +171,54 @@ static FloatParts parts_default_nan(float_status *status)
     };
 }
 
+static void parts128_default_nan(FloatParts128 *p, float_status *status)
+{
+    /*
+     * Extrapolate from the choices made by parts64_default_nan to fill
+     * in the quad-floating format.  If the low bit is set, assume we
+     * want to set all non-snan bits.
+     */
+    FloatParts64 p64;
+    parts64_default_nan(&p64, status);
+
+    *p = (FloatParts128) {
+        .cls = float_class_qnan,
+        .sign = p64.sign,
+        .exp = INT_MAX,
+        .frac_hi = p64.frac,
+        .frac_lo = -(p64.frac & 1)
+    };
+}
+
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN from a signalling NaN for the deconstructed
 | floating-point parts.
 *----------------------------------------------------------------------------*/
 
-static FloatParts parts_silence_nan(FloatParts a, float_status *status)
+static uint64_t parts_silence_nan_frac(uint64_t frac, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    g_assert_not_reached();
-#elif defined(TARGET_HPPA)
-    a.frac &= ~(1ULL << (DECOMPOSED_BINARY_POINT - 1));
-    a.frac |= 1ULL << (DECOMPOSED_BINARY_POINT - 2);
-#else
+    g_assert(!no_signaling_nans(status));
+
+    /* The only snan_bit_is_one target without default_nan_mode is HPPA. */
     if (snan_bit_is_one(status)) {
-        return parts_default_nan(status);
+        frac &= ~(1ULL << (DECOMPOSED_BINARY_POINT - 1));
+        frac |= 1ULL << (DECOMPOSED_BINARY_POINT - 2);
     } else {
-        a.frac |= 1ULL << (DECOMPOSED_BINARY_POINT - 1);
+        frac |= 1ULL << (DECOMPOSED_BINARY_POINT - 1);
     }
-#endif
-    a.cls = float_class_qnan;
-    return a;
+    return frac;
+}
+
+static void parts64_silence_nan(FloatParts64 *p, float_status *status)
+{
+    p->frac = parts_silence_nan_frac(p->frac, status);
+    p->cls = float_class_qnan;
+}
+
+static void parts128_silence_nan(FloatParts128 *p, float_status *status)
+{
+    p->frac_hi = parts_silence_nan_frac(p->frac_hi, status);
+    p->cls = float_class_qnan;
 }
 
 /*----------------------------------------------------------------------------
@@ -220,139 +255,118 @@ floatx80 floatx80_default_nan(float_status *status)
 const floatx80 floatx80_infinity
     = make_floatx80_init(floatx80_infinity_high, floatx80_infinity_low);
 
-/*----------------------------------------------------------------------------
-| Raises the exceptions specified by `flags'.  Floating-point traps can be
-| defined here if desired.  It is currently not possible for such a trap
-| to substitute a result value.  If traps are not implemented, this routine
-| should be simply `float_exception_flags |= flags;'.
-*----------------------------------------------------------------------------*/
-
-void float_raise(uint8_t flags, float_status *status)
-{
-    status->float_exception_flags |= flags;
-}
-
-/*----------------------------------------------------------------------------
-| Internal canonical NaN format.
-*----------------------------------------------------------------------------*/
-typedef struct {
-    flag sign;
-    uint64_t high, low;
-} commonNaNT;
-
 /*----------------------------------------------------------------------------
 | Returns 1 if the half-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float16_is_quiet_nan(float16 a_, float_status *status)
+bool float16_is_quiet_nan(float16 a_, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return float16_is_any_nan(a_);
-#else
-    uint16_t a = float16_val(a_);
-    if (snan_bit_is_one(status)) {
-        return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
+    if (no_signaling_nans(status)) {
+        return float16_is_any_nan(a_);
     } else {
-        return ((a & ~0x8000) >= 0x7C80);
+        uint16_t a = float16_val(a_);
+        if (snan_bit_is_one(status)) {
+            return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
+        } else {
+
+            return ((a >> 9) & 0x3F) == 0x3F;
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the half-precision floating-point value `a' is a signaling
+| Returns 1 if the bfloat16 value `a' is a quiet
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float16_is_signaling_nan(float16 a_, float_status *status)
+bool bfloat16_is_quiet_nan(bfloat16 a_, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return 0;
-#else
-    uint16_t a = float16_val(a_);
-    if (snan_bit_is_one(status)) {
-        return ((a & ~0x8000) >= 0x7C80);
+    if (no_signaling_nans(status)) {
+        return bfloat16_is_any_nan(a_);
     } else {
-        return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
+        uint16_t a = a_;
+        if (snan_bit_is_one(status)) {
+            return (((a >> 6) & 0x1FF) == 0x1FE) && (a & 0x3F);
+        } else {
+            return ((a >> 6) & 0x1FF) == 0x1FF;
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is a quiet
+| Returns 1 if the half-precision floating-point value `a' is a signaling
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float32_is_quiet_nan(float32 a_, float_status *status)
+bool float16_is_signaling_nan(float16 a_, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return float32_is_any_nan(a_);
-#else
-    uint32_t a = float32_val(a_);
-    if (snan_bit_is_one(status)) {
-        return (((a >> 22) & 0x1FF) == 0x1FE) && (a & 0x003FFFFF);
+    if (no_signaling_nans(status)) {
+        return 0;
     } else {
-        return ((uint32_t)(a << 1) >= 0xFF800000);
+        uint16_t a = float16_val(a_);
+        if (snan_bit_is_one(status)) {
+            return ((a >> 9) & 0x3F) == 0x3F;
+        } else {
+            return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is a signaling
+| Returns 1 if the bfloat16 value `a' is a signaling
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float32_is_signaling_nan(float32 a_, float_status *status)
+bool bfloat16_is_signaling_nan(bfloat16 a_, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return 0;
-#else
-    uint32_t a = float32_val(a_);
-    if (snan_bit_is_one(status)) {
-        return ((uint32_t)(a << 1) >= 0xFF800000);
+    if (no_signaling_nans(status)) {
+        return 0;
     } else {
-        return (((a >> 22) & 0x1FF) == 0x1FE) && (a & 0x003FFFFF);
+        uint16_t a = a_;
+        if (snan_bit_is_one(status)) {
+            return ((a >> 6) & 0x1FF) == 0x1FF;
+        } else {
+            return (((a >> 6) & 0x1FF) == 0x1FE) && (a & 0x3F);
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
-| Returns the result of converting the single-precision floating-point NaN
-| `a' to the canonical NaN format.  If `a' is a signaling NaN, the invalid
-| exception is raised.
+| Returns 1 if the single-precision floating-point value `a' is a quiet
+| NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-static commonNaNT float32ToCommonNaN(float32 a, float_status *status)
+bool float32_is_quiet_nan(float32 a_, float_status *status)
 {
-    commonNaNT z;
-
-    if (float32_is_signaling_nan(a, status)) {
-        float_raise(float_flag_invalid, status);
+    if (no_signaling_nans(status)) {
+        return float32_is_any_nan(a_);
+    } else {
+        uint32_t a = float32_val(a_);
+        if (snan_bit_is_one(status)) {
+            return (((a >> 22) & 0x1FF) == 0x1FE) && (a & 0x003FFFFF);
+        } else {
+            return ((uint32_t)(a << 1) >= 0xFF800000);
+        }
     }
-    z.sign = float32_val(a) >> 31;
-    z.low = 0;
-    z.high = ((uint64_t)float32_val(a)) << 41;
-    return z;
 }
 
 /*----------------------------------------------------------------------------
-| Returns the result of converting the canonical NaN `a' to the single-
-| precision floating-point format.
+| Returns 1 if the single-precision floating-point value `a' is a signaling
+| NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-static float32 commonNaNToFloat32(commonNaNT a, float_status *status)
+bool float32_is_signaling_nan(float32 a_, float_status *status)
 {
-    uint32_t mantissa = a.high >> 41;
-
-    if (status->default_nan_mode) {
-        return float32_default_nan(status);
-    }
-
-    if (mantissa) {
-        return make_float32(
-            (((uint32_t)a.sign) << 31) | 0x7F800000 | (a.high >> 41));
+    if (no_signaling_nans(status)) {
+        return 0;
     } else {
-        return float32_default_nan(status);
+        uint32_t a = float32_val(a_);
+        if (snan_bit_is_one(status)) {
+            return ((uint32_t)(a << 1) >= 0xFF800000);
+        } else {
+            return (((a >> 22) & 0x1FF) == 0x1FE) && (a & 0x003FFFFF);
+        }
     }
 }
 
@@ -374,9 +388,10 @@ static float32 commonNaNToFloat32(commonNaNT a, float_status *status)
 *----------------------------------------------------------------------------*/
 
 static int pickNaN(FloatClass a_cls, FloatClass b_cls,
-                   flag aIsLargerSignificand)
+                   bool aIsLargerSignificand, float_status *status)
 {
-#if defined(TARGET_ARM) || defined(TARGET_MIPS) || defined(TARGET_HPPA)
+#if defined(TARGET_ARM) || defined(TARGET_MIPS) || defined(TARGET_HPPA) || \
+    defined(TARGET_LOONGARCH64) || defined(TARGET_S390X)
     /* ARM mandated NaN propagation rules (see FPProcessNaNs()), take
      * the first of:
      *  1. A if it is signaling
@@ -407,7 +422,7 @@ static int pickNaN(FloatClass a_cls, FloatClass b_cls,
     } else {
         return 1;
     }
-#elif defined(TARGET_PPC) || defined(TARGET_XTENSA) || defined(TARGET_M68K)
+#elif defined(TARGET_PPC) || defined(TARGET_M68K)
     /* PowerPC propagation rules:
      *  1. A if it sNaN or qNaN
      *  2. B if it sNaN or qNaN
@@ -432,6 +447,24 @@ static int pickNaN(FloatClass a_cls, FloatClass b_cls,
     } else {
         return 1;
     }
+#elif defined(TARGET_XTENSA)
+    /*
+     * Xtensa has two NaN propagation modes.
+     * Which one is active is controlled by float_status::use_first_nan.
+     */
+    if (status->use_first_nan) {
+        if (is_nan(a_cls)) {
+            return 0;
+        } else {
+            return 1;
+        }
+    } else {
+        if (is_nan(b_cls)) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
 #else
     /* This implements x87 NaN propagation rules:
      * SNaN + QNaN => return the QNaN
@@ -474,7 +507,7 @@ static int pickNaNMulAdd(FloatClass a_cls, FloatClass b_cls, FloatClass c_cls,
      * the default NaN
      */
     if (infzero && is_qnan(c_cls)) {
-        float_raise(float_flag_invalid, status);
+        float_raise(float_flag_invalid | float_flag_invalid_imz, status);
         return 3;
     }
 
@@ -501,7 +534,7 @@ static int pickNaNMulAdd(FloatClass a_cls, FloatClass b_cls, FloatClass c_cls,
          * case sets InvalidOp and returns the default NaN
          */
         if (infzero) {
-            float_raise(float_flag_invalid, status);
+            float_raise(float_flag_invalid | float_flag_invalid_imz, status);
             return 3;
         }
         /* Prefer sNaN over qNaN, in the a, b, c order. */
@@ -524,7 +557,7 @@ static int pickNaNMulAdd(FloatClass a_cls, FloatClass b_cls, FloatClass c_cls,
          * case sets InvalidOp and returns the input value 'c'
          */
         if (infzero) {
-            float_raise(float_flag_invalid, status);
+            float_raise(float_flag_invalid | float_flag_invalid_imz, status);
             return 2;
         }
         /* Prefer sNaN over qNaN, in the c, a, b order. */
@@ -542,13 +575,36 @@ static int pickNaNMulAdd(FloatClass a_cls, FloatClass b_cls, FloatClass c_cls,
             return 1;
         }
     }
+#elif defined(TARGET_LOONGARCH64)
+    /*
+     * For LoongArch systems that conform to IEEE754-2008, the (inf,zero,nan)
+     * case sets InvalidOp and returns the input value 'c'
+     */
+    if (infzero) {
+        float_raise(float_flag_invalid | float_flag_invalid_imz, status);
+        return 2;
+    }
+    /* Prefer sNaN over qNaN, in the c, a, b order. */
+    if (is_snan(c_cls)) {
+        return 2;
+    } else if (is_snan(a_cls)) {
+        return 0;
+    } else if (is_snan(b_cls)) {
+        return 1;
+    } else if (is_qnan(c_cls)) {
+        return 2;
+    } else if (is_qnan(a_cls)) {
+        return 0;
+    } else {
+        return 1;
+    }
 #elif defined(TARGET_PPC)
     /* For PPC, the (inf,zero,qnan) case sets InvalidOp, but we prefer
      * to return an input NaN if we have one (ie c) rather than generating
      * a default NaN
      */
     if (infzero) {
-        float_raise(float_flag_invalid, status);
+        float_raise(float_flag_invalid | float_flag_invalid_imz, status);
         return 2;
     }
 
@@ -562,6 +618,38 @@ static int pickNaNMulAdd(FloatClass a_cls, FloatClass b_cls, FloatClass c_cls,
     } else {
         return 1;
     }
+#elif defined(TARGET_RISCV)
+    /* For RISC-V, InvalidOp is set when multiplicands are Inf and zero */
+    if (infzero) {
+        float_raise(float_flag_invalid | float_flag_invalid_imz, status);
+    }
+    return 3; /* default NaN */
+#elif defined(TARGET_XTENSA)
+    /*
+     * For Xtensa, the (inf,zero,nan) case sets InvalidOp and returns
+     * an input NaN if we have one (ie c).
+     */
+    if (infzero) {
+        float_raise(float_flag_invalid | float_flag_invalid_imz, status);
+        return 2;
+    }
+    if (status->use_first_nan) {
+        if (is_nan(a_cls)) {
+            return 0;
+        } else if (is_nan(b_cls)) {
+            return 1;
+        } else {
+            return 2;
+        }
+    } else {
+        if (is_nan(c_cls)) {
+            return 2;
+        } else if (is_nan(b_cls)) {
+            return 1;
+        } else {
+            return 0;
+        }
+    }
 #else
     /* A default implementation: prefer a to b to c.
      * This is unlikely to actually match any real implementation.
@@ -576,80 +664,24 @@ static int pickNaNMulAdd(FloatClass a_cls, FloatClass b_cls, FloatClass c_cls,
 #endif
 }
 
-/*----------------------------------------------------------------------------
-| Takes two single-precision floating-point values `a' and `b', one of which
-| is a NaN, and returns the appropriate NaN result.  If either `a' or `b' is a
-| signaling NaN, the invalid exception is raised.
-*----------------------------------------------------------------------------*/
-
-static float32 propagateFloat32NaN(float32 a, float32 b, float_status *status)
-{
-    flag aIsLargerSignificand;
-    uint32_t av, bv;
-    FloatClass a_cls, b_cls;
-
-    /* This is not complete, but is good enough for pickNaN.  */
-    a_cls = (!float32_is_any_nan(a)
-             ? float_class_normal
-             : float32_is_signaling_nan(a, status)
-             ? float_class_snan
-             : float_class_qnan);
-    b_cls = (!float32_is_any_nan(b)
-             ? float_class_normal
-             : float32_is_signaling_nan(b, status)
-             ? float_class_snan
-             : float_class_qnan);
-
-    av = float32_val(a);
-    bv = float32_val(b);
-
-    if (is_snan(a_cls) || is_snan(b_cls)) {
-        float_raise(float_flag_invalid, status);
-    }
-
-    if (status->default_nan_mode) {
-        return float32_default_nan(status);
-    }
-
-    if ((uint32_t)(av << 1) < (uint32_t)(bv << 1)) {
-        aIsLargerSignificand = 0;
-    } else if ((uint32_t)(bv << 1) < (uint32_t)(av << 1)) {
-        aIsLargerSignificand = 1;
-    } else {
-        aIsLargerSignificand = (av < bv) ? 1 : 0;
-    }
-
-    if (pickNaN(a_cls, b_cls, aIsLargerSignificand)) {
-        if (is_snan(b_cls)) {
-            return float32_silence_nan(b, status);
-        }
-        return b;
-    } else {
-        if (is_snan(a_cls)) {
-            return float32_silence_nan(a, status);
-        }
-        return a;
-    }
-}
-
 /*----------------------------------------------------------------------------
 | Returns 1 if the double-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float64_is_quiet_nan(float64 a_, float_status *status)
+bool float64_is_quiet_nan(float64 a_, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return float64_is_any_nan(a_);
-#else
-    uint64_t a = float64_val(a_);
-    if (snan_bit_is_one(status)) {
-        return (((a >> 51) & 0xFFF) == 0xFFE)
-            && (a & 0x0007FFFFFFFFFFFFULL);
+    if (no_signaling_nans(status)) {
+        return float64_is_any_nan(a_);
     } else {
-        return ((a << 1) >= 0xFFF0000000000000ULL);
+        uint64_t a = float64_val(a_);
+        if (snan_bit_is_one(status)) {
+            return (((a >> 51) & 0xFFF) == 0xFFE)
+                && (a & 0x0007FFFFFFFFFFFFULL);
+        } else {
+            return ((a << 1) >= 0xFFF0000000000000ULL);
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
@@ -657,116 +689,18 @@ int float64_is_quiet_nan(float64 a_, float_status *status)
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float64_is_signaling_nan(float64 a_, float_status *status)
+bool float64_is_signaling_nan(float64 a_, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return 0;
-#else
-    uint64_t a = float64_val(a_);
-    if (snan_bit_is_one(status)) {
-        return ((a << 1) >= 0xFFF0000000000000ULL);
-    } else {
-        return (((a >> 51) & 0xFFF) == 0xFFE)
-            && (a & UINT64_C(0x0007FFFFFFFFFFFF));
-    }
-#endif
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the double-precision floating-point NaN
-| `a' to the canonical NaN format.  If `a' is a signaling NaN, the invalid
-| exception is raised.
-*----------------------------------------------------------------------------*/
-
-static commonNaNT float64ToCommonNaN(float64 a, float_status *status)
-{
-    commonNaNT z;
-
-    if (float64_is_signaling_nan(a, status)) {
-        float_raise(float_flag_invalid, status);
-    }
-    z.sign = float64_val(a) >> 63;
-    z.low = 0;
-    z.high = float64_val(a) << 12;
-    return z;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the canonical NaN `a' to the double-
-| precision floating-point format.
-*----------------------------------------------------------------------------*/
-
-static float64 commonNaNToFloat64(commonNaNT a, float_status *status)
-{
-    uint64_t mantissa = a.high >> 12;
-
-    if (status->default_nan_mode) {
-        return float64_default_nan(status);
-    }
-
-    if (mantissa) {
-        return make_float64(
-              (((uint64_t) a.sign) << 63)
-            | UINT64_C(0x7FF0000000000000)
-            | (a.high >> 12));
-    } else {
-        return float64_default_nan(status);
-    }
-}
-
-/*----------------------------------------------------------------------------
-| Takes two double-precision floating-point values `a' and `b', one of which
-| is a NaN, and returns the appropriate NaN result.  If either `a' or `b' is a
-| signaling NaN, the invalid exception is raised.
-*----------------------------------------------------------------------------*/
-
-static float64 propagateFloat64NaN(float64 a, float64 b, float_status *status)
-{
-    flag aIsLargerSignificand;
-    uint64_t av, bv;
-    FloatClass a_cls, b_cls;
-
-    /* This is not complete, but is good enough for pickNaN.  */
-    a_cls = (!float64_is_any_nan(a)
-             ? float_class_normal
-             : float64_is_signaling_nan(a, status)
-             ? float_class_snan
-             : float_class_qnan);
-    b_cls = (!float64_is_any_nan(b)
-             ? float_class_normal
-             : float64_is_signaling_nan(b, status)
-             ? float_class_snan
-             : float_class_qnan);
-
-    av = float64_val(a);
-    bv = float64_val(b);
-
-    if (is_snan(a_cls) || is_snan(b_cls)) {
-        float_raise(float_flag_invalid, status);
-    }
-
-    if (status->default_nan_mode) {
-        return float64_default_nan(status);
-    }
-
-    if ((uint64_t)(av << 1) < (uint64_t)(bv << 1)) {
-        aIsLargerSignificand = 0;
-    } else if ((uint64_t)(bv << 1) < (uint64_t)(av << 1)) {
-        aIsLargerSignificand = 1;
-    } else {
-        aIsLargerSignificand = (av < bv) ? 1 : 0;
-    }
-
-    if (pickNaN(a_cls, b_cls, aIsLargerSignificand)) {
-        if (is_snan(b_cls)) {
-            return float64_silence_nan(b, status);
-        }
-        return b;
+    if (no_signaling_nans(status)) {
+        return 0;
     } else {
-        if (is_snan(a_cls)) {
-            return float64_silence_nan(a, status);
+        uint64_t a = float64_val(a_);
+        if (snan_bit_is_one(status)) {
+            return ((a << 1) >= 0xFFF0000000000000ULL);
+        } else {
+            return (((a >> 51) & 0xFFF) == 0xFFE)
+                && (a & UINT64_C(0x0007FFFFFFFFFFFF));
         }
-        return a;
     }
 }
 
@@ -778,21 +712,21 @@ static float64 propagateFloat64NaN(float64 a, float64 b, float_status *status)
 
 int floatx80_is_quiet_nan(floatx80 a, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return floatx80_is_any_nan(a);
-#else
-    if (snan_bit_is_one(status)) {
-        uint64_t aLow;
-
-        aLow = a.low & ~0x4000000000000000ULL;
-        return ((a.high & 0x7FFF) == 0x7FFF)
-            && (aLow << 1)
-            && (a.low == aLow);
+    if (no_signaling_nans(status)) {
+        return floatx80_is_any_nan(a);
     } else {
-        return ((a.high & 0x7FFF) == 0x7FFF)
-            && (UINT64_C(0x8000000000000000) <= ((uint64_t)(a.low << 1)));
+        if (snan_bit_is_one(status)) {
+            uint64_t aLow;
+
+            aLow = a.low & ~0x4000000000000000ULL;
+            return ((a.high & 0x7FFF) == 0x7FFF)
+                && (aLow << 1)
+                && (a.low == aLow);
+        } else {
+            return ((a.high & 0x7FFF) == 0x7FFF)
+                && (UINT64_C(0x8000000000000000) <= ((uint64_t)(a.low << 1)));
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
@@ -803,21 +737,21 @@ int floatx80_is_quiet_nan(floatx80 a, float_status *status)
 
 int floatx80_is_signaling_nan(floatx80 a, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return 0;
-#else
-    if (snan_bit_is_one(status)) {
-        return ((a.high & 0x7FFF) == 0x7FFF)
-            && ((a.low << 1) >= 0x8000000000000000ULL);
+    if (no_signaling_nans(status)) {
+        return 0;
     } else {
-        uint64_t aLow;
+        if (snan_bit_is_one(status)) {
+            return ((a.high & 0x7FFF) == 0x7FFF)
+                && ((a.low << 1) >= 0x8000000000000000ULL);
+        } else {
+            uint64_t aLow;
 
-        aLow = a.low & ~UINT64_C(0x4000000000000000);
-        return ((a.high & 0x7FFF) == 0x7FFF)
-            && (uint64_t)(aLow << 1)
-            && (a.low == aLow);
+            aLow = a.low & ~UINT64_C(0x4000000000000000);
+            return ((a.high & 0x7FFF) == 0x7FFF)
+                && (uint64_t)(aLow << 1)
+                && (a.low == aLow);
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
@@ -833,55 +767,6 @@ floatx80 floatx80_silence_nan(floatx80 a, float_status *status)
     return a;
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point NaN `a' to the canonical NaN format.  If `a' is a signaling NaN, the
-| invalid exception is raised.
-*----------------------------------------------------------------------------*/
-
-static commonNaNT floatx80ToCommonNaN(floatx80 a, float_status *status)
-{
-    floatx80 dflt;
-    commonNaNT z;
-
-    if (floatx80_is_signaling_nan(a, status)) {
-        float_raise(float_flag_invalid, status);
-    }
-    if (a.low >> 63) {
-        z.sign = a.high >> 15;
-        z.low = 0;
-        z.high = a.low << 1;
-    } else {
-        dflt = floatx80_default_nan(status);
-        z.sign = dflt.high >> 15;
-        z.low = 0;
-        z.high = dflt.low << 1;
-    }
-    return z;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the canonical NaN `a' to the extended
-| double-precision floating-point format.
-*----------------------------------------------------------------------------*/
-
-static floatx80 commonNaNToFloatx80(commonNaNT a, float_status *status)
-{
-    floatx80 z;
-
-    if (status->default_nan_mode) {
-        return floatx80_default_nan(status);
-    }
-
-    if (a.high >> 1) {
-        z.low = UINT64_C(0x8000000000000000) | a.high >> 1;
-        z.high = (((uint16_t)a.sign) << 15) | 0x7FFF;
-    } else {
-        z = floatx80_default_nan(status);
-    }
-    return z;
-}
-
 /*----------------------------------------------------------------------------
 | Takes two extended double-precision floating-point values `a' and `b', one
 | of which is a NaN, and returns the appropriate NaN result.  If either `a' or
@@ -890,7 +775,7 @@ static floatx80 commonNaNToFloatx80(commonNaNT a, float_status *status)
 
 floatx80 propagateFloatx80NaN(floatx80 a, floatx80 b, float_status *status)
 {
-    flag aIsLargerSignificand;
+    bool aIsLargerSignificand;
     FloatClass a_cls, b_cls;
 
     /* This is not complete, but is good enough for pickNaN.  */
@@ -921,7 +806,7 @@ floatx80 propagateFloatx80NaN(floatx80 a, floatx80 b, float_status *status)
         aIsLargerSignificand = (a.high < b.high) ? 1 : 0;
     }
 
-    if (pickNaN(a_cls, b_cls, aIsLargerSignificand)) {
+    if (pickNaN(a_cls, b_cls, aIsLargerSignificand, status)) {
         if (is_snan(b_cls)) {
             return floatx80_silence_nan(b, status);
         }
@@ -939,19 +824,19 @@ floatx80 propagateFloatx80NaN(floatx80 a, floatx80 b, float_status *status)
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float128_is_quiet_nan(float128 a, float_status *status)
+bool float128_is_quiet_nan(float128 a, float_status *status)
 {
-#ifdef NO_SIGNALING_NANS
-    return float128_is_any_nan(a);
-#else
-    if (snan_bit_is_one(status)) {
-        return (((a.high >> 47) & 0xFFFF) == 0xFFFE)
-            && (a.low || (a.high & 0x00007FFFFFFFFFFFULL));
+    if (no_signaling_nans(status)) {
+        return float128_is_any_nan(a);
     } else {
-        return ((a.high << 1) >= 0xFFFF000000000000ULL)
-            && (a.low || (a.high & 0x0000FFFFFFFFFFFFULL));
+        if (snan_bit_is_one(status)) {
+            return (((a.high >> 47) & 0xFFFF) == 0xFFFE)
+                && (a.low || (a.high & 0x00007FFFFFFFFFFFULL));
+        } else {
+            return ((a.high << 1) >= 0xFFFF000000000000ULL)
+                && (a.low || (a.high & 0x0000FFFFFFFFFFFFULL));
+        }
     }
-#endif
 }
 
 /*----------------------------------------------------------------------------
@@ -959,125 +844,17 @@ int float128_is_quiet_nan(float128 a, float_status *status)
 | signaling NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float128_is_signaling_nan(float128 a, float_status *status)
-{
-#ifdef NO_SIGNALING_NANS
-    return 0;
-#else
-    if (snan_bit_is_one(status)) {
-        return ((a.high << 1) >= 0xFFFF000000000000ULL)
-            && (a.low || (a.high & 0x0000FFFFFFFFFFFFULL));
-    } else {
-        return (((a.high >> 47) & 0xFFFF) == 0xFFFE)
-            && (a.low || (a.high & UINT64_C(0x00007FFFFFFFFFFF)));
-    }
-#endif
-}
-
-/*----------------------------------------------------------------------------
-| Returns a quiet NaN from a signalling NaN for the quadruple-precision
-| floating point value `a'.
-*----------------------------------------------------------------------------*/
-
-float128 float128_silence_nan(float128 a, float_status *status)
-{
-#ifdef NO_SIGNALING_NANS
-    g_assert_not_reached();
-#else
-    if (snan_bit_is_one(status)) {
-        return float128_default_nan(status);
-    } else {
-        a.high |= UINT64_C(0x0000800000000000);
-        return a;
-    }
-#endif
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point NaN
-| `a' to the canonical NaN format.  If `a' is a signaling NaN, the invalid
-| exception is raised.
-*----------------------------------------------------------------------------*/
-
-static commonNaNT float128ToCommonNaN(float128 a, float_status *status)
-{
-    commonNaNT z;
-
-    if (float128_is_signaling_nan(a, status)) {
-        float_raise(float_flag_invalid, status);
-    }
-    z.sign = a.high >> 63;
-    shortShift128Left(a.high, a.low, 16, &z.high, &z.low);
-    return z;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the canonical NaN `a' to the quadruple-
-| precision floating-point format.
-*----------------------------------------------------------------------------*/
-
-static float128 commonNaNToFloat128(commonNaNT a, float_status *status)
-{
-    float128 z;
-
-    if (status->default_nan_mode) {
-        return float128_default_nan(status);
-    }
-
-    shift128Right(a.high, a.low, 16, &z.high, &z.low);
-    z.high |= (((uint64_t)a.sign) << 63) | UINT64_C(0x7FFF000000000000);
-    return z;
-}
-
-/*----------------------------------------------------------------------------
-| Takes two quadruple-precision floating-point values `a' and `b', one of
-| which is a NaN, and returns the appropriate NaN result.  If either `a' or
-| `b' is a signaling NaN, the invalid exception is raised.
-*----------------------------------------------------------------------------*/
-
-static float128 propagateFloat128NaN(float128 a, float128 b,
-                                     float_status *status)
+bool float128_is_signaling_nan(float128 a, float_status *status)
 {
-    flag aIsLargerSignificand;
-    FloatClass a_cls, b_cls;
-
-    /* This is not complete, but is good enough for pickNaN.  */
-    a_cls = (!float128_is_any_nan(a)
-             ? float_class_normal
-             : float128_is_signaling_nan(a, status)
-             ? float_class_snan
-             : float_class_qnan);
-    b_cls = (!float128_is_any_nan(b)
-             ? float_class_normal
-             : float128_is_signaling_nan(b, status)
-             ? float_class_snan
-             : float_class_qnan);
-
-    if (is_snan(a_cls) || is_snan(b_cls)) {
-        float_raise(float_flag_invalid, status);
-    }
-
-    if (status->default_nan_mode) {
-        return float128_default_nan(status);
-    }
-
-    if (lt128(a.high << 1, a.low, b.high << 1, b.low)) {
-        aIsLargerSignificand = 0;
-    } else if (lt128(b.high << 1, b.low, a.high << 1, a.low)) {
-        aIsLargerSignificand = 1;
-    } else {
-        aIsLargerSignificand = (a.high < b.high) ? 1 : 0;
-    }
-
-    if (pickNaN(a_cls, b_cls, aIsLargerSignificand)) {
-        if (is_snan(b_cls)) {
-            return float128_silence_nan(b, status);
-        }
-        return b;
+    if (no_signaling_nans(status)) {
+        return 0;
     } else {
-        if (is_snan(a_cls)) {
-            return float128_silence_nan(a, status);
+        if (snan_bit_is_one(status)) {
+            return ((a.high << 1) >= 0xFFFF000000000000ULL)
+                && (a.low || (a.high & 0x0000FFFFFFFFFFFFULL));
+        } else {
+            return (((a.high >> 47) & 0xFFFF) == 0xFFFE)
+                && (a.low || (a.high & UINT64_C(0x00007FFFFFFFFFFF)));
         }
-        return a;
     }
 }
diff --git a/qemu/fpu/softfloat.c b/qemu/fpu/softfloat.c
index 0e7938dc1c..9f128a2ee6 100644
--- a/qemu/fpu/softfloat.c
+++ b/qemu/fpu/softfloat.c
@@ -132,7 +132,7 @@ this code that are retained.
         if (unlikely(soft_t ## _is_denormal(*a))) {                     \
             *a = soft_t ## _set_sign(soft_t ## _zero,                   \
                                      soft_t ## _is_neg(*a));            \
-            s->float_exception_flags |= float_flag_input_denormal;      \
+            float_raise(float_flag_input_denormal, s);                  \
         }                                                               \
     }
 
@@ -227,12 +227,9 @@ GEN_INPUT_FLUSH3(float64_input_flush3, float64)
 # endif
 # define QEMU_NO_HARDFLOAT 1
 # define QEMU_SOFTFLOAT_ATTR QEMU_FLATTEN
-#elif !defined(_MSC_VER)
+#else
 # define QEMU_NO_HARDFLOAT 0
 # define QEMU_SOFTFLOAT_ATTR QEMU_FLATTEN __attribute__((noinline))
-#else // MSVC
-# define QEMU_NO_HARDFLOAT 0
-# define QEMU_SOFTFLOAT_ATTR
 #endif
 
 static inline bool can_use_fpu(const float_status *s)
@@ -342,12 +339,10 @@ static inline bool f64_is_inf(union_float64 a)
     return float64_is_infinity(a.s);
 }
 
-/* Note: @fast_test and @post can be NULL */
 static inline float32
 float32_gen2(float32 xa, float32 xb, float_status *s,
              hard_f32_op2_fn hard, soft_f32_op2_fn soft,
-             f32_check_fn pre, f32_check_fn post,
-             f32_check_fn fast_test, soft_f32_op2_fn fast_op)
+             f32_check_fn pre, f32_check_fn post)
 {
     union_float32 ua, ub, ur;
 
@@ -362,17 +357,12 @@ float32_gen2(float32 xa, float32 xb, float_status *s,
     if (unlikely(!pre(ua, ub))) {
         goto soft;
     }
-    if (fast_test && fast_test(ua, ub)) {
-        return fast_op(ua.s, ub.s, s);
-    }
 
     ur.h = hard(ua.h, ub.h);
     if (unlikely(f32_is_inf(ur))) {
-        s->float_exception_flags |= float_flag_overflow;
-    } else if (unlikely(fabsf(ur.h) <= FLT_MIN)) {
-        if (post == NULL || post(ua, ub)) {
-            goto soft;
-        }
+        float_raise(float_flag_overflow, s);
+    } else if (unlikely(fabsf(ur.h) <= FLT_MIN) && post(ua, ub)) {
+        goto soft;
     }
     return ur.s;
 
@@ -383,8 +373,7 @@ float32_gen2(float32 xa, float32 xb, float_status *s,
 static inline float64
 float64_gen2(float64 xa, float64 xb, float_status *s,
              hard_f64_op2_fn hard, soft_f64_op2_fn soft,
-             f64_check_fn pre, f64_check_fn post,
-             f64_check_fn fast_test, soft_f64_op2_fn fast_op)
+             f64_check_fn pre, f64_check_fn post)
 {
     union_float64 ua, ub, ur;
 
@@ -399,17 +388,12 @@ float64_gen2(float64 xa, float64 xb, float_status *s,
     if (unlikely(!pre(ua, ub))) {
         goto soft;
     }
-    if (fast_test && fast_test(ua, ub)) {
-        return fast_op(ua.s, ub.s, s);
-    }
 
     ur.h = hard(ua.h, ub.h);
     if (unlikely(f64_is_inf(ur))) {
-        s->float_exception_flags |= float_flag_overflow;
-    } else if (unlikely(fabs(ur.h) <= DBL_MIN)) {
-        if (post == NULL || post(ua, ub)) {
-            goto soft;
-        }
+        float_raise(float_flag_overflow, s);
+    } else if (unlikely(fabs(ur.h) <= DBL_MIN) && post(ua, ub)) {
+        goto soft;
     }
     return ur.s;
 
@@ -417,66 +401,11 @@ float64_gen2(float64 xa, float64 xb, float_status *s,
     return soft(ua.s, ub.s, s);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the fraction bits of the single-precision floating-point value `a'.
-*----------------------------------------------------------------------------*/
-
-static inline uint32_t extractFloat32Frac(float32 a)
-{
-    return float32_val(a) & 0x007FFFFF;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the exponent bits of the single-precision floating-point value `a'.
-*----------------------------------------------------------------------------*/
-
-static inline int extractFloat32Exp(float32 a)
-{
-    return (float32_val(a) >> 23) & 0xFF;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the sign bit of the single-precision floating-point value `a'.
-*----------------------------------------------------------------------------*/
-
-static inline flag extractFloat32Sign(float32 a)
-{
-    return float32_val(a) >> 31;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the fraction bits of the double-precision floating-point value `a'.
-*----------------------------------------------------------------------------*/
-
-static inline uint64_t extractFloat64Frac(float64 a)
-{
-    return float64_val(a) & UINT64_C(0x000FFFFFFFFFFFFF);
-}
-
-/*----------------------------------------------------------------------------
-| Returns the exponent bits of the double-precision floating-point value `a'.
-*----------------------------------------------------------------------------*/
-
-static inline int extractFloat64Exp(float64 a)
-{
-    return (float64_val(a) >> 52) & 0x7FF;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the sign bit of the double-precision floating-point value `a'.
-*----------------------------------------------------------------------------*/
-
-static inline flag extractFloat64Sign(float64 a)
-{
-    return float64_val(a) >> 63;
-}
-
 /*
  * Classify a floating point number. Everything above float_class_qnan
  * is a NaN so cls >= float_class_qnan is any NaN.
  */
 
-#ifndef _MSC_VER
 typedef enum __attribute__ ((__packed__)) {
     float_class_unclassified,
     float_class_zero,
@@ -485,68 +414,98 @@ typedef enum __attribute__ ((__packed__)) {
     float_class_qnan,  /* all NaNs from here */
     float_class_snan,
 } FloatClass;
-#else
-__pragma(pack(push, 1))
-typedef enum {
-    float_class_unclassified,
-    float_class_zero,
-    float_class_normal,
-    float_class_inf,
-    float_class_qnan,  /* all NaNs from here */
-    float_class_snan,
-} FloatClass;
-__pragma(pack(pop))
-#endif
+
+#define float_cmask(bit)  (1u << (bit))
+
+enum {
+    float_cmask_zero    = float_cmask(float_class_zero),
+    float_cmask_normal  = float_cmask(float_class_normal),
+    float_cmask_inf     = float_cmask(float_class_inf),
+    float_cmask_qnan    = float_cmask(float_class_qnan),
+    float_cmask_snan    = float_cmask(float_class_snan),
+
+    float_cmask_infzero = float_cmask_zero | float_cmask_inf,
+    float_cmask_anynan  = float_cmask_qnan | float_cmask_snan,
+};
+
+/* Flags for parts_minmax. */
+enum {
+    /* Set for minimum; clear for maximum. */
+    minmax_ismin = 1,
+    /* Set for the IEEE 754-2008 minNum() and maxNum() operations. */
+    minmax_isnum = 2,
+    /* Set for the IEEE 754-2008 minNumMag() and minNumMag() operations. */
+    minmax_ismag = 4,
+    /*
+     * Set for the IEEE 754-2019 minimumNumber() and maximumNumber()
+     * operations.
+     */
+    minmax_isnumber = 8,
+};
 
 /* Simple helpers for checking if, or what kind of, NaN we have */
-#ifndef _MSC_VER
 static inline __attribute__((unused)) bool is_nan(FloatClass c)
-#else
-static inline bool is_nan(FloatClass c)
-#endif
 {
     return unlikely(c >= float_class_qnan);
 }
 
-#ifndef _MSC_VER
 static inline __attribute__((unused)) bool is_snan(FloatClass c)
-#else
-static inline bool is_snan(FloatClass c)
-#endif
 {
     return c == float_class_snan;
 }
 
-#ifndef _MSC_VER
 static inline __attribute__((unused)) bool is_qnan(FloatClass c)
-#else
-static inline bool is_qnan(FloatClass c)
-#endif
 {
     return c == float_class_qnan;
 }
 
 /*
- * Structure holding all of the decomposed parts of a float. The
- * exponent is unbiased and the fraction is normalized. All
- * calculations are done with a 64 bit fraction and then rounded as
- * appropriate for the final format.
+ * Structure holding all of the decomposed parts of a float.
+ * The exponent is unbiased and the fraction is normalized.
  *
- * Thanks to the packed FloatClass a decent compiler should be able to
- * fit the whole structure into registers and avoid using the stack
- * for parameter passing.
+ * The fraction words are stored in big-endian word ordering,
+ * so that truncation from a larger format to a smaller format
+ * can be done simply by ignoring subsequent elements.
  */
 
 typedef struct {
-    uint64_t frac;
-    int32_t  exp;
     FloatClass cls;
     bool sign;
-} FloatParts;
+    int32_t exp;
+    union {
+        /* Routines that know the structure may reference the singular name. */
+        uint64_t frac;
+        /*
+         * Routines expanded with multiple structures reference "hi" and "lo"
+         * depending on the operation.  In FloatParts64, "hi" and "lo" are
+         * both the same word and aliased here.
+         */
+        uint64_t frac_hi;
+        uint64_t frac_lo;
+    };
+} FloatParts64;
+
+typedef struct {
+    FloatClass cls;
+    bool sign;
+    int32_t exp;
+    uint64_t frac_hi;
+    uint64_t frac_lo;
+} FloatParts128;
 
-#define DECOMPOSED_BINARY_POINT    (64 - 2)
+typedef struct {
+    FloatClass cls;
+    bool sign;
+    int32_t exp;
+    uint64_t frac_hi;
+    uint64_t frac_hm;  /* high-middle */
+    uint64_t frac_lm;  /* low-middle */
+    uint64_t frac_lo;
+} FloatParts256;
+
+/* These apply to the most significant word of each FloatPartsN. */
+#define DECOMPOSED_BINARY_POINT    63
 #define DECOMPOSED_IMPLICIT_BIT    (1ull << DECOMPOSED_BINARY_POINT)
-#define DECOMPOSED_OVERFLOW_BIT    (DECOMPOSED_IMPLICIT_BIT << 1)
 
 /* Structure holding all of the relevant parameters for a format.
  *   exp_size: the size of the exponent field
@@ -555,36 +514,33 @@ typedef struct {
  *   frac_size: the size of the fraction field
  *   frac_shift: shift to normalise the fraction with DECOMPOSED_BINARY_POINT
  * The following are computed based the size of fraction
- *   frac_lsb: least significant bit of fraction
- *   frac_lsbm1: the bit below the least significant bit (for rounding)
- *   round_mask/roundeven_mask: masks used for rounding
+ *   round_mask: bits below lsb which must be rounded
  * The following optional modifiers are available:
  *   arm_althp: handle ARM Alternative Half Precision
  */
 typedef struct {
     int exp_size;
     int exp_bias;
+    int exp_re_bias;
     int exp_max;
     int frac_size;
     int frac_shift;
-    uint64_t frac_lsb;
-    uint64_t frac_lsbm1;
-    uint64_t round_mask;
-    uint64_t roundeven_mask;
     bool arm_althp;
+    uint64_t round_mask;
 } FloatFmt;
 
 /* Expand fields based on the size of exponent and fraction */
-#define FLOAT_PARAMS(E, F)                                           \
-    .exp_size       = E,                                             \
-    .exp_bias       = ((1 << E) - 1) >> 1,                           \
-    .exp_max        = (1 << E) - 1,                                  \
-    .frac_size      = F,                                             \
-    .frac_shift     = DECOMPOSED_BINARY_POINT - F,                   \
-    .frac_lsb       = 1ull << (DECOMPOSED_BINARY_POINT - F),         \
-    .frac_lsbm1     = 1ull << ((DECOMPOSED_BINARY_POINT - F) - 1),   \
-    .round_mask     = (1ull << (DECOMPOSED_BINARY_POINT - F)) - 1,   \
-    .roundeven_mask = (2ull << (DECOMPOSED_BINARY_POINT - F)) - 1
+#define FLOAT_PARAMS_(E)                                \
+    .exp_size       = E,                                \
+    .exp_bias       = ((1 << E) - 1) >> 1,              \
+    .exp_re_bias    = (1 << (E - 1)) + (1 << (E - 2)),  \
+    .exp_max        = (1 << E) - 1
+
+#define FLOAT_PARAMS(E, F)                              \
+    FLOAT_PARAMS_(E),                                   \
+    .frac_size      = F,                                \
+    .frac_shift     = (-F - 1) & 63,                    \
+    .round_mask     = (1ull << ((-F - 1) & 63)) - 1
 
 static const FloatFmt float16_params = {
     FLOAT_PARAMS(5, 10)
@@ -595,6 +551,10 @@ static const FloatFmt float16_params_ahp = {
     .arm_althp = true
 };
 
+static const FloatFmt bfloat16_params = {
+    FLOAT_PARAMS(8, 7)
+};
+
 static const FloatFmt float32_params = {
     FLOAT_PARAMS(8, 23)
 };
@@ -603,55 +563,123 @@ static const FloatFmt float64_params = {
     FLOAT_PARAMS(11, 52)
 };
 
+static const FloatFmt float128_params = {
+    FLOAT_PARAMS(15, 112)
+};
+
+#define FLOATX80_PARAMS(R)              \
+    FLOAT_PARAMS_(15),                  \
+    .frac_size = R == 64 ? 63 : R,      \
+    .frac_shift = 0,                    \
+    .round_mask = R == 64 ? -1 : (1ull << ((-R - 1) & 63)) - 1
+
+static const FloatFmt floatx80_params[3] = {
+    [floatx80_precision_s] = { FLOATX80_PARAMS(23) },
+    [floatx80_precision_d] = { FLOATX80_PARAMS(52) },
+    [floatx80_precision_x] = { FLOATX80_PARAMS(64) },
+};
+
 /* Unpack a float to parts, but do not canonicalize.  */
-static inline FloatParts unpack_raw(FloatFmt fmt, uint64_t raw)
+static void unpack_raw64(FloatParts64 *r, const FloatFmt *fmt, uint64_t raw)
 {
-    const int sign_pos = fmt.frac_size + fmt.exp_size;
+    const int f_size = fmt->frac_size;
+    const int e_size = fmt->exp_size;
 
-    return (FloatParts) {
+    *r = (FloatParts64) {
         .cls = float_class_unclassified,
-        .sign = extract64(raw, sign_pos, 1),
-        .exp = extract64(raw, fmt.frac_size, fmt.exp_size),
-        .frac = extract64(raw, 0, fmt.frac_size),
+        .sign = extract64(raw, f_size + e_size, 1),
+        .exp = extract64(raw, f_size, e_size),
+        .frac = extract64(raw, 0, f_size)
     };
 }
 
-static inline FloatParts float16_unpack_raw(float16 f)
+static inline void float16_unpack_raw(FloatParts64 *p, float16 f)
+{
+    unpack_raw64(p, &float16_params, f);
+}
+
+static inline void bfloat16_unpack_raw(FloatParts64 *p, bfloat16 f)
+{
+    unpack_raw64(p, &bfloat16_params, f);
+}
+
+static inline void float32_unpack_raw(FloatParts64 *p, float32 f)
+{
+    unpack_raw64(p, &float32_params, f);
+}
+
+static inline void float64_unpack_raw(FloatParts64 *p, float64 f)
 {
-    return unpack_raw(float16_params, f);
+    unpack_raw64(p, &float64_params, f);
 }
 
-static inline FloatParts float32_unpack_raw(float32 f)
+static void floatx80_unpack_raw(FloatParts128 *p, floatx80 f)
 {
-    return unpack_raw(float32_params, f);
+    *p = (FloatParts128) {
+        .cls = float_class_unclassified,
+        .sign = extract32(f.high, 15, 1),
+        .exp = extract32(f.high, 0, 15),
+        .frac_hi = f.low
+    };
 }
 
-static inline FloatParts float64_unpack_raw(float64 f)
+static void float128_unpack_raw(FloatParts128 *p, float128 f)
 {
-    return unpack_raw(float64_params, f);
+    const int f_size = float128_params.frac_size - 64;
+    const int e_size = float128_params.exp_size;
+
+    *p = (FloatParts128) {
+        .cls = float_class_unclassified,
+        .sign = extract64(f.high, f_size + e_size, 1),
+        .exp = extract64(f.high, f_size, e_size),
+        .frac_hi = extract64(f.high, 0, f_size),
+        .frac_lo = f.low,
+    };
 }
 
 /* Pack a float from parts, but do not canonicalize.  */
-static inline uint64_t pack_raw(FloatFmt fmt, FloatParts p)
+static uint64_t pack_raw64(const FloatParts64 *p, const FloatFmt *fmt)
+{
+    const int f_size = fmt->frac_size;
+    const int e_size = fmt->exp_size;
+    uint64_t ret;
+
+    ret = (uint64_t)p->sign << (f_size + e_size);
+    ret = deposit64(ret, f_size, e_size, p->exp);
+    ret = deposit64(ret, 0, f_size, p->frac);
+    return ret;
+}
+
+static inline float16 float16_pack_raw(const FloatParts64 *p)
+{
+    return make_float16(pack_raw64(p, &float16_params));
+}
+
+static inline bfloat16 bfloat16_pack_raw(const FloatParts64 *p)
 {
-    const int sign_pos = fmt.frac_size + fmt.exp_size;
-    uint64_t ret = deposit64(p.frac, fmt.frac_size, fmt.exp_size, p.exp);
-    return deposit64(ret, sign_pos, 1, p.sign);
+    return pack_raw64(p, &bfloat16_params);
 }
 
-static inline float16 float16_pack_raw(FloatParts p)
+static inline float32 float32_pack_raw(const FloatParts64 *p)
 {
-    return make_float16(pack_raw(float16_params, p));
+    return make_float32(pack_raw64(p, &float32_params));
 }
 
-static inline float32 float32_pack_raw(FloatParts p)
+static inline float64 float64_pack_raw(const FloatParts64 *p)
 {
-    return make_float32(pack_raw(float32_params, p));
+    return make_float64(pack_raw64(p, &float64_params));
 }
 
-static inline float64 float64_pack_raw(FloatParts p)
+static float128 float128_pack_raw(const FloatParts128 *p)
 {
-    return make_float64(pack_raw(float64_params, p));
+    const int f_size = float128_params.frac_size - 64;
+    const int e_size = float128_params.exp_size;
+    uint64_t hi;
+
+    hi = (uint64_t)p->sign << (f_size + e_size);
+    hi = deposit64(hi, f_size, e_size, p->exp);
+    hi = deposit64(hi, 0, f_size, p->frac_hi);
+    return make_float128(hi, p->frac_lo);
 }
 
 /*----------------------------------------------------------------------------
@@ -662,886 +690,1517 @@ static inline float64 float64_pack_raw(FloatParts p)
 | are propagated from function inputs to output.  These details are target-
 | specific.
 *----------------------------------------------------------------------------*/
-#include "softfloat-specialize.inc.c"
+#include "softfloat-specialize.c.inc"
 
-/* Canonicalize EXP and FRAC, setting CLS.  */
-static FloatParts sf_canonicalize(FloatParts part, const FloatFmt *parm,
-                                  float_status *status)
-{
-    if (part.exp == parm->exp_max && !parm->arm_althp) {
-        if (part.frac == 0) {
-            part.cls = float_class_inf;
-        } else {
-            part.frac <<= parm->frac_shift;
-            part.cls = (parts_is_snan_frac(part.frac, status)
-                        ? float_class_snan : float_class_qnan);
-        }
-    } else if (part.exp == 0) {
-        if (likely(part.frac == 0)) {
-            part.cls = float_class_zero;
-        } else if (status->flush_inputs_to_zero) {
-            float_raise(float_flag_input_denormal, status);
-            part.cls = float_class_zero;
-            part.frac = 0;
-        } else {
-            int shift = clz64(part.frac) - 1;
-            part.cls = float_class_normal;
-            part.exp = parm->frac_shift - parm->exp_bias - shift + 1;
-            part.frac <<= shift;
-        }
-    } else {
-        part.cls = float_class_normal;
-        part.exp -= parm->exp_bias;
-        part.frac = DECOMPOSED_IMPLICIT_BIT + (part.frac << parm->frac_shift);
-    }
-    return part;
-}
+#define PARTS_GENERIC_64_128(NAME, P) \
+    _Generic((P), FloatParts64 *: parts64_##NAME, \
+                  FloatParts128 *: parts128_##NAME)
 
-/* Round and uncanonicalize a floating-point number by parts. There
- * are FRAC_SHIFT bits that may require rounding at the bottom of the
- * fraction; these bits will be removed. The exponent will be biased
- * by EXP_BIAS and must be bounded by [EXP_MAX-1, 0].
- */
+#define PARTS_GENERIC_64_128_256(NAME, P) \
+    _Generic((P), FloatParts64 *: parts64_##NAME, \
+                  FloatParts128 *: parts128_##NAME, \
+                  FloatParts256 *: parts256_##NAME)
 
-static FloatParts round_canonical(FloatParts p, float_status *s,
-                                  const FloatFmt *parm)
-{
-    const uint64_t frac_lsb = parm->frac_lsb;
-    const uint64_t frac_lsbm1 = parm->frac_lsbm1;
-    const uint64_t round_mask = parm->round_mask;
-    const uint64_t roundeven_mask = parm->roundeven_mask;
-    const int exp_max = parm->exp_max;
-    const int frac_shift = parm->frac_shift;
-    uint64_t frac, inc = 0;
-    int exp, flags = 0;
-    bool overflow_norm = false;
+#define parts_default_nan(P, S)    PARTS_GENERIC_64_128(default_nan, P)(P, S)
+#define parts_silence_nan(P, S)    PARTS_GENERIC_64_128(silence_nan, P)(P, S)
 
-    frac = p.frac;
-    exp = p.exp;
+static void parts64_return_nan(FloatParts64 *a, float_status *s);
+static void parts128_return_nan(FloatParts128 *a, float_status *s);
 
-    switch (p.cls) {
-    case float_class_normal:
-        switch (s->float_rounding_mode) {
-        case float_round_nearest_even:
-            overflow_norm = false;
-            inc = ((frac & roundeven_mask) != frac_lsbm1 ? frac_lsbm1 : 0);
-            break;
-        case float_round_ties_away:
-            overflow_norm = false;
-            inc = frac_lsbm1;
-            break;
-        case float_round_to_zero:
-            overflow_norm = true;
-            inc = 0;
-            break;
-        case float_round_up:
-            inc = p.sign ? 0 : round_mask;
-            overflow_norm = p.sign;
-            break;
-        case float_round_down:
-            inc = p.sign ? round_mask : 0;
-            overflow_norm = !p.sign;
-            break;
-        case float_round_to_odd:
-            overflow_norm = true;
-            inc = frac & frac_lsb ? 0 : round_mask;
-            break;
-        default:
-            g_assert_not_reached();
-            break;
-        }
+#define parts_return_nan(P, S)     PARTS_GENERIC_64_128(return_nan, P)(P, S)
 
-        exp += parm->exp_bias;
-        if (likely(exp > 0)) {
-            if (frac & round_mask) {
-                flags |= float_flag_inexact;
-                frac += inc;
-                if (frac & DECOMPOSED_OVERFLOW_BIT) {
-                    frac >>= 1;
-                    exp++;
-                }
-            }
-            frac >>= frac_shift;
-
-            if (parm->arm_althp) {
-                /* ARM Alt HP eschews Inf and NaN for a wider exponent.  */
-                if (unlikely(exp > exp_max)) {
-                    /* Overflow.  Return the maximum normal.  */
-                    flags = float_flag_invalid;
-                    exp = exp_max;
-                    frac = -1;
-                }
-            } else if (unlikely(exp >= exp_max)) {
-                flags |= float_flag_overflow | float_flag_inexact;
-                if (overflow_norm) {
-                    exp = exp_max - 1;
-                    frac = -1;
-                } else {
-                    p.cls = float_class_inf;
-                    goto do_inf;
-                }
-            }
-        } else if (s->flush_to_zero) {
-            flags |= float_flag_output_denormal;
-            p.cls = float_class_zero;
-            goto do_zero;
-        } else {
-            bool is_tiny = (s->float_detect_tininess
-                            == float_tininess_before_rounding)
-                        || (exp < 0)
-                        || !((frac + inc) & DECOMPOSED_OVERFLOW_BIT);
-
-            shift64RightJamming(frac, 1 - exp, &frac);
-            if (frac & round_mask) {
-                /* Need to recompute round-to-even.  */
-                switch (s->float_rounding_mode) {
-                case float_round_nearest_even:
-                    inc = ((frac & roundeven_mask) != frac_lsbm1
-                           ? frac_lsbm1 : 0);
-                    break;
-                case float_round_to_odd:
-                    inc = frac & frac_lsb ? 0 : round_mask;
-                    break;
-                }
-                flags |= float_flag_inexact;
-                frac += inc;
-            }
+static FloatParts64 *parts64_pick_nan(FloatParts64 *a, FloatParts64 *b,
+                                      float_status *s);
+static FloatParts128 *parts128_pick_nan(FloatParts128 *a, FloatParts128 *b,
+                                        float_status *s);
 
-            exp = (frac & DECOMPOSED_IMPLICIT_BIT ? 1 : 0);
-            frac >>= frac_shift;
+#define parts_pick_nan(A, B, S)    PARTS_GENERIC_64_128(pick_nan, A)(A, B, S)
 
-            if (is_tiny && (flags & float_flag_inexact)) {
-                flags |= float_flag_underflow;
-            }
-            if (exp == 0 && frac == 0) {
-                p.cls = float_class_zero;
-            }
-        }
-        break;
+static FloatParts64 *parts64_pick_nan_muladd(FloatParts64 *a, FloatParts64 *b,
+                                             FloatParts64 *c, float_status *s,
+                                             int ab_mask, int abc_mask);
+static FloatParts128 *parts128_pick_nan_muladd(FloatParts128 *a,
+                                               FloatParts128 *b,
+                                               FloatParts128 *c,
+                                               float_status *s,
+                                               int ab_mask, int abc_mask);
 
-    case float_class_zero:
-    do_zero:
-        exp = 0;
-        frac = 0;
-        break;
+#define parts_pick_nan_muladd(A, B, C, S, ABM, ABCM) \
+    PARTS_GENERIC_64_128(pick_nan_muladd, A)(A, B, C, S, ABM, ABCM)
 
-    case float_class_inf:
-    do_inf:
-        assert(!parm->arm_althp);
-        exp = exp_max;
-        frac = 0;
-        break;
+static void parts64_canonicalize(FloatParts64 *p, float_status *status,
+                                 const FloatFmt *fmt);
+static void parts128_canonicalize(FloatParts128 *p, float_status *status,
+                                  const FloatFmt *fmt);
 
-    case float_class_qnan:
-    case float_class_snan:
-        assert(!parm->arm_althp);
-        exp = exp_max;
-        frac >>= parm->frac_shift;
-        break;
+#define parts_canonicalize(A, S, F) \
+    PARTS_GENERIC_64_128(canonicalize, A)(A, S, F)
 
-    default:
-        g_assert_not_reached();
-        break;
-    }
+static void parts64_uncanon_normal(FloatParts64 *p, float_status *status,
+                                   const FloatFmt *fmt);
+static void parts128_uncanon_normal(FloatParts128 *p, float_status *status,
+                                    const FloatFmt *fmt);
 
-    float_raise(flags, s);
-    p.exp = exp;
-    p.frac = frac;
-    return p;
-}
+#define parts_uncanon_normal(A, S, F) \
+    PARTS_GENERIC_64_128(uncanon_normal, A)(A, S, F)
+
+static void parts64_uncanon(FloatParts64 *p, float_status *status,
+                            const FloatFmt *fmt);
+static void parts128_uncanon(FloatParts128 *p, float_status *status,
+                             const FloatFmt *fmt);
+
+#define parts_uncanon(A, S, F) \
+    PARTS_GENERIC_64_128(uncanon, A)(A, S, F)
+
+static void parts64_add_normal(FloatParts64 *a, FloatParts64 *b);
+static void parts128_add_normal(FloatParts128 *a, FloatParts128 *b);
+static void parts256_add_normal(FloatParts256 *a, FloatParts256 *b);
+
+#define parts_add_normal(A, B) \
+    PARTS_GENERIC_64_128_256(add_normal, A)(A, B)
+
+static bool parts64_sub_normal(FloatParts64 *a, FloatParts64 *b);
+static bool parts128_sub_normal(FloatParts128 *a, FloatParts128 *b);
+static bool parts256_sub_normal(FloatParts256 *a, FloatParts256 *b);
+
+#define parts_sub_normal(A, B) \
+    PARTS_GENERIC_64_128_256(sub_normal, A)(A, B)
+
+static FloatParts64 *parts64_addsub(FloatParts64 *a, FloatParts64 *b,
+                                    float_status *s, bool subtract);
+static FloatParts128 *parts128_addsub(FloatParts128 *a, FloatParts128 *b,
+                                      float_status *s, bool subtract);
+
+#define parts_addsub(A, B, S, Z) \
+    PARTS_GENERIC_64_128(addsub, A)(A, B, S, Z)
+
+static FloatParts64 *parts64_mul(FloatParts64 *a, FloatParts64 *b,
+                                 float_status *s);
+static FloatParts128 *parts128_mul(FloatParts128 *a, FloatParts128 *b,
+                                   float_status *s);
+
+#define parts_mul(A, B, S) \
+    PARTS_GENERIC_64_128(mul, A)(A, B, S)
+
+static FloatParts64 *parts64_muladd(FloatParts64 *a, FloatParts64 *b,
+                                    FloatParts64 *c, int flags,
+                                    float_status *s);
+static FloatParts128 *parts128_muladd(FloatParts128 *a, FloatParts128 *b,
+                                      FloatParts128 *c, int flags,
+                                      float_status *s);
+
+#define parts_muladd(A, B, C, Z, S) \
+    PARTS_GENERIC_64_128(muladd, A)(A, B, C, Z, S)
+
+static FloatParts64 *parts64_div(FloatParts64 *a, FloatParts64 *b,
+                                 float_status *s);
+static FloatParts128 *parts128_div(FloatParts128 *a, FloatParts128 *b,
+                                   float_status *s);
+
+#define parts_div(A, B, S) \
+    PARTS_GENERIC_64_128(div, A)(A, B, S)
+
+static FloatParts64 *parts64_modrem(FloatParts64 *a, FloatParts64 *b,
+                                    uint64_t *mod_quot, float_status *s);
+static FloatParts128 *parts128_modrem(FloatParts128 *a, FloatParts128 *b,
+                                      uint64_t *mod_quot, float_status *s);
+
+#define parts_modrem(A, B, Q, S) \
+    PARTS_GENERIC_64_128(modrem, A)(A, B, Q, S)
+
+static void parts64_sqrt(FloatParts64 *a, float_status *s, const FloatFmt *f);
+static void parts128_sqrt(FloatParts128 *a, float_status *s, const FloatFmt *f);
+
+#define parts_sqrt(A, S, F) \
+    PARTS_GENERIC_64_128(sqrt, A)(A, S, F)
+
+static bool parts64_round_to_int_normal(FloatParts64 *a, FloatRoundMode rm,
+                                        int scale, int frac_size);
+static bool parts128_round_to_int_normal(FloatParts128 *a, FloatRoundMode r,
+                                         int scale, int frac_size);
+
+#define parts_round_to_int_normal(A, R, C, F) \
+    PARTS_GENERIC_64_128(round_to_int_normal, A)(A, R, C, F)
+
+static void parts64_round_to_int(FloatParts64 *a, FloatRoundMode rm,
+                                 int scale, float_status *s,
+                                 const FloatFmt *fmt);
+static void parts128_round_to_int(FloatParts128 *a, FloatRoundMode r,
+                                  int scale, float_status *s,
+                                  const FloatFmt *fmt);
+
+#define parts_round_to_int(A, R, C, S, F) \
+    PARTS_GENERIC_64_128(round_to_int, A)(A, R, C, S, F)
+
+static int64_t parts64_float_to_sint(FloatParts64 *p, FloatRoundMode rmode,
+                                     int scale, int64_t min, int64_t max,
+                                     float_status *s);
+static int64_t parts128_float_to_sint(FloatParts128 *p, FloatRoundMode rmode,
+                                     int scale, int64_t min, int64_t max,
+                                     float_status *s);
+
+#define parts_float_to_sint(P, R, Z, MN, MX, S) \
+    PARTS_GENERIC_64_128(float_to_sint, P)(P, R, Z, MN, MX, S)
+
+static uint64_t parts64_float_to_uint(FloatParts64 *p, FloatRoundMode rmode,
+                                      int scale, uint64_t max,
+                                      float_status *s);
+static uint64_t parts128_float_to_uint(FloatParts128 *p, FloatRoundMode rmode,
+                                       int scale, uint64_t max,
+                                       float_status *s);
+
+#define parts_float_to_uint(P, R, Z, M, S) \
+    PARTS_GENERIC_64_128(float_to_uint, P)(P, R, Z, M, S)
+
+static void parts64_sint_to_float(FloatParts64 *p, int64_t a,
+                                  int scale, float_status *s);
+static void parts128_sint_to_float(FloatParts128 *p, int64_t a,
+                                   int scale, float_status *s);
+
+#define parts_sint_to_float(P, I, Z, S) \
+    PARTS_GENERIC_64_128(sint_to_float, P)(P, I, Z, S)
+
+static void parts64_uint_to_float(FloatParts64 *p, uint64_t a,
+                                  int scale, float_status *s);
+static void parts128_uint_to_float(FloatParts128 *p, uint64_t a,
+                                   int scale, float_status *s);
+
+#define parts_uint_to_float(P, I, Z, S) \
+    PARTS_GENERIC_64_128(uint_to_float, P)(P, I, Z, S)
+
+static FloatParts64 *parts64_minmax(FloatParts64 *a, FloatParts64 *b,
+                                    float_status *s, int flags);
+static FloatParts128 *parts128_minmax(FloatParts128 *a, FloatParts128 *b,
+                                      float_status *s, int flags);
+
+#define parts_minmax(A, B, S, F) \
+    PARTS_GENERIC_64_128(minmax, A)(A, B, S, F)
+
+static FloatRelation parts64_compare(FloatParts64 *a, FloatParts64 *b,
+                                     float_status *s, bool q);
+static FloatRelation parts128_compare(FloatParts128 *a, FloatParts128 *b,
+                                      float_status *s, bool q);
+
+#define parts_compare(A, B, S, Q) \
+    PARTS_GENERIC_64_128(compare, A)(A, B, S, Q)
+
+static void parts64_scalbn(FloatParts64 *a, int n, float_status *s);
+static void parts128_scalbn(FloatParts128 *a, int n, float_status *s);
+
+#define parts_scalbn(A, N, S) \
+    PARTS_GENERIC_64_128(scalbn, A)(A, N, S)
+
+static void parts64_log2(FloatParts64 *a, float_status *s, const FloatFmt *f);
+static void parts128_log2(FloatParts128 *a, float_status *s, const FloatFmt *f);
+
+#define parts_log2(A, S, F) \
+    PARTS_GENERIC_64_128(log2, A)(A, S, F)
+
+/*
+ * Helper functions for softfloat-parts.c.inc, per-size operations.
+ */
+
+#define FRAC_GENERIC_64_128(NAME, P) \
+    _Generic((P), FloatParts64 *: frac64_##NAME, \
+                  FloatParts128 *: frac128_##NAME)
 
-/* Explicit FloatFmt version */
-static FloatParts float16a_unpack_canonical(float16 f, float_status *s,
-                                            const FloatFmt *params)
+#define FRAC_GENERIC_64_128_256(NAME, P) \
+    _Generic((P), FloatParts64 *: frac64_##NAME, \
+                  FloatParts128 *: frac128_##NAME, \
+                  FloatParts256 *: frac256_##NAME)
+
+static bool frac64_add(FloatParts64 *r, FloatParts64 *a, FloatParts64 *b)
 {
-    return sf_canonicalize(float16_unpack_raw(f), params, s);
+    return uadd64_overflow(a->frac, b->frac, &r->frac);
 }
 
-static FloatParts float16_unpack_canonical(float16 f, float_status *s)
+static bool frac128_add(FloatParts128 *r, FloatParts128 *a, FloatParts128 *b)
 {
-    return float16a_unpack_canonical(f, s, &float16_params);
+    bool c = 0;
+    r->frac_lo = uadd64_carry(a->frac_lo, b->frac_lo, &c);
+    r->frac_hi = uadd64_carry(a->frac_hi, b->frac_hi, &c);
+    return c;
 }
 
-static float16 float16a_round_pack_canonical(FloatParts p, float_status *s,
-                                             const FloatFmt *params)
+static bool frac256_add(FloatParts256 *r, FloatParts256 *a, FloatParts256 *b)
 {
-    return float16_pack_raw(round_canonical(p, s, params));
+    bool c = 0;
+    r->frac_lo = uadd64_carry(a->frac_lo, b->frac_lo, &c);
+    r->frac_lm = uadd64_carry(a->frac_lm, b->frac_lm, &c);
+    r->frac_hm = uadd64_carry(a->frac_hm, b->frac_hm, &c);
+    r->frac_hi = uadd64_carry(a->frac_hi, b->frac_hi, &c);
+    return c;
 }
 
-static float16 float16_round_pack_canonical(FloatParts p, float_status *s)
+#define frac_add(R, A, B)  FRAC_GENERIC_64_128_256(add, R)(R, A, B)
+
+static bool frac64_addi(FloatParts64 *r, FloatParts64 *a, uint64_t c)
 {
-    return float16a_round_pack_canonical(p, s, &float16_params);
+    return uadd64_overflow(a->frac, c, &r->frac);
 }
 
-static FloatParts float32_unpack_canonical(float32 f, float_status *s)
+static bool frac128_addi(FloatParts128 *r, FloatParts128 *a, uint64_t c)
 {
-    return sf_canonicalize(float32_unpack_raw(f), &float32_params, s);
+    c = uadd64_overflow(a->frac_lo, c, &r->frac_lo);
+    return uadd64_overflow(a->frac_hi, c, &r->frac_hi);
 }
 
-static float32 float32_round_pack_canonical(FloatParts p, float_status *s)
+#define frac_addi(R, A, C)  FRAC_GENERIC_64_128(addi, R)(R, A, C)
+
+static void frac64_allones(FloatParts64 *a)
 {
-    return float32_pack_raw(round_canonical(p, s, &float32_params));
+    a->frac = -1;
 }
 
-static FloatParts float64_unpack_canonical(float64 f, float_status *s)
+static void frac128_allones(FloatParts128 *a)
 {
-    return sf_canonicalize(float64_unpack_raw(f), &float64_params, s);
+    a->frac_hi = a->frac_lo = -1;
 }
 
-static float64 float64_round_pack_canonical(FloatParts p, float_status *s)
+#define frac_allones(A)  FRAC_GENERIC_64_128(allones, A)(A)
+
+static FloatRelation frac64_cmp(FloatParts64 *a, FloatParts64 *b)
 {
-    return float64_pack_raw(round_canonical(p, s, &float64_params));
+    return (a->frac == b->frac ? float_relation_equal
+            : a->frac < b->frac ? float_relation_less
+            : float_relation_greater);
 }
 
-static FloatParts return_nan(FloatParts a, float_status *s)
+static FloatRelation frac128_cmp(FloatParts128 *a, FloatParts128 *b)
 {
-    switch (a.cls) {
-    case float_class_snan:
-        s->float_exception_flags |= float_flag_invalid;
-        a = parts_silence_nan(a, s);
-        /* fall through */
-    case float_class_qnan:
-        if (s->default_nan_mode) {
-            return parts_default_nan(s);
+    uint64_t ta = a->frac_hi, tb = b->frac_hi;
+    if (ta == tb) {
+        ta = a->frac_lo, tb = b->frac_lo;
+        if (ta == tb) {
+            return float_relation_equal;
         }
-        break;
-
-    default:
-        g_assert_not_reached();
-        break;
     }
-    return a;
+    return ta < tb ? float_relation_less : float_relation_greater;
 }
 
-static FloatParts pick_nan(FloatParts a, FloatParts b, float_status *s)
+#define frac_cmp(A, B)  FRAC_GENERIC_64_128(cmp, A)(A, B)
+
+static void frac64_clear(FloatParts64 *a)
 {
-    if (is_snan(a.cls) || is_snan(b.cls)) {
-        s->float_exception_flags |= float_flag_invalid;
-    }
+    a->frac = 0;
+}
+
+static void frac128_clear(FloatParts128 *a)
+{
+    a->frac_hi = a->frac_lo = 0;
+}
 
-    if (s->default_nan_mode) {
-        return parts_default_nan(s);
+#define frac_clear(A)  FRAC_GENERIC_64_128(clear, A)(A)
+
+static bool frac64_div(FloatParts64 *a, FloatParts64 *b)
+{
+    uint64_t n1, n0, r, q;
+    bool ret;
+
+    /*
+     * We want a 2*N / N-bit division to produce exactly an N-bit
+     * result, so that we do not lose any precision and so that we
+     * do not have to renormalize afterward.  If A.frac < B.frac,
+     * then division would produce an (N-1)-bit result; shift A left
+     * by one to produce the an N-bit result, and return true to
+     * decrement the exponent to match.
+     *
+     * The udiv_qrnnd algorithm that we're using requires normalization,
+     * i.e. the msb of the denominator must be set, which is already true.
+     */
+    ret = a->frac < b->frac;
+    if (ret) {
+        n0 = a->frac;
+        n1 = 0;
     } else {
-        if (pickNaN(a.cls, b.cls,
-                    a.frac > b.frac ||
-                    (a.frac == b.frac && a.sign < b.sign))) {
-            a = b;
-        }
-        if (is_snan(a.cls)) {
-            return parts_silence_nan(a, s);
-        }
+        n0 = a->frac >> 1;
+        n1 = a->frac << 63;
     }
-    return a;
+    q = udiv_qrnnd(&r, n0, n1, b->frac);
+
+    /* Set lsb if there is a remainder, to set inexact. */
+    a->frac = q | (r != 0);
+
+    return ret;
 }
 
-static FloatParts pick_nan_muladd(FloatParts a, FloatParts b, FloatParts c,
-                                  bool inf_zero, float_status *s)
+static bool frac128_div(FloatParts128 *a, FloatParts128 *b)
 {
-    int which;
+    uint64_t q0, q1, a0, a1, b0, b1;
+    uint64_t r0, r1, r2, r3, t0, t1, t2, t3;
+    bool ret = false;
 
-    if (is_snan(a.cls) || is_snan(b.cls) || is_snan(c.cls)) {
-        s->float_exception_flags |= float_flag_invalid;
+    a0 = a->frac_hi, a1 = a->frac_lo;
+    b0 = b->frac_hi, b1 = b->frac_lo;
+
+    ret = lt128(a0, a1, b0, b1);
+    if (!ret) {
+        a1 = shr_double(a0, a1, 1);
+        a0 = a0 >> 1;
     }
 
-    which = pickNaNMulAdd(a.cls, b.cls, c.cls, inf_zero, s);
+    /* Use 128/64 -> 64 division as estimate for 192/128 -> 128 division. */
+    q0 = estimateDiv128To64(a0, a1, b0);
 
-    if (s->default_nan_mode) {
-        /* Note that this check is after pickNaNMulAdd so that function
-         * has an opportunity to set the Invalid flag.
-         */
-        which = 3;
+    /*
+     * Estimate is high because B1 was not included (unless B1 == 0).
+     * Reduce quotient and increase remainder until remainder is non-negative.
+     * This loop will execute 0 to 2 times.
+     */
+    mul128By64To192(b0, b1, q0, &t0, &t1, &t2);
+    sub192(a0, a1, 0, t0, t1, t2, &r0, &r1, &r2);
+    while (r0 != 0) {
+        q0--;
+        add192(r0, r1, r2, 0, b0, b1, &r0, &r1, &r2);
     }
 
-    switch (which) {
-    case 0:
-        break;
-    case 1:
-        a = b;
-        break;
-    case 2:
-        a = c;
-        break;
-    case 3:
-        return parts_default_nan(s);
-    default:
-        g_assert_not_reached();
-        break;
+    /* Repeat using the remainder, producing a second word of quotient. */
+    q1 = estimateDiv128To64(r1, r2, b0);
+    mul128By64To192(b0, b1, q1, &t1, &t2, &t3);
+    sub192(r1, r2, 0, t1, t2, t3, &r1, &r2, &r3);
+    while (r1 != 0) {
+        q1--;
+        add192(r1, r2, r3, 0, b0, b1, &r1, &r2, &r3);
     }
 
-    if (is_snan(a.cls)) {
-        return parts_silence_nan(a, s);
-    }
-    return a;
+    /* Any remainder indicates inexact; set sticky bit. */
+    q1 |= (r2 | r3) != 0;
+
+    a->frac_hi = q0;
+    a->frac_lo = q1;
+    return ret;
 }
 
-/*
- * Returns the result of adding or subtracting the values of the
- * floating-point values `a' and `b'. The operation is performed
- * according to the IEC/IEEE Standard for Binary Floating-Point
- * Arithmetic.
- */
+#define frac_div(A, B)  FRAC_GENERIC_64_128(div, A)(A, B)
 
-static FloatParts addsub_floats(FloatParts a, FloatParts b, bool subtract,
-                                float_status *s)
+static bool frac64_eqz(FloatParts64 *a)
 {
-    bool a_sign = a.sign;
-    bool b_sign = b.sign ^ subtract;
-
-    if (a_sign != b_sign) {
-        /* Subtraction */
-
-        if (a.cls == float_class_normal && b.cls == float_class_normal) {
-            if (a.exp > b.exp || (a.exp == b.exp && a.frac >= b.frac)) {
-                shift64RightJamming(b.frac, a.exp - b.exp, &b.frac);
-                a.frac = a.frac - b.frac;
-            } else {
-                shift64RightJamming(a.frac, b.exp - a.exp, &a.frac);
-                a.frac = b.frac - a.frac;
-                a.exp = b.exp;
-                a_sign ^= 1;
-            }
+    return a->frac == 0;
+}
 
-            if (a.frac == 0) {
-                a.cls = float_class_zero;
-                a.sign = s->float_rounding_mode == float_round_down;
-            } else {
-                int shift = clz64(a.frac) - 1;
-                a.frac = a.frac << shift;
-                a.exp = a.exp - shift;
-                a.sign = a_sign;
-            }
-            return a;
-        }
-        if (is_nan(a.cls) || is_nan(b.cls)) {
-            return pick_nan(a, b, s);
-        }
-        if (a.cls == float_class_inf) {
-            if (b.cls == float_class_inf) {
-                float_raise(float_flag_invalid, s);
-                return parts_default_nan(s);
-            }
-            return a;
-        }
-        if (a.cls == float_class_zero && b.cls == float_class_zero) {
-            a.sign = s->float_rounding_mode == float_round_down;
-            return a;
-        }
-        if (a.cls == float_class_zero || b.cls == float_class_inf) {
-            b.sign = a_sign ^ 1;
-            return b;
-        }
-        if (b.cls == float_class_zero) {
-            return a;
-        }
-    } else {
-        /* Addition */
-        if (a.cls == float_class_normal && b.cls == float_class_normal) {
-            if (a.exp > b.exp) {
-                shift64RightJamming(b.frac, a.exp - b.exp, &b.frac);
-            } else if (a.exp < b.exp) {
-                shift64RightJamming(a.frac, b.exp - a.exp, &a.frac);
-                a.exp = b.exp;
-            }
-            a.frac += b.frac;
-            if (a.frac & DECOMPOSED_OVERFLOW_BIT) {
-                shift64RightJamming(a.frac, 1, &a.frac);
-                a.exp += 1;
-            }
-            return a;
-        }
-        if (is_nan(a.cls) || is_nan(b.cls)) {
-            return pick_nan(a, b, s);
-        }
-        if (a.cls == float_class_inf || b.cls == float_class_zero) {
-            return a;
-        }
-        if (b.cls == float_class_inf || a.cls == float_class_zero) {
-            b.sign = b_sign;
-            return b;
-        }
-    }
-
-    g_assert_not_reached();
-    return a;
+static bool frac128_eqz(FloatParts128 *a)
+{
+    return (a->frac_hi | a->frac_lo) == 0;
 }
 
-/*
- * Returns the result of adding or subtracting the floating-point
- * values `a' and `b'. The operation is performed according to the
- * IEC/IEEE Standard for Binary Floating-Point Arithmetic.
- */
+#define frac_eqz(A)  FRAC_GENERIC_64_128(eqz, A)(A)
 
-float16 QEMU_FLATTEN float16_add(float16 a, float16 b, float_status *status)
+static void frac64_mulw(FloatParts128 *r, FloatParts64 *a, FloatParts64 *b)
 {
-    FloatParts pa = float16_unpack_canonical(a, status);
-    FloatParts pb = float16_unpack_canonical(b, status);
-    FloatParts pr = addsub_floats(pa, pb, false, status);
+    mulu64(&r->frac_lo, &r->frac_hi, a->frac, b->frac);
+}
 
-    return float16_round_pack_canonical(pr, status);
+static void frac128_mulw(FloatParts256 *r, FloatParts128 *a, FloatParts128 *b)
+{
+    mul128To256(a->frac_hi, a->frac_lo, b->frac_hi, b->frac_lo,
+                &r->frac_hi, &r->frac_hm, &r->frac_lm, &r->frac_lo);
 }
 
-float16 QEMU_FLATTEN float16_sub(float16 a, float16 b, float_status *status)
+#define frac_mulw(R, A, B)  FRAC_GENERIC_64_128(mulw, A)(R, A, B)
+
+static void frac64_neg(FloatParts64 *a)
 {
-    FloatParts pa = float16_unpack_canonical(a, status);
-    FloatParts pb = float16_unpack_canonical(b, status);
-    FloatParts pr = addsub_floats(pa, pb, true, status);
+    a->frac = -a->frac;
+}
 
-    return float16_round_pack_canonical(pr, status);
+static void frac128_neg(FloatParts128 *a)
+{
+    bool c = 0;
+    a->frac_lo = usub64_borrow(0, a->frac_lo, &c);
+    a->frac_hi = usub64_borrow(0, a->frac_hi, &c);
 }
 
-static float32 QEMU_SOFTFLOAT_ATTR
-soft_f32_addsub(float32 a, float32 b, bool subtract, float_status *status)
+static void frac256_neg(FloatParts256 *a)
 {
-    FloatParts pa = float32_unpack_canonical(a, status);
-    FloatParts pb = float32_unpack_canonical(b, status);
-    FloatParts pr = addsub_floats(pa, pb, subtract, status);
+    bool c = 0;
+    a->frac_lo = usub64_borrow(0, a->frac_lo, &c);
+    a->frac_lm = usub64_borrow(0, a->frac_lm, &c);
+    a->frac_hm = usub64_borrow(0, a->frac_hm, &c);
+    a->frac_hi = usub64_borrow(0, a->frac_hi, &c);
+}
 
-    return float32_round_pack_canonical(pr, status);
+#define frac_neg(A)  FRAC_GENERIC_64_128_256(neg, A)(A)
+
+static int frac64_normalize(FloatParts64 *a)
+{
+    if (a->frac) {
+        int shift = clz64(a->frac);
+        a->frac <<= shift;
+        return shift;
+    }
+    return 64;
 }
 
-static inline float32 soft_f32_add(float32 a, float32 b, float_status *status)
+static int frac128_normalize(FloatParts128 *a)
 {
-    return soft_f32_addsub(a, b, false, status);
+    if (a->frac_hi) {
+        int shl = clz64(a->frac_hi);
+        a->frac_hi = shl_double(a->frac_hi, a->frac_lo, shl);
+        a->frac_lo <<= shl;
+        return shl;
+    } else if (a->frac_lo) {
+        int shl = clz64(a->frac_lo);
+        a->frac_hi = a->frac_lo << shl;
+        a->frac_lo = 0;
+        return shl + 64;
+    }
+    return 128;
 }
 
-static inline float32 soft_f32_sub(float32 a, float32 b, float_status *status)
+static int frac256_normalize(FloatParts256 *a)
 {
-    return soft_f32_addsub(a, b, true, status);
+    uint64_t a0 = a->frac_hi, a1 = a->frac_hm;
+    uint64_t a2 = a->frac_lm, a3 = a->frac_lo;
+    int ret, shl;
+
+    if (likely(a0)) {
+        shl = clz64(a0);
+        if (shl == 0) {
+            return 0;
+        }
+        ret = shl;
+    } else {
+        if (a1) {
+            ret = 64;
+            a0 = a1, a1 = a2, a2 = a3, a3 = 0;
+        } else if (a2) {
+            ret = 128;
+            a0 = a2, a1 = a3, a2 = 0, a3 = 0;
+        } else if (a3) {
+            ret = 192;
+            a0 = a3, a1 = 0, a2 = 0, a3 = 0;
+        } else {
+            ret = 256;
+            a0 = 0, a1 = 0, a2 = 0, a3 = 0;
+            goto done;
+        }
+        shl = clz64(a0);
+        if (shl == 0) {
+            goto done;
+        }
+        ret += shl;
+    }
+
+    a0 = shl_double(a0, a1, shl);
+    a1 = shl_double(a1, a2, shl);
+    a2 = shl_double(a2, a3, shl);
+    a3 <<= shl;
+
+ done:
+    a->frac_hi = a0;
+    a->frac_hm = a1;
+    a->frac_lm = a2;
+    a->frac_lo = a3;
+    return ret;
 }
 
-static float64 QEMU_SOFTFLOAT_ATTR
-soft_f64_addsub(float64 a, float64 b, bool subtract, float_status *status)
+#define frac_normalize(A)  FRAC_GENERIC_64_128_256(normalize, A)(A)
+
+static void frac64_modrem(FloatParts64 *a, FloatParts64 *b, uint64_t *mod_quot)
 {
-    FloatParts pa = float64_unpack_canonical(a, status);
-    FloatParts pb = float64_unpack_canonical(b, status);
-    FloatParts pr = addsub_floats(pa, pb, subtract, status);
+    uint64_t a0, a1, b0, t0, t1, q, quot;
+    int exp_diff = a->exp - b->exp;
+    int shift;
 
-    return float64_round_pack_canonical(pr, status);
+    a0 = a->frac;
+    a1 = 0;
+
+    if (exp_diff < -1) {
+        if (mod_quot) {
+            *mod_quot = 0;
+        }
+        return;
+    }
+    if (exp_diff == -1) {
+        a0 >>= 1;
+        exp_diff = 0;
+    }
+
+    b0 = b->frac;
+    quot = q = b0 <= a0;
+    if (q) {
+        a0 -= b0;
+    }
+
+    exp_diff -= 64;
+    while (exp_diff > 0) {
+        q = estimateDiv128To64(a0, a1, b0);
+        q = q > 2 ? q - 2 : 0;
+        mul64To128(b0, q, &t0, &t1);
+        sub128(a0, a1, t0, t1, &a0, &a1);
+        shortShift128Left(a0, a1, 62, &a0, &a1);
+        exp_diff -= 62;
+        quot = (quot << 62) + q;
+    }
+
+    exp_diff += 64;
+    if (exp_diff > 0) {
+        q = estimateDiv128To64(a0, a1, b0);
+        q = q > 2 ? (q - 2) >> (64 - exp_diff) : 0;
+        mul64To128(b0, q << (64 - exp_diff), &t0, &t1);
+        sub128(a0, a1, t0, t1, &a0, &a1);
+        shortShift128Left(0, b0, 64 - exp_diff, &t0, &t1);
+        while (le128(t0, t1, a0, a1)) {
+            ++q;
+            sub128(a0, a1, t0, t1, &a0, &a1);
+        }
+        quot = (exp_diff < 64 ? quot << exp_diff : 0) + q;
+    } else {
+        t0 = b0;
+        t1 = 0;
+    }
+
+    if (mod_quot) {
+        *mod_quot = quot;
+    } else {
+        sub128(t0, t1, a0, a1, &t0, &t1);
+        if (lt128(t0, t1, a0, a1) ||
+            (eq128(t0, t1, a0, a1) && (q & 1))) {
+            a0 = t0;
+            a1 = t1;
+            a->sign = !a->sign;
+        }
+    }
+
+    if (likely(a0)) {
+        shift = clz64(a0);
+        shortShift128Left(a0, a1, shift, &a0, &a1);
+    } else if (likely(a1)) {
+        shift = clz64(a1);
+        a0 = a1 << shift;
+        a1 = 0;
+        shift += 64;
+    } else {
+        a->cls = float_class_zero;
+        return;
+    }
+
+    a->exp = b->exp + exp_diff - shift;
+    a->frac = a0 | (a1 != 0);
 }
 
-static inline float64 soft_f64_add(float64 a, float64 b, float_status *status)
+static void frac128_modrem(FloatParts128 *a, FloatParts128 *b,
+                           uint64_t *mod_quot)
 {
-    return soft_f64_addsub(a, b, false, status);
+    uint64_t a0, a1, a2, b0, b1, t0, t1, t2, q, quot;
+    int exp_diff = a->exp - b->exp;
+    int shift;
+
+    a0 = a->frac_hi;
+    a1 = a->frac_lo;
+    a2 = 0;
+
+    if (exp_diff < -1) {
+        if (mod_quot) {
+            *mod_quot = 0;
+        }
+        return;
+    }
+    if (exp_diff == -1) {
+        shift128Right(a0, a1, 1, &a0, &a1);
+        exp_diff = 0;
+    }
+
+    b0 = b->frac_hi;
+    b1 = b->frac_lo;
+
+    quot = q = le128(b0, b1, a0, a1);
+    if (q) {
+        sub128(a0, a1, b0, b1, &a0, &a1);
+    }
+
+    exp_diff -= 64;
+    while (exp_diff > 0) {
+        q = estimateDiv128To64(a0, a1, b0);
+        q = q > 4 ? q - 4 : 0;
+        mul128By64To192(b0, b1, q, &t0, &t1, &t2);
+        sub192(a0, a1, a2, t0, t1, t2, &a0, &a1, &a2);
+        shortShift192Left(a0, a1, a2, 61, &a0, &a1, &a2);
+        exp_diff -= 61;
+        quot = (quot << 61) + q;
+    }
+
+    exp_diff += 64;
+    if (exp_diff > 0) {
+        q = estimateDiv128To64(a0, a1, b0);
+        q = q > 4 ? (q - 4) >> (64 - exp_diff) : 0;
+        mul128By64To192(b0, b1, q << (64 - exp_diff), &t0, &t1, &t2);
+        sub192(a0, a1, a2, t0, t1, t2, &a0, &a1, &a2);
+        shortShift192Left(0, b0, b1, 64 - exp_diff, &t0, &t1, &t2);
+        while (le192(t0, t1, t2, a0, a1, a2)) {
+            ++q;
+            sub192(a0, a1, a2, t0, t1, t2, &a0, &a1, &a2);
+        }
+        quot = (exp_diff < 64 ? quot << exp_diff : 0) + q;
+    } else {
+        t0 = b0;
+        t1 = b1;
+        t2 = 0;
+    }
+
+    if (mod_quot) {
+        *mod_quot = quot;
+    } else {
+        sub192(t0, t1, t2, a0, a1, a2, &t0, &t1, &t2);
+        if (lt192(t0, t1, t2, a0, a1, a2) ||
+            (eq192(t0, t1, t2, a0, a1, a2) && (q & 1))) {
+            a0 = t0;
+            a1 = t1;
+            a2 = t2;
+            a->sign = !a->sign;
+        }
+    }
+
+    if (likely(a0)) {
+        shift = clz64(a0);
+        shortShift192Left(a0, a1, a2, shift, &a0, &a1, &a2);
+    } else if (likely(a1)) {
+        shift = clz64(a1);
+        shortShift128Left(a1, a2, shift, &a0, &a1);
+        a2 = 0;
+        shift += 64;
+    } else if (likely(a2)) {
+        shift = clz64(a2);
+        a0 = a2 << shift;
+        a1 = a2 = 0;
+        shift += 128;
+    } else {
+        a->cls = float_class_zero;
+        return;
+    }
+
+    a->exp = b->exp + exp_diff - shift;
+    a->frac_hi = a0;
+    a->frac_lo = a1 | (a2 != 0);
 }
 
-static inline float64 soft_f64_sub(float64 a, float64 b, float_status *status)
+#define frac_modrem(A, B, Q)  FRAC_GENERIC_64_128(modrem, A)(A, B, Q)
+
+static void frac64_shl(FloatParts64 *a, int c)
 {
-    return soft_f64_addsub(a, b, true, status);
+    a->frac <<= c;
 }
 
-static float hard_f32_add(float a, float b)
+static void frac128_shl(FloatParts128 *a, int c)
 {
-    return a + b;
+    uint64_t a0 = a->frac_hi, a1 = a->frac_lo;
+
+    if (c & 64) {
+        a0 = a1, a1 = 0;
+    }
+
+    c &= 63;
+    if (c) {
+        a0 = shl_double(a0, a1, c);
+        a1 = a1 << c;
+    }
+
+    a->frac_hi = a0;
+    a->frac_lo = a1;
 }
 
-static float hard_f32_sub(float a, float b)
+#define frac_shl(A, C)  FRAC_GENERIC_64_128(shl, A)(A, C)
+
+static void frac64_shr(FloatParts64 *a, int c)
 {
-    return a - b;
+    a->frac >>= c;
 }
 
-static double hard_f64_add(double a, double b)
+static void frac128_shr(FloatParts128 *a, int c)
 {
-    return a + b;
+    uint64_t a0 = a->frac_hi, a1 = a->frac_lo;
+
+    if (c & 64) {
+        a1 = a0, a0 = 0;
+    }
+
+    c &= 63;
+    if (c) {
+        a1 = shr_double(a0, a1, c);
+        a0 = a0 >> c;
+    }
+
+    a->frac_hi = a0;
+    a->frac_lo = a1;
 }
 
-static double hard_f64_sub(double a, double b)
+#define frac_shr(A, C)  FRAC_GENERIC_64_128(shr, A)(A, C)
+
+static void frac64_shrjam(FloatParts64 *a, int c)
 {
-    return a - b;
+    uint64_t a0 = a->frac;
+
+    if (likely(c != 0)) {
+        if (likely(c < 64)) {
+            a0 = (a0 >> c) | (shr_double(a0, 0, c) != 0);
+        } else {
+            a0 = a0 != 0;
+        }
+        a->frac = a0;
+    }
 }
 
-static bool f32_addsub_post(union_float32 a, union_float32 b)
+static void frac128_shrjam(FloatParts128 *a, int c)
 {
-    if (QEMU_HARDFLOAT_2F32_USE_FP) {
-        return !(fpclassify(a.h) == FP_ZERO && fpclassify(b.h) == FP_ZERO);
+    uint64_t a0 = a->frac_hi, a1 = a->frac_lo;
+    uint64_t sticky = 0;
+
+    if (unlikely(c == 0)) {
+        return;
+    } else if (likely(c < 64)) {
+        /* nothing */
+    } else if (likely(c < 128)) {
+        sticky = a1;
+        a1 = a0;
+        a0 = 0;
+        c &= 63;
+        if (c == 0) {
+            goto done;
+        }
+    } else {
+        sticky = a0 | a1;
+        a0 = a1 = 0;
+        goto done;
     }
-    return !(float32_is_zero(a.s) && float32_is_zero(b.s));
+
+    sticky |= shr_double(a1, 0, c);
+    a1 = shr_double(a0, a1, c);
+    a0 = a0 >> c;
+
+ done:
+    a->frac_lo = a1 | (sticky != 0);
+    a->frac_hi = a0;
 }
 
-static bool f64_addsub_post(union_float64 a, union_float64 b)
+static void frac256_shrjam(FloatParts256 *a, int c)
 {
-    if (QEMU_HARDFLOAT_2F64_USE_FP) {
-        return !(fpclassify(a.h) == FP_ZERO && fpclassify(b.h) == FP_ZERO);
+    uint64_t a0 = a->frac_hi, a1 = a->frac_hm;
+    uint64_t a2 = a->frac_lm, a3 = a->frac_lo;
+    uint64_t sticky = 0;
+
+    if (unlikely(c == 0)) {
+        return;
+    } else if (likely(c < 64)) {
+        /* nothing */
+    } else if (likely(c < 256)) {
+        if (unlikely(c & 128)) {
+            sticky |= a2 | a3;
+            a3 = a1, a2 = a0, a1 = 0, a0 = 0;
+        }
+        if (unlikely(c & 64)) {
+            sticky |= a3;
+            a3 = a2, a2 = a1, a1 = a0, a0 = 0;
+        }
+        c &= 63;
+        if (c == 0) {
+            goto done;
+        }
     } else {
-        return !(float64_is_zero(a.s) && float64_is_zero(b.s));
+        sticky = a0 | a1 | a2 | a3;
+        a0 = a1 = a2 = a3 = 0;
+        goto done;
     }
+
+    sticky |= shr_double(a3, 0, c);
+    a3 = shr_double(a2, a3, c);
+    a2 = shr_double(a1, a2, c);
+    a1 = shr_double(a0, a1, c);
+    a0 = a0 >> c;
+
+ done:
+    a->frac_lo = a3 | (sticky != 0);
+    a->frac_lm = a2;
+    a->frac_hm = a1;
+    a->frac_hi = a0;
 }
 
-static float32 float32_addsub(float32 a, float32 b, float_status *s,
-                              hard_f32_op2_fn hard, soft_f32_op2_fn soft)
+#define frac_shrjam(A, C)  FRAC_GENERIC_64_128_256(shrjam, A)(A, C)
+
+static bool frac64_sub(FloatParts64 *r, FloatParts64 *a, FloatParts64 *b)
 {
-    return float32_gen2(a, b, s, hard, soft,
-                        f32_is_zon2, f32_addsub_post, NULL, NULL);
+    return usub64_overflow(a->frac, b->frac, &r->frac);
 }
 
-static float64 float64_addsub(float64 a, float64 b, float_status *s,
-                              hard_f64_op2_fn hard, soft_f64_op2_fn soft)
+static bool frac128_sub(FloatParts128 *r, FloatParts128 *a, FloatParts128 *b)
 {
-    return float64_gen2(a, b, s, hard, soft,
-                        f64_is_zon2, f64_addsub_post, NULL, NULL);
+    bool c = 0;
+    r->frac_lo = usub64_borrow(a->frac_lo, b->frac_lo, &c);
+    r->frac_hi = usub64_borrow(a->frac_hi, b->frac_hi, &c);
+    return c;
 }
 
-float32 QEMU_FLATTEN
-float32_add(float32 a, float32 b, float_status *s)
+static bool frac256_sub(FloatParts256 *r, FloatParts256 *a, FloatParts256 *b)
 {
-    return float32_addsub(a, b, s, hard_f32_add, soft_f32_add);
+    bool c = 0;
+    r->frac_lo = usub64_borrow(a->frac_lo, b->frac_lo, &c);
+    r->frac_lm = usub64_borrow(a->frac_lm, b->frac_lm, &c);
+    r->frac_hm = usub64_borrow(a->frac_hm, b->frac_hm, &c);
+    r->frac_hi = usub64_borrow(a->frac_hi, b->frac_hi, &c);
+    return c;
 }
 
-float32 QEMU_FLATTEN
-float32_sub(float32 a, float32 b, float_status *s)
+#define frac_sub(R, A, B)  FRAC_GENERIC_64_128_256(sub, R)(R, A, B)
+
+static void frac64_truncjam(FloatParts64 *r, FloatParts128 *a)
 {
-    return float32_addsub(a, b, s, hard_f32_sub, soft_f32_sub);
+    r->frac = a->frac_hi | (a->frac_lo != 0);
 }
 
-float64 QEMU_FLATTEN
-float64_add(float64 a, float64 b, float_status *s)
+static void frac128_truncjam(FloatParts128 *r, FloatParts256 *a)
 {
-    return float64_addsub(a, b, s, hard_f64_add, soft_f64_add);
+    r->frac_hi = a->frac_hi;
+    r->frac_lo = a->frac_hm | ((a->frac_lm | a->frac_lo) != 0);
 }
 
-float64 QEMU_FLATTEN
-float64_sub(float64 a, float64 b, float_status *s)
+#define frac_truncjam(R, A)  FRAC_GENERIC_64_128(truncjam, R)(R, A)
+
+static void frac64_widen(FloatParts128 *r, FloatParts64 *a)
 {
-    return float64_addsub(a, b, s, hard_f64_sub, soft_f64_sub);
+    r->frac_hi = a->frac;
+    r->frac_lo = 0;
+}
+
+static void frac128_widen(FloatParts256 *r, FloatParts128 *a)
+{
+    r->frac_hi = a->frac_hi;
+    r->frac_hm = a->frac_lo;
+    r->frac_lm = 0;
+    r->frac_lo = 0;
 }
 
+#define frac_widen(A, B)  FRAC_GENERIC_64_128(widen, B)(A, B)
+
 /*
- * Returns the result of multiplying the floating-point values `a' and
- * `b'. The operation is performed according to the IEC/IEEE Standard
- * for Binary Floating-Point Arithmetic.
+ * Reciprocal sqrt table.  1 bit of exponent, 6-bits of mantessa.
+ * From https://git.musl-libc.org/cgit/musl/tree/src/math/sqrt_data.c
+ * and thus MIT licenced.
  */
+static const uint16_t rsqrt_tab[128] = {
+    0xb451, 0xb2f0, 0xb196, 0xb044, 0xaef9, 0xadb6, 0xac79, 0xab43,
+    0xaa14, 0xa8eb, 0xa7c8, 0xa6aa, 0xa592, 0xa480, 0xa373, 0xa26b,
+    0xa168, 0xa06a, 0x9f70, 0x9e7b, 0x9d8a, 0x9c9d, 0x9bb5, 0x9ad1,
+    0x99f0, 0x9913, 0x983a, 0x9765, 0x9693, 0x95c4, 0x94f8, 0x9430,
+    0x936b, 0x92a9, 0x91ea, 0x912e, 0x9075, 0x8fbe, 0x8f0a, 0x8e59,
+    0x8daa, 0x8cfe, 0x8c54, 0x8bac, 0x8b07, 0x8a64, 0x89c4, 0x8925,
+    0x8889, 0x87ee, 0x8756, 0x86c0, 0x862b, 0x8599, 0x8508, 0x8479,
+    0x83ec, 0x8361, 0x82d8, 0x8250, 0x81c9, 0x8145, 0x80c2, 0x8040,
+    0xff02, 0xfd0e, 0xfb25, 0xf947, 0xf773, 0xf5aa, 0xf3ea, 0xf234,
+    0xf087, 0xeee3, 0xed47, 0xebb3, 0xea27, 0xe8a3, 0xe727, 0xe5b2,
+    0xe443, 0xe2dc, 0xe17a, 0xe020, 0xdecb, 0xdd7d, 0xdc34, 0xdaf1,
+    0xd9b3, 0xd87b, 0xd748, 0xd61a, 0xd4f1, 0xd3cd, 0xd2ad, 0xd192,
+    0xd07b, 0xcf69, 0xce5b, 0xcd51, 0xcc4a, 0xcb48, 0xca4a, 0xc94f,
+    0xc858, 0xc764, 0xc674, 0xc587, 0xc49d, 0xc3b7, 0xc2d4, 0xc1f4,
+    0xc116, 0xc03c, 0xbf65, 0xbe90, 0xbdbe, 0xbcef, 0xbc23, 0xbb59,
+    0xba91, 0xb9cc, 0xb90a, 0xb84a, 0xb78c, 0xb6d0, 0xb617, 0xb560,
+};
 
-static FloatParts mul_floats(FloatParts a, FloatParts b, float_status *s)
-{
-    bool sign = a.sign ^ b.sign;
+#define partsN(NAME)   glue(glue(glue(parts,N),_),NAME)
+#define FloatPartsN    glue(FloatParts,N)
+#define FloatPartsW    glue(FloatParts,W)
 
-    if (a.cls == float_class_normal && b.cls == float_class_normal) {
-        uint64_t hi, lo;
-        int exp = a.exp + b.exp;
+#define N 64
+#define W 128
 
-        mul64To128(a.frac, b.frac, &hi, &lo);
-        shift128RightJamming(hi, lo, DECOMPOSED_BINARY_POINT, &hi, &lo);
-        if (lo & DECOMPOSED_OVERFLOW_BIT) {
-            shift64RightJamming(lo, 1, &lo);
-            exp += 1;
-        }
+#include "softfloat-parts-addsub.c.inc"
+#include "softfloat-parts.c.inc"
 
-        /* Re-use a */
-        a.exp = exp;
-        a.sign = sign;
-        a.frac = lo;
-        return a;
-    }
-    /* handle all the NaN cases */
-    if (is_nan(a.cls) || is_nan(b.cls)) {
-        return pick_nan(a, b, s);
-    }
-    /* Inf * Zero == NaN */
-    if ((a.cls == float_class_inf && b.cls == float_class_zero) ||
-        (a.cls == float_class_zero && b.cls == float_class_inf)) {
-        s->float_exception_flags |= float_flag_invalid;
-        return parts_default_nan(s);
-    }
-    /* Multiply by 0 or Inf */
-    if (a.cls == float_class_inf || a.cls == float_class_zero) {
-        a.sign = sign;
-        return a;
-    }
-    if (b.cls == float_class_inf || b.cls == float_class_zero) {
-        b.sign = sign;
-        return b;
-    }
+#undef  N
+#undef  W
+#define N 128
+#define W 256
 
-    g_assert_not_reached();
-    return a;
-}
+#include "softfloat-parts-addsub.c.inc"
+#include "softfloat-parts.c.inc"
 
-float16 QEMU_FLATTEN float16_mul(float16 a, float16 b, float_status *status)
-{
-    FloatParts pa = float16_unpack_canonical(a, status);
-    FloatParts pb = float16_unpack_canonical(b, status);
-    FloatParts pr = mul_floats(pa, pb, status);
+#undef  N
+#undef  W
+#define N            256
 
-    return float16_round_pack_canonical(pr, status);
-}
+#include "softfloat-parts-addsub.c.inc"
 
-static float32 QEMU_SOFTFLOAT_ATTR
-soft_f32_mul(float32 a, float32 b, float_status *status)
-{
-    FloatParts pa = float32_unpack_canonical(a, status);
-    FloatParts pb = float32_unpack_canonical(b, status);
-    FloatParts pr = mul_floats(pa, pb, status);
+#undef  N
+#undef  W
+#undef  partsN
+#undef  FloatPartsN
+#undef  FloatPartsW
 
-    return float32_round_pack_canonical(pr, status);
-}
+/*
+ * Pack/unpack routines with a specific FloatFmt.
+ */
 
-static float64 QEMU_SOFTFLOAT_ATTR
-soft_f64_mul(float64 a, float64 b, float_status *status)
+static void float16a_unpack_canonical(FloatParts64 *p, float16 f,
+                                      float_status *s, const FloatFmt *params)
 {
-    FloatParts pa = float64_unpack_canonical(a, status);
-    FloatParts pb = float64_unpack_canonical(b, status);
-    FloatParts pr = mul_floats(pa, pb, status);
-
-    return float64_round_pack_canonical(pr, status);
+    float16_unpack_raw(p, f);
+    parts_canonicalize(p, s, params);
 }
 
-static float hard_f32_mul(float a, float b)
+static void float16_unpack_canonical(FloatParts64 *p, float16 f,
+                                     float_status *s)
 {
-    return a * b;
+    float16a_unpack_canonical(p, f, s, &float16_params);
 }
 
-static double hard_f64_mul(double a, double b)
+static void bfloat16_unpack_canonical(FloatParts64 *p, bfloat16 f,
+                                      float_status *s)
 {
-    return a * b;
+    bfloat16_unpack_raw(p, f);
+    parts_canonicalize(p, s, &bfloat16_params);
 }
 
-static bool f32_mul_fast_test(union_float32 a, union_float32 b)
+static float16 float16a_round_pack_canonical(FloatParts64 *p,
+                                             float_status *s,
+                                             const FloatFmt *params)
 {
-    return float32_is_zero(a.s) || float32_is_zero(b.s);
+    parts_uncanon(p, s, params);
+    return float16_pack_raw(p);
 }
 
-static bool f64_mul_fast_test(union_float64 a, union_float64 b)
+static float16 float16_round_pack_canonical(FloatParts64 *p,
+                                            float_status *s)
 {
-    return float64_is_zero(a.s) || float64_is_zero(b.s);
+    return float16a_round_pack_canonical(p, s, &float16_params);
 }
 
-static float32 f32_mul_fast_op(float32 a, float32 b, float_status *s)
+static bfloat16 bfloat16_round_pack_canonical(FloatParts64 *p,
+                                              float_status *s)
 {
-    bool signbit = float32_is_neg(a) ^ float32_is_neg(b);
-
-    return float32_set_sign(float32_zero, signbit);
+    parts_uncanon(p, s, &bfloat16_params);
+    return bfloat16_pack_raw(p);
 }
 
-static float64 f64_mul_fast_op(float64 a, float64 b, float_status *s)
+static void float32_unpack_canonical(FloatParts64 *p, float32 f,
+                                     float_status *s)
 {
-    bool signbit = float64_is_neg(a) ^ float64_is_neg(b);
+    float32_unpack_raw(p, f);
+    parts_canonicalize(p, s, &float32_params);
+}
 
-    return float64_set_sign(float64_zero, signbit);
+static float32 float32_round_pack_canonical(FloatParts64 *p,
+                                            float_status *s)
+{
+    parts_uncanon(p, s, &float32_params);
+    return float32_pack_raw(p);
 }
 
-float32 QEMU_FLATTEN
-float32_mul(float32 a, float32 b, float_status *s)
+static void float64_unpack_canonical(FloatParts64 *p, float64 f,
+                                     float_status *s)
 {
-    return float32_gen2(a, b, s, hard_f32_mul, soft_f32_mul,
-                        f32_is_zon2, NULL, f32_mul_fast_test, f32_mul_fast_op);
+    float64_unpack_raw(p, f);
+    parts_canonicalize(p, s, &float64_params);
 }
 
-float64 QEMU_FLATTEN
-float64_mul(float64 a, float64 b, float_status *s)
+static float64 float64_round_pack_canonical(FloatParts64 *p,
+                                            float_status *s)
 {
-    return float64_gen2(a, b, s, hard_f64_mul, soft_f64_mul,
-                        f64_is_zon2, NULL, f64_mul_fast_test, f64_mul_fast_op);
+    parts_uncanon(p, s, &float64_params);
+    return float64_pack_raw(p);
 }
 
-/*
- * Returns the result of multiplying the floating-point values `a' and
- * `b' then adding 'c', with no intermediate rounding step after the
- * multiplication. The operation is performed according to the
- * IEC/IEEE Standard for Binary Floating-Point Arithmetic 754-2008.
- * The flags argument allows the caller to select negation of the
- * addend, the intermediate product, or the final result. (The
- * difference between this and having the caller do a separate
- * negation is that negating externally will flip the sign bit on
- * NaNs.)
- */
+static float64 float64r32_round_pack_canonical(FloatParts64 *p,
+                                               float_status *s)
+{
+    parts_uncanon(p, s, &float32_params);
 
-static FloatParts muladd_floats(FloatParts a, FloatParts b, FloatParts c,
-                                int flags, float_status *s)
-{
-    bool inf_zero = ((1 << a.cls) | (1 << b.cls)) ==
-                    ((1 << float_class_inf) | (1 << float_class_zero));
-    bool p_sign;
-    bool sign_flip = flags & float_muladd_negate_result;
-    FloatClass p_class;
-    uint64_t hi, lo;
-    int p_exp;
-
-    /* It is implementation-defined whether the cases of (0,inf,qnan)
-     * and (inf,0,qnan) raise InvalidOperation or not (and what QNaN
-     * they return if they do), so we have to hand this information
-     * off to the target-specific pick-a-NaN routine.
+    /*
+     * In parts_uncanon, we placed the fraction for float32 at the lsb.
+     * We need to adjust the fraction higher so that the least N bits are
+     * zero, and the fraction is adjacent to the float64 implicit bit.
      */
-    if (is_nan(a.cls) || is_nan(b.cls) || is_nan(c.cls)) {
-        return pick_nan_muladd(a, b, c, inf_zero, s);
+    switch (p->cls) {
+    case float_class_normal:
+        if (unlikely(p->exp == 0)) {
+            /*
+             * The result is denormal for float32, but can be represented
+             * in normalized form for float64.  Adjust, per canonicalize.
+             */
+            int shift = frac_normalize(p);
+            p->exp = (float32_params.frac_shift -
+                      float32_params.exp_bias - shift + 1 +
+                      float64_params.exp_bias);
+            frac_shr(p, float64_params.frac_shift);
+        } else {
+            frac_shl(p, float32_params.frac_shift - float64_params.frac_shift);
+            p->exp += float64_params.exp_bias - float32_params.exp_bias;
+        }
+        break;
+    case float_class_snan:
+    case float_class_qnan:
+        frac_shl(p, float32_params.frac_shift - float64_params.frac_shift);
+        p->exp = float64_params.exp_max;
+        break;
+    case float_class_inf:
+        p->exp = float64_params.exp_max;
+        break;
+    case float_class_zero:
+        break;
+    default:
+        g_assert_not_reached();
     }
 
-    if (inf_zero) {
-        s->float_exception_flags |= float_flag_invalid;
-        return parts_default_nan(s);
-    }
+    return float64_pack_raw(p);
+}
 
-    if (flags & float_muladd_negate_c) {
-        c.sign ^= 1;
-    }
+static void float128_unpack_canonical(FloatParts128 *p, float128 f,
+                                      float_status *s)
+{
+    float128_unpack_raw(p, f);
+    parts_canonicalize(p, s, &float128_params);
+}
+
+static float128 float128_round_pack_canonical(FloatParts128 *p,
+                                              float_status *s)
+{
+    parts_uncanon(p, s, &float128_params);
+    return float128_pack_raw(p);
+}
 
-    p_sign = a.sign ^ b.sign;
+/* Returns false if the encoding is invalid. */
+static bool floatx80_unpack_canonical(FloatParts128 *p, floatx80 f,
+                                      float_status *s)
+{
+    /* Ensure rounding precision is set before beginning. */
+    switch (s->floatx80_rounding_precision) {
+    case floatx80_precision_x:
+    case floatx80_precision_d:
+    case floatx80_precision_s:
+        break;
+    default:
+        g_assert_not_reached();
+    }
 
-    if (flags & float_muladd_negate_product) {
-        p_sign ^= 1;
+    if (unlikely(floatx80_invalid_encoding(f))) {
+        float_raise(float_flag_invalid, s);
+        return false;
     }
 
-    if (a.cls == float_class_inf || b.cls == float_class_inf) {
-        p_class = float_class_inf;
-    } else if (a.cls == float_class_zero || b.cls == float_class_zero) {
-        p_class = float_class_zero;
+    floatx80_unpack_raw(p, f);
+
+    if (likely(p->exp != floatx80_params[floatx80_precision_x].exp_max)) {
+        parts_canonicalize(p, s, &floatx80_params[floatx80_precision_x]);
     } else {
-        p_class = float_class_normal;
+        /* The explicit integer bit is ignored, after invalid checks. */
+        p->frac_hi &= MAKE_64BIT_MASK(0, 63);
+        p->cls = (p->frac_hi == 0 ? float_class_inf
+                  : parts_is_snan_frac(p->frac_hi, s)
+                  ? float_class_snan : float_class_qnan);
     }
+    return true;
+}
 
-    if (c.cls == float_class_inf) {
-        if (p_class == float_class_inf && p_sign != c.sign) {
-            s->float_exception_flags |= float_flag_invalid;
-            return parts_default_nan(s);
-        } else {
-            a.cls = float_class_inf;
-            a.sign = c.sign ^ sign_flip;
-            return a;
-        }
-    }
+static floatx80 floatx80_round_pack_canonical(FloatParts128 *p,
+                                              float_status *s)
+{
+    const FloatFmt *fmt = &floatx80_params[s->floatx80_rounding_precision];
+    uint64_t frac;
+    int exp;
 
-    if (p_class == float_class_inf) {
-        a.cls = float_class_inf;
-        a.sign = p_sign ^ sign_flip;
-        return a;
-    }
+    switch (p->cls) {
+    case float_class_normal:
+        if (s->floatx80_rounding_precision == floatx80_precision_x) {
+            parts_uncanon_normal(p, s, fmt);
+            frac = p->frac_hi;
+            exp = p->exp;
+        } else {
+            FloatParts64 p64;
 
-    if (p_class == float_class_zero) {
-        if (c.cls == float_class_zero) {
-            if (p_sign != c.sign) {
-                p_sign = s->float_rounding_mode == float_round_down;
-            }
-            c.sign = p_sign;
-        } else if (flags & float_muladd_halve_result) {
-            c.exp -= 1;
+            p64.sign = p->sign;
+            p64.exp = p->exp;
+            frac_truncjam(&p64, p);
+            parts_uncanon_normal(&p64, s, fmt);
+            frac = p64.frac;
+            exp = p64.exp;
         }
-        c.sign ^= sign_flip;
-        return c;
-    }
+        if (exp != fmt->exp_max) {
+            break;
+        }
+        /* rounded to inf -- fall through to set frac correctly */
 
-    /* a & b should be normals now... */
-    assert(a.cls == float_class_normal &&
-           b.cls == float_class_normal);
+    case float_class_inf:
+        /* x86 and m68k differ in the setting of the integer bit. */
+        frac = floatx80_infinity_low;
+        exp = fmt->exp_max;
+        break;
 
-    p_exp = a.exp + b.exp;
+    case float_class_zero:
+        frac = 0;
+        exp = 0;
+        break;
 
-    /* Multiply of 2 62-bit numbers produces a (2*62) == 124-bit
-     * result.
-     */
-    mul64To128(a.frac, b.frac, &hi, &lo);
-    /* binary point now at bit 124 */
+    case float_class_snan:
+    case float_class_qnan:
+        /* NaNs have the integer bit set. */
+        frac = p->frac_hi | (1ull << 63);
+        exp = fmt->exp_max;
+        break;
 
-    /* check for overflow */
-    if (hi & (1ULL << (DECOMPOSED_BINARY_POINT * 2 + 1 - 64))) {
-        shift128RightJamming(hi, lo, 1, &hi, &lo);
-        p_exp += 1;
+    default:
+        g_assert_not_reached();
     }
 
-    /* + add/sub */
-    if (c.cls == float_class_zero) {
-        /* move binary point back to 62 */
-        shift128RightJamming(hi, lo, DECOMPOSED_BINARY_POINT, &hi, &lo);
-    } else {
-        int exp_diff = p_exp - c.exp;
-        if (p_sign == c.sign) {
-            /* Addition */
-            if (exp_diff <= 0) {
-                shift128RightJamming(hi, lo,
-                                     DECOMPOSED_BINARY_POINT - exp_diff,
-                                     &hi, &lo);
-                lo += c.frac;
-                p_exp = c.exp;
-            } else {
-                uint64_t c_hi, c_lo;
-                /* shift c to the same binary point as the product (124) */
-                c_hi = c.frac >> 2;
-                c_lo = 0;
-                shift128RightJamming(c_hi, c_lo,
-                                     exp_diff,
-                                     &c_hi, &c_lo);
-                add128(hi, lo, c_hi, c_lo, &hi, &lo);
-                /* move binary point back to 62 */
-                shift128RightJamming(hi, lo, DECOMPOSED_BINARY_POINT, &hi, &lo);
-            }
+    return packFloatx80(p->sign, exp, frac);
+}
 
-            if (lo & DECOMPOSED_OVERFLOW_BIT) {
-                shift64RightJamming(lo, 1, &lo);
-                p_exp += 1;
-            }
+/*
+ * Addition and subtraction
+ */
 
-        } else {
-            /* Subtraction */
-            uint64_t c_hi, c_lo;
-            /* make C binary point match product at bit 124 */
-            c_hi = c.frac >> 2;
-            c_lo = 0;
-
-            if (exp_diff <= 0) {
-                shift128RightJamming(hi, lo, -exp_diff, &hi, &lo);
-                if (exp_diff == 0
-                    &&
-                    (hi > c_hi || (hi == c_hi && lo >= c_lo))) {
-                    sub128(hi, lo, c_hi, c_lo, &hi, &lo);
-                } else {
-                    sub128(c_hi, c_lo, hi, lo, &hi, &lo);
-                    p_sign ^= 1;
-                    p_exp = c.exp;
-                }
-            } else {
-                shift128RightJamming(c_hi, c_lo,
-                                     exp_diff,
-                                     &c_hi, &c_lo);
-                sub128(hi, lo, c_hi, c_lo, &hi, &lo);
-            }
+static float16 QEMU_FLATTEN
+float16_addsub(float16 a, float16 b, float_status *status, bool subtract)
+{
+    FloatParts64 pa, pb, *pr;
 
-            if (hi == 0 && lo == 0) {
-                a.cls = float_class_zero;
-                a.sign = s->float_rounding_mode == float_round_down;
-                a.sign ^= sign_flip;
-                return a;
-            } else {
-                int shift;
-                if (hi != 0) {
-                    shift = clz64(hi);
-                } else {
-                    shift = clz64(lo) + 64;
-                }
-                /* Normalizing to a binary point of 124 is the
-                   correct adjust for the exponent.  However since we're
-                   shifting, we might as well put the binary point back
-                   at 62 where we really want it.  Therefore shift as
-                   if we're leaving 1 bit at the top of the word, but
-                   adjust the exponent as if we're leaving 3 bits.  */
-                shift -= 1;
-                if (shift >= 64) {
-                    lo = lo << (shift - 64);
-                } else {
-                    hi = (hi << shift) | (lo >> (64 - shift));
-                    lo = hi | ((lo << shift) != 0);
-                }
-                p_exp -= shift - 2;
-            }
-        }
+    float16_unpack_canonical(&pa, a, status);
+    float16_unpack_canonical(&pb, b, status);
+    pr = parts_addsub(&pa, &pb, status, subtract);
+
+    return float16_round_pack_canonical(pr, status);
+}
+
+float16 float16_add(float16 a, float16 b, float_status *status)
+{
+    return float16_addsub(a, b, status, false);
+}
+
+float16 float16_sub(float16 a, float16 b, float_status *status)
+{
+    return float16_addsub(a, b, status, true);
+}
+
+static float32 QEMU_SOFTFLOAT_ATTR
+soft_f32_addsub(float32 a, float32 b, float_status *status, bool subtract)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float32_unpack_canonical(&pa, a, status);
+    float32_unpack_canonical(&pb, b, status);
+    pr = parts_addsub(&pa, &pb, status, subtract);
+
+    return float32_round_pack_canonical(pr, status);
+}
+
+static float32 soft_f32_add(float32 a, float32 b, float_status *status)
+{
+    return soft_f32_addsub(a, b, status, false);
+}
+
+static float32 soft_f32_sub(float32 a, float32 b, float_status *status)
+{
+    return soft_f32_addsub(a, b, status, true);
+}
+
+static float64 QEMU_SOFTFLOAT_ATTR
+soft_f64_addsub(float64 a, float64 b, float_status *status, bool subtract)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    pr = parts_addsub(&pa, &pb, status, subtract);
+
+    return float64_round_pack_canonical(pr, status);
+}
+
+static float64 soft_f64_add(float64 a, float64 b, float_status *status)
+{
+    return soft_f64_addsub(a, b, status, false);
+}
+
+static float64 soft_f64_sub(float64 a, float64 b, float_status *status)
+{
+    return soft_f64_addsub(a, b, status, true);
+}
+
+static float hard_f32_add(float a, float b)
+{
+    return a + b;
+}
+
+static float hard_f32_sub(float a, float b)
+{
+    return a - b;
+}
+
+static double hard_f64_add(double a, double b)
+{
+    return a + b;
+}
+
+static double hard_f64_sub(double a, double b)
+{
+    return a - b;
+}
+
+static bool f32_addsubmul_post(union_float32 a, union_float32 b)
+{
+    if (QEMU_HARDFLOAT_2F32_USE_FP) {
+        return !(fpclassify(a.h) == FP_ZERO && fpclassify(b.h) == FP_ZERO);
     }
+    return !(float32_is_zero(a.s) && float32_is_zero(b.s));
+}
 
-    if (flags & float_muladd_halve_result) {
-        p_exp -= 1;
+static bool f64_addsubmul_post(union_float64 a, union_float64 b)
+{
+    if (QEMU_HARDFLOAT_2F64_USE_FP) {
+        return !(fpclassify(a.h) == FP_ZERO && fpclassify(b.h) == FP_ZERO);
+    } else {
+        return !(float64_is_zero(a.s) && float64_is_zero(b.s));
     }
+}
+
+static float32 float32_addsub(float32 a, float32 b, float_status *s,
+                              hard_f32_op2_fn hard, soft_f32_op2_fn soft)
+{
+    return float32_gen2(a, b, s, hard, soft,
+                        f32_is_zon2, f32_addsubmul_post);
+}
+
+static float64 float64_addsub(float64 a, float64 b, float_status *s,
+                              hard_f64_op2_fn hard, soft_f64_op2_fn soft)
+{
+    return float64_gen2(a, b, s, hard, soft,
+                        f64_is_zon2, f64_addsubmul_post);
+}
 
-    /* finally prepare our result */
-    a.cls = float_class_normal;
-    a.sign = p_sign ^ sign_flip;
-    a.exp = p_exp;
-    a.frac = lo;
+float32 QEMU_FLATTEN
+float32_add(float32 a, float32 b, float_status *s)
+{
+    return float32_addsub(a, b, s, hard_f32_add, soft_f32_add);
+}
 
-    return a;
+float32 QEMU_FLATTEN
+float32_sub(float32 a, float32 b, float_status *s)
+{
+    return float32_addsub(a, b, s, hard_f32_sub, soft_f32_sub);
+}
+
+float64 QEMU_FLATTEN
+float64_add(float64 a, float64 b, float_status *s)
+{
+    return float64_addsub(a, b, s, hard_f64_add, soft_f64_add);
+}
+
+float64 QEMU_FLATTEN
+float64_sub(float64 a, float64 b, float_status *s)
+{
+    return float64_addsub(a, b, s, hard_f64_sub, soft_f64_sub);
+}
+
+static float64 float64r32_addsub(float64 a, float64 b, float_status *status,
+                                 bool subtract)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    pr = parts_addsub(&pa, &pb, status, subtract);
+
+    return float64r32_round_pack_canonical(pr, status);
+}
+
+float64 float64r32_add(float64 a, float64 b, float_status *status)
+{
+    return float64r32_addsub(a, b, status, false);
+}
+
+float64 float64r32_sub(float64 a, float64 b, float_status *status)
+{
+    return float64r32_addsub(a, b, status, true);
+}
+
+static bfloat16 QEMU_FLATTEN
+bfloat16_addsub(bfloat16 a, bfloat16 b, float_status *status, bool subtract)
+{
+    FloatParts64 pa, pb, *pr;
+
+    bfloat16_unpack_canonical(&pa, a, status);
+    bfloat16_unpack_canonical(&pb, b, status);
+    pr = parts_addsub(&pa, &pb, status, subtract);
+
+    return bfloat16_round_pack_canonical(pr, status);
 }
 
+bfloat16 bfloat16_add(bfloat16 a, bfloat16 b, float_status *status)
+{
+    return bfloat16_addsub(a, b, status, false);
+}
+
+bfloat16 bfloat16_sub(bfloat16 a, bfloat16 b, float_status *status)
+{
+    return bfloat16_addsub(a, b, status, true);
+}
+
+static float128 QEMU_FLATTEN
+float128_addsub(float128 a, float128 b, float_status *status, bool subtract)
+{
+    FloatParts128 pa, pb, *pr;
+
+    float128_unpack_canonical(&pa, a, status);
+    float128_unpack_canonical(&pb, b, status);
+    pr = parts_addsub(&pa, &pb, status, subtract);
+
+    return float128_round_pack_canonical(pr, status);
+}
+
+float128 float128_add(float128 a, float128 b, float_status *status)
+{
+    return float128_addsub(a, b, status, false);
+}
+
+float128 float128_sub(float128 a, float128 b, float_status *status)
+{
+    return float128_addsub(a, b, status, true);
+}
+
+static floatx80 QEMU_FLATTEN
+floatx80_addsub(floatx80 a, floatx80 b, float_status *status, bool subtract)
+{
+    FloatParts128 pa, pb, *pr;
+
+    if (!floatx80_unpack_canonical(&pa, a, status) ||
+        !floatx80_unpack_canonical(&pb, b, status)) {
+        return floatx80_default_nan(status);
+    }
+
+    pr = parts_addsub(&pa, &pb, status, subtract);
+    return floatx80_round_pack_canonical(pr, status);
+}
+
+floatx80 floatx80_add(floatx80 a, floatx80 b, float_status *status)
+{
+    return floatx80_addsub(a, b, status, false);
+}
+
+floatx80 floatx80_sub(floatx80 a, floatx80 b, float_status *status)
+{
+    return floatx80_addsub(a, b, status, true);
+}
+
+/*
+ * Multiplication
+ */
+
+float16 QEMU_FLATTEN float16_mul(float16 a, float16 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float16_unpack_canonical(&pa, a, status);
+    float16_unpack_canonical(&pb, b, status);
+    pr = parts_mul(&pa, &pb, status);
+
+    return float16_round_pack_canonical(pr, status);
+}
+
+static float32 QEMU_SOFTFLOAT_ATTR
+soft_f32_mul(float32 a, float32 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float32_unpack_canonical(&pa, a, status);
+    float32_unpack_canonical(&pb, b, status);
+    pr = parts_mul(&pa, &pb, status);
+
+    return float32_round_pack_canonical(pr, status);
+}
+
+static float64 QEMU_SOFTFLOAT_ATTR
+soft_f64_mul(float64 a, float64 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    pr = parts_mul(&pa, &pb, status);
+
+    return float64_round_pack_canonical(pr, status);
+}
+
+static float hard_f32_mul(float a, float b)
+{
+    return a * b;
+}
+
+static double hard_f64_mul(double a, double b)
+{
+    return a * b;
+}
+
+float32 QEMU_FLATTEN
+float32_mul(float32 a, float32 b, float_status *s)
+{
+    return float32_gen2(a, b, s, hard_f32_mul, soft_f32_mul,
+                        f32_is_zon2, f32_addsubmul_post);
+}
+
+float64 QEMU_FLATTEN
+float64_mul(float64 a, float64 b, float_status *s)
+{
+    return float64_gen2(a, b, s, hard_f64_mul, soft_f64_mul,
+                        f64_is_zon2, f64_addsubmul_post);
+}
+
+float64 float64r32_mul(float64 a, float64 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    pr = parts_mul(&pa, &pb, status);
+
+    return float64r32_round_pack_canonical(pr, status);
+}
+
+bfloat16 QEMU_FLATTEN
+bfloat16_mul(bfloat16 a, bfloat16 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
+
+    bfloat16_unpack_canonical(&pa, a, status);
+    bfloat16_unpack_canonical(&pb, b, status);
+    pr = parts_mul(&pa, &pb, status);
+
+    return bfloat16_round_pack_canonical(pr, status);
+}
+
+float128 QEMU_FLATTEN
+float128_mul(float128 a, float128 b, float_status *status)
+{
+    FloatParts128 pa, pb, *pr;
+
+    float128_unpack_canonical(&pa, a, status);
+    float128_unpack_canonical(&pb, b, status);
+    pr = parts_mul(&pa, &pb, status);
+
+    return float128_round_pack_canonical(pr, status);
+}
+
+floatx80 QEMU_FLATTEN
+floatx80_mul(floatx80 a, floatx80 b, float_status *status)
+{
+    FloatParts128 pa, pb, *pr;
+
+    if (!floatx80_unpack_canonical(&pa, a, status) ||
+        !floatx80_unpack_canonical(&pb, b, status)) {
+        return floatx80_default_nan(status);
+    }
+
+    pr = parts_mul(&pa, &pb, status);
+    return floatx80_round_pack_canonical(pr, status);
+}
+
+/*
+ * Fused multiply-add
+ */
+
 float16 QEMU_FLATTEN float16_muladd(float16 a, float16 b, float16 c,
-                                                int flags, float_status *status)
+                                    int flags, float_status *status)
 {
-    FloatParts pa = float16_unpack_canonical(a, status);
-    FloatParts pb = float16_unpack_canonical(b, status);
-    FloatParts pc = float16_unpack_canonical(c, status);
-    FloatParts pr = muladd_floats(pa, pb, pc, flags, status);
+    FloatParts64 pa, pb, pc, *pr;
+
+    float16_unpack_canonical(&pa, a, status);
+    float16_unpack_canonical(&pb, b, status);
+    float16_unpack_canonical(&pc, c, status);
+    pr = parts_muladd(&pa, &pb, &pc, flags, status);
 
     return float16_round_pack_canonical(pr, status);
 }
@@ -1550,10 +2209,12 @@ static float32 QEMU_SOFTFLOAT_ATTR
 soft_f32_muladd(float32 a, float32 b, float32 c, int flags,
                 float_status *status)
 {
-    FloatParts pa = float32_unpack_canonical(a, status);
-    FloatParts pb = float32_unpack_canonical(b, status);
-    FloatParts pc = float32_unpack_canonical(c, status);
-    FloatParts pr = muladd_floats(pa, pb, pc, flags, status);
+    FloatParts64 pa, pb, pc, *pr;
+
+    float32_unpack_canonical(&pa, a, status);
+    float32_unpack_canonical(&pb, b, status);
+    float32_unpack_canonical(&pc, c, status);
+    pr = parts_muladd(&pa, &pb, &pc, flags, status);
 
     return float32_round_pack_canonical(pr, status);
 }
@@ -1562,10 +2223,12 @@ static float64 QEMU_SOFTFLOAT_ATTR
 soft_f64_muladd(float64 a, float64 b, float64 c, int flags,
                 float_status *status)
 {
-    FloatParts pa = float64_unpack_canonical(a, status);
-    FloatParts pb = float64_unpack_canonical(b, status);
-    FloatParts pc = float64_unpack_canonical(c, status);
-    FloatParts pr = muladd_floats(pa, pb, pc, flags, status);
+    FloatParts64 pa, pb, pc, *pr;
+
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    float64_unpack_canonical(&pc, c, status);
+    pr = parts_muladd(&pa, &pb, &pc, flags, status);
 
     return float64_round_pack_canonical(pr, status);
 }
@@ -1627,7 +2290,7 @@ float32_muladd(float32 xa, float32 xb, float32 xc, int flags, float_status *s)
         ur.h = fmaf(ua.h, ub.h, uc.h);
 
         if (unlikely(f32_is_inf(ur))) {
-            s->float_exception_flags |= float_flag_overflow;
+            float_raise(float_flag_overflow, s);
         } else if (unlikely(fabsf(ur.h) <= FLT_MIN)) {
             ua = ua_orig;
             uc = uc_orig;
@@ -1698,7 +2361,7 @@ float64_muladd(float64 xa, float64 xb, float64 xc, int flags, float_status *s)
         ur.h = fma(ua.h, ub.h, uc.h);
 
         if (unlikely(f64_is_inf(ur))) {
-            s->float_exception_flags |= float_flag_overflow;
+            float_raise(float_flag_overflow, s);
         } else if (unlikely(fabs(ur.h) <= FLT_MIN)) {
             ua = ua_orig;
             uc = uc_orig;
@@ -1714,112 +2377,80 @@ float64_muladd(float64 xa, float64 xb, float64 xc, int flags, float_status *s)
     return soft_f64_muladd(ua.s, ub.s, uc.s, flags, s);
 }
 
-/*
- * Returns the result of dividing the floating-point value `a' by the
- * corresponding value `b'. The operation is performed according to
- * the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
- */
-
-static FloatParts div_floats(FloatParts a, FloatParts b, float_status *s)
+float64 float64r32_muladd(float64 a, float64 b, float64 c,
+                          int flags, float_status *status)
 {
-    bool sign = a.sign ^ b.sign;
-
-    if (a.cls == float_class_normal && b.cls == float_class_normal) {
-        uint64_t n0, n1, q, r;
-        int exp = a.exp - b.exp;
+    FloatParts64 pa, pb, pc, *pr;
 
-        /*
-         * We want a 2*N / N-bit division to produce exactly an N-bit
-         * result, so that we do not lose any precision and so that we
-         * do not have to renormalize afterward.  If A.frac < B.frac,
-         * then division would produce an (N-1)-bit result; shift A left
-         * by one to produce the an N-bit result, and decrement the
-         * exponent to match.
-         *
-         * The udiv_qrnnd algorithm that we're using requires normalization,
-         * i.e. the msb of the denominator must be set.  Since we know that
-         * DECOMPOSED_BINARY_POINT is msb-1, the inputs must be shifted left
-         * by one (more), and the remainder must be shifted right by one.
-         */
-        if (a.frac < b.frac) {
-            exp -= 1;
-            shift128Left(0, a.frac, DECOMPOSED_BINARY_POINT + 2, &n1, &n0);
-        } else {
-            shift128Left(0, a.frac, DECOMPOSED_BINARY_POINT + 1, &n1, &n0);
-        }
-        q = udiv_qrnnd(&r, n1, n0, b.frac << 1);
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    float64_unpack_canonical(&pc, c, status);
+    pr = parts_muladd(&pa, &pb, &pc, flags, status);
 
-        /*
-         * Set lsb if there is a remainder, to set inexact.
-         * As mentioned above, to find the actual value of the remainder we
-         * would need to shift right, but (1) we are only concerned about
-         * non-zero-ness, and (2) the remainder will always be even because
-         * both inputs to the division primitive are even.
-         */
-        a.frac = q | (r != 0);
-        a.sign = sign;
-        a.exp = exp;
-        return a;
-    }
-    /* handle all the NaN cases */
-    if (is_nan(a.cls) || is_nan(b.cls)) {
-        return pick_nan(a, b, s);
-    }
-    /* 0/0 or Inf/Inf */
-    if (a.cls == b.cls
-        &&
-        (a.cls == float_class_inf || a.cls == float_class_zero)) {
-        s->float_exception_flags |= float_flag_invalid;
-        return parts_default_nan(s);
-    }
-    /* Inf / x or 0 / x */
-    if (a.cls == float_class_inf || a.cls == float_class_zero) {
-        a.sign = sign;
-        return a;
-    }
-    /* Div 0 => Inf */
-    if (b.cls == float_class_zero) {
-        s->float_exception_flags |= float_flag_divbyzero;
-        a.cls = float_class_inf;
-        a.sign = sign;
-        return a;
-    }
-    /* Div by Inf */
-    if (b.cls == float_class_inf) {
-        a.cls = float_class_zero;
-        a.sign = sign;
-        return a;
-    }
-
-    g_assert_not_reached();
-    return a;
+    return float64r32_round_pack_canonical(pr, status);
 }
 
-float16 float16_div(float16 a, float16 b, float_status *status)
+bfloat16 QEMU_FLATTEN bfloat16_muladd(bfloat16 a, bfloat16 b, bfloat16 c,
+                                      int flags, float_status *status)
 {
-    FloatParts pa = float16_unpack_canonical(a, status);
-    FloatParts pb = float16_unpack_canonical(b, status);
-    FloatParts pr = div_floats(pa, pb, status);
+    FloatParts64 pa, pb, pc, *pr;
 
-    return float16_round_pack_canonical(pr, status);
+    bfloat16_unpack_canonical(&pa, a, status);
+    bfloat16_unpack_canonical(&pb, b, status);
+    bfloat16_unpack_canonical(&pc, c, status);
+    pr = parts_muladd(&pa, &pb, &pc, flags, status);
+
+    return bfloat16_round_pack_canonical(pr, status);
 }
 
-static float32 QEMU_SOFTFLOAT_ATTR
-soft_f32_div(float32 a, float32 b, float_status *status)
+float128 QEMU_FLATTEN float128_muladd(float128 a, float128 b, float128 c,
+                                      int flags, float_status *status)
 {
-    FloatParts pa = float32_unpack_canonical(a, status);
-    FloatParts pb = float32_unpack_canonical(b, status);
-    FloatParts pr = div_floats(pa, pb, status);
+    FloatParts128 pa, pb, pc, *pr;
 
-    return float32_round_pack_canonical(pr, status);
+    float128_unpack_canonical(&pa, a, status);
+    float128_unpack_canonical(&pb, b, status);
+    float128_unpack_canonical(&pc, c, status);
+    pr = parts_muladd(&pa, &pb, &pc, flags, status);
+
+    return float128_round_pack_canonical(pr, status);
+}
+
+/*
+ * Division
+ */
+
+float16 float16_div(float16 a, float16 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float16_unpack_canonical(&pa, a, status);
+    float16_unpack_canonical(&pb, b, status);
+    pr = parts_div(&pa, &pb, status);
+
+    return float16_round_pack_canonical(pr, status);
+}
+
+static float32 QEMU_SOFTFLOAT_ATTR
+soft_f32_div(float32 a, float32 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
+
+    float32_unpack_canonical(&pa, a, status);
+    float32_unpack_canonical(&pb, b, status);
+    pr = parts_div(&pa, &pb, status);
+
+    return float32_round_pack_canonical(pr, status);
 }
 
 static float64 QEMU_SOFTFLOAT_ATTR
 soft_f64_div(float64 a, float64 b, float_status *status)
 {
-    FloatParts pa = float64_unpack_canonical(a, status);
-    FloatParts pb = float64_unpack_canonical(b, status);
-    FloatParts pr = div_floats(pa, pb, status);
+    FloatParts64 pa, pb, *pr;
+
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    pr = parts_div(&pa, &pb, status);
 
     return float64_round_pack_canonical(pr, status);
 }
@@ -1872,6296 +2503,2661 @@ float32 QEMU_FLATTEN
 float32_div(float32 a, float32 b, float_status *s)
 {
     return float32_gen2(a, b, s, hard_f32_div, soft_f32_div,
-                        f32_div_pre, f32_div_post, NULL, NULL);
+                        f32_div_pre, f32_div_post);
 }
 
 float64 QEMU_FLATTEN
 float64_div(float64 a, float64 b, float_status *s)
 {
     return float64_gen2(a, b, s, hard_f64_div, soft_f64_div,
-                        f64_div_pre, f64_div_post, NULL, NULL);
+                        f64_div_pre, f64_div_post);
 }
 
-/*
- * Float to Float conversions
- *
- * Returns the result of converting one float format to another. The
- * conversion is performed according to the IEC/IEEE Standard for
- * Binary Floating-Point Arithmetic.
- *
- * The float_to_float helper only needs to take care of raising
- * invalid exceptions and handling the conversion on NaNs.
- */
+float64 float64r32_div(float64 a, float64 b, float_status *status)
+{
+    FloatParts64 pa, pb, *pr;
 
-static FloatParts float_to_float(FloatParts a, const FloatFmt *dstf,
-                                 float_status *s)
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    pr = parts_div(&pa, &pb, status);
+
+    return float64r32_round_pack_canonical(pr, status);
+}
+
+bfloat16 QEMU_FLATTEN
+bfloat16_div(bfloat16 a, bfloat16 b, float_status *status)
 {
-    if (dstf->arm_althp) {
-        switch (a.cls) {
-        case float_class_qnan:
-        case float_class_snan:
-            /* There is no NaN in the destination format.  Raise Invalid
-             * and return a zero with the sign of the input NaN.
-             */
-            s->float_exception_flags |= float_flag_invalid;
-            a.cls = float_class_zero;
-            a.frac = 0;
-            a.exp = 0;
-            break;
+    FloatParts64 pa, pb, *pr;
 
-        case float_class_inf:
-            /* There is no Inf in the destination format.  Raise Invalid
-             * and return the maximum normal with the correct sign.
-             */
-            s->float_exception_flags |= float_flag_invalid;
-            a.cls = float_class_normal;
-            a.exp = dstf->exp_max;
-            a.frac = ((1ull << dstf->frac_size) - 1) << dstf->frac_shift;
-            break;
+    bfloat16_unpack_canonical(&pa, a, status);
+    bfloat16_unpack_canonical(&pb, b, status);
+    pr = parts_div(&pa, &pb, status);
 
-        default:
-            break;
-        }
-    } else if (is_nan(a.cls)) {
-        if (is_snan(a.cls)) {
-            s->float_exception_flags |= float_flag_invalid;
-            a = parts_silence_nan(a, s);
-        }
-        if (s->default_nan_mode) {
-            return parts_default_nan(s);
-        }
-    }
-    return a;
+    return bfloat16_round_pack_canonical(pr, status);
 }
 
-float32 float16_to_float32(float16 a, bool ieee, float_status *s)
+float128 QEMU_FLATTEN
+float128_div(float128 a, float128 b, float_status *status)
 {
-    const FloatFmt *fmt16 = ieee ? &float16_params : &float16_params_ahp;
-    FloatParts p = float16a_unpack_canonical(a, s, fmt16);
-    FloatParts pr = float_to_float(p, &float32_params, s);
-    return float32_round_pack_canonical(pr, s);
+    FloatParts128 pa, pb, *pr;
+
+    float128_unpack_canonical(&pa, a, status);
+    float128_unpack_canonical(&pb, b, status);
+    pr = parts_div(&pa, &pb, status);
+
+    return float128_round_pack_canonical(pr, status);
 }
 
-float64 float16_to_float64(float16 a, bool ieee, float_status *s)
+floatx80 floatx80_div(floatx80 a, floatx80 b, float_status *status)
 {
-    const FloatFmt *fmt16 = ieee ? &float16_params : &float16_params_ahp;
-    FloatParts p = float16a_unpack_canonical(a, s, fmt16);
-    FloatParts pr = float_to_float(p, &float64_params, s);
-    return float64_round_pack_canonical(pr, s);
+    FloatParts128 pa, pb, *pr;
+
+    if (!floatx80_unpack_canonical(&pa, a, status) ||
+        !floatx80_unpack_canonical(&pb, b, status)) {
+        return floatx80_default_nan(status);
+    }
+
+    pr = parts_div(&pa, &pb, status);
+    return floatx80_round_pack_canonical(pr, status);
 }
 
-float16 float32_to_float16(float32 a, bool ieee, float_status *s)
+/*
+ * Remainder
+ */
+
+float32 float32_rem(float32 a, float32 b, float_status *status)
 {
-    const FloatFmt *fmt16 = ieee ? &float16_params : &float16_params_ahp;
-    FloatParts p = float32_unpack_canonical(a, s);
-    FloatParts pr = float_to_float(p, fmt16, s);
-    return float16a_round_pack_canonical(pr, s, fmt16);
+    FloatParts64 pa, pb, *pr;
+
+    float32_unpack_canonical(&pa, a, status);
+    float32_unpack_canonical(&pb, b, status);
+    pr = parts_modrem(&pa, &pb, NULL, status);
+
+    return float32_round_pack_canonical(pr, status);
 }
 
-static float64 QEMU_SOFTFLOAT_ATTR
-soft_float32_to_float64(float32 a, float_status *s)
+float64 float64_rem(float64 a, float64 b, float_status *status)
 {
-    FloatParts p = float32_unpack_canonical(a, s);
-    FloatParts pr = float_to_float(p, &float64_params, s);
-    return float64_round_pack_canonical(pr, s);
+    FloatParts64 pa, pb, *pr;
+
+    float64_unpack_canonical(&pa, a, status);
+    float64_unpack_canonical(&pb, b, status);
+    pr = parts_modrem(&pa, &pb, NULL, status);
+
+    return float64_round_pack_canonical(pr, status);
 }
 
-float64 float32_to_float64(float32 a, float_status *s)
+float128 float128_rem(float128 a, float128 b, float_status *status)
 {
-    if (likely(float32_is_normal(a))) {
-        /* Widening conversion can never produce inexact results.  */
-        union_float32 uf;
-        union_float64 ud;
-        uf.s = a;
-        ud.h = uf.h;
-        return ud.s;
-    } else if (float32_is_zero(a)) {
-        return float64_set_sign(float64_zero, float32_is_neg(a));
-    } else {
-        return soft_float32_to_float64(a, s);
+    FloatParts128 pa, pb, *pr;
+
+    float128_unpack_canonical(&pa, a, status);
+    float128_unpack_canonical(&pb, b, status);
+    pr = parts_modrem(&pa, &pb, NULL, status);
+
+    return float128_round_pack_canonical(pr, status);
+}
+
+/*
+ * Returns the remainder of the extended double-precision floating-point value
+ * `a' with respect to the corresponding value `b'.
+ * If 'mod' is false, the operation is performed according to the IEC/IEEE
+ * Standard for Binary Floating-Point Arithmetic.  If 'mod' is true, return
+ * the remainder based on truncating the quotient toward zero instead and
+ * *quotient is set to the low 64 bits of the absolute value of the integer
+ * quotient.
+ */
+floatx80 floatx80_modrem(floatx80 a, floatx80 b, bool mod,
+                         uint64_t *quotient, float_status *status)
+{
+    FloatParts128 pa, pb, *pr;
+
+    *quotient = 0;
+    if (!floatx80_unpack_canonical(&pa, a, status) ||
+        !floatx80_unpack_canonical(&pb, b, status)) {
+        return floatx80_default_nan(status);
     }
+    pr = parts_modrem(&pa, &pb, mod ? quotient : NULL, status);
+
+    return floatx80_round_pack_canonical(pr, status);
 }
 
-float16 float64_to_float16(float64 a, bool ieee, float_status *s)
+floatx80 floatx80_rem(floatx80 a, floatx80 b, float_status *status)
 {
-    const FloatFmt *fmt16 = ieee ? &float16_params : &float16_params_ahp;
-    FloatParts p = float64_unpack_canonical(a, s);
-    FloatParts pr = float_to_float(p, fmt16, s);
-    return float16a_round_pack_canonical(pr, s, fmt16);
+    uint64_t quotient;
+    return floatx80_modrem(a, b, false, &quotient, status);
 }
 
-float32 float64_to_float32(float64 a, float_status *s)
+floatx80 floatx80_mod(floatx80 a, floatx80 b, float_status *status)
 {
-    FloatParts p = float64_unpack_canonical(a, s);
-    FloatParts pr = float_to_float(p, &float32_params, s);
-    return float32_round_pack_canonical(pr, s);
+    uint64_t quotient;
+    return floatx80_modrem(a, b, true, &quotient, status);
 }
 
 /*
- * Rounds the floating-point value `a' to an integer, and returns the
- * result as a floating-point value. The operation is performed
- * according to the IEC/IEEE Standard for Binary Floating-Point
- * Arithmetic.
+ * Float to Float conversions
+ *
+ * Returns the result of converting one float format to another. The
+ * conversion is performed according to the IEC/IEEE Standard for
+ * Binary Floating-Point Arithmetic.
+ *
+ * Usually this only needs to take care of raising invalid exceptions
+ * and handling the conversion on NaNs.
  */
 
-static FloatParts round_to_int(FloatParts a, int rmode,
-                               int scale, float_status *s)
+static void parts_float_to_ahp(FloatParts64 *a, float_status *s)
 {
-    switch (a.cls) {
-    case float_class_qnan:
+    switch (a->cls) {
     case float_class_snan:
-        return return_nan(a, s);
+        float_raise(float_flag_invalid_snan, s);
+        /* fall through */
+    case float_class_qnan:
+        /*
+         * There is no NaN in the destination format.  Raise Invalid
+         * and return a zero with the sign of the input NaN.
+         */
+        float_raise(float_flag_invalid, s);
+        a->cls = float_class_zero;
+        break;
 
-    case float_class_zero:
     case float_class_inf:
-        /* already "integral" */
+        /*
+         * There is no Inf in the destination format.  Raise Invalid
+         * and return the maximum normal with the correct sign.
+         */
+        float_raise(float_flag_invalid, s);
+        a->cls = float_class_normal;
+        a->exp = float16_params_ahp.exp_max;
+        a->frac = MAKE_64BIT_MASK(float16_params_ahp.frac_shift,
+                                  float16_params_ahp.frac_size + 1);
         break;
 
     case float_class_normal:
-        scale = MIN(MAX(scale, -0x10000), 0x10000);
-        a.exp += scale;
-
-        if (a.exp >= DECOMPOSED_BINARY_POINT) {
-            /* already integral */
-            break;
-        }
-        if (a.exp < 0) {
-            bool one = false;
-            /* all fractional */
-            s->float_exception_flags |= float_flag_inexact;
-            switch (rmode) {
-            case float_round_nearest_even:
-                one = a.exp == -1 && a.frac > DECOMPOSED_IMPLICIT_BIT;
-                break;
-            case float_round_ties_away:
-                one = a.exp == -1 && a.frac >= DECOMPOSED_IMPLICIT_BIT;
-                break;
-            case float_round_to_zero:
-                one = false;
-                break;
-            case float_round_up:
-                one = !a.sign;
-                break;
-            case float_round_down:
-                one = a.sign;
-                break;
-            case float_round_to_odd:
-                one = true;
-                break;
-            default:
-                g_assert_not_reached();
-                break;
-            }
-
-            if (one) {
-                a.frac = DECOMPOSED_IMPLICIT_BIT;
-                a.exp = 0;
-            } else {
-                a.cls = float_class_zero;
-            }
-        } else {
-            uint64_t frac_lsb = DECOMPOSED_IMPLICIT_BIT >> a.exp;
-            uint64_t frac_lsbm1 = frac_lsb >> 1;
-            uint64_t rnd_even_mask = (frac_lsb - 1) | frac_lsb;
-            uint64_t rnd_mask = rnd_even_mask >> 1;
-            uint64_t inc = 0;
-
-            switch (rmode) {
-            case float_round_nearest_even:
-                inc = ((a.frac & rnd_even_mask) != frac_lsbm1 ? frac_lsbm1 : 0);
-                break;
-            case float_round_ties_away:
-                inc = frac_lsbm1;
-                break;
-            case float_round_to_zero:
-                inc = 0;
-                break;
-            case float_round_up:
-                inc = a.sign ? 0 : rnd_mask;
-                break;
-            case float_round_down:
-                inc = a.sign ? rnd_mask : 0;
-                break;
-            case float_round_to_odd:
-                inc = a.frac & frac_lsb ? 0 : rnd_mask;
-                break;
-            default:
-                g_assert_not_reached();
-                break;
-            }
-
-            if (a.frac & rnd_mask) {
-                s->float_exception_flags |= float_flag_inexact;
-                a.frac += inc;
-                a.frac &= ~rnd_mask;
-                if (a.frac & DECOMPOSED_OVERFLOW_BIT) {
-                    a.frac >>= 1;
-                    a.exp++;
-                }
-            }
-        }
+    case float_class_zero:
         break;
+
     default:
         g_assert_not_reached();
     }
-    return a;
 }
 
-float16 float16_round_to_int(float16 a, float_status *s)
+static void parts64_float_to_float(FloatParts64 *a, float_status *s)
 {
-    FloatParts pa = float16_unpack_canonical(a, s);
-    FloatParts pr = round_to_int(pa, s->float_rounding_mode, 0, s);
-    return float16_round_pack_canonical(pr, s);
+    if (is_nan(a->cls)) {
+        parts_return_nan(a, s);
+    }
 }
 
-float32 float32_round_to_int(float32 a, float_status *s)
+static void parts128_float_to_float(FloatParts128 *a, float_status *s)
 {
-    FloatParts pa = float32_unpack_canonical(a, s);
-    FloatParts pr = round_to_int(pa, s->float_rounding_mode, 0, s);
-    return float32_round_pack_canonical(pr, s);
+    if (is_nan(a->cls)) {
+        parts_return_nan(a, s);
+    }
 }
 
-float64 float64_round_to_int(float64 a, float_status *s)
+#define parts_float_to_float(P, S) \
+    PARTS_GENERIC_64_128(float_to_float, P)(P, S)
+
+static void parts_float_to_float_narrow(FloatParts64 *a, FloatParts128 *b,
+                                        float_status *s)
 {
-    FloatParts pa = float64_unpack_canonical(a, s);
-    FloatParts pr = round_to_int(pa, s->float_rounding_mode, 0, s);
-    return float64_round_pack_canonical(pr, s);
-}
+    a->cls = b->cls;
+    a->sign = b->sign;
+    a->exp = b->exp;
 
-/*
- * Returns the result of converting the floating-point value `a' to
- * the two's complement integer format. The conversion is performed
- * according to the IEC/IEEE Standard for Binary Floating-Point
- * Arithmetic---which means in particular that the conversion is
- * rounded according to the current rounding mode. If `a' is a NaN,
- * the largest positive integer is returned. Otherwise, if the
- * conversion overflows, the largest integer with the same sign as `a'
- * is returned.
-*/
+    if (a->cls == float_class_normal) {
+        frac_truncjam(a, b);
+    } else if (is_nan(a->cls)) {
+        /* Discard the low bits of the NaN. */
+        a->frac = b->frac_hi;
+        parts_return_nan(a, s);
+    }
+}
 
-static int64_t round_to_int_and_pack(FloatParts in, int rmode, int scale,
-                                     int64_t min, int64_t max,
-                                     float_status *s)
+static void parts_float_to_float_widen(FloatParts128 *a, FloatParts64 *b,
+                                       float_status *s)
 {
-    uint64_t r;
-    int orig_flags = get_float_exception_flags(s);
-    FloatParts p = round_to_int(in, rmode, scale, s);
+    a->cls = b->cls;
+    a->sign = b->sign;
+    a->exp = b->exp;
+    frac_widen(a, b);
 
-    switch (p.cls) {
-    case float_class_snan:
-    case float_class_qnan:
-        s->float_exception_flags = orig_flags | float_flag_invalid;
-        return max;
-    case float_class_inf:
-        s->float_exception_flags = orig_flags | float_flag_invalid;
-        return p.sign ? min : max;
-    case float_class_zero:
-        return 0;
-    case float_class_normal:
-        if (p.exp < DECOMPOSED_BINARY_POINT) {
-            r = p.frac >> (DECOMPOSED_BINARY_POINT - p.exp);
-        } else if (p.exp - DECOMPOSED_BINARY_POINT < 2) {
-            r = p.frac << (p.exp - DECOMPOSED_BINARY_POINT);
-        } else {
-            r = UINT64_MAX;
-        }
-        if (p.sign) {
-#ifdef _MSC_VER
-            if (r <=  0ULL - (uint64_t)min) {
-                return (0ULL - r);
-#else
-            if (r <= -(uint64_t) min) {
-                return -r;
-#endif
-            } else {
-                s->float_exception_flags = orig_flags | float_flag_invalid;
-                return min;
-            }
-        } else {
-            if (r <= max) {
-                return r;
-            } else {
-                s->float_exception_flags = orig_flags | float_flag_invalid;
-                return max;
-            }
-        }
-    default:
-        g_assert_not_reached();
-        return max;
+    if (is_nan(a->cls)) {
+        parts_return_nan(a, s);
     }
 }
 
-int16_t float16_to_int16_scalbn(float16 a, int rmode, int scale,
-                                float_status *s)
+float32 float16_to_float32(float16 a, bool ieee, float_status *s)
 {
-    return round_to_int_and_pack(float16_unpack_canonical(a, s),
-                                 rmode, scale, INT16_MIN, INT16_MAX, s);
+    const FloatFmt *fmt16 = ieee ? &float16_params : &float16_params_ahp;
+    FloatParts64 p;
+
+    float16a_unpack_canonical(&p, a, s, fmt16);
+    parts_float_to_float(&p, s);
+    return float32_round_pack_canonical(&p, s);
 }
 
-int32_t float16_to_int32_scalbn(float16 a, int rmode, int scale,
-                                float_status *s)
+float64 float16_to_float64(float16 a, bool ieee, float_status *s)
 {
-    return round_to_int_and_pack(float16_unpack_canonical(a, s),
-                                 rmode, scale, INT32_MIN, INT32_MAX, s);
+    const FloatFmt *fmt16 = ieee ? &float16_params : &float16_params_ahp;
+    FloatParts64 p;
+
+    float16a_unpack_canonical(&p, a, s, fmt16);
+    parts_float_to_float(&p, s);
+    return float64_round_pack_canonical(&p, s);
 }
 
-int64_t float16_to_int64_scalbn(float16 a, int rmode, int scale,
-                                float_status *s)
+float16 float32_to_float16(float32 a, bool ieee, float_status *s)
 {
-    return round_to_int_and_pack(float16_unpack_canonical(a, s),
-                                 rmode, scale, INT64_MIN, INT64_MAX, s);
+    FloatParts64 p;
+    const FloatFmt *fmt;
+
+    float32_unpack_canonical(&p, a, s);
+    if (ieee) {
+        parts_float_to_float(&p, s);
+        fmt = &float16_params;
+    } else {
+        parts_float_to_ahp(&p, s);
+        fmt = &float16_params_ahp;
+    }
+    return float16a_round_pack_canonical(&p, s, fmt);
 }
 
-int16_t float32_to_int16_scalbn(float32 a, int rmode, int scale,
-                                float_status *s)
+static float64 QEMU_SOFTFLOAT_ATTR
+soft_float32_to_float64(float32 a, float_status *s)
 {
-    return round_to_int_and_pack(float32_unpack_canonical(a, s),
-                                 rmode, scale, INT16_MIN, INT16_MAX, s);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    parts_float_to_float(&p, s);
+    return float64_round_pack_canonical(&p, s);
 }
 
-int32_t float32_to_int32_scalbn(float32 a, int rmode, int scale,
-                                float_status *s)
+float64 float32_to_float64(float32 a, float_status *s)
 {
-    return round_to_int_and_pack(float32_unpack_canonical(a, s),
-                                 rmode, scale, INT32_MIN, INT32_MAX, s);
+    if (likely(float32_is_normal(a))) {
+        /* Widening conversion can never produce inexact results.  */
+        union_float32 uf;
+        union_float64 ud;
+        uf.s = a;
+        ud.h = uf.h;
+        return ud.s;
+    } else if (float32_is_zero(a)) {
+        return float64_set_sign(float64_zero, float32_is_neg(a));
+    } else {
+        return soft_float32_to_float64(a, s);
+    }
 }
 
-int64_t float32_to_int64_scalbn(float32 a, int rmode, int scale,
-                                float_status *s)
+float16 float64_to_float16(float64 a, bool ieee, float_status *s)
 {
-    return round_to_int_and_pack(float32_unpack_canonical(a, s),
-                                 rmode, scale, INT64_MIN, INT64_MAX, s);
+    FloatParts64 p;
+    const FloatFmt *fmt;
+
+    float64_unpack_canonical(&p, a, s);
+    if (ieee) {
+        parts_float_to_float(&p, s);
+        fmt = &float16_params;
+    } else {
+        parts_float_to_ahp(&p, s);
+        fmt = &float16_params_ahp;
+    }
+    return float16a_round_pack_canonical(&p, s, fmt);
 }
 
-int16_t float64_to_int16_scalbn(float64 a, int rmode, int scale,
-                                float_status *s)
+float32 float64_to_float32(float64 a, float_status *s)
 {
-    return round_to_int_and_pack(float64_unpack_canonical(a, s),
-                                 rmode, scale, INT16_MIN, INT16_MAX, s);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    parts_float_to_float(&p, s);
+    return float32_round_pack_canonical(&p, s);
 }
 
-int32_t float64_to_int32_scalbn(float64 a, int rmode, int scale,
-                                float_status *s)
+float32 bfloat16_to_float32(bfloat16 a, float_status *s)
 {
-    return round_to_int_and_pack(float64_unpack_canonical(a, s),
-                                 rmode, scale, INT32_MIN, INT32_MAX, s);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    parts_float_to_float(&p, s);
+    return float32_round_pack_canonical(&p, s);
 }
 
-int64_t float64_to_int64_scalbn(float64 a, int rmode, int scale,
-                                float_status *s)
+float64 bfloat16_to_float64(bfloat16 a, float_status *s)
 {
-    return round_to_int_and_pack(float64_unpack_canonical(a, s),
-                                 rmode, scale, INT64_MIN, INT64_MAX, s);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    parts_float_to_float(&p, s);
+    return float64_round_pack_canonical(&p, s);
 }
 
-int16_t float16_to_int16(float16 a, float_status *s)
+bfloat16 float32_to_bfloat16(float32 a, float_status *s)
 {
-    return float16_to_int16_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    parts_float_to_float(&p, s);
+    return bfloat16_round_pack_canonical(&p, s);
 }
 
-int32_t float16_to_int32(float16 a, float_status *s)
+bfloat16 float64_to_bfloat16(float64 a, float_status *s)
 {
-    return float16_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    parts_float_to_float(&p, s);
+    return bfloat16_round_pack_canonical(&p, s);
 }
 
-int64_t float16_to_int64(float16 a, float_status *s)
+float32 float128_to_float32(float128 a, float_status *s)
 {
-    return float16_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p64;
+    FloatParts128 p128;
+
+    float128_unpack_canonical(&p128, a, s);
+    parts_float_to_float_narrow(&p64, &p128, s);
+    return float32_round_pack_canonical(&p64, s);
 }
 
-int16_t float32_to_int16(float32 a, float_status *s)
+float64 float128_to_float64(float128 a, float_status *s)
 {
-    return float32_to_int16_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p64;
+    FloatParts128 p128;
+
+    float128_unpack_canonical(&p128, a, s);
+    parts_float_to_float_narrow(&p64, &p128, s);
+    return float64_round_pack_canonical(&p64, s);
 }
 
-int32_t float32_to_int32(float32 a, float_status *s)
+float128 float32_to_float128(float32 a, float_status *s)
 {
-    return float32_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p64;
+    FloatParts128 p128;
+
+    float32_unpack_canonical(&p64, a, s);
+    parts_float_to_float_widen(&p128, &p64, s);
+    return float128_round_pack_canonical(&p128, s);
 }
 
-int64_t float32_to_int64(float32 a, float_status *s)
+float128 float64_to_float128(float64 a, float_status *s)
 {
-    return float32_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p64;
+    FloatParts128 p128;
+
+    float64_unpack_canonical(&p64, a, s);
+    parts_float_to_float_widen(&p128, &p64, s);
+    return float128_round_pack_canonical(&p128, s);
 }
 
-int16_t float64_to_int16(float64 a, float_status *s)
+float32 floatx80_to_float32(floatx80 a, float_status *s)
 {
-    return float64_to_int16_scalbn(a, s->float_rounding_mode, 0, s);
-}
+    FloatParts64 p64;
+    FloatParts128 p128;
 
-int32_t float64_to_int32(float64 a, float_status *s)
+    if (floatx80_unpack_canonical(&p128, a, s)) {
+        parts_float_to_float_narrow(&p64, &p128, s);
+    } else {
+        parts_default_nan(&p64, s);
+    }
+    return float32_round_pack_canonical(&p64, s);
+}
+
+float64 floatx80_to_float64(floatx80 a, float_status *s)
 {
-    return float64_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p64;
+    FloatParts128 p128;
+
+    if (floatx80_unpack_canonical(&p128, a, s)) {
+        parts_float_to_float_narrow(&p64, &p128, s);
+    } else {
+        parts_default_nan(&p64, s);
+    }
+    return float64_round_pack_canonical(&p64, s);
 }
 
-int64_t float64_to_int64(float64 a, float_status *s)
+float128 floatx80_to_float128(floatx80 a, float_status *s)
 {
-    return float64_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts128 p;
+
+    if (floatx80_unpack_canonical(&p, a, s)) {
+        parts_float_to_float(&p, s);
+    } else {
+        parts_default_nan(&p, s);
+    }
+    return float128_round_pack_canonical(&p, s);
 }
 
-int16_t float16_to_int16_round_to_zero(float16 a, float_status *s)
+floatx80 float32_to_floatx80(float32 a, float_status *s)
 {
-    return float16_to_int16_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts64 p64;
+    FloatParts128 p128;
+
+    float32_unpack_canonical(&p64, a, s);
+    parts_float_to_float_widen(&p128, &p64, s);
+    return floatx80_round_pack_canonical(&p128, s);
 }
 
-int32_t float16_to_int32_round_to_zero(float16 a, float_status *s)
+floatx80 float64_to_floatx80(float64 a, float_status *s)
 {
-    return float16_to_int32_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts64 p64;
+    FloatParts128 p128;
+
+    float64_unpack_canonical(&p64, a, s);
+    parts_float_to_float_widen(&p128, &p64, s);
+    return floatx80_round_pack_canonical(&p128, s);
 }
 
-int64_t float16_to_int64_round_to_zero(float16 a, float_status *s)
+floatx80 float128_to_floatx80(float128 a, float_status *s)
 {
-    return float16_to_int64_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts128 p;
+
+    float128_unpack_canonical(&p, a, s);
+    parts_float_to_float(&p, s);
+    return floatx80_round_pack_canonical(&p, s);
 }
 
-int16_t float32_to_int16_round_to_zero(float32 a, float_status *s)
+/*
+ * Round to integral value
+ */
+
+float16 float16_round_to_int(float16 a, float_status *s)
 {
-    return float32_to_int16_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts64 p;
+
+    float16_unpack_canonical(&p, a, s);
+    parts_round_to_int(&p, s->float_rounding_mode, 0, s, &float16_params);
+    return float16_round_pack_canonical(&p, s);
 }
 
-int32_t float32_to_int32_round_to_zero(float32 a, float_status *s)
+float32 float32_round_to_int(float32 a, float_status *s)
 {
-    return float32_to_int32_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    parts_round_to_int(&p, s->float_rounding_mode, 0, s, &float32_params);
+    return float32_round_pack_canonical(&p, s);
 }
 
-int64_t float32_to_int64_round_to_zero(float32 a, float_status *s)
+float64 float64_round_to_int(float64 a, float_status *s)
 {
-    return float32_to_int64_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    parts_round_to_int(&p, s->float_rounding_mode, 0, s, &float64_params);
+    return float64_round_pack_canonical(&p, s);
 }
 
-int16_t float64_to_int16_round_to_zero(float64 a, float_status *s)
+bfloat16 bfloat16_round_to_int(bfloat16 a, float_status *s)
 {
-    return float64_to_int16_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    parts_round_to_int(&p, s->float_rounding_mode, 0, s, &bfloat16_params);
+    return bfloat16_round_pack_canonical(&p, s);
 }
 
-int32_t float64_to_int32_round_to_zero(float64 a, float_status *s)
+float128 float128_round_to_int(float128 a, float_status *s)
 {
-    return float64_to_int32_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts128 p;
+
+    float128_unpack_canonical(&p, a, s);
+    parts_round_to_int(&p, s->float_rounding_mode, 0, s, &float128_params);
+    return float128_round_pack_canonical(&p, s);
 }
 
-int64_t float64_to_int64_round_to_zero(float64 a, float_status *s)
+floatx80 floatx80_round_to_int(floatx80 a, float_status *status)
 {
-    return float64_to_int64_scalbn(a, float_round_to_zero, 0, s);
+    FloatParts128 p;
+
+    if (!floatx80_unpack_canonical(&p, a, status)) {
+        return floatx80_default_nan(status);
+    }
+
+    parts_round_to_int(&p, status->float_rounding_mode, 0, status,
+                       &floatx80_params[status->floatx80_rounding_precision]);
+    return floatx80_round_pack_canonical(&p, status);
 }
 
 /*
- *  Returns the result of converting the floating-point value `a' to
- *  the unsigned integer format. The conversion is performed according
- *  to the IEC/IEEE Standard for Binary Floating-Point
- *  Arithmetic---which means in particular that the conversion is
- *  rounded according to the current rounding mode. If `a' is a NaN,
- *  the largest unsigned integer is returned. Otherwise, if the
- *  conversion overflows, the largest unsigned integer is returned. If
- *  the 'a' is negative, the result is rounded and zero is returned;
- *  values that do not round to zero will raise the inexact exception
- *  flag.
+ * Floating-point to signed integer conversions
  */
 
-static uint64_t round_to_uint_and_pack(FloatParts in, int rmode, int scale,
-                                       uint64_t max, float_status *s)
+int8_t float16_to_int8_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                              float_status *s)
 {
-    int orig_flags = get_float_exception_flags(s);
-    FloatParts p = round_to_int(in, rmode, scale, s);
-    uint64_t r;
+    FloatParts64 p;
 
-    switch (p.cls) {
-    case float_class_snan:
-    case float_class_qnan:
-        s->float_exception_flags = orig_flags | float_flag_invalid;
-        return max;
-    case float_class_inf:
-        s->float_exception_flags = orig_flags | float_flag_invalid;
-        return p.sign ? 0 : max;
-    case float_class_zero:
-        return 0;
-    case float_class_normal:
-        if (p.sign) {
-            s->float_exception_flags = orig_flags | float_flag_invalid;
-            return 0;
-        }
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT8_MIN, INT8_MAX, s);
+}
 
-        if (p.exp < DECOMPOSED_BINARY_POINT) {
-            r = p.frac >> (DECOMPOSED_BINARY_POINT - p.exp);
-        } else if (p.exp - DECOMPOSED_BINARY_POINT < 2) {
-            r = p.frac << (p.exp - DECOMPOSED_BINARY_POINT);
-        } else {
-            s->float_exception_flags = orig_flags | float_flag_invalid;
-            return max;
-        }
+int16_t float16_to_int16_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
+{
+    FloatParts64 p;
 
-        /* For uint64 this will never trip, but if p.exp is too large
-         * to shift a decomposed fraction we shall have exited via the
-         * 3rd leg above.
-         */
-        if (r > max) {
-            s->float_exception_flags = orig_flags | float_flag_invalid;
-            return max;
-        }
-        return r;
-    default:
-        g_assert_not_reached();
-        return max;
-    }
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT16_MIN, INT16_MAX, s);
 }
 
-uint16_t float16_to_uint16_scalbn(float16 a, int rmode, int scale,
-                                  float_status *s)
+int32_t float16_to_int32_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float16_unpack_canonical(a, s),
-                                  rmode, scale, UINT16_MAX, s);
+    FloatParts64 p;
+
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT32_MIN, INT32_MAX, s);
 }
 
-uint32_t float16_to_uint32_scalbn(float16 a, int rmode, int scale,
-                                  float_status *s)
+int64_t float16_to_int64_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float16_unpack_canonical(a, s),
-                                  rmode, scale, UINT32_MAX, s);
+    FloatParts64 p;
+
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s);
 }
 
-uint64_t float16_to_uint64_scalbn(float16 a, int rmode, int scale,
-                                  float_status *s)
+int16_t float32_to_int16_scalbn(float32 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float16_unpack_canonical(a, s),
-                                  rmode, scale, UINT64_MAX, s);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT16_MIN, INT16_MAX, s);
 }
 
-uint16_t float32_to_uint16_scalbn(float32 a, int rmode, int scale,
-                                  float_status *s)
+int32_t float32_to_int32_scalbn(float32 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float32_unpack_canonical(a, s),
-                                  rmode, scale, UINT16_MAX, s);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT32_MIN, INT32_MAX, s);
 }
 
-uint32_t float32_to_uint32_scalbn(float32 a, int rmode, int scale,
-                                  float_status *s)
+int64_t float32_to_int64_scalbn(float32 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float32_unpack_canonical(a, s),
-                                  rmode, scale, UINT32_MAX, s);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s);
 }
 
-uint64_t float32_to_uint64_scalbn(float32 a, int rmode, int scale,
-                                  float_status *s)
+int16_t float64_to_int16_scalbn(float64 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float32_unpack_canonical(a, s),
-                                  rmode, scale, UINT64_MAX, s);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT16_MIN, INT16_MAX, s);
 }
 
-uint16_t float64_to_uint16_scalbn(float64 a, int rmode, int scale,
-                                  float_status *s)
+int32_t float64_to_int32_scalbn(float64 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float64_unpack_canonical(a, s),
-                                  rmode, scale, UINT16_MAX, s);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT32_MIN, INT32_MAX, s);
 }
 
-uint32_t float64_to_uint32_scalbn(float64 a, int rmode, int scale,
-                                  float_status *s)
+int64_t float64_to_int64_scalbn(float64 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    return round_to_uint_and_pack(float64_unpack_canonical(a, s),
-                                  rmode, scale, UINT32_MAX, s);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s);
 }
 
-uint64_t float64_to_uint64_scalbn(float64 a, int rmode, int scale,
-                                  float_status *s)
+int16_t bfloat16_to_int16_scalbn(bfloat16 a, FloatRoundMode rmode, int scale,
+                                 float_status *s)
 {
-    return round_to_uint_and_pack(float64_unpack_canonical(a, s),
-                                  rmode, scale, UINT64_MAX, s);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT16_MIN, INT16_MAX, s);
 }
 
-uint16_t float16_to_uint16(float16 a, float_status *s)
+int32_t bfloat16_to_int32_scalbn(bfloat16 a, FloatRoundMode rmode, int scale,
+                                 float_status *s)
 {
-    return float16_to_uint16_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT32_MIN, INT32_MAX, s);
 }
 
-uint32_t float16_to_uint32(float16 a, float_status *s)
+int64_t bfloat16_to_int64_scalbn(bfloat16 a, FloatRoundMode rmode, int scale,
+                                 float_status *s)
 {
-    return float16_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s);
 }
 
-uint64_t float16_to_uint64(float16 a, float_status *s)
+static int32_t float128_to_int32_scalbn(float128 a, FloatRoundMode rmode,
+                                        int scale, float_status *s)
 {
-    return float16_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts128 p;
+
+    float128_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT32_MIN, INT32_MAX, s);
 }
 
-uint16_t float32_to_uint16(float32 a, float_status *s)
+static int64_t float128_to_int64_scalbn(float128 a, FloatRoundMode rmode,
+                                        int scale, float_status *s)
 {
-    return float32_to_uint16_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts128 p;
+
+    float128_unpack_canonical(&p, a, s);
+    return parts_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s);
 }
 
-uint32_t float32_to_uint32(float32 a, float_status *s)
+static Int128 float128_to_int128_scalbn(float128 a, FloatRoundMode rmode,
+                                        int scale, float_status *s)
 {
-    return float32_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
+    int flags = 0;
+    Int128 r;
+    FloatParts128 p;
+
+    float128_unpack_canonical(&p, a, s);
+
+    switch (p.cls) {
+    case float_class_snan:
+        flags |= float_flag_invalid_snan;
+        /* fall through */
+    case float_class_qnan:
+        flags |= float_flag_invalid;
+        r = UINT128_MAX;
+        break;
+
+    case float_class_inf:
+        flags = float_flag_invalid | float_flag_invalid_cvti;
+        r = p.sign ? INT128_MIN : INT128_MAX;
+        break;
+
+    case float_class_zero:
+        return int128_zero();
+
+    case float_class_normal:
+        if (parts_round_to_int_normal(&p, rmode, scale, 128 - 2)) {
+            flags = float_flag_inexact;
+        }
+
+        if (p.exp < 127) {
+            int shift = 127 - p.exp;
+            r = int128_urshift(int128_make128(p.frac_lo, p.frac_hi), shift);
+            if (p.sign) {
+                r = int128_neg(r);
+            }
+        } else if (p.exp == 127 && p.sign && p.frac_lo == 0 &&
+                   p.frac_hi == DECOMPOSED_IMPLICIT_BIT) {
+            r = INT128_MIN;
+        } else {
+            flags = float_flag_invalid | float_flag_invalid_cvti;
+            r = p.sign ? INT128_MIN : INT128_MAX;
+        }
+        break;
+
+    default:
+        g_assert_not_reached();
+    }
+
+    float_raise(flags, s);
+    return r;
 }
 
-uint64_t float32_to_uint64(float32 a, float_status *s)
+static int32_t floatx80_to_int32_scalbn(floatx80 a, FloatRoundMode rmode,
+                                        int scale, float_status *s)
 {
-    return float32_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts128 p;
+
+    if (!floatx80_unpack_canonical(&p, a, s)) {
+        parts_default_nan(&p, s);
+    }
+    return parts_float_to_sint(&p, rmode, scale, INT32_MIN, INT32_MAX, s);
 }
 
-uint16_t float64_to_uint16(float64 a, float_status *s)
+static int64_t floatx80_to_int64_scalbn(floatx80 a, FloatRoundMode rmode,
+                                        int scale, float_status *s)
 {
-    return float64_to_uint16_scalbn(a, s->float_rounding_mode, 0, s);
+    FloatParts128 p;
+
+    if (!floatx80_unpack_canonical(&p, a, s)) {
+        parts_default_nan(&p, s);
+    }
+    return parts_float_to_sint(&p, rmode, scale, INT64_MIN, INT64_MAX, s);
 }
 
-uint32_t float64_to_uint32(float64 a, float_status *s)
+int8_t float16_to_int8(float16 a, float_status *s)
 {
-    return float64_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
+    return float16_to_int8_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint64_t float64_to_uint64(float64 a, float_status *s)
+int16_t float16_to_int16(float16 a, float_status *s)
 {
-    return float64_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
+    return float16_to_int16_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint16_t float16_to_uint16_round_to_zero(float16 a, float_status *s)
+int32_t float16_to_int32(float16 a, float_status *s)
 {
-    return float16_to_uint16_scalbn(a, float_round_to_zero, 0, s);
+    return float16_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint32_t float16_to_uint32_round_to_zero(float16 a, float_status *s)
+int64_t float16_to_int64(float16 a, float_status *s)
 {
-    return float16_to_uint32_scalbn(a, float_round_to_zero, 0, s);
+    return float16_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint64_t float16_to_uint64_round_to_zero(float16 a, float_status *s)
+int16_t float32_to_int16(float32 a, float_status *s)
 {
-    return float16_to_uint64_scalbn(a, float_round_to_zero, 0, s);
+    return float32_to_int16_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint16_t float32_to_uint16_round_to_zero(float32 a, float_status *s)
+int32_t float32_to_int32(float32 a, float_status *s)
 {
-    return float32_to_uint16_scalbn(a, float_round_to_zero, 0, s);
+    return float32_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint32_t float32_to_uint32_round_to_zero(float32 a, float_status *s)
+int64_t float32_to_int64(float32 a, float_status *s)
 {
-    return float32_to_uint32_scalbn(a, float_round_to_zero, 0, s);
+    return float32_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint64_t float32_to_uint64_round_to_zero(float32 a, float_status *s)
+int16_t float64_to_int16(float64 a, float_status *s)
 {
-    return float32_to_uint64_scalbn(a, float_round_to_zero, 0, s);
+    return float64_to_int16_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint16_t float64_to_uint16_round_to_zero(float64 a, float_status *s)
+int32_t float64_to_int32(float64 a, float_status *s)
 {
-    return float64_to_uint16_scalbn(a, float_round_to_zero, 0, s);
+    return float64_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint32_t float64_to_uint32_round_to_zero(float64 a, float_status *s)
+int64_t float64_to_int64(float64 a, float_status *s)
 {
-    return float64_to_uint32_scalbn(a, float_round_to_zero, 0, s);
+    return float64_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-uint64_t float64_to_uint64_round_to_zero(float64 a, float_status *s)
+int32_t float128_to_int32(float128 a, float_status *s)
 {
-    return float64_to_uint64_scalbn(a, float_round_to_zero, 0, s);
+    return float128_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-/*
- * Integer to float conversions
- *
- * Returns the result of converting the two's complement integer `a'
- * to the floating-point format. The conversion is performed according
- * to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
- */
-
-static FloatParts int_to_float(int64_t a, int scale, float_status *status)
+int64_t float128_to_int64(float128 a, float_status *s)
 {
-    FloatParts r = { .sign = false };
-
-    if (a == 0) {
-        r.cls = float_class_zero;
-    } else {
-        uint64_t f = a;
-        int shift;
+    return float128_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-        r.cls = float_class_normal;
-        if (a < 0) {
-#ifdef _MSC_VER
-            f = 0ULL - f;
-#else
-            f = -f;
-#endif
-            r.sign = true;
-        }
-        shift = clz64(f) - 1;
-        scale = MIN(MAX(scale, -0x10000), 0x10000);
+Int128 float128_to_int128(float128 a, float_status *s)
+{
+    return float128_to_int128_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-        r.exp = DECOMPOSED_BINARY_POINT - shift + scale;
-        r.frac = (shift < 0 ? DECOMPOSED_IMPLICIT_BIT : f << shift);
-    }
+int32_t floatx80_to_int32(floatx80 a, float_status *s)
+{
+    return floatx80_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    return r;
+int64_t floatx80_to_int64(floatx80 a, float_status *s)
+{
+    return floatx80_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-float16 int64_to_float16_scalbn(int64_t a, int scale, float_status *status)
+int16_t float16_to_int16_round_to_zero(float16 a, float_status *s)
 {
-    FloatParts pa = int_to_float(a, scale, status);
-    return float16_round_pack_canonical(pa, status);
+    return float16_to_int16_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float16 int32_to_float16_scalbn(int32_t a, int scale, float_status *status)
+int32_t float16_to_int32_round_to_zero(float16 a, float_status *s)
 {
-    return int64_to_float16_scalbn(a, scale, status);
+    return float16_to_int32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float16 int16_to_float16_scalbn(int16_t a, int scale, float_status *status)
+int64_t float16_to_int64_round_to_zero(float16 a, float_status *s)
 {
-    return int64_to_float16_scalbn(a, scale, status);
+    return float16_to_int64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float16 int64_to_float16(int64_t a, float_status *status)
+int16_t float32_to_int16_round_to_zero(float32 a, float_status *s)
 {
-    return int64_to_float16_scalbn(a, 0, status);
+    return float32_to_int16_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float16 int32_to_float16(int32_t a, float_status *status)
+int32_t float32_to_int32_round_to_zero(float32 a, float_status *s)
 {
-    return int64_to_float16_scalbn(a, 0, status);
+    return float32_to_int32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float16 int16_to_float16(int16_t a, float_status *status)
+int64_t float32_to_int64_round_to_zero(float32 a, float_status *s)
 {
-    return int64_to_float16_scalbn(a, 0, status);
+    return float32_to_int64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float32 int64_to_float32_scalbn(int64_t a, int scale, float_status *status)
+int16_t float64_to_int16_round_to_zero(float64 a, float_status *s)
 {
-    FloatParts pa = int_to_float(a, scale, status);
-    return float32_round_pack_canonical(pa, status);
+    return float64_to_int16_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float32 int32_to_float32_scalbn(int32_t a, int scale, float_status *status)
+int32_t float64_to_int32_round_to_zero(float64 a, float_status *s)
 {
-    return int64_to_float32_scalbn(a, scale, status);
+    return float64_to_int32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float32 int16_to_float32_scalbn(int16_t a, int scale, float_status *status)
+int64_t float64_to_int64_round_to_zero(float64 a, float_status *s)
 {
-    return int64_to_float32_scalbn(a, scale, status);
+    return float64_to_int64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float32 int64_to_float32(int64_t a, float_status *status)
+int32_t float128_to_int32_round_to_zero(float128 a, float_status *s)
 {
-    return int64_to_float32_scalbn(a, 0, status);
+    return float128_to_int32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float32 int32_to_float32(int32_t a, float_status *status)
+int64_t float128_to_int64_round_to_zero(float128 a, float_status *s)
 {
-    return int64_to_float32_scalbn(a, 0, status);
+    return float128_to_int64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float32 int16_to_float32(int16_t a, float_status *status)
+Int128 float128_to_int128_round_to_zero(float128 a, float_status *s)
 {
-    return int64_to_float32_scalbn(a, 0, status);
+    return float128_to_int128_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float64 int64_to_float64_scalbn(int64_t a, int scale, float_status *status)
+int32_t floatx80_to_int32_round_to_zero(floatx80 a, float_status *s)
 {
-    FloatParts pa = int_to_float(a, scale, status);
-    return float64_round_pack_canonical(pa, status);
+    return floatx80_to_int32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float64 int32_to_float64_scalbn(int32_t a, int scale, float_status *status)
+int64_t floatx80_to_int64_round_to_zero(floatx80 a, float_status *s)
 {
-    return int64_to_float64_scalbn(a, scale, status);
+    return floatx80_to_int64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float64 int16_to_float64_scalbn(int16_t a, int scale, float_status *status)
+int16_t bfloat16_to_int16(bfloat16 a, float_status *s)
 {
-    return int64_to_float64_scalbn(a, scale, status);
+    return bfloat16_to_int16_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-float64 int64_to_float64(int64_t a, float_status *status)
+int32_t bfloat16_to_int32(bfloat16 a, float_status *s)
 {
-    return int64_to_float64_scalbn(a, 0, status);
+    return bfloat16_to_int32_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-float64 int32_to_float64(int32_t a, float_status *status)
+int64_t bfloat16_to_int64(bfloat16 a, float_status *s)
 {
-    return int64_to_float64_scalbn(a, 0, status);
+    return bfloat16_to_int64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-float64 int16_to_float64(int16_t a, float_status *status)
+int16_t bfloat16_to_int16_round_to_zero(bfloat16 a, float_status *s)
 {
-    return int64_to_float64_scalbn(a, 0, status);
+    return bfloat16_to_int16_scalbn(a, float_round_to_zero, 0, s);
 }
 
+int32_t bfloat16_to_int32_round_to_zero(bfloat16 a, float_status *s)
+{
+    return bfloat16_to_int32_scalbn(a, float_round_to_zero, 0, s);
+}
+
+int64_t bfloat16_to_int64_round_to_zero(bfloat16 a, float_status *s)
+{
+    return bfloat16_to_int64_scalbn(a, float_round_to_zero, 0, s);
+}
 
 /*
- * Unsigned Integer to float conversions
- *
- * Returns the result of converting the unsigned integer `a' to the
- * floating-point format. The conversion is performed according to the
- * IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+ * Floating-point to unsigned integer conversions
  */
 
-static FloatParts uint_to_float(uint64_t a, int scale, float_status *status)
+uint8_t float16_to_uint8_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                                float_status *s)
 {
-    FloatParts r = { .sign = false };
-
-    if (a == 0) {
-        r.cls = float_class_zero;
-    } else {
-        scale = MIN(MAX(scale, -0x10000), 0x10000);
-        r.cls = float_class_normal;
-        if ((int64_t)a < 0) {
-            r.exp = DECOMPOSED_BINARY_POINT + 1 + scale;
-            shift64RightJamming(a, 1, &a);
-            r.frac = a;
-        } else {
-            int shift = clz64(a) - 1;
-            r.exp = DECOMPOSED_BINARY_POINT - shift + scale;
-            r.frac = a << shift;
-        }
-    }
+    FloatParts64 p;
 
-    return r;
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT8_MAX, s);
 }
 
-float16 uint64_to_float16_scalbn(uint64_t a, int scale, float_status *status)
+uint16_t float16_to_uint16_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    FloatParts pa = uint_to_float(a, scale, status);
-    return float16_round_pack_canonical(pa, status);
-}
+    FloatParts64 p;
 
-float16 uint32_to_float16_scalbn(uint32_t a, int scale, float_status *status)
-{
-    return uint64_to_float16_scalbn(a, scale, status);
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT16_MAX, s);
 }
 
-float16 uint16_to_float16_scalbn(uint16_t a, int scale, float_status *status)
+uint32_t float16_to_uint32_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    return uint64_to_float16_scalbn(a, scale, status);
-}
+    FloatParts64 p;
 
-float16 uint64_to_float16(uint64_t a, float_status *status)
-{
-    return uint64_to_float16_scalbn(a, 0, status);
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT32_MAX, s);
 }
 
-float16 uint32_to_float16(uint32_t a, float_status *status)
+uint64_t float16_to_uint64_scalbn(float16 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    return uint64_to_float16_scalbn(a, 0, status);
-}
+    FloatParts64 p;
 
-float16 uint16_to_float16(uint16_t a, float_status *status)
-{
-    return uint64_to_float16_scalbn(a, 0, status);
+    float16_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT64_MAX, s);
 }
 
-float32 uint64_to_float32_scalbn(uint64_t a, int scale, float_status *status)
+uint16_t float32_to_uint16_scalbn(float32 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    FloatParts pa = uint_to_float(a, scale, status);
-    return float32_round_pack_canonical(pa, status);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT16_MAX, s);
 }
 
-float32 uint32_to_float32_scalbn(uint32_t a, int scale, float_status *status)
+uint32_t float32_to_uint32_scalbn(float32 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    return uint64_to_float32_scalbn(a, scale, status);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT32_MAX, s);
 }
 
-float32 uint16_to_float32_scalbn(uint16_t a, int scale, float_status *status)
+uint64_t float32_to_uint64_scalbn(float32 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    return uint64_to_float32_scalbn(a, scale, status);
+    FloatParts64 p;
+
+    float32_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT64_MAX, s);
 }
 
-float32 uint64_to_float32(uint64_t a, float_status *status)
+uint16_t float64_to_uint16_scalbn(float64 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    return uint64_to_float32_scalbn(a, 0, status);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT16_MAX, s);
 }
 
-float32 uint32_to_float32(uint32_t a, float_status *status)
+uint32_t float64_to_uint32_scalbn(float64 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    return uint64_to_float32_scalbn(a, 0, status);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT32_MAX, s);
 }
 
-float32 uint16_to_float32(uint16_t a, float_status *status)
+uint64_t float64_to_uint64_scalbn(float64 a, FloatRoundMode rmode, int scale,
+                                  float_status *s)
 {
-    return uint64_to_float32_scalbn(a, 0, status);
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT64_MAX, s);
 }
 
-float64 uint64_to_float64_scalbn(uint64_t a, int scale, float_status *status)
+uint16_t bfloat16_to_uint16_scalbn(bfloat16 a, FloatRoundMode rmode,
+                                   int scale, float_status *s)
 {
-    FloatParts pa = uint_to_float(a, scale, status);
-    return float64_round_pack_canonical(pa, status);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT16_MAX, s);
 }
 
-float64 uint32_to_float64_scalbn(uint32_t a, int scale, float_status *status)
+uint32_t bfloat16_to_uint32_scalbn(bfloat16 a, FloatRoundMode rmode,
+                                   int scale, float_status *s)
 {
-    return uint64_to_float64_scalbn(a, scale, status);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT32_MAX, s);
 }
 
-float64 uint16_to_float64_scalbn(uint16_t a, int scale, float_status *status)
+uint64_t bfloat16_to_uint64_scalbn(bfloat16 a, FloatRoundMode rmode,
+                                   int scale, float_status *s)
 {
-    return uint64_to_float64_scalbn(a, scale, status);
+    FloatParts64 p;
+
+    bfloat16_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT64_MAX, s);
 }
 
-float64 uint64_to_float64(uint64_t a, float_status *status)
+static uint32_t float128_to_uint32_scalbn(float128 a, FloatRoundMode rmode,
+                                          int scale, float_status *s)
 {
-    return uint64_to_float64_scalbn(a, 0, status);
+    FloatParts128 p;
+
+    float128_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT32_MAX, s);
 }
 
-float64 uint32_to_float64(uint32_t a, float_status *status)
+static uint64_t float128_to_uint64_scalbn(float128 a, FloatRoundMode rmode,
+                                          int scale, float_status *s)
 {
-    return uint64_to_float64_scalbn(a, 0, status);
+    FloatParts128 p;
+
+    float128_unpack_canonical(&p, a, s);
+    return parts_float_to_uint(&p, rmode, scale, UINT64_MAX, s);
 }
 
-float64 uint16_to_float64(uint16_t a, float_status *status)
+static Int128 float128_to_uint128_scalbn(float128 a, FloatRoundMode rmode,
+                                         int scale, float_status *s)
 {
-    return uint64_to_float64_scalbn(a, 0, status);
-}
+    int flags = 0;
+    Int128 r;
+    FloatParts128 p;
 
-/* Float Min/Max */
-/* min() and max() functions. These can't be implemented as
- * 'compare and pick one input' because that would mishandle
- * NaNs and +0 vs -0.
- *
- * minnum() and maxnum() functions. These are similar to the min()
- * and max() functions but if one of the arguments is a QNaN and
- * the other is numerical then the numerical argument is returned.
- * SNaNs will get quietened before being returned.
- * minnum() and maxnum correspond to the IEEE 754-2008 minNum()
- * and maxNum() operations. min() and max() are the typical min/max
- * semantics provided by many CPUs which predate that specification.
- *
- * minnummag() and maxnummag() functions correspond to minNumMag()
- * and minNumMag() from the IEEE-754 2008.
- */
-static FloatParts minmax_floats(FloatParts a, FloatParts b, bool ismin,
-                                bool ieee, bool ismag, float_status *s)
-{
-    if (unlikely(is_nan(a.cls) || is_nan(b.cls))) {
-        if (ieee) {
-            /* Takes two floating-point values `a' and `b', one of
-             * which is a NaN, and returns the appropriate NaN
-             * result. If either `a' or `b' is a signaling NaN,
-             * the invalid exception is raised.
-             */
-            if (is_snan(a.cls) || is_snan(b.cls)) {
-                return pick_nan(a, b, s);
-            } else if (is_nan(a.cls) && !is_nan(b.cls)) {
-                return b;
-            } else if (is_nan(b.cls) && !is_nan(a.cls)) {
-                return a;
-            }
-        }
-        return pick_nan(a, b, s);
-    } else {
-        int a_exp = 0, b_exp = 0;
+    float128_unpack_canonical(&p, a, s);
 
-        switch (a.cls) {
-        case float_class_normal:
-            a_exp = a.exp;
-            break;
-        case float_class_inf:
-            a_exp = INT_MAX;
-            break;
-        case float_class_zero:
-            a_exp = INT_MIN;
-            break;
-        default:
-            g_assert_not_reached();
-            break;
-        }
-        switch (b.cls) {
-        case float_class_normal:
-            b_exp = b.exp;
-            break;
-        case float_class_inf:
-            b_exp = INT_MAX;
-            break;
-        case float_class_zero:
-            b_exp = INT_MIN;
-            break;
-        default:
-            g_assert_not_reached();
-            break;
-        }
+    switch (p.cls) {
+    case float_class_snan:
+        flags |= float_flag_invalid_snan;
+        /* fall through */
+    case float_class_qnan:
+        flags |= float_flag_invalid;
+        r = UINT128_MAX;
+        break;
+
+    case float_class_inf:
+        flags = float_flag_invalid | float_flag_invalid_cvti;
+        r = p.sign ? int128_zero() : UINT128_MAX;
+        break;
 
-        if (ismag && (a_exp != b_exp || a.frac != b.frac)) {
-            bool a_less = a_exp < b_exp;
-            if (a_exp == b_exp) {
-                a_less = a.frac < b.frac;
+    case float_class_zero:
+        return int128_zero();
+
+    case float_class_normal:
+        if (parts_round_to_int_normal(&p, rmode, scale, 128 - 2)) {
+            flags = float_flag_inexact;
+            if (p.cls == float_class_zero) {
+                r = int128_zero();
+                break;
             }
-            return a_less ^ ismin ? b : a;
         }
 
-        if (a.sign == b.sign) {
-            bool a_less = a_exp < b_exp;
-            if (a_exp == b_exp) {
-                a_less = a.frac < b.frac;
-            }
-            return a.sign ^ a_less ^ ismin ? b : a;
+        if (p.sign) {
+            flags = float_flag_invalid | float_flag_invalid_cvti;
+            r = int128_zero();
+        } else if (p.exp <= 127) {
+            int shift = 127 - p.exp;
+            r = int128_urshift(int128_make128(p.frac_lo, p.frac_hi), shift);
         } else {
-            return a.sign ^ ismin ? b : a;
+            flags = float_flag_invalid | float_flag_invalid_cvti;
+            r = UINT128_MAX;
         }
-    }
-}
+        break;
 
-#define MINMAX(sz, name, ismin, isiee, ismag)                           \
-float ## sz float ## sz ## _ ## name(float ## sz a, float ## sz b,      \
-                                     float_status *s)                   \
-{                                                                       \
-    FloatParts pa = float ## sz ## _unpack_canonical(a, s);             \
-    FloatParts pb = float ## sz ## _unpack_canonical(b, s);             \
-    FloatParts pr = minmax_floats(pa, pb, ismin, isiee, ismag, s);      \
-                                                                        \
-    return float ## sz ## _round_pack_canonical(pr, s);                 \
-}
-
-MINMAX(16, min, true, false, false)
-MINMAX(16, minnum, true, true, false)
-MINMAX(16, minnummag, true, true, true)
-MINMAX(16, max, false, false, false)
-MINMAX(16, maxnum, false, true, false)
-MINMAX(16, maxnummag, false, true, true)
-
-MINMAX(32, min, true, false, false)
-MINMAX(32, minnum, true, true, false)
-MINMAX(32, minnummag, true, true, true)
-MINMAX(32, max, false, false, false)
-MINMAX(32, maxnum, false, true, false)
-MINMAX(32, maxnummag, false, true, true)
-
-MINMAX(64, min, true, false, false)
-MINMAX(64, minnum, true, true, false)
-MINMAX(64, minnummag, true, true, true)
-MINMAX(64, max, false, false, false)
-MINMAX(64, maxnum, false, true, false)
-MINMAX(64, maxnummag, false, true, true)
-
-#undef MINMAX
-
-/* Floating point compare */
-static int compare_floats(FloatParts a, FloatParts b, bool is_quiet,
-                          float_status *s)
-{
-    if (is_nan(a.cls) || is_nan(b.cls)) {
-        if (!is_quiet ||
-            a.cls == float_class_snan ||
-            b.cls == float_class_snan) {
-            s->float_exception_flags |= float_flag_invalid;
-        }
-        return float_relation_unordered;
+    default:
+        g_assert_not_reached();
     }
 
-    if (a.cls == float_class_zero) {
-        if (b.cls == float_class_zero) {
-            return float_relation_equal;
-        }
-        return b.sign ? float_relation_greater : float_relation_less;
-    } else if (b.cls == float_class_zero) {
-        return a.sign ? float_relation_less : float_relation_greater;
-    }
+    float_raise(flags, s);
+    return r;
+}
 
-    /* The only really important thing about infinity is its sign. If
-     * both are infinities the sign marks the smallest of the two.
-     */
-    if (a.cls == float_class_inf) {
-        if ((b.cls == float_class_inf) && (a.sign == b.sign)) {
-            return float_relation_equal;
-        }
-        return a.sign ? float_relation_less : float_relation_greater;
-    } else if (b.cls == float_class_inf) {
-        return b.sign ? float_relation_greater : float_relation_less;
-    }
+uint8_t float16_to_uint8(float16 a, float_status *s)
+{
+    return float16_to_uint8_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    if (a.sign != b.sign) {
-        return a.sign ? float_relation_less : float_relation_greater;
-    }
+uint16_t float16_to_uint16(float16 a, float_status *s)
+{
+    return float16_to_uint16_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    if (a.exp == b.exp) {
-        if (a.frac == b.frac) {
-            return float_relation_equal;
-        }
-        if (a.sign) {
-            return a.frac > b.frac ?
-                float_relation_less : float_relation_greater;
-        } else {
-            return a.frac > b.frac ?
-                float_relation_greater : float_relation_less;
-        }
-    } else {
-        if (a.sign) {
-            return a.exp > b.exp ? float_relation_less : float_relation_greater;
-        } else {
-            return a.exp > b.exp ? float_relation_greater : float_relation_less;
-        }
-    }
+uint32_t float16_to_uint32(float16 a, float_status *s)
+{
+    return float16_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-#define COMPARE(name, attr, sz)                                         \
-static int attr                                                         \
-name(float ## sz a, float ## sz b, bool is_quiet, float_status *s)      \
-{                                                                       \
-    FloatParts pa = float ## sz ## _unpack_canonical(a, s);             \
-    FloatParts pb = float ## sz ## _unpack_canonical(b, s);             \
-    return compare_floats(pa, pb, is_quiet, s);                         \
+uint64_t float16_to_uint64(float16 a, float_status *s)
+{
+    return float16_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-COMPARE(soft_f16_compare, QEMU_FLATTEN, 16)
-COMPARE(soft_f32_compare, QEMU_SOFTFLOAT_ATTR, 32)
-COMPARE(soft_f64_compare, QEMU_SOFTFLOAT_ATTR, 64)
+uint16_t float32_to_uint16(float32 a, float_status *s)
+{
+    return float32_to_uint16_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-#undef COMPARE
+uint32_t float32_to_uint32(float32 a, float_status *s)
+{
+    return float32_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-int float16_compare(float16 a, float16 b, float_status *s)
+uint64_t float32_to_uint64(float32 a, float_status *s)
 {
-    return soft_f16_compare(a, b, false, s);
+    return float32_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-int float16_compare_quiet(float16 a, float16 b, float_status *s)
+uint16_t float64_to_uint16(float64 a, float_status *s)
 {
-    return soft_f16_compare(a, b, true, s);
+    return float64_to_uint16_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-static int QEMU_FLATTEN
-f32_compare(float32 xa, float32 xb, bool is_quiet, float_status *s)
+uint32_t float64_to_uint32(float64 a, float_status *s)
 {
-    union_float32 ua, ub;
+    return float64_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    ua.s = xa;
-    ub.s = xb;
+uint64_t float64_to_uint64(float64 a, float_status *s)
+{
+    return float64_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    if (QEMU_NO_HARDFLOAT) {
-        goto soft;
-    }
+uint32_t float128_to_uint32(float128 a, float_status *s)
+{
+    return float128_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    float32_input_flush2(&ua.s, &ub.s, s);
-    if (isgreaterequal(ua.h, ub.h)) {
-        if (isgreater(ua.h, ub.h)) {
-            return float_relation_greater;
-        }
-        return float_relation_equal;
-    }
-    if (likely(isless(ua.h, ub.h))) {
-        return float_relation_less;
-    }
-    /* The only condition remaining is unordered.
-     * Fall through to set flags.
-     */
- soft:
-    return soft_f32_compare(ua.s, ub.s, is_quiet, s);
+uint64_t float128_to_uint64(float128 a, float_status *s)
+{
+    return float128_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-int float32_compare(float32 a, float32 b, float_status *s)
+Int128 float128_to_uint128(float128 a, float_status *s)
 {
-    return f32_compare(a, b, false, s);
+    return float128_to_uint128_scalbn(a, s->float_rounding_mode, 0, s);
 }
 
-int float32_compare_quiet(float32 a, float32 b, float_status *s)
+uint16_t float16_to_uint16_round_to_zero(float16 a, float_status *s)
 {
-    return f32_compare(a, b, true, s);
+    return float16_to_uint16_scalbn(a, float_round_to_zero, 0, s);
 }
 
-static int QEMU_FLATTEN
-f64_compare(float64 xa, float64 xb, bool is_quiet, float_status *s)
+uint32_t float16_to_uint32_round_to_zero(float16 a, float_status *s)
 {
-    union_float64 ua, ub;
+    return float16_to_uint32_scalbn(a, float_round_to_zero, 0, s);
+}
 
-    ua.s = xa;
-    ub.s = xb;
+uint64_t float16_to_uint64_round_to_zero(float16 a, float_status *s)
+{
+    return float16_to_uint64_scalbn(a, float_round_to_zero, 0, s);
+}
 
-    if (QEMU_NO_HARDFLOAT) {
-        goto soft;
-    }
+uint16_t float32_to_uint16_round_to_zero(float32 a, float_status *s)
+{
+    return float32_to_uint16_scalbn(a, float_round_to_zero, 0, s);
+}
 
-    float64_input_flush2(&ua.s, &ub.s, s);
-    if (isgreaterequal(ua.h, ub.h)) {
-        if (isgreater(ua.h, ub.h)) {
-            return float_relation_greater;
-        }
-        return float_relation_equal;
-    }
-    if (likely(isless(ua.h, ub.h))) {
-        return float_relation_less;
-    }
-    /* The only condition remaining is unordered.
-     * Fall through to set flags.
-     */
- soft:
-    return soft_f64_compare(ua.s, ub.s, is_quiet, s);
+uint32_t float32_to_uint32_round_to_zero(float32 a, float_status *s)
+{
+    return float32_to_uint32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-int float64_compare(float64 a, float64 b, float_status *s)
+uint64_t float32_to_uint64_round_to_zero(float32 a, float_status *s)
 {
-    return f64_compare(a, b, false, s);
+    return float32_to_uint64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-int float64_compare_quiet(float64 a, float64 b, float_status *s)
+uint16_t float64_to_uint16_round_to_zero(float64 a, float_status *s)
 {
-    return f64_compare(a, b, true, s);
+    return float64_to_uint16_scalbn(a, float_round_to_zero, 0, s);
 }
 
-/* Multiply A by 2 raised to the power N.  */
-static FloatParts scalbn_decomposed(FloatParts a, int n, float_status *s)
+uint32_t float64_to_uint32_round_to_zero(float64 a, float_status *s)
 {
-    if (unlikely(is_nan(a.cls))) {
-        return return_nan(a, s);
-    }
-    if (a.cls == float_class_normal) {
-        /* The largest float type (even though not supported by FloatParts)
-         * is float128, which has a 15 bit exponent.  Bounding N to 16 bits
-         * still allows rounding to infinity, without allowing overflow
-         * within the int32_t that backs FloatParts.exp.
-         */
-        n = MIN(MAX(n, -0x10000), 0x10000);
-        a.exp += n;
-    }
-    return a;
+    return float64_to_uint32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float16 float16_scalbn(float16 a, int n, float_status *status)
+uint64_t float64_to_uint64_round_to_zero(float64 a, float_status *s)
 {
-    FloatParts pa = float16_unpack_canonical(a, status);
-    FloatParts pr = scalbn_decomposed(pa, n, status);
-    return float16_round_pack_canonical(pr, status);
+    return float64_to_uint64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float32 float32_scalbn(float32 a, int n, float_status *status)
+uint32_t float128_to_uint32_round_to_zero(float128 a, float_status *s)
 {
-    FloatParts pa = float32_unpack_canonical(a, status);
-    FloatParts pr = scalbn_decomposed(pa, n, status);
-    return float32_round_pack_canonical(pr, status);
+    return float128_to_uint32_scalbn(a, float_round_to_zero, 0, s);
 }
 
-float64 float64_scalbn(float64 a, int n, float_status *status)
+uint64_t float128_to_uint64_round_to_zero(float128 a, float_status *s)
 {
-    FloatParts pa = float64_unpack_canonical(a, status);
-    FloatParts pr = scalbn_decomposed(pa, n, status);
-    return float64_round_pack_canonical(pr, status);
+    return float128_to_uint64_scalbn(a, float_round_to_zero, 0, s);
 }
 
-/*
- * Square Root
- *
- * The old softfloat code did an approximation step before zeroing in
- * on the final result. However for simpleness we just compute the
- * square root by iterating down from the implicit bit to enough extra
- * bits to ensure we get a correctly rounded result.
- *
- * This does mean however the calculation is slower than before,
- * especially for 64 bit floats.
- */
+Int128 float128_to_uint128_round_to_zero(float128 a, float_status *s)
+{
+    return float128_to_uint128_scalbn(a, float_round_to_zero, 0, s);
+}
 
-static FloatParts sqrt_float(FloatParts a, float_status *s, const FloatFmt *p)
+uint16_t bfloat16_to_uint16(bfloat16 a, float_status *s)
 {
-    uint64_t a_frac, r_frac, s_frac;
-    int bit, last_bit;
+    return bfloat16_to_uint16_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    if (is_nan(a.cls)) {
-        return return_nan(a, s);
-    }
-    if (a.cls == float_class_zero) {
-        return a;  /* sqrt(+-0) = +-0 */
-    }
-    if (a.sign) {
-        s->float_exception_flags |= float_flag_invalid;
-        return parts_default_nan(s);
-    }
-    if (a.cls == float_class_inf) {
-        return a;  /* sqrt(+inf) = +inf */
-    }
+uint32_t bfloat16_to_uint32(bfloat16 a, float_status *s)
+{
+    return bfloat16_to_uint32_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    assert(a.cls == float_class_normal);
+uint64_t bfloat16_to_uint64(bfloat16 a, float_status *s)
+{
+    return bfloat16_to_uint64_scalbn(a, s->float_rounding_mode, 0, s);
+}
 
-    /* We need two overflow bits at the top. Adding room for that is a
-     * right shift. If the exponent is odd, we can discard the low bit
-     * by multiplying the fraction by 2; that's a left shift. Combine
-     * those and we shift right if the exponent is even.
-     */
-    a_frac = a.frac;
-    if (!(a.exp & 1)) {
-        a_frac >>= 1;
-    }
-    a.exp >>= 1;
+uint16_t bfloat16_to_uint16_round_to_zero(bfloat16 a, float_status *s)
+{
+    return bfloat16_to_uint16_scalbn(a, float_round_to_zero, 0, s);
+}
 
-    /* Bit-by-bit computation of sqrt.  */
-    r_frac = 0;
-    s_frac = 0;
+uint32_t bfloat16_to_uint32_round_to_zero(bfloat16 a, float_status *s)
+{
+    return bfloat16_to_uint32_scalbn(a, float_round_to_zero, 0, s);
+}
 
-    /* Iterate from implicit bit down to the 3 extra bits to compute a
-     * properly rounded result. Remember we've inserted one more bit
-     * at the top, so these positions are one less.
-     */
-    bit = DECOMPOSED_BINARY_POINT - 1;
-    last_bit = MAX(p->frac_shift - 4, 0);
-    do {
-        uint64_t q = 1ULL << bit;
-        uint64_t t_frac = s_frac + q;
-        if (t_frac <= a_frac) {
-            s_frac = t_frac + q;
-            a_frac -= t_frac;
-            r_frac += q;
-        }
-        a_frac <<= 1;
-    } while (--bit >= last_bit);
+uint64_t bfloat16_to_uint64_round_to_zero(bfloat16 a, float_status *s)
+{
+    return bfloat16_to_uint64_scalbn(a, float_round_to_zero, 0, s);
+}
 
-    /* Undo the right shift done above. If there is any remaining
-     * fraction, the result is inexact. Set the sticky bit.
-     */
-    a.frac = (r_frac << 1) + (a_frac != 0);
+/*
+ * Signed integer to floating-point conversions
+ */
 
-    return a;
+float16 int64_to_float16_scalbn(int64_t a, int scale, float_status *status)
+{
+    FloatParts64 p;
+
+    parts_sint_to_float(&p, a, scale, status);
+    return float16_round_pack_canonical(&p, status);
 }
 
-float16 QEMU_FLATTEN float16_sqrt(float16 a, float_status *status)
+float16 int32_to_float16_scalbn(int32_t a, int scale, float_status *status)
 {
-    FloatParts pa = float16_unpack_canonical(a, status);
-    FloatParts pr = sqrt_float(pa, status, &float16_params);
-    return float16_round_pack_canonical(pr, status);
+    return int64_to_float16_scalbn(a, scale, status);
 }
 
-static float32 QEMU_SOFTFLOAT_ATTR
-soft_f32_sqrt(float32 a, float_status *status)
+float16 int16_to_float16_scalbn(int16_t a, int scale, float_status *status)
 {
-    FloatParts pa = float32_unpack_canonical(a, status);
-    FloatParts pr = sqrt_float(pa, status, &float32_params);
-    return float32_round_pack_canonical(pr, status);
+    return int64_to_float16_scalbn(a, scale, status);
 }
 
-static float64 QEMU_SOFTFLOAT_ATTR
-soft_f64_sqrt(float64 a, float_status *status)
+float16 int64_to_float16(int64_t a, float_status *status)
 {
-    FloatParts pa = float64_unpack_canonical(a, status);
-    FloatParts pr = sqrt_float(pa, status, &float64_params);
-    return float64_round_pack_canonical(pr, status);
+    return int64_to_float16_scalbn(a, 0, status);
 }
 
-float32 QEMU_FLATTEN float32_sqrt(float32 xa, float_status *s)
+float16 int32_to_float16(int32_t a, float_status *status)
 {
-    union_float32 ua, ur;
-
-    ua.s = xa;
-    if (unlikely(!can_use_fpu(s))) {
-        goto soft;
-    }
+    return int64_to_float16_scalbn(a, 0, status);
+}
 
-    float32_input_flush1(&ua.s, s);
-    if (QEMU_HARDFLOAT_1F32_USE_FP) {
-        if (unlikely(!(fpclassify(ua.h) == FP_NORMAL ||
-                       fpclassify(ua.h) == FP_ZERO) ||
-                     signbit(ua.h))) {
-            goto soft;
-        }
-    } else if (unlikely(!float32_is_zero_or_normal(ua.s) ||
-                        float32_is_neg(ua.s))) {
-        goto soft;
-    }
-    ur.h = sqrtf(ua.h);
-    return ur.s;
+float16 int16_to_float16(int16_t a, float_status *status)
+{
+    return int64_to_float16_scalbn(a, 0, status);
+}
 
- soft:
-    return soft_f32_sqrt(ua.s, s);
+float16 int8_to_float16(int8_t a, float_status *status)
+{
+    return int64_to_float16_scalbn(a, 0, status);
 }
 
-float64 QEMU_FLATTEN float64_sqrt(float64 xa, float_status *s)
+float32 int64_to_float32_scalbn(int64_t a, int scale, float_status *status)
 {
-    union_float64 ua, ur;
+    FloatParts64 p;
 
-    ua.s = xa;
-    if (unlikely(!can_use_fpu(s))) {
-        goto soft;
+    /* Without scaling, there are no overflow concerns. */
+    if (likely(scale == 0) && can_use_fpu(status)) {
+        union_float32 ur;
+        ur.h = a;
+        return ur.s;
     }
 
-    float64_input_flush1(&ua.s, s);
-    if (QEMU_HARDFLOAT_1F64_USE_FP) {
-        if (unlikely(!(fpclassify(ua.h) == FP_NORMAL ||
-                       fpclassify(ua.h) == FP_ZERO) ||
-                     signbit(ua.h))) {
-            goto soft;
-        }
-    } else if (unlikely(!float64_is_zero_or_normal(ua.s) ||
-                        float64_is_neg(ua.s))) {
-        goto soft;
-    }
-    ur.h = sqrt(ua.h);
-    return ur.s;
+    parts64_sint_to_float(&p, a, scale, status);
+    return float32_round_pack_canonical(&p, status);
+}
 
- soft:
-    return soft_f64_sqrt(ua.s, s);
+float32 int32_to_float32_scalbn(int32_t a, int scale, float_status *status)
+{
+    return int64_to_float32_scalbn(a, scale, status);
 }
 
-/*----------------------------------------------------------------------------
-| The pattern for a default generated NaN.
-*----------------------------------------------------------------------------*/
+float32 int16_to_float32_scalbn(int16_t a, int scale, float_status *status)
+{
+    return int64_to_float32_scalbn(a, scale, status);
+}
 
-float16 float16_default_nan(float_status *status)
+float32 int64_to_float32(int64_t a, float_status *status)
 {
-    FloatParts p = parts_default_nan(status);
-    p.frac >>= float16_params.frac_shift;
-    return float16_pack_raw(p);
+    return int64_to_float32_scalbn(a, 0, status);
 }
 
-float32 float32_default_nan(float_status *status)
+float32 int32_to_float32(int32_t a, float_status *status)
 {
-    FloatParts p = parts_default_nan(status);
-    p.frac >>= float32_params.frac_shift;
-    return float32_pack_raw(p);
+    return int64_to_float32_scalbn(a, 0, status);
 }
 
-float64 float64_default_nan(float_status *status)
+float32 int16_to_float32(int16_t a, float_status *status)
 {
-    FloatParts p = parts_default_nan(status);
-    p.frac >>= float64_params.frac_shift;
-    return float64_pack_raw(p);
+    return int64_to_float32_scalbn(a, 0, status);
 }
 
-float128 float128_default_nan(float_status *status)
+float64 int64_to_float64_scalbn(int64_t a, int scale, float_status *status)
 {
-    FloatParts p = parts_default_nan(status);
-    float128 r;
+    FloatParts64 p;
 
-    /* Extrapolate from the choices made by parts_default_nan to fill
-     * in the quad-floating format.  If the low bit is set, assume we
-     * want to set all non-snan bits.
-     */
-#ifdef _MSC_VER
-    r.low = 0ULL - (p.frac & 1);
-#else
-    r.low = -(p.frac & 1);
-#endif
-    r.high = p.frac >> (DECOMPOSED_BINARY_POINT - 48);
-    r.high |= UINT64_C(0x7FFF000000000000);
-    r.high |= (uint64_t)p.sign << 63;
+    /* Without scaling, there are no overflow concerns. */
+    if (likely(scale == 0) && can_use_fpu(status)) {
+        union_float64 ur;
+        ur.h = a;
+        return ur.s;
+    }
 
-    return r;
+    parts_sint_to_float(&p, a, scale, status);
+    return float64_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns a quiet NaN from a signalling NaN for the floating point value `a'.
-*----------------------------------------------------------------------------*/
-
-float16 float16_silence_nan(float16 a, float_status *status)
+float64 int32_to_float64_scalbn(int32_t a, int scale, float_status *status)
 {
-    FloatParts p = float16_unpack_raw(a);
-    p.frac <<= float16_params.frac_shift;
-    p = parts_silence_nan(p, status);
-    p.frac >>= float16_params.frac_shift;
-    return float16_pack_raw(p);
+    return int64_to_float64_scalbn(a, scale, status);
 }
 
-float32 float32_silence_nan(float32 a, float_status *status)
+float64 int16_to_float64_scalbn(int16_t a, int scale, float_status *status)
 {
-    FloatParts p = float32_unpack_raw(a);
-    p.frac <<= float32_params.frac_shift;
-    p = parts_silence_nan(p, status);
-    p.frac >>= float32_params.frac_shift;
-    return float32_pack_raw(p);
+    return int64_to_float64_scalbn(a, scale, status);
 }
 
-float64 float64_silence_nan(float64 a, float_status *status)
+float64 int64_to_float64(int64_t a, float_status *status)
 {
-    FloatParts p = float64_unpack_raw(a);
-    p.frac <<= float64_params.frac_shift;
-    p = parts_silence_nan(p, status);
-    p.frac >>= float64_params.frac_shift;
-    return float64_pack_raw(p);
+    return int64_to_float64_scalbn(a, 0, status);
 }
 
+float64 int32_to_float64(int32_t a, float_status *status)
+{
+    return int64_to_float64_scalbn(a, 0, status);
+}
 
-/*----------------------------------------------------------------------------
-| If `a' is denormal and we are in flush-to-zero mode then set the
-| input-denormal exception and return zero. Otherwise just return the value.
-*----------------------------------------------------------------------------*/
+float64 int16_to_float64(int16_t a, float_status *status)
+{
+    return int64_to_float64_scalbn(a, 0, status);
+}
 
-static bool parts_squash_denormal(FloatParts p, float_status *status)
+bfloat16 int64_to_bfloat16_scalbn(int64_t a, int scale, float_status *status)
 {
-    if (p.exp == 0 && p.frac != 0) {
-        float_raise(float_flag_input_denormal, status);
-        return true;
-    }
+    FloatParts64 p;
 
-    return false;
+    parts_sint_to_float(&p, a, scale, status);
+    return bfloat16_round_pack_canonical(&p, status);
 }
 
-float16 float16_squash_input_denormal(float16 a, float_status *status)
+bfloat16 int32_to_bfloat16_scalbn(int32_t a, int scale, float_status *status)
 {
-    if (status->flush_inputs_to_zero) {
-        FloatParts p = float16_unpack_raw(a);
-        if (parts_squash_denormal(p, status)) {
-            return float16_set_sign(float16_zero, p.sign);
-        }
-    }
-    return a;
+    return int64_to_bfloat16_scalbn(a, scale, status);
 }
 
-float32 float32_squash_input_denormal(float32 a, float_status *status)
+bfloat16 int16_to_bfloat16_scalbn(int16_t a, int scale, float_status *status)
 {
-    if (status->flush_inputs_to_zero) {
-        FloatParts p = float32_unpack_raw(a);
-        if (parts_squash_denormal(p, status)) {
-            return float32_set_sign(float32_zero, p.sign);
-        }
-    }
-    return a;
+    return int64_to_bfloat16_scalbn(a, scale, status);
 }
 
-float64 float64_squash_input_denormal(float64 a, float_status *status)
+bfloat16 int64_to_bfloat16(int64_t a, float_status *status)
 {
-    if (status->flush_inputs_to_zero) {
-        FloatParts p = float64_unpack_raw(a);
-        if (parts_squash_denormal(p, status)) {
-            return float64_set_sign(float64_zero, p.sign);
-        }
-    }
-    return a;
+    return int64_to_bfloat16_scalbn(a, 0, status);
 }
 
-/*----------------------------------------------------------------------------
-| Takes a 64-bit fixed-point value `absZ' with binary point between bits 6
-| and 7, and returns the properly rounded 32-bit integer corresponding to the
-| input.  If `zSign' is 1, the input is negated before being converted to an
-| integer.  Bit 63 of `absZ' must be zero.  Ordinarily, the fixed-point input
-| is simply rounded to an integer, with the inexact exception raised if the
-| input cannot be represented exactly as an integer.  However, if the fixed-
-| point input is too large, the invalid exception is raised and the largest
-| positive or negative integer is returned.
-*----------------------------------------------------------------------------*/
+bfloat16 int32_to_bfloat16(int32_t a, float_status *status)
+{
+    return int64_to_bfloat16_scalbn(a, 0, status);
+}
 
-static int32_t roundAndPackInt32(flag zSign, uint64_t absZ, float_status *status)
+bfloat16 int16_to_bfloat16(int16_t a, float_status *status)
 {
-    int8_t roundingMode;
-    flag roundNearestEven;
-    int8_t roundIncrement, roundBits;
-    int32_t z;
+    return int64_to_bfloat16_scalbn(a, 0, status);
+}
 
-    roundingMode = status->float_rounding_mode;
-    roundNearestEven = ( roundingMode == float_round_nearest_even );
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        roundIncrement = 0x40;
-        break;
-    case float_round_to_zero:
-        roundIncrement = 0;
-        break;
-    case float_round_up:
-        roundIncrement = zSign ? 0 : 0x7f;
-        break;
-    case float_round_down:
-        roundIncrement = zSign ? 0x7f : 0;
-        break;
-    case float_round_to_odd:
-        roundIncrement = absZ & 0x80 ? 0 : 0x7f;
-        break;
-    default:
-        abort();
-    }
-    roundBits = absZ & 0x7F;
-    absZ = ( absZ + roundIncrement )>>7;
-    absZ &= ~ ( ( ( roundBits ^ 0x40 ) == 0 ) & roundNearestEven );
-    z = absZ;
-    if ( zSign ) z = - z;
-    if ( ( absZ>>32 ) || ( z && ( ( z < 0 ) ^ zSign ) ) ) {
-        float_raise(float_flag_invalid, status);
-        return zSign ? INT32_MIN : INT32_MAX;
-    }
-    if (roundBits) {
-        status->float_exception_flags |= float_flag_inexact;
+float128 int128_to_float128(Int128 a, float_status *status)
+{
+    FloatParts128 p = { };
+    int shift;
+
+    if (int128_nz(a)) {
+        p.cls = float_class_normal;
+        if (!int128_nonneg(a)) {
+            p.sign = true;
+            a = int128_neg(a);
+        }
+
+        shift = clz64(int128_gethi(a));
+        if (shift == 64) {
+            shift += clz64(int128_getlo(a));
+        }
+
+        p.exp = 127 - shift;
+        a = int128_lshift(a, shift);
+
+        p.frac_hi = int128_gethi(a);
+        p.frac_lo = int128_getlo(a);
+    } else {
+        p.cls = float_class_zero;
     }
-    return z;
 
+    return float128_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Takes the 128-bit fixed-point value formed by concatenating `absZ0' and
-| `absZ1', with binary point between bits 63 and 64 (between the input words),
-| and returns the properly rounded 64-bit integer corresponding to the input.
-| If `zSign' is 1, the input is negated before being converted to an integer.
-| Ordinarily, the fixed-point input is simply rounded to an integer, with
-| the inexact exception raised if the input cannot be represented exactly as
-| an integer.  However, if the fixed-point input is too large, the invalid
-| exception is raised and the largest positive or negative integer is
-| returned.
-*----------------------------------------------------------------------------*/
+float128 int64_to_float128(int64_t a, float_status *status)
+{
+    FloatParts128 p;
+
+    parts_sint_to_float(&p, a, 0, status);
+    return float128_round_pack_canonical(&p, status);
+}
 
-static int64_t roundAndPackInt64(flag zSign, uint64_t absZ0, uint64_t absZ1,
-                               float_status *status)
+float128 int32_to_float128(int32_t a, float_status *status)
 {
-    int8_t roundingMode;
-    flag roundNearestEven, increment;
-    int64_t z;
+    return int64_to_float128(a, status);
+}
 
-    roundingMode = status->float_rounding_mode;
-    roundNearestEven = ( roundingMode == float_round_nearest_even );
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        increment = ((int64_t) absZ1 < 0);
-        break;
-    case float_round_to_zero:
-        increment = 0;
-        break;
-    case float_round_up:
-        increment = !zSign && absZ1;
-        break;
-    case float_round_down:
-        increment = zSign && absZ1;
-        break;
-    case float_round_to_odd:
-        increment = !(absZ0 & 1) && absZ1;
-        break;
-    default:
-        abort();
-    }
-    if ( increment ) {
-        ++absZ0;
-        if ( absZ0 == 0 ) goto overflow;
-        absZ0 &= ~ ( ( (uint64_t) ( absZ1<<1 ) == 0 ) & roundNearestEven );
-    }
-    z = absZ0;
-    if ( zSign ) z = - z;
-    if ( z && ( ( z < 0 ) ^ zSign ) ) {
- overflow:
-        float_raise(float_flag_invalid, status);
-        return zSign ? INT64_MIN : INT64_MAX;
-    }
-    if (absZ1) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    return z;
+floatx80 int64_to_floatx80(int64_t a, float_status *status)
+{
+    FloatParts128 p;
 
+    parts_sint_to_float(&p, a, 0, status);
+    return floatx80_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Takes the 128-bit fixed-point value formed by concatenating `absZ0' and
-| `absZ1', with binary point between bits 63 and 64 (between the input words),
-| and returns the properly rounded 64-bit unsigned integer corresponding to the
-| input.  Ordinarily, the fixed-point input is simply rounded to an integer,
-| with the inexact exception raised if the input cannot be represented exactly
-| as an integer.  However, if the fixed-point input is too large, the invalid
-| exception is raised and the largest unsigned integer is returned.
-*----------------------------------------------------------------------------*/
+floatx80 int32_to_floatx80(int32_t a, float_status *status)
+{
+    return int64_to_floatx80(a, status);
+}
+
+/*
+ * Unsigned Integer to floating-point conversions
+ */
 
-static int64_t roundAndPackUint64(flag zSign, uint64_t absZ0,
-                                uint64_t absZ1, float_status *status)
+float16 uint64_to_float16_scalbn(uint64_t a, int scale, float_status *status)
 {
-    int8_t roundingMode;
-    flag roundNearestEven, increment;
+    FloatParts64 p;
 
-    roundingMode = status->float_rounding_mode;
-    roundNearestEven = (roundingMode == float_round_nearest_even);
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        increment = ((int64_t)absZ1 < 0);
-        break;
-    case float_round_to_zero:
-        increment = 0;
-        break;
-    case float_round_up:
-        increment = !zSign && absZ1;
-        break;
-    case float_round_down:
-        increment = zSign && absZ1;
-        break;
-    case float_round_to_odd:
-        increment = !(absZ0 & 1) && absZ1;
-        break;
-    default:
-        abort();
-    }
-    if (increment) {
-        ++absZ0;
-        if (absZ0 == 0) {
-            float_raise(float_flag_invalid, status);
-            return UINT64_MAX;
-        }
-        absZ0 &= ~(((uint64_t)(absZ1<<1) == 0) & roundNearestEven);
-    }
+    parts_uint_to_float(&p, a, scale, status);
+    return float16_round_pack_canonical(&p, status);
+}
 
-    if (zSign && absZ0) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
+float16 uint32_to_float16_scalbn(uint32_t a, int scale, float_status *status)
+{
+    return uint64_to_float16_scalbn(a, scale, status);
+}
+
+float16 uint16_to_float16_scalbn(uint16_t a, int scale, float_status *status)
+{
+    return uint64_to_float16_scalbn(a, scale, status);
+}
+
+float16 uint64_to_float16(uint64_t a, float_status *status)
+{
+    return uint64_to_float16_scalbn(a, 0, status);
+}
+
+float16 uint32_to_float16(uint32_t a, float_status *status)
+{
+    return uint64_to_float16_scalbn(a, 0, status);
+}
+
+float16 uint16_to_float16(uint16_t a, float_status *status)
+{
+    return uint64_to_float16_scalbn(a, 0, status);
+}
+
+float16 uint8_to_float16(uint8_t a, float_status *status)
+{
+    return uint64_to_float16_scalbn(a, 0, status);
+}
+
+float32 uint64_to_float32_scalbn(uint64_t a, int scale, float_status *status)
+{
+    FloatParts64 p;
 
-    if (absZ1) {
-        status->float_exception_flags |= float_flag_inexact;
+    /* Without scaling, there are no overflow concerns. */
+    if (likely(scale == 0) && can_use_fpu(status)) {
+        union_float32 ur;
+        ur.h = a;
+        return ur.s;
     }
-    return absZ0;
+
+    parts_uint_to_float(&p, a, scale, status);
+    return float32_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Normalizes the subnormal single-precision floating-point value represented
-| by the denormalized significand `aSig'.  The normalized exponent and
-| significand are stored at the locations pointed to by `zExpPtr' and
-| `zSigPtr', respectively.
-*----------------------------------------------------------------------------*/
+float32 uint32_to_float32_scalbn(uint32_t a, int scale, float_status *status)
+{
+    return uint64_to_float32_scalbn(a, scale, status);
+}
 
-static void
- normalizeFloat32Subnormal(uint32_t aSig, int *zExpPtr, uint32_t *zSigPtr)
+float32 uint16_to_float32_scalbn(uint16_t a, int scale, float_status *status)
 {
-    int8_t shiftCount;
+    return uint64_to_float32_scalbn(a, scale, status);
+}
 
-    shiftCount = clz32(aSig) - 8;
-    *zSigPtr = aSig<<shiftCount;
-    *zExpPtr = 1 - shiftCount;
+float32 uint64_to_float32(uint64_t a, float_status *status)
+{
+    return uint64_to_float32_scalbn(a, 0, status);
+}
 
+float32 uint32_to_float32(uint32_t a, float_status *status)
+{
+    return uint64_to_float32_scalbn(a, 0, status);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
-| and significand `zSig', and returns the proper single-precision floating-
-| point value corresponding to the abstract input.  Ordinarily, the abstract
-| value is simply rounded and packed into the single-precision format, with
-| the inexact exception raised if the abstract input cannot be represented
-| exactly.  However, if the abstract value is too large, the overflow and
-| inexact exceptions are raised and an infinity or maximal finite value is
-| returned.  If the abstract value is too small, the input value is rounded to
-| a subnormal number, and the underflow and inexact exceptions are raised if
-| the abstract input cannot be represented exactly as a subnormal single-
-| precision floating-point number.
-|     The input significand `zSig' has its binary point between bits 30
-| and 29, which is 7 bits to the left of the usual location.  This shifted
-| significand must be normalized or smaller.  If `zSig' is not normalized,
-| `zExp' must be 0; in that case, the result returned is a subnormal number,
-| and it must not require rounding.  In the usual case that `zSig' is
-| normalized, `zExp' must be 1 less than the ``true'' floating-point exponent.
-| The handling of underflow and overflow follows the IEC/IEEE Standard for
-| Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
+float32 uint16_to_float32(uint16_t a, float_status *status)
+{
+    return uint64_to_float32_scalbn(a, 0, status);
+}
 
-static float32 roundAndPackFloat32(flag zSign, int zExp, uint32_t zSig,
-                                   float_status *status)
+float64 uint64_to_float64_scalbn(uint64_t a, int scale, float_status *status)
 {
-    int8_t roundingMode;
-    flag roundNearestEven;
-    int8_t roundIncrement, roundBits;
-    flag isTiny;
+    FloatParts64 p;
 
-    roundingMode = status->float_rounding_mode;
-    roundNearestEven = ( roundingMode == float_round_nearest_even );
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        roundIncrement = 0x40;
-        break;
-    case float_round_to_zero:
-        roundIncrement = 0;
-        break;
-    case float_round_up:
-        roundIncrement = zSign ? 0 : 0x7f;
-        break;
-    case float_round_down:
-        roundIncrement = zSign ? 0x7f : 0;
-        break;
-    case float_round_to_odd:
-        roundIncrement = zSig & 0x80 ? 0 : 0x7f;
-        break;
-    default:
-        abort();
-        break;
-    }
-    roundBits = zSig & 0x7F;
-    if ( 0xFD <= (uint16_t) zExp ) {
-        if (    ( 0xFD < zExp )
-             || (    ( zExp == 0xFD )
-                  && ( (int32_t) ( zSig + roundIncrement ) < 0 ) )
-           ) {
-            bool overflow_to_inf = roundingMode != float_round_to_odd &&
-                                   roundIncrement != 0;
-            float_raise(float_flag_overflow | float_flag_inexact, status);
-            return packFloat32(zSign, 0xFF, -!overflow_to_inf);
-        }
-        if ( zExp < 0 ) {
-            if (status->flush_to_zero) {
-                float_raise(float_flag_output_denormal, status);
-                return packFloat32(zSign, 0, 0);
-            }
-            isTiny =
-                (status->float_detect_tininess
-                 == float_tininess_before_rounding)
-                || ( zExp < -1 )
-                || ( zSig + roundIncrement < 0x80000000 );
-            shift32RightJamming( zSig, - zExp, &zSig );
-            zExp = 0;
-            roundBits = zSig & 0x7F;
-            if (isTiny && roundBits) {
-                float_raise(float_flag_underflow, status);
-            }
-            if (roundingMode == float_round_to_odd) {
-                /*
-                 * For round-to-odd case, the roundIncrement depends on
-                 * zSig which just changed.
-                 */
-                roundIncrement = zSig & 0x80 ? 0 : 0x7f;
-            }
-        }
-    }
-    if (roundBits) {
-        status->float_exception_flags |= float_flag_inexact;
+    /* Without scaling, there are no overflow concerns. */
+    if (likely(scale == 0) && can_use_fpu(status)) {
+        union_float64 ur;
+        ur.h = a;
+        return ur.s;
     }
-    zSig = ( zSig + roundIncrement )>>7;
-    zSig &= ~ ( ( ( roundBits ^ 0x40 ) == 0 ) & roundNearestEven );
-    if ( zSig == 0 ) zExp = 0;
-    return packFloat32( zSign, zExp, zSig );
 
+    parts_uint_to_float(&p, a, scale, status);
+    return float64_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
-| and significand `zSig', and returns the proper single-precision floating-
-| point value corresponding to the abstract input.  This routine is just like
-| `roundAndPackFloat32' except that `zSig' does not have to be normalized.
-| Bit 31 of `zSig' must be zero, and `zExp' must be 1 less than the ``true''
-| floating-point exponent.
-*----------------------------------------------------------------------------*/
+float64 uint32_to_float64_scalbn(uint32_t a, int scale, float_status *status)
+{
+    return uint64_to_float64_scalbn(a, scale, status);
+}
 
-static float32
- normalizeRoundAndPackFloat32(flag zSign, int zExp, uint32_t zSig,
-                              float_status *status)
+float64 uint16_to_float64_scalbn(uint16_t a, int scale, float_status *status)
 {
-    int8_t shiftCount;
+    return uint64_to_float64_scalbn(a, scale, status);
+}
 
-    shiftCount = clz32(zSig) - 1;
-    return roundAndPackFloat32(zSign, zExp - shiftCount, zSig<<shiftCount,
-                               status);
+float64 uint64_to_float64(uint64_t a, float_status *status)
+{
+    return uint64_to_float64_scalbn(a, 0, status);
+}
 
+float64 uint32_to_float64(uint32_t a, float_status *status)
+{
+    return uint64_to_float64_scalbn(a, 0, status);
 }
 
-/*----------------------------------------------------------------------------
-| Normalizes the subnormal double-precision floating-point value represented
-| by the denormalized significand `aSig'.  The normalized exponent and
-| significand are stored at the locations pointed to by `zExpPtr' and
-| `zSigPtr', respectively.
-*----------------------------------------------------------------------------*/
+float64 uint16_to_float64(uint16_t a, float_status *status)
+{
+    return uint64_to_float64_scalbn(a, 0, status);
+}
 
-static void
- normalizeFloat64Subnormal(uint64_t aSig, int *zExpPtr, uint64_t *zSigPtr)
+bfloat16 uint64_to_bfloat16_scalbn(uint64_t a, int scale, float_status *status)
 {
-    int8_t shiftCount;
+    FloatParts64 p;
 
-    shiftCount = clz64(aSig) - 11;
-    *zSigPtr = aSig<<shiftCount;
-    *zExpPtr = 1 - shiftCount;
+    parts_uint_to_float(&p, a, scale, status);
+    return bfloat16_round_pack_canonical(&p, status);
+}
 
+bfloat16 uint32_to_bfloat16_scalbn(uint32_t a, int scale, float_status *status)
+{
+    return uint64_to_bfloat16_scalbn(a, scale, status);
 }
 
-/*----------------------------------------------------------------------------
-| Packs the sign `zSign', exponent `zExp', and significand `zSig' into a
-| double-precision floating-point value, returning the result.  After being
-| shifted into the proper positions, the three fields are simply added
-| together to form the result.  This means that any integer portion of `zSig'
-| will be added into the exponent.  Since a properly normalized significand
-| will have an integer portion equal to 1, the `zExp' input should be 1 less
-| than the desired result exponent whenever `zSig' is a complete, normalized
-| significand.
-*----------------------------------------------------------------------------*/
+bfloat16 uint16_to_bfloat16_scalbn(uint16_t a, int scale, float_status *status)
+{
+    return uint64_to_bfloat16_scalbn(a, scale, status);
+}
 
-static inline float64 packFloat64(flag zSign, int zExp, uint64_t zSig)
+bfloat16 uint64_to_bfloat16(uint64_t a, float_status *status)
 {
+    return uint64_to_bfloat16_scalbn(a, 0, status);
+}
 
-    return make_float64(
-        ( ( (uint64_t) zSign )<<63 ) + ( ( (uint64_t) zExp )<<52 ) + zSig);
+bfloat16 uint32_to_bfloat16(uint32_t a, float_status *status)
+{
+    return uint64_to_bfloat16_scalbn(a, 0, status);
+}
 
+bfloat16 uint16_to_bfloat16(uint16_t a, float_status *status)
+{
+    return uint64_to_bfloat16_scalbn(a, 0, status);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
-| and significand `zSig', and returns the proper double-precision floating-
-| point value corresponding to the abstract input.  Ordinarily, the abstract
-| value is simply rounded and packed into the double-precision format, with
-| the inexact exception raised if the abstract input cannot be represented
-| exactly.  However, if the abstract value is too large, the overflow and
-| inexact exceptions are raised and an infinity or maximal finite value is
-| returned.  If the abstract value is too small, the input value is rounded to
-| a subnormal number, and the underflow and inexact exceptions are raised if
-| the abstract input cannot be represented exactly as a subnormal double-
-| precision floating-point number.
-|     The input significand `zSig' has its binary point between bits 62
-| and 61, which is 10 bits to the left of the usual location.  This shifted
-| significand must be normalized or smaller.  If `zSig' is not normalized,
-| `zExp' must be 0; in that case, the result returned is a subnormal number,
-| and it must not require rounding.  In the usual case that `zSig' is
-| normalized, `zExp' must be 1 less than the ``true'' floating-point exponent.
-| The handling of underflow and overflow follows the IEC/IEEE Standard for
-| Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
+float128 uint64_to_float128(uint64_t a, float_status *status)
+{
+    FloatParts128 p;
+
+    parts_uint_to_float(&p, a, 0, status);
+    return float128_round_pack_canonical(&p, status);
+}
 
-static float64 roundAndPackFloat64(flag zSign, int zExp, uint64_t zSig,
-                                   float_status *status)
+float128 uint128_to_float128(Int128 a, float_status *status)
 {
-    int8_t roundingMode;
-    flag roundNearestEven;
-    int roundIncrement, roundBits;
-    flag isTiny;
+    FloatParts128 p = { };
+    int shift;
 
-    roundingMode = status->float_rounding_mode;
-    roundNearestEven = ( roundingMode == float_round_nearest_even );
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        roundIncrement = 0x200;
-        break;
-    case float_round_to_zero:
-        roundIncrement = 0;
-        break;
-    case float_round_up:
-        roundIncrement = zSign ? 0 : 0x3ff;
-        break;
-    case float_round_down:
-        roundIncrement = zSign ? 0x3ff : 0;
-        break;
-    case float_round_to_odd:
-        roundIncrement = (zSig & 0x400) ? 0 : 0x3ff;
-        break;
-    default:
-        abort();
-    }
-    roundBits = zSig & 0x3FF;
-    if ( 0x7FD <= (uint16_t) zExp ) {
-        if (    ( 0x7FD < zExp )
-             || (    ( zExp == 0x7FD )
-                  && ( (int64_t) ( zSig + roundIncrement ) < 0 ) )
-           ) {
-            bool overflow_to_inf = roundingMode != float_round_to_odd &&
-                                   roundIncrement != 0;
-            float_raise(float_flag_overflow | float_flag_inexact, status);
-            return packFloat64(zSign, 0x7FF, -(!overflow_to_inf));
-        }
-        if ( zExp < 0 ) {
-            if (status->flush_to_zero) {
-                float_raise(float_flag_output_denormal, status);
-                return packFloat64(zSign, 0, 0);
-            }
-            isTiny =
-                   (status->float_detect_tininess
-                    == float_tininess_before_rounding)
-                || ( zExp < -1 )
-                || ( zSig + roundIncrement < UINT64_C(0x8000000000000000) );
-            shift64RightJamming( zSig, - zExp, &zSig );
-            zExp = 0;
-            roundBits = zSig & 0x3FF;
-            if (isTiny && roundBits) {
-                float_raise(float_flag_underflow, status);
-            }
-            if (roundingMode == float_round_to_odd) {
-                /*
-                 * For round-to-odd case, the roundIncrement depends on
-                 * zSig which just changed.
-                 */
-                roundIncrement = (zSig & 0x400) ? 0 : 0x3ff;
-            }
+    if (int128_nz(a)) {
+        p.cls = float_class_normal;
+
+        shift = clz64(int128_gethi(a));
+        if (shift == 64) {
+            shift += clz64(int128_getlo(a));
         }
+
+        p.exp = 127 - shift;
+        a = int128_lshift(a, shift);
+
+        p.frac_hi = int128_gethi(a);
+        p.frac_lo = int128_getlo(a);
+    } else {
+        p.cls = float_class_zero;
     }
-    if (roundBits) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    zSig = ( zSig + roundIncrement )>>10;
-    zSig &= ~ ( ( ( roundBits ^ 0x200 ) == 0 ) & roundNearestEven );
-    if ( zSig == 0 ) zExp = 0;
-    return packFloat64( zSign, zExp, zSig );
 
+    return float128_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
-| and significand `zSig', and returns the proper double-precision floating-
-| point value corresponding to the abstract input.  This routine is just like
-| `roundAndPackFloat64' except that `zSig' does not have to be normalized.
-| Bit 63 of `zSig' must be zero, and `zExp' must be 1 less than the ``true''
-| floating-point exponent.
-*----------------------------------------------------------------------------*/
+/*
+ * Minimum and maximum
+ */
 
-static float64
- normalizeRoundAndPackFloat64(flag zSign, int zExp, uint64_t zSig,
-                              float_status *status)
+static float16 float16_minmax(float16 a, float16 b, float_status *s, int flags)
 {
-    int8_t shiftCount;
+    FloatParts64 pa, pb, *pr;
 
-    shiftCount = clz64(zSig) - 1;
-    return roundAndPackFloat64(zSign, zExp - shiftCount, zSig<<shiftCount,
-                               status);
+    float16_unpack_canonical(&pa, a, s);
+    float16_unpack_canonical(&pb, b, s);
+    pr = parts_minmax(&pa, &pb, s, flags);
 
+    return float16_round_pack_canonical(pr, s);
 }
 
-/*----------------------------------------------------------------------------
-| Normalizes the subnormal extended double-precision floating-point value
-| represented by the denormalized significand `aSig'.  The normalized exponent
-| and significand are stored at the locations pointed to by `zExpPtr' and
-| `zSigPtr', respectively.
-*----------------------------------------------------------------------------*/
+static bfloat16 bfloat16_minmax(bfloat16 a, bfloat16 b,
+                                float_status *s, int flags)
+{
+    FloatParts64 pa, pb, *pr;
 
-void normalizeFloatx80Subnormal(uint64_t aSig, int32_t *zExpPtr,
-                                uint64_t *zSigPtr)
+    bfloat16_unpack_canonical(&pa, a, s);
+    bfloat16_unpack_canonical(&pb, b, s);
+    pr = parts_minmax(&pa, &pb, s, flags);
+
+    return bfloat16_round_pack_canonical(pr, s);
+}
+
+static float32 float32_minmax(float32 a, float32 b, float_status *s, int flags)
 {
-    int8_t shiftCount;
+    FloatParts64 pa, pb, *pr;
 
-    shiftCount = clz64(aSig);
-    *zSigPtr = aSig<<shiftCount;
-    *zExpPtr = 1 - shiftCount;
+    float32_unpack_canonical(&pa, a, s);
+    float32_unpack_canonical(&pb, b, s);
+    pr = parts_minmax(&pa, &pb, s, flags);
+
+    return float32_round_pack_canonical(pr, s);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
-| and extended significand formed by the concatenation of `zSig0' and `zSig1',
-| and returns the proper extended double-precision floating-point value
-| corresponding to the abstract input.  Ordinarily, the abstract value is
-| rounded and packed into the extended double-precision format, with the
-| inexact exception raised if the abstract input cannot be represented
-| exactly.  However, if the abstract value is too large, the overflow and
-| inexact exceptions are raised and an infinity or maximal finite value is
-| returned.  If the abstract value is too small, the input value is rounded to
-| a subnormal number, and the underflow and inexact exceptions are raised if
-| the abstract input cannot be represented exactly as a subnormal extended
-| double-precision floating-point number.
-|     If `roundingPrecision' is 32 or 64, the result is rounded to the same
-| number of bits as single or double precision, respectively.  Otherwise, the
-| result is rounded to the full precision of the extended double-precision
-| format.
-|     The input significand must be normalized or smaller.  If the input
-| significand is not normalized, `zExp' must be 0; in that case, the result
-| returned is a subnormal number, and it must not require rounding.  The
-| handling of underflow and overflow follows the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
+static float64 float64_minmax(float64 a, float64 b, float_status *s, int flags)
+{
+    FloatParts64 pa, pb, *pr;
 
-floatx80 roundAndPackFloatx80(int8_t roundingPrecision, flag zSign,
-                              int32_t zExp, uint64_t zSig0, uint64_t zSig1,
-                              float_status *status)
+    float64_unpack_canonical(&pa, a, s);
+    float64_unpack_canonical(&pb, b, s);
+    pr = parts_minmax(&pa, &pb, s, flags);
+
+    return float64_round_pack_canonical(pr, s);
+}
+
+static float128 float128_minmax(float128 a, float128 b,
+                                float_status *s, int flags)
 {
-    int8_t roundingMode;
-    flag roundNearestEven, increment, isTiny;
-    int64_t roundIncrement, roundMask, roundBits;
+    FloatParts128 pa, pb, *pr;
 
-    roundingMode = status->float_rounding_mode;
-    roundNearestEven = ( roundingMode == float_round_nearest_even );
-    if ( roundingPrecision == 80 ) goto precision80;
-    if ( roundingPrecision == 64 ) {
-        roundIncrement = UINT64_C(0x0000000000000400);
-        roundMask = UINT64_C(0x00000000000007FF);
-    }
-    else if ( roundingPrecision == 32 ) {
-        roundIncrement = UINT64_C(0x0000008000000000);
-        roundMask = UINT64_C(0x000000FFFFFFFFFF);
-    }
-    else {
-        goto precision80;
-    }
-    zSig0 |= ( zSig1 != 0 );
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        break;
-    case float_round_to_zero:
-        roundIncrement = 0;
-        break;
-    case float_round_up:
-        roundIncrement = zSign ? 0 : roundMask;
-        break;
-    case float_round_down:
-        roundIncrement = zSign ? roundMask : 0;
-        break;
-    default:
-        abort();
-    }
-    roundBits = zSig0 & roundMask;
-    if ( 0x7FFD <= (uint32_t) ( zExp - 1 ) ) {
-        if (    ( 0x7FFE < zExp )
-             || ( ( zExp == 0x7FFE ) && ( zSig0 + roundIncrement < zSig0 ) )
-           ) {
-            goto overflow;
-        }
-        if ( zExp <= 0 ) {
-            if (status->flush_to_zero) {
-                float_raise(float_flag_output_denormal, status);
-                return packFloatx80(zSign, 0, 0);
-            }
-            isTiny =
-                   (status->float_detect_tininess
-                    == float_tininess_before_rounding)
-                || ( zExp < 0 )
-                || ( zSig0 <= zSig0 + roundIncrement );
-            shift64RightJamming( zSig0, 1 - zExp, &zSig0 );
-            zExp = 0;
-            roundBits = zSig0 & roundMask;
-            if (isTiny && roundBits) {
-                float_raise(float_flag_underflow, status);
-            }
-            if (roundBits) {
-                status->float_exception_flags |= float_flag_inexact;
-            }
-            zSig0 += roundIncrement;
-            if ( (int64_t) zSig0 < 0 ) zExp = 1;
-            roundIncrement = roundMask + 1;
-            if ( roundNearestEven && ( roundBits<<1 == roundIncrement ) ) {
-                roundMask |= roundIncrement;
-            }
-            zSig0 &= ~ roundMask;
-            return packFloatx80( zSign, zExp, zSig0 );
-        }
-    }
-    if (roundBits) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    zSig0 += roundIncrement;
-    if ( zSig0 < roundIncrement ) {
-        ++zExp;
-        zSig0 = UINT64_C(0x8000000000000000);
-    }
-    roundIncrement = roundMask + 1;
-    if ( roundNearestEven && ( roundBits<<1 == roundIncrement ) ) {
-        roundMask |= roundIncrement;
-    }
-    zSig0 &= ~ roundMask;
-    if ( zSig0 == 0 ) zExp = 0;
-    return packFloatx80( zSign, zExp, zSig0 );
- precision80:
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        increment = ((int64_t)zSig1 < 0);
-        break;
-    case float_round_to_zero:
-        increment = 0;
-        break;
-    case float_round_up:
-        increment = !zSign && zSig1;
-        break;
-    case float_round_down:
-        increment = zSign && zSig1;
-        break;
-    default:
-        abort();
-    }
-    if ( 0x7FFD <= (uint32_t) ( zExp - 1 ) ) {
-        if (    ( 0x7FFE < zExp )
-             || (    ( zExp == 0x7FFE )
-                  && ( zSig0 == UINT64_C(0xFFFFFFFFFFFFFFFF) )
-                  && increment
-                )
-           ) {
-            roundMask = 0;
- overflow:
-            float_raise(float_flag_overflow | float_flag_inexact, status);
-            if (    ( roundingMode == float_round_to_zero )
-                 || ( zSign && ( roundingMode == float_round_up ) )
-                 || ( ! zSign && ( roundingMode == float_round_down ) )
-               ) {
-                return packFloatx80( zSign, 0x7FFE, ~ roundMask );
-            }
-            return packFloatx80(zSign,
-                                floatx80_infinity_high,
-                                floatx80_infinity_low);
-        }
-        if ( zExp <= 0 ) {
-            isTiny =
-                   (status->float_detect_tininess
-                    == float_tininess_before_rounding)
-                || ( zExp < 0 )
-                || ! increment
-                || ( zSig0 < UINT64_C(0xFFFFFFFFFFFFFFFF) );
-            shift64ExtraRightJamming( zSig0, zSig1, 1 - zExp, &zSig0, &zSig1 );
-            zExp = 0;
-            if (isTiny && zSig1) {
-                float_raise(float_flag_underflow, status);
-            }
-            if (zSig1) {
-                status->float_exception_flags |= float_flag_inexact;
-            }
-            switch (roundingMode) {
-            case float_round_nearest_even:
-            case float_round_ties_away:
-                increment = ((int64_t)zSig1 < 0);
-                break;
-            case float_round_to_zero:
-                increment = 0;
-                break;
-            case float_round_up:
-                increment = !zSign && zSig1;
-                break;
-            case float_round_down:
-                increment = zSign && zSig1;
-                break;
-            default:
-                abort();
-            }
-            if ( increment ) {
-                ++zSig0;
-                zSig0 &=
-                    ~ ( ( (uint64_t) ( zSig1<<1 ) == 0 ) & roundNearestEven );
-                if ( (int64_t) zSig0 < 0 ) zExp = 1;
-            }
-            return packFloatx80( zSign, zExp, zSig0 );
-        }
-    }
-    if (zSig1) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    if ( increment ) {
-        ++zSig0;
-        if ( zSig0 == 0 ) {
-            ++zExp;
-            zSig0 = UINT64_C(0x8000000000000000);
-        }
-        else {
-            zSig0 &= ~ ( ( (uint64_t) ( zSig1<<1 ) == 0 ) & roundNearestEven );
-        }
-    }
-    else {
-        if ( zSig0 == 0 ) zExp = 0;
-    }
-    return packFloatx80( zSign, zExp, zSig0 );
+    float128_unpack_canonical(&pa, a, s);
+    float128_unpack_canonical(&pb, b, s);
+    pr = parts_minmax(&pa, &pb, s, flags);
 
+    return float128_round_pack_canonical(pr, s);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent
-| `zExp', and significand formed by the concatenation of `zSig0' and `zSig1',
-| and returns the proper extended double-precision floating-point value
-| corresponding to the abstract input.  This routine is just like
-| `roundAndPackFloatx80' except that the input significand does not have to be
-| normalized.
-*----------------------------------------------------------------------------*/
+#define MINMAX_1(type, name, flags) \
+    type type##_##name(type a, type b, float_status *s) \
+    { return type##_minmax(a, b, s, flags); }
 
-floatx80 normalizeRoundAndPackFloatx80(int8_t roundingPrecision,
-                                       flag zSign, int32_t zExp,
-                                       uint64_t zSig0, uint64_t zSig1,
-                                       float_status *status)
-{
-    int8_t shiftCount;
+#define MINMAX_2(type) \
+    MINMAX_1(type, max, 0)                                                \
+    MINMAX_1(type, maxnum, minmax_isnum)                                  \
+    MINMAX_1(type, maxnummag, minmax_isnum | minmax_ismag)                \
+    MINMAX_1(type, maximum_number, minmax_isnumber)                       \
+    MINMAX_1(type, min, minmax_ismin)                                     \
+    MINMAX_1(type, minnum, minmax_ismin | minmax_isnum)                   \
+    MINMAX_1(type, minnummag, minmax_ismin | minmax_isnum | minmax_ismag) \
+    MINMAX_1(type, minimum_number, minmax_ismin | minmax_isnumber)        \
 
-    if ( zSig0 == 0 ) {
-        zSig0 = zSig1;
-        zSig1 = 0;
-        zExp -= 64;
-    }
-    shiftCount = clz64(zSig0);
-    shortShift128Left( zSig0, zSig1, shiftCount, &zSig0, &zSig1 );
-    zExp -= shiftCount;
-    return roundAndPackFloatx80(roundingPrecision, zSign, zExp,
-                                zSig0, zSig1, status);
+MINMAX_2(float16)
+MINMAX_2(bfloat16)
+MINMAX_2(float32)
+MINMAX_2(float64)
+MINMAX_2(float128)
 
-}
+#undef MINMAX_1
+#undef MINMAX_2
 
-/*----------------------------------------------------------------------------
-| Returns the least-significant 64 fraction bits of the quadruple-precision
-| floating-point value `a'.
-*----------------------------------------------------------------------------*/
+/*
+ * Floating point compare
+ */
 
-static inline uint64_t extractFloat128Frac1( float128 a )
+static FloatRelation QEMU_FLATTEN
+float16_do_compare(float16 a, float16 b, float_status *s, bool is_quiet)
 {
+    FloatParts64 pa, pb;
 
-    return a.low;
-
+    float16_unpack_canonical(&pa, a, s);
+    float16_unpack_canonical(&pb, b, s);
+    return parts_compare(&pa, &pb, s, is_quiet);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the most-significant 48 fraction bits of the quadruple-precision
-| floating-point value `a'.
-*----------------------------------------------------------------------------*/
+FloatRelation float16_compare(float16 a, float16 b, float_status *s)
+{
+    return float16_do_compare(a, b, s, false);
+}
 
-static inline uint64_t extractFloat128Frac0( float128 a )
+FloatRelation float16_compare_quiet(float16 a, float16 b, float_status *s)
 {
+    return float16_do_compare(a, b, s, true);
+}
 
-    return a.high & UINT64_C(0x0000FFFFFFFFFFFF);
+static FloatRelation QEMU_SOFTFLOAT_ATTR
+float32_do_compare(float32 a, float32 b, float_status *s, bool is_quiet)
+{
+    FloatParts64 pa, pb;
 
+    float32_unpack_canonical(&pa, a, s);
+    float32_unpack_canonical(&pb, b, s);
+    return parts_compare(&pa, &pb, s, is_quiet);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the exponent bits of the quadruple-precision floating-point value
-| `a'.
-*----------------------------------------------------------------------------*/
-
-static inline int32_t extractFloat128Exp( float128 a )
+static FloatRelation QEMU_FLATTEN
+float32_hs_compare(float32 xa, float32 xb, float_status *s, bool is_quiet)
 {
+    union_float32 ua, ub;
+
+    ua.s = xa;
+    ub.s = xb;
 
-    return ( a.high>>48 ) & 0x7FFF;
+    if (QEMU_NO_HARDFLOAT) {
+        goto soft;
+    }
 
+    float32_input_flush2(&ua.s, &ub.s, s);
+    if (isgreaterequal(ua.h, ub.h)) {
+        if (isgreater(ua.h, ub.h)) {
+            return float_relation_greater;
+        }
+        return float_relation_equal;
+    }
+    if (likely(isless(ua.h, ub.h))) {
+        return float_relation_less;
+    }
+    /*
+     * The only condition remaining is unordered.
+     * Fall through to set flags.
+     */
+ soft:
+    return float32_do_compare(ua.s, ub.s, s, is_quiet);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the sign bit of the quadruple-precision floating-point value `a'.
-*----------------------------------------------------------------------------*/
+FloatRelation float32_compare(float32 a, float32 b, float_status *s)
+{
+    return float32_hs_compare(a, b, s, false);
+}
 
-static inline flag extractFloat128Sign( float128 a )
+FloatRelation float32_compare_quiet(float32 a, float32 b, float_status *s)
 {
+    return float32_hs_compare(a, b, s, true);
+}
 
-    return a.high>>63;
+static FloatRelation QEMU_SOFTFLOAT_ATTR
+float64_do_compare(float64 a, float64 b, float_status *s, bool is_quiet)
+{
+    FloatParts64 pa, pb;
 
+    float64_unpack_canonical(&pa, a, s);
+    float64_unpack_canonical(&pb, b, s);
+    return parts_compare(&pa, &pb, s, is_quiet);
 }
 
-/*----------------------------------------------------------------------------
-| Normalizes the subnormal quadruple-precision floating-point value
-| represented by the denormalized significand formed by the concatenation of
-| `aSig0' and `aSig1'.  The normalized exponent is stored at the location
-| pointed to by `zExpPtr'.  The most significant 49 bits of the normalized
-| significand are stored at the location pointed to by `zSig0Ptr', and the
-| least significant 64 bits of the normalized significand are stored at the
-| location pointed to by `zSig1Ptr'.
-*----------------------------------------------------------------------------*/
-
-static void
- normalizeFloat128Subnormal(
-     uint64_t aSig0,
-     uint64_t aSig1,
-     int32_t *zExpPtr,
-     uint64_t *zSig0Ptr,
-     uint64_t *zSig1Ptr
- )
+static FloatRelation QEMU_FLATTEN
+float64_hs_compare(float64 xa, float64 xb, float_status *s, bool is_quiet)
 {
-    int8_t shiftCount;
+    union_float64 ua, ub;
 
-    if ( aSig0 == 0 ) {
-        shiftCount = clz64(aSig1) - 15;
-        if ( shiftCount < 0 ) {
-            *zSig0Ptr = aSig1>>( - shiftCount );
-            *zSig1Ptr = aSig1<<( shiftCount & 63 );
-        }
-        else {
-            *zSig0Ptr = aSig1<<shiftCount;
-            *zSig1Ptr = 0;
+    ua.s = xa;
+    ub.s = xb;
+
+    if (QEMU_NO_HARDFLOAT) {
+        goto soft;
+    }
+
+    float64_input_flush2(&ua.s, &ub.s, s);
+    if (isgreaterequal(ua.h, ub.h)) {
+        if (isgreater(ua.h, ub.h)) {
+            return float_relation_greater;
         }
-        *zExpPtr = - shiftCount - 63;
+        return float_relation_equal;
     }
-    else {
-        shiftCount = clz64(aSig0) - 15;
-        shortShift128Left( aSig0, aSig1, shiftCount, zSig0Ptr, zSig1Ptr );
-        *zExpPtr = 1 - shiftCount;
+    if (likely(isless(ua.h, ub.h))) {
+        return float_relation_less;
     }
+    /*
+     * The only condition remaining is unordered.
+     * Fall through to set flags.
+     */
+ soft:
+    return float64_do_compare(ua.s, ub.s, s, is_quiet);
+}
 
+FloatRelation float64_compare(float64 a, float64 b, float_status *s)
+{
+    return float64_hs_compare(a, b, s, false);
 }
 
-/*----------------------------------------------------------------------------
-| Packs the sign `zSign', the exponent `zExp', and the significand formed
-| by the concatenation of `zSig0' and `zSig1' into a quadruple-precision
-| floating-point value, returning the result.  After being shifted into the
-| proper positions, the three fields `zSign', `zExp', and `zSig0' are simply
-| added together to form the most significant 32 bits of the result.  This
-| means that any integer portion of `zSig0' will be added into the exponent.
-| Since a properly normalized significand will have an integer portion equal
-| to 1, the `zExp' input should be 1 less than the desired result exponent
-| whenever `zSig0' and `zSig1' concatenated form a complete, normalized
-| significand.
-*----------------------------------------------------------------------------*/
+FloatRelation float64_compare_quiet(float64 a, float64 b, float_status *s)
+{
+    return float64_hs_compare(a, b, s, true);
+}
 
-static inline float128
- packFloat128( flag zSign, int32_t zExp, uint64_t zSig0, uint64_t zSig1 )
+static FloatRelation QEMU_FLATTEN
+bfloat16_do_compare(bfloat16 a, bfloat16 b, float_status *s, bool is_quiet)
 {
-    float128 z;
+    FloatParts64 pa, pb;
 
-    z.low = zSig1;
-    z.high = ( ( (uint64_t) zSign )<<63 ) + ( ( (uint64_t) zExp )<<48 ) + zSig0;
-    return z;
+    bfloat16_unpack_canonical(&pa, a, s);
+    bfloat16_unpack_canonical(&pb, b, s);
+    return parts_compare(&pa, &pb, s, is_quiet);
+}
 
+FloatRelation bfloat16_compare(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_do_compare(a, b, s, false);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
-| and extended significand formed by the concatenation of `zSig0', `zSig1',
-| and `zSig2', and returns the proper quadruple-precision floating-point value
-| corresponding to the abstract input.  Ordinarily, the abstract value is
-| simply rounded and packed into the quadruple-precision format, with the
-| inexact exception raised if the abstract input cannot be represented
-| exactly.  However, if the abstract value is too large, the overflow and
-| inexact exceptions are raised and an infinity or maximal finite value is
-| returned.  If the abstract value is too small, the input value is rounded to
-| a subnormal number, and the underflow and inexact exceptions are raised if
-| the abstract input cannot be represented exactly as a subnormal quadruple-
-| precision floating-point number.
-|     The input significand must be normalized or smaller.  If the input
-| significand is not normalized, `zExp' must be 0; in that case, the result
-| returned is a subnormal number, and it must not require rounding.  In the
-| usual case that the input significand is normalized, `zExp' must be 1 less
-| than the ``true'' floating-point exponent.  The handling of underflow and
-| overflow follows the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
+FloatRelation bfloat16_compare_quiet(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_do_compare(a, b, s, true);
+}
 
-static float128 roundAndPackFloat128(flag zSign, int32_t zExp,
-                                     uint64_t zSig0, uint64_t zSig1,
-                                     uint64_t zSig2, float_status *status)
+static FloatRelation QEMU_FLATTEN
+float128_do_compare(float128 a, float128 b, float_status *s, bool is_quiet)
 {
-    int8_t roundingMode;
-    flag roundNearestEven, increment, isTiny;
+    FloatParts128 pa, pb;
 
-    roundingMode = status->float_rounding_mode;
-    roundNearestEven = ( roundingMode == float_round_nearest_even );
-    switch (roundingMode) {
-    case float_round_nearest_even:
-    case float_round_ties_away:
-        increment = ((int64_t)zSig2 < 0);
-        break;
-    case float_round_to_zero:
-        increment = 0;
-        break;
-    case float_round_up:
-        increment = !zSign && zSig2;
-        break;
-    case float_round_down:
-        increment = zSign && zSig2;
-        break;
-    case float_round_to_odd:
-        increment = !(zSig1 & 0x1) && zSig2;
-        break;
-    default:
-        abort();
-    }
-    if ( 0x7FFD <= (uint32_t) zExp ) {
-        if (    ( 0x7FFD < zExp )
-             || (    ( zExp == 0x7FFD )
-                  && eq128(
-                         UINT64_C(0x0001FFFFFFFFFFFF),
-                         UINT64_C(0xFFFFFFFFFFFFFFFF),
-                         zSig0,
-                         zSig1
-                     )
-                  && increment
-                )
-           ) {
-            float_raise(float_flag_overflow | float_flag_inexact, status);
-            if (    ( roundingMode == float_round_to_zero )
-                 || ( zSign && ( roundingMode == float_round_up ) )
-                 || ( ! zSign && ( roundingMode == float_round_down ) )
-                 || (roundingMode == float_round_to_odd)
-               ) {
-                return
-                    packFloat128(
-                        zSign,
-                        0x7FFE,
-                        UINT64_C(0x0000FFFFFFFFFFFF),
-                        UINT64_C(0xFFFFFFFFFFFFFFFF)
-                    );
-            }
-            return packFloat128( zSign, 0x7FFF, 0, 0 );
-        }
-        if ( zExp < 0 ) {
-            if (status->flush_to_zero) {
-                float_raise(float_flag_output_denormal, status);
-                return packFloat128(zSign, 0, 0, 0);
-            }
-            isTiny =
-                   (status->float_detect_tininess
-                    == float_tininess_before_rounding)
-                || ( zExp < -1 )
-                || ! increment
-                || lt128(
-                       zSig0,
-                       zSig1,
-                       UINT64_C(0x0001FFFFFFFFFFFF),
-                       UINT64_C(0xFFFFFFFFFFFFFFFF)
-                   );
-            shift128ExtraRightJamming(
-                zSig0, zSig1, zSig2, - zExp, &zSig0, &zSig1, &zSig2 );
-            zExp = 0;
-            if (isTiny && zSig2) {
-                float_raise(float_flag_underflow, status);
-            }
-            switch (roundingMode) {
-            case float_round_nearest_even:
-            case float_round_ties_away:
-                increment = ((int64_t)zSig2 < 0);
-                break;
-            case float_round_to_zero:
-                increment = 0;
-                break;
-            case float_round_up:
-                increment = !zSign && zSig2;
-                break;
-            case float_round_down:
-                increment = zSign && zSig2;
-                break;
-            case float_round_to_odd:
-                increment = !(zSig1 & 0x1) && zSig2;
-                break;
-            default:
-                abort();
-            }
-        }
-    }
-    if (zSig2) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    if ( increment ) {
-        add128( zSig0, zSig1, 0, 1, &zSig0, &zSig1 );
-        zSig1 &= ~ ( ( zSig2 + zSig2 == 0 ) & roundNearestEven );
-    }
-    else {
-        if ( ( zSig0 | zSig1 ) == 0 ) zExp = 0;
-    }
-    return packFloat128( zSign, zExp, zSig0, zSig1 );
+    float128_unpack_canonical(&pa, a, s);
+    float128_unpack_canonical(&pb, b, s);
+    return parts_compare(&pa, &pb, s, is_quiet);
+}
 
+FloatRelation float128_compare(float128 a, float128 b, float_status *s)
+{
+    return float128_do_compare(a, b, s, false);
 }
 
-/*----------------------------------------------------------------------------
-| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
-| and significand formed by the concatenation of `zSig0' and `zSig1', and
-| returns the proper quadruple-precision floating-point value corresponding
-| to the abstract input.  This routine is just like `roundAndPackFloat128'
-| except that the input significand has fewer bits and does not have to be
-| normalized.  In all cases, `zExp' must be 1 less than the ``true'' floating-
-| point exponent.
-*----------------------------------------------------------------------------*/
+FloatRelation float128_compare_quiet(float128 a, float128 b, float_status *s)
+{
+    return float128_do_compare(a, b, s, true);
+}
 
-static float128 normalizeRoundAndPackFloat128(flag zSign, int32_t zExp,
-                                              uint64_t zSig0, uint64_t zSig1,
-                                              float_status *status)
+static FloatRelation QEMU_FLATTEN
+floatx80_do_compare(floatx80 a, floatx80 b, float_status *s, bool is_quiet)
 {
-    int8_t shiftCount;
-    uint64_t zSig2;
+    FloatParts128 pa, pb;
 
-    if ( zSig0 == 0 ) {
-        zSig0 = zSig1;
-        zSig1 = 0;
-        zExp -= 64;
-    }
-    shiftCount = clz64(zSig0) - 15;
-    if ( 0 <= shiftCount ) {
-        zSig2 = 0;
-        shortShift128Left( zSig0, zSig1, shiftCount, &zSig0, &zSig1 );
-    }
-    else {
-        shift128ExtraRightJamming(
-            zSig0, zSig1, 0, - shiftCount, &zSig0, &zSig1, &zSig2 );
+    if (!floatx80_unpack_canonical(&pa, a, s) ||
+        !floatx80_unpack_canonical(&pb, b, s)) {
+        return float_relation_unordered;
     }
-    zExp -= shiftCount;
-    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
+    return parts_compare(&pa, &pb, s, is_quiet);
+}
 
+FloatRelation floatx80_compare(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_do_compare(a, b, s, false);
 }
 
+FloatRelation floatx80_compare_quiet(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_do_compare(a, b, s, true);
+}
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the 32-bit two's complement integer `a'
-| to the extended double-precision floating-point format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
+/*
+ * Scale by 2**N
+ */
 
-floatx80 int32_to_floatx80(int32_t a, float_status *status)
+float16 float16_scalbn(float16 a, int n, float_status *status)
 {
-    flag zSign;
-    uint32_t absA;
-    int8_t shiftCount;
-    uint64_t zSig;
-
-    if ( a == 0 ) return packFloatx80( 0, 0, 0 );
-    zSign = ( a < 0 );
-    absA = zSign ? - a : a;
-    shiftCount = clz32(absA) + 32;
-    zSig = absA;
-    return packFloatx80( zSign, 0x403E - shiftCount, zSig<<shiftCount );
+    FloatParts64 p;
 
+    float16_unpack_canonical(&p, a, status);
+    parts_scalbn(&p, n, status);
+    return float16_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the 32-bit two's complement integer `a' to
-| the quadruple-precision floating-point format.  The conversion is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 int32_to_float128(int32_t a, float_status *status)
+float32 float32_scalbn(float32 a, int n, float_status *status)
 {
-    flag zSign;
-    uint32_t absA;
-    int8_t shiftCount;
-    uint64_t zSig0;
-
-    if ( a == 0 ) return packFloat128( 0, 0, 0, 0 );
-    zSign = ( a < 0 );
-    absA = zSign ? - a : a;
-    shiftCount = clz32(absA) + 17;
-    zSig0 = absA;
-    return packFloat128( zSign, 0x402E - shiftCount, zSig0<<shiftCount, 0 );
+    FloatParts64 p;
 
+    float32_unpack_canonical(&p, a, status);
+    parts_scalbn(&p, n, status);
+    return float32_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the 64-bit two's complement integer `a'
-| to the extended double-precision floating-point format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 int64_to_floatx80(int64_t a, float_status *status)
+float64 float64_scalbn(float64 a, int n, float_status *status)
 {
-    flag zSign;
-    uint64_t absA;
-    int8_t shiftCount;
-
-    if ( a == 0 ) return packFloatx80( 0, 0, 0 );
-    zSign = ( a < 0 );
-    absA = zSign ? - a : a;
-    shiftCount = clz64(absA);
-    return packFloatx80( zSign, 0x403E - shiftCount, absA<<shiftCount );
+    FloatParts64 p;
 
+    float64_unpack_canonical(&p, a, status);
+    parts_scalbn(&p, n, status);
+    return float64_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the 64-bit two's complement integer `a' to
-| the quadruple-precision floating-point format.  The conversion is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 int64_to_float128(int64_t a, float_status *status)
+bfloat16 bfloat16_scalbn(bfloat16 a, int n, float_status *status)
 {
-    flag zSign;
-    uint64_t absA;
-    int8_t shiftCount;
-    int32_t zExp;
-    uint64_t zSig0, zSig1;
-
-    if ( a == 0 ) return packFloat128( 0, 0, 0, 0 );
-    zSign = ( a < 0 );
-    absA = zSign ? - a : a;
-    shiftCount = clz64(absA) + 49;
-    zExp = 0x406E - shiftCount;
-    if ( 64 <= shiftCount ) {
-        zSig1 = 0;
-        zSig0 = absA;
-        shiftCount -= 64;
-    }
-    else {
-        zSig1 = absA;
-        zSig0 = 0;
-    }
-    shortShift128Left( zSig0, zSig1, shiftCount, &zSig0, &zSig1 );
-    return packFloat128( zSign, zExp, zSig0, zSig1 );
+    FloatParts64 p;
 
+    bfloat16_unpack_canonical(&p, a, status);
+    parts_scalbn(&p, n, status);
+    return bfloat16_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the 64-bit unsigned integer `a'
-| to the quadruple-precision floating-point format.  The conversion is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
+float128 float128_scalbn(float128 a, int n, float_status *status)
+{
+    FloatParts128 p;
 
-float128 uint64_to_float128(uint64_t a, float_status *status)
+    float128_unpack_canonical(&p, a, status);
+    parts_scalbn(&p, n, status);
+    return float128_round_pack_canonical(&p, status);
+}
+
+floatx80 floatx80_scalbn(floatx80 a, int n, float_status *status)
 {
-    if (a == 0) {
-        return float128_zero;
+    FloatParts128 p;
+
+    if (!floatx80_unpack_canonical(&p, a, status)) {
+        return floatx80_default_nan(status);
     }
-    return normalizeRoundAndPackFloat128(0, 0x406E, 0, a, status);
+    parts_scalbn(&p, n, status);
+    return floatx80_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the single-precision floating-point value
-| `a' to the extended double-precision floating-point format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
+/*
+ * Square Root
+ */
 
-floatx80 float32_to_floatx80(float32 a, float_status *status)
+float16 QEMU_FLATTEN float16_sqrt(float16 a, float_status *status)
 {
-    flag aSign;
-    int aExp;
-    uint32_t aSig;
+    FloatParts64 p;
 
-    a = float32_squash_input_denormal(a, status);
-    aSig = extractFloat32Frac( a );
-    aExp = extractFloat32Exp( a );
-    aSign = extractFloat32Sign( a );
-    if ( aExp == 0xFF ) {
-        if (aSig) {
-            return commonNaNToFloatx80(float32ToCommonNaN(a, status), status);
-        }
-        return packFloatx80(aSign,
-                            floatx80_infinity_high,
-                            floatx80_infinity_low);
-    }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloatx80( aSign, 0, 0 );
-        normalizeFloat32Subnormal( aSig, &aExp, &aSig );
-    }
-    aSig |= 0x00800000;
-    return packFloatx80( aSign, aExp + 0x3F80, ( (uint64_t) aSig )<<40 );
+    float16_unpack_canonical(&p, a, status);
+    parts_sqrt(&p, status, &float16_params);
+    return float16_round_pack_canonical(&p, status);
+}
+
+static float32 QEMU_SOFTFLOAT_ATTR
+soft_f32_sqrt(float32 a, float_status *status)
+{
+    FloatParts64 p;
 
+    float32_unpack_canonical(&p, a, status);
+    parts_sqrt(&p, status, &float32_params);
+    return float32_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the single-precision floating-point value
-| `a' to the double-precision floating-point format.  The conversion is
-| performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
+static float64 QEMU_SOFTFLOAT_ATTR
+soft_f64_sqrt(float64 a, float_status *status)
+{
+    FloatParts64 p;
+
+    float64_unpack_canonical(&p, a, status);
+    parts_sqrt(&p, status, &float64_params);
+    return float64_round_pack_canonical(&p, status);
+}
 
-float128 float32_to_float128(float32 a, float_status *status)
+float32 QEMU_FLATTEN float32_sqrt(float32 xa, float_status *s)
 {
-    flag aSign;
-    int aExp;
-    uint32_t aSig;
+    union_float32 ua, ur;
 
-    a = float32_squash_input_denormal(a, status);
-    aSig = extractFloat32Frac( a );
-    aExp = extractFloat32Exp( a );
-    aSign = extractFloat32Sign( a );
-    if ( aExp == 0xFF ) {
-        if (aSig) {
-            return commonNaNToFloat128(float32ToCommonNaN(a, status), status);
-        }
-        return packFloat128( aSign, 0x7FFF, 0, 0 );
+    ua.s = xa;
+    if (unlikely(!can_use_fpu(s))) {
+        goto soft;
     }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloat128( aSign, 0, 0, 0 );
-        normalizeFloat32Subnormal( aSig, &aExp, &aSig );
-        --aExp;
+
+    float32_input_flush1(&ua.s, s);
+    if (QEMU_HARDFLOAT_1F32_USE_FP) {
+        if (unlikely(!(fpclassify(ua.h) == FP_NORMAL ||
+                       fpclassify(ua.h) == FP_ZERO) ||
+                     signbit(ua.h))) {
+            goto soft;
+        }
+    } else if (unlikely(!float32_is_zero_or_normal(ua.s) ||
+                        float32_is_neg(ua.s))) {
+        goto soft;
     }
-    return packFloat128( aSign, aExp + 0x3F80, ( (uint64_t) aSig )<<25, 0 );
+    ur.h = sqrtf(ua.h);
+    return ur.s;
 
+ soft:
+    return soft_f32_sqrt(ua.s, s);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the remainder of the single-precision floating-point value `a'
-| with respect to the corresponding value `b'.  The operation is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float32 float32_rem(float32 a, float32 b, float_status *status)
+float64 QEMU_FLATTEN float64_sqrt(float64 xa, float_status *s)
 {
-    flag aSign, zSign;
-    int aExp, bExp, expDiff;
-    uint32_t aSig, bSig;
-    uint32_t q;
-    uint64_t aSig64, bSig64, q64;
-    uint32_t alternateASig;
-    int32_t sigMean;
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
-
-    aSig = extractFloat32Frac( a );
-    aExp = extractFloat32Exp( a );
-    aSign = extractFloat32Sign( a );
-    bSig = extractFloat32Frac( b );
-    bExp = extractFloat32Exp( b );
-    if ( aExp == 0xFF ) {
-        if ( aSig || ( ( bExp == 0xFF ) && bSig ) ) {
-            return propagateFloat32NaN(a, b, status);
-        }
-        float_raise(float_flag_invalid, status);
-        return float32_default_nan(status);
-    }
-    if ( bExp == 0xFF ) {
-        if (bSig) {
-            return propagateFloat32NaN(a, b, status);
-        }
-        return a;
+    union_float64 ua, ur;
+
+    ua.s = xa;
+    if (unlikely(!can_use_fpu(s))) {
+        goto soft;
     }
-    if ( bExp == 0 ) {
-        if ( bSig == 0 ) {
-            float_raise(float_flag_invalid, status);
-            return float32_default_nan(status);
-        }
-        normalizeFloat32Subnormal( bSig, &bExp, &bSig );
-    }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return a;
-        normalizeFloat32Subnormal( aSig, &aExp, &aSig );
-    }
-    expDiff = aExp - bExp;
-    aSig |= 0x00800000;
-    bSig |= 0x00800000;
-    if ( expDiff < 32 ) {
-        aSig <<= 8;
-        bSig <<= 8;
-        if ( expDiff < 0 ) {
-            if ( expDiff < -1 ) return a;
-            aSig >>= 1;
-        }
-        q = ( bSig <= aSig );
-        if ( q ) aSig -= bSig;
-        if ( 0 < expDiff ) {
-            q = ( ( (uint64_t) aSig )<<32 ) / bSig;
-            q >>= 32 - expDiff;
-            bSig >>= 2;
-            aSig = ( ( aSig>>1 )<<( expDiff - 1 ) ) - bSig * q;
-        }
-        else {
-            aSig >>= 2;
-            bSig >>= 2;
+
+    float64_input_flush1(&ua.s, s);
+    if (QEMU_HARDFLOAT_1F64_USE_FP) {
+        if (unlikely(!(fpclassify(ua.h) == FP_NORMAL ||
+                       fpclassify(ua.h) == FP_ZERO) ||
+                     signbit(ua.h))) {
+            goto soft;
         }
+    } else if (unlikely(!float64_is_zero_or_normal(ua.s) ||
+                        float64_is_neg(ua.s))) {
+        goto soft;
     }
-    else {
-        if ( bSig <= aSig ) aSig -= bSig;
-        aSig64 = ( (uint64_t) aSig )<<40;
-        bSig64 = ( (uint64_t) bSig )<<40;
-        expDiff -= 64;
-        while ( 0 < expDiff ) {
-            q64 = estimateDiv128To64( aSig64, 0, bSig64 );
-            q64 = ( 2 < q64 ) ? q64 - 2 : 0;
-#ifdef _MSC_VER
-            aSig64 = 0ULL - ( ( bSig * q64 )<<38 );
-#else
-            aSig64 = - ( ( bSig * q64 )<<38 );
-#endif
-            expDiff -= 62;
-        }
-        expDiff += 64;
-        q64 = estimateDiv128To64( aSig64, 0, bSig64 );
-        q64 = ( 2 < q64 ) ? q64 - 2 : 0;
-        q = q64>>( 64 - expDiff );
-        bSig <<= 6;
-        aSig = ( ( aSig64>>33 )<<( expDiff - 1 ) ) - bSig * q;
-    }
-    do {
-        alternateASig = aSig;
-        ++q;
-        aSig -= bSig;
-    } while ( 0 <= (int32_t) aSig );
-    sigMean = aSig + alternateASig;
-    if ( ( sigMean < 0 ) || ( ( sigMean == 0 ) && ( q & 1 ) ) ) {
-        aSig = alternateASig;
-    }
-    zSign = ( (int32_t) aSig < 0 );
-#ifdef _MSC_VER
-    if ( zSign ) aSig = 0ULL - aSig;
-#else
-    if ( zSign ) aSig = - aSig;
-#endif
-    return normalizeRoundAndPackFloat32(aSign ^ zSign, bExp, aSig, status);
-}
+    ur.h = sqrt(ua.h);
+    return ur.s;
 
+ soft:
+    return soft_f64_sqrt(ua.s, s);
+}
 
+float64 float64r32_sqrt(float64 a, float_status *status)
+{
+    FloatParts64 p;
 
-/*----------------------------------------------------------------------------
-| Returns the binary exponential of the single-precision floating-point value
-| `a'. The operation is performed according to the IEC/IEEE Standard for
-| Binary Floating-Point Arithmetic.
-|
-| Uses the following identities:
-|
-| 1. -------------------------------------------------------------------------
-|      x    x*ln(2)
-|     2  = e
-|
-| 2. -------------------------------------------------------------------------
-|                      2     3     4     5           n
-|      x        x     x     x     x     x           x
-|     e  = 1 + --- + --- + --- + --- + --- + ... + --- + ...
-|               1!    2!    3!    4!    5!          n!
-*----------------------------------------------------------------------------*/
+    float64_unpack_canonical(&p, a, status);
+    parts_sqrt(&p, status, &float64_params);
+    return float64r32_round_pack_canonical(&p, status);
+}
 
-static const float64 float32_exp2_coefficients[15] =
+bfloat16 QEMU_FLATTEN bfloat16_sqrt(bfloat16 a, float_status *status)
 {
-    const_float64( 0x3ff0000000000000ll ), /*  1 */
-    const_float64( 0x3fe0000000000000ll ), /*  2 */
-    const_float64( 0x3fc5555555555555ll ), /*  3 */
-    const_float64( 0x3fa5555555555555ll ), /*  4 */
-    const_float64( 0x3f81111111111111ll ), /*  5 */
-    const_float64( 0x3f56c16c16c16c17ll ), /*  6 */
-    const_float64( 0x3f2a01a01a01a01all ), /*  7 */
-    const_float64( 0x3efa01a01a01a01all ), /*  8 */
-    const_float64( 0x3ec71de3a556c734ll ), /*  9 */
-    const_float64( 0x3e927e4fb7789f5cll ), /* 10 */
-    const_float64( 0x3e5ae64567f544e4ll ), /* 11 */
-    const_float64( 0x3e21eed8eff8d898ll ), /* 12 */
-    const_float64( 0x3de6124613a86d09ll ), /* 13 */
-    const_float64( 0x3da93974a8c07c9dll ), /* 14 */
-    const_float64( 0x3d6ae7f3e733b81fll ), /* 15 */
-};
+    FloatParts64 p;
 
-float32 float32_exp2(float32 a, float_status *status)
+    bfloat16_unpack_canonical(&p, a, status);
+    parts_sqrt(&p, status, &bfloat16_params);
+    return bfloat16_round_pack_canonical(&p, status);
+}
+
+float128 QEMU_FLATTEN float128_sqrt(float128 a, float_status *status)
 {
-    flag aSign;
-    int aExp;
-    uint32_t aSig;
-    float64 r, x, xn;
-    int i;
-    a = float32_squash_input_denormal(a, status);
+    FloatParts128 p;
 
-    aSig = extractFloat32Frac( a );
-    aExp = extractFloat32Exp( a );
-    aSign = extractFloat32Sign( a );
+    float128_unpack_canonical(&p, a, status);
+    parts_sqrt(&p, status, &float128_params);
+    return float128_round_pack_canonical(&p, status);
+}
 
-    if ( aExp == 0xFF) {
-        if (aSig) {
-            return propagateFloat32NaN(a, float32_zero, status);
-        }
-        return (aSign) ? float32_zero : a;
-    }
-    if (aExp == 0) {
-        if (aSig == 0) return float32_one;
+floatx80 floatx80_sqrt(floatx80 a, float_status *s)
+{
+    FloatParts128 p;
+
+    if (!floatx80_unpack_canonical(&p, a, s)) {
+        return floatx80_default_nan(s);
     }
+    parts_sqrt(&p, s, &floatx80_params[s->floatx80_rounding_precision]);
+    return floatx80_round_pack_canonical(&p, s);
+}
 
-    float_raise(float_flag_inexact, status);
+/*
+ * log2
+ */
+float32 float32_log2(float32 a, float_status *status)
+{
+    FloatParts64 p;
 
-    /* ******************************* */
-    /* using float64 for approximation */
-    /* ******************************* */
-    x = float32_to_float64(a, status);
-    x = float64_mul(x, float64_ln2, status);
-
-    xn = x;
-    r = float64_one;
-    for (i = 0 ; i < 15 ; i++) {
-        float64 f;
-
-        f = float64_mul(xn, float32_exp2_coefficients[i], status);
-        r = float64_add(r, f, status);
-
-        xn = float64_mul(xn, x, status);
-    }
-
-    return float64_to_float32(r, status);
+    float32_unpack_canonical(&p, a, status);
+    parts_log2(&p, status, &float32_params);
+    return float32_round_pack_canonical(&p, status);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the binary log of the single-precision floating-point value `a'.
-| The operation is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-float32 float32_log2(float32 a, float_status *status)
+float64 float64_log2(float64 a, float_status *status)
 {
-    flag aSign, zSign;
-    int aExp;
-    uint32_t aSig, zSig, i;
-
-    a = float32_squash_input_denormal(a, status);
-    aSig = extractFloat32Frac( a );
-    aExp = extractFloat32Exp( a );
-    aSign = extractFloat32Sign( a );
-
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloat32( 1, 0xFF, 0 );
-        normalizeFloat32Subnormal( aSig, &aExp, &aSig );
-    }
-    if ( aSign ) {
-        float_raise(float_flag_invalid, status);
-        return float32_default_nan(status);
-    }
-    if ( aExp == 0xFF ) {
-        if (aSig) {
-            return propagateFloat32NaN(a, float32_zero, status);
-        }
-        return a;
-    }
-
-    aExp -= 0x7F;
-    aSig |= 0x00800000;
-    zSign = aExp < 0;
-    zSig = aExp << 23;
-
-    for (i = 1 << 22; i > 0; i >>= 1) {
-        aSig = ( (uint64_t)aSig * aSig ) >> 23;
-        if ( aSig & 0x01000000 ) {
-            aSig >>= 1;
-            zSig |= i;
-        }
-    }
-
-    if ( zSign )
-#ifdef _MSC_VER
-        zSig = 0 - zSig;
-#else
-        zSig = -zSig;
-#endif
+    FloatParts64 p;
 
-    return normalizeRoundAndPackFloat32(zSign, 0x85, zSig, status);
+    float64_unpack_canonical(&p, a, status);
+    parts_log2(&p, status, &float64_params);
+    return float64_round_pack_canonical(&p, status);
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is equal to
-| the corresponding value `b', and 0 otherwise.  The invalid exception is
-| raised if either operand is a NaN.  Otherwise, the comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+| The pattern for a default generated NaN.
 *----------------------------------------------------------------------------*/
 
-int float32_eq(float32 a, float32 b, float_status *status)
+float16 float16_default_nan(float_status *status)
 {
-    uint32_t av, bv;
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
+    FloatParts64 p;
 
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    av = float32_val(a);
-    bv = float32_val(b);
-    return ( av == bv ) || ( (uint32_t) ( ( av | bv )<<1 ) == 0 );
+    parts_default_nan(&p, status);
+    p.frac >>= float16_params.frac_shift;
+    return float16_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is less than
-| or equal to the corresponding value `b', and 0 otherwise.  The invalid
-| exception is raised if either operand is a NaN.  The comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float32_le(float32 a, float32 b, float_status *status)
+float32 float32_default_nan(float_status *status)
 {
-    flag aSign, bSign;
-    uint32_t av, bv;
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
-
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloat32Sign( a );
-    bSign = extractFloat32Sign( b );
-    av = float32_val(a);
-    bv = float32_val(b);
-    if ( aSign != bSign ) return aSign || ( (uint32_t) ( ( av | bv )<<1 ) == 0 );
-    return ( av == bv ) || ( aSign ^ ( av < bv ) );
+    FloatParts64 p;
 
+    parts_default_nan(&p, status);
+    p.frac >>= float32_params.frac_shift;
+    return float32_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is less than
-| the corresponding value `b', and 0 otherwise.  The invalid exception is
-| raised if either operand is a NaN.  The comparison is performed according
-| to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float32_lt(float32 a, float32 b, float_status *status)
+float64 float64_default_nan(float_status *status)
 {
-    flag aSign, bSign;
-    uint32_t av, bv;
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
-
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloat32Sign( a );
-    bSign = extractFloat32Sign( b );
-    av = float32_val(a);
-    bv = float32_val(b);
-    if ( aSign != bSign ) return aSign && ( (uint32_t) ( ( av | bv )<<1 ) != 0 );
-    return ( av != bv ) && ( aSign ^ ( av < bv ) );
+    FloatParts64 p;
 
+    parts_default_nan(&p, status);
+    p.frac >>= float64_params.frac_shift;
+    return float64_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point values `a' and `b' cannot
-| be compared, and 0 otherwise.  The invalid exception is raised if either
-| operand is a NaN.  The comparison is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float32_unordered(float32 a, float32 b, float_status *status)
+float128 float128_default_nan(float_status *status)
 {
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
+    FloatParts128 p;
 
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 1;
-    }
-    return 0;
+    parts_default_nan(&p, status);
+    frac_shr(&p, float128_params.frac_shift);
+    return float128_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is equal to
-| the corresponding value `b', and 0 otherwise.  Quiet NaNs do not cause an
-| exception.  The comparison is performed according to the IEC/IEEE Standard
-| for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float32_eq_quiet(float32 a, float32 b, float_status *status)
+bfloat16 bfloat16_default_nan(float_status *status)
 {
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
+    FloatParts64 p;
 
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        if (float32_is_signaling_nan(a, status)
-         || float32_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    return ( float32_val(a) == float32_val(b) ) ||
-            ( (uint32_t) ( ( float32_val(a) | float32_val(b) )<<1 ) == 0 );
+    parts_default_nan(&p, status);
+    p.frac >>= bfloat16_params.frac_shift;
+    return bfloat16_pack_raw(&p);
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is less than or
-| equal to the corresponding value `b', and 0 otherwise.  Quiet NaNs do not
-| cause an exception.  Otherwise, the comparison is performed according to the
-| IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+| Returns a quiet NaN from a signalling NaN for the floating point value `a'.
 *----------------------------------------------------------------------------*/
 
-int float32_le_quiet(float32 a, float32 b, float_status *status)
+float16 float16_silence_nan(float16 a, float_status *status)
 {
-    flag aSign, bSign;
-    uint32_t av, bv;
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
-
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        if (float32_is_signaling_nan(a, status)
-         || float32_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    aSign = extractFloat32Sign( a );
-    bSign = extractFloat32Sign( b );
-    av = float32_val(a);
-    bv = float32_val(b);
-    if ( aSign != bSign ) return aSign || ( (uint32_t) ( ( av | bv )<<1 ) == 0 );
-    return ( av == bv ) || ( aSign ^ ( av < bv ) );
+    FloatParts64 p;
 
+    float16_unpack_raw(&p, a);
+    p.frac <<= float16_params.frac_shift;
+    parts_silence_nan(&p, status);
+    p.frac >>= float16_params.frac_shift;
+    return float16_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point value `a' is less than
-| the corresponding value `b', and 0 otherwise.  Quiet NaNs do not cause an
-| exception.  Otherwise, the comparison is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float32_lt_quiet(float32 a, float32 b, float_status *status)
+float32 float32_silence_nan(float32 a, float_status *status)
 {
-    flag aSign, bSign;
-    uint32_t av, bv;
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
-
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        if (float32_is_signaling_nan(a, status)
-         || float32_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    aSign = extractFloat32Sign( a );
-    bSign = extractFloat32Sign( b );
-    av = float32_val(a);
-    bv = float32_val(b);
-    if ( aSign != bSign ) return aSign && ( (uint32_t) ( ( av | bv )<<1 ) != 0 );
-    return ( av != bv ) && ( aSign ^ ( av < bv ) );
+    FloatParts64 p;
 
+    float32_unpack_raw(&p, a);
+    p.frac <<= float32_params.frac_shift;
+    parts_silence_nan(&p, status);
+    p.frac >>= float32_params.frac_shift;
+    return float32_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the single-precision floating-point values `a' and `b' cannot
-| be compared, and 0 otherwise.  Quiet NaNs do not cause an exception.  The
-| comparison is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float32_unordered_quiet(float32 a, float32 b, float_status *status)
+float64 float64_silence_nan(float64 a, float_status *status)
 {
-    a = float32_squash_input_denormal(a, status);
-    b = float32_squash_input_denormal(b, status);
+    FloatParts64 p;
 
-    if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
-         || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
-       ) {
-        if (float32_is_signaling_nan(a, status)
-         || float32_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 1;
-    }
-    return 0;
+    float64_unpack_raw(&p, a);
+    p.frac <<= float64_params.frac_shift;
+    parts_silence_nan(&p, status);
+    p.frac >>= float64_params.frac_shift;
+    return float64_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the double-precision floating-point value
-| `a' to the extended double-precision floating-point format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 float64_to_floatx80(float64 a, float_status *status)
+bfloat16 bfloat16_silence_nan(bfloat16 a, float_status *status)
 {
-    flag aSign;
-    int aExp;
-    uint64_t aSig;
-
-    a = float64_squash_input_denormal(a, status);
-    aSig = extractFloat64Frac( a );
-    aExp = extractFloat64Exp( a );
-    aSign = extractFloat64Sign( a );
-    if ( aExp == 0x7FF ) {
-        if (aSig) {
-            return commonNaNToFloatx80(float64ToCommonNaN(a, status), status);
-        }
-        return packFloatx80(aSign,
-                            floatx80_infinity_high,
-                            floatx80_infinity_low);
-    }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloatx80( aSign, 0, 0 );
-        normalizeFloat64Subnormal( aSig, &aExp, &aSig );
-    }
-    return
-        packFloatx80(
-            aSign, aExp + 0x3C00, (aSig | UINT64_C(0x0010000000000000)) << 11);
+    FloatParts64 p;
 
+    bfloat16_unpack_raw(&p, a);
+    p.frac <<= bfloat16_params.frac_shift;
+    parts_silence_nan(&p, status);
+    p.frac >>= bfloat16_params.frac_shift;
+    return bfloat16_pack_raw(&p);
 }
 
-/*----------------------------------------------------------------------------
-| Returns the result of converting the double-precision floating-point value
-| `a' to the quadruple-precision floating-point format.  The conversion is
-| performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float64_to_float128(float64 a, float_status *status)
+float128 float128_silence_nan(float128 a, float_status *status)
 {
-    flag aSign;
-    int aExp;
-    uint64_t aSig, zSig0, zSig1;
-
-    a = float64_squash_input_denormal(a, status);
-    aSig = extractFloat64Frac( a );
-    aExp = extractFloat64Exp( a );
-    aSign = extractFloat64Sign( a );
-    if ( aExp == 0x7FF ) {
-        if (aSig) {
-            return commonNaNToFloat128(float64ToCommonNaN(a, status), status);
-        }
-        return packFloat128( aSign, 0x7FFF, 0, 0 );
-    }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloat128( aSign, 0, 0, 0 );
-        normalizeFloat64Subnormal( aSig, &aExp, &aSig );
-        --aExp;
-    }
-    shift128Right( aSig, 0, 4, &zSig0, &zSig1 );
-    return packFloat128( aSign, aExp + 0x3C00, zSig0, zSig1 );
+    FloatParts128 p;
 
+    float128_unpack_raw(&p, a);
+    frac_shl(&p, float128_params.frac_shift);
+    parts_silence_nan(&p, status);
+    frac_shr(&p, float128_params.frac_shift);
+    return float128_pack_raw(&p);
 }
 
-
 /*----------------------------------------------------------------------------
-| Returns the remainder of the double-precision floating-point value `a'
-| with respect to the corresponding value `b'.  The operation is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+| If `a' is denormal and we are in flush-to-zero mode then set the
+| input-denormal exception and return zero. Otherwise just return the value.
 *----------------------------------------------------------------------------*/
 
-float64 float64_rem(float64 a, float64 b, float_status *status)
+static bool parts_squash_denormal(FloatParts64 p, float_status *status)
 {
-    flag aSign, zSign;
-    int aExp, bExp, expDiff;
-    uint64_t aSig, bSig;
-    uint64_t q, alternateASig;
-    int64_t sigMean;
-
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
-    aSig = extractFloat64Frac( a );
-    aExp = extractFloat64Exp( a );
-    aSign = extractFloat64Sign( a );
-    bSig = extractFloat64Frac( b );
-    bExp = extractFloat64Exp( b );
-    if ( aExp == 0x7FF ) {
-        if ( aSig || ( ( bExp == 0x7FF ) && bSig ) ) {
-            return propagateFloat64NaN(a, b, status);
-        }
-        float_raise(float_flag_invalid, status);
-        return float64_default_nan(status);
-    }
-    if ( bExp == 0x7FF ) {
-        if (bSig) {
-            return propagateFloat64NaN(a, b, status);
-        }
-        return a;
-    }
-    if ( bExp == 0 ) {
-        if ( bSig == 0 ) {
-            float_raise(float_flag_invalid, status);
-            return float64_default_nan(status);
-        }
-        normalizeFloat64Subnormal( bSig, &bExp, &bSig );
-    }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return a;
-        normalizeFloat64Subnormal( aSig, &aExp, &aSig );
-    }
-    expDiff = aExp - bExp;
-    aSig = (aSig | UINT64_C(0x0010000000000000)) << 11;
-    bSig = (bSig | UINT64_C(0x0010000000000000)) << 11;
-    if ( expDiff < 0 ) {
-        if ( expDiff < -1 ) return a;
-        aSig >>= 1;
-    }
-    q = ( bSig <= aSig );
-    if ( q ) aSig -= bSig;
-    expDiff -= 64;
-    while ( 0 < expDiff ) {
-        q = estimateDiv128To64( aSig, 0, bSig );
-        q = ( 2 < q ) ? q - 2 : 0;
-#ifdef _MSC_VER
-        aSig = 0ULL - ( ( bSig>>2 ) * q );
-#else
-        aSig = - ( ( bSig>>2 ) * q );
-#endif
-        expDiff -= 62;
-    }
-    expDiff += 64;
-    if ( 0 < expDiff ) {
-        q = estimateDiv128To64( aSig, 0, bSig );
-        q = ( 2 < q ) ? q - 2 : 0;
-        q >>= 64 - expDiff;
-        bSig >>= 2;
-        aSig = ( ( aSig>>1 )<<( expDiff - 1 ) ) - bSig * q;
+    if (p.exp == 0 && p.frac != 0) {
+        float_raise(float_flag_input_denormal, status);
+        return true;
     }
-    else {
-        aSig >>= 2;
-        bSig >>= 2;
-    }
-    do {
-        alternateASig = aSig;
-        ++q;
-        aSig -= bSig;
-    } while ( 0 <= (int64_t) aSig );
-    sigMean = aSig + alternateASig;
-    if ( ( sigMean < 0 ) || ( ( sigMean == 0 ) && ( q & 1 ) ) ) {
-        aSig = alternateASig;
-    }
-    zSign = ( (int64_t) aSig < 0 );
-#ifdef _MSC_VER
-    if ( zSign ) aSig = 0 - aSig;
-#else
-    if ( zSign ) aSig = - aSig;
-#endif
-    return normalizeRoundAndPackFloat64(aSign ^ zSign, bExp, aSig, status);
 
+    return false;
 }
 
-/*----------------------------------------------------------------------------
-| Returns the binary log of the double-precision floating-point value `a'.
-| The operation is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-float64 float64_log2(float64 a, float_status *status)
+float16 float16_squash_input_denormal(float16 a, float_status *status)
 {
-    flag aSign, zSign;
-    int aExp;
-    uint64_t aSig, aSig0, aSig1, zSig, i;
-    a = float64_squash_input_denormal(a, status);
-
-    aSig = extractFloat64Frac( a );
-    aExp = extractFloat64Exp( a );
-    aSign = extractFloat64Sign( a );
+    if (status->flush_inputs_to_zero) {
+        FloatParts64 p;
 
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloat64( 1, 0x7FF, 0 );
-        normalizeFloat64Subnormal( aSig, &aExp, &aSig );
-    }
-    if ( aSign ) {
-        float_raise(float_flag_invalid, status);
-        return float64_default_nan(status);
-    }
-    if ( aExp == 0x7FF ) {
-        if (aSig) {
-            return propagateFloat64NaN(a, float64_zero, status);
-        }
-        return a;
-    }
-
-    aExp -= 0x3FF;
-    aSig |= UINT64_C(0x0010000000000000);
-    zSign = aExp < 0;
-    zSig = (uint64_t)aExp << 52;
-    for (i = 1LL << 51; i > 0; i >>= 1) {
-        mul64To128( aSig, aSig, &aSig0, &aSig1 );
-        aSig = ( aSig0 << 12 ) | ( aSig1 >> 52 );
-        if ( aSig & UINT64_C(0x0020000000000000) ) {
-            aSig >>= 1;
-            zSig |= i;
+        float16_unpack_raw(&p, a);
+        if (parts_squash_denormal(p, status)) {
+            return float16_set_sign(float16_zero, p.sign);
         }
     }
-
-    if ( zSign )
-#ifdef _MSC_VER
-        zSig = 0 - zSig;
-#else
-        zSig = -zSig;
-#endif
-    return normalizeRoundAndPackFloat64(zSign, 0x408, zSig, status);
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point value `a' is equal to the
-| corresponding value `b', and 0 otherwise.  The invalid exception is raised
-| if either operand is a NaN.  Otherwise, the comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float64_eq(float64 a, float64 b, float_status *status)
-{
-    uint64_t av, bv;
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
-
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    av = float64_val(a);
-    bv = float64_val(b);
-    return ( av == bv ) || ( (uint64_t) ( ( av | bv )<<1 ) == 0 );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point value `a' is less than or
-| equal to the corresponding value `b', and 0 otherwise.  The invalid
-| exception is raised if either operand is a NaN.  The comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float64_le(float64 a, float64 b, float_status *status)
-{
-    flag aSign, bSign;
-    uint64_t av, bv;
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
-
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloat64Sign( a );
-    bSign = extractFloat64Sign( b );
-    av = float64_val(a);
-    bv = float64_val(b);
-    if ( aSign != bSign ) return aSign || ( (uint64_t) ( ( av | bv )<<1 ) == 0 );
-    return ( av == bv ) || ( aSign ^ ( av < bv ) );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point value `a' is less than
-| the corresponding value `b', and 0 otherwise.  The invalid exception is
-| raised if either operand is a NaN.  The comparison is performed according
-| to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float64_lt(float64 a, float64 b, float_status *status)
-{
-    flag aSign, bSign;
-    uint64_t av, bv;
-
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloat64Sign( a );
-    bSign = extractFloat64Sign( b );
-    av = float64_val(a);
-    bv = float64_val(b);
-    if ( aSign != bSign ) return aSign && ( (uint64_t) ( ( av | bv )<<1 ) != 0 );
-    return ( av != bv ) && ( aSign ^ ( av < bv ) );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point values `a' and `b' cannot
-| be compared, and 0 otherwise.  The invalid exception is raised if either
-| operand is a NaN.  The comparison is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float64_unordered(float64 a, float64 b, float_status *status)
-{
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
-
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 1;
-    }
-    return 0;
+    return a;
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point value `a' is equal to the
-| corresponding value `b', and 0 otherwise.  Quiet NaNs do not cause an
-| exception.The comparison is performed according to the IEC/IEEE Standard
-| for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float64_eq_quiet(float64 a, float64 b, float_status *status)
+float32 float32_squash_input_denormal(float32 a, float_status *status)
 {
-    uint64_t av, bv;
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
+    if (status->flush_inputs_to_zero) {
+        FloatParts64 p;
 
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        if (float64_is_signaling_nan(a, status)
-         || float64_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
+        float32_unpack_raw(&p, a);
+        if (parts_squash_denormal(p, status)) {
+            return float32_set_sign(float32_zero, p.sign);
         }
-        return 0;
     }
-    av = float64_val(a);
-    bv = float64_val(b);
-    return ( av == bv ) || ( (uint64_t) ( ( av | bv )<<1 ) == 0 );
-
+    return a;
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point value `a' is less than or
-| equal to the corresponding value `b', and 0 otherwise.  Quiet NaNs do not
-| cause an exception.  Otherwise, the comparison is performed according to the
-| IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float64_le_quiet(float64 a, float64 b, float_status *status)
+float64 float64_squash_input_denormal(float64 a, float_status *status)
 {
-    flag aSign, bSign;
-    uint64_t av, bv;
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
+    if (status->flush_inputs_to_zero) {
+        FloatParts64 p;
 
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        if (float64_is_signaling_nan(a, status)
-         || float64_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
+        float64_unpack_raw(&p, a);
+        if (parts_squash_denormal(p, status)) {
+            return float64_set_sign(float64_zero, p.sign);
         }
-        return 0;
     }
-    aSign = extractFloat64Sign( a );
-    bSign = extractFloat64Sign( b );
-    av = float64_val(a);
-    bv = float64_val(b);
-    if ( aSign != bSign ) return aSign || ( (uint64_t) ( ( av | bv )<<1 ) == 0 );
-    return ( av == bv ) || ( aSign ^ ( av < bv ) );
-
+    return a;
 }
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point value `a' is less than
-| the corresponding value `b', and 0 otherwise.  Quiet NaNs do not cause an
-| exception.  Otherwise, the comparison is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float64_lt_quiet(float64 a, float64 b, float_status *status)
+bfloat16 bfloat16_squash_input_denormal(bfloat16 a, float_status *status)
 {
-    flag aSign, bSign;
-    uint64_t av, bv;
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
+    if (status->flush_inputs_to_zero) {
+        FloatParts64 p;
 
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        if (float64_is_signaling_nan(a, status)
-         || float64_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
+        bfloat16_unpack_raw(&p, a);
+        if (parts_squash_denormal(p, status)) {
+            return bfloat16_set_sign(bfloat16_zero, p.sign);
         }
-        return 0;
     }
-    aSign = extractFloat64Sign( a );
-    bSign = extractFloat64Sign( b );
-    av = float64_val(a);
-    bv = float64_val(b);
-    if ( aSign != bSign ) return aSign && ( (uint64_t) ( ( av | bv )<<1 ) != 0 );
-    return ( av != bv ) && ( aSign ^ ( av < bv ) );
-
+    return a;
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the double-precision floating-point values `a' and `b' cannot
-| be compared, and 0 otherwise.  Quiet NaNs do not cause an exception.  The
-| comparison is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
+| Normalizes the subnormal extended double-precision floating-point value
+| represented by the denormalized significand `aSig'.  The normalized exponent
+| and significand are stored at the locations pointed to by `zExpPtr' and
+| `zSigPtr', respectively.
 *----------------------------------------------------------------------------*/
 
-int float64_unordered_quiet(float64 a, float64 b, float_status *status)
+void normalizeFloatx80Subnormal(uint64_t aSig, int32_t *zExpPtr,
+                                uint64_t *zSigPtr)
 {
-    a = float64_squash_input_denormal(a, status);
-    b = float64_squash_input_denormal(b, status);
+    int8_t shiftCount;
 
-    if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
-         || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
-       ) {
-        if (float64_is_signaling_nan(a, status)
-         || float64_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 1;
-    }
-    return 0;
+    shiftCount = clz64(aSig);
+    *zSigPtr = aSig<<shiftCount;
+    *zExpPtr = 1 - shiftCount;
 }
 
 /*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point value `a' to the 32-bit two's complement integer format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic---which means in particular that the conversion
-| is rounded according to the current rounding mode.  If `a' is a NaN, the
-| largest positive integer is returned.  Otherwise, if the conversion
-| overflows, the largest integer with the same sign as `a' is returned.
+| Takes an abstract floating-point value having sign `zSign', exponent `zExp',
+| and extended significand formed by the concatenation of `zSig0' and `zSig1',
+| and returns the proper extended double-precision floating-point value
+| corresponding to the abstract input.  Ordinarily, the abstract value is
+| rounded and packed into the extended double-precision format, with the
+| inexact exception raised if the abstract input cannot be represented
+| exactly.  However, if the abstract value is too large, the overflow and
+| inexact exceptions are raised and an infinity or maximal finite value is
+| returned.  If the abstract value is too small, the input value is rounded to
+| a subnormal number, and the underflow and inexact exceptions are raised if
+| the abstract input cannot be represented exactly as a subnormal extended
+| double-precision floating-point number.
+|     If `roundingPrecision' is floatx80_precision_s or floatx80_precision_d,
+| the result is rounded to the same number of bits as single or double
+| precision, respectively.  Otherwise, the result is rounded to the full
+| precision of the extended double-precision format.
+|     The input significand must be normalized or smaller.  If the input
+| significand is not normalized, `zExp' must be 0; in that case, the result
+| returned is a subnormal number, and it must not require rounding.  The
+| handling of underflow and overflow follows the IEC/IEEE Standard for Binary
+| Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int32_t floatx80_to_int32(floatx80 a, float_status *status)
+floatx80 roundAndPackFloatx80(FloatX80RoundPrec roundingPrecision, bool zSign,
+                              int32_t zExp, uint64_t zSig0, uint64_t zSig1,
+                              float_status *status)
 {
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig;
+    FloatRoundMode roundingMode;
+    bool roundNearestEven, increment, isTiny;
+    int64_t roundIncrement, roundMask, roundBits;
 
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return 1 << 31;
+    roundingMode = status->float_rounding_mode;
+    roundNearestEven = ( roundingMode == float_round_nearest_even );
+    switch (roundingPrecision) {
+    case floatx80_precision_x:
+        goto precision80;
+    case floatx80_precision_d:
+        roundIncrement = UINT64_C(0x0000000000000400);
+        roundMask = UINT64_C(0x00000000000007FF);
+        break;
+    case floatx80_precision_s:
+        roundIncrement = UINT64_C(0x0000008000000000);
+        roundMask = UINT64_C(0x000000FFFFFFFFFF);
+        break;
+    default:
+        g_assert_not_reached();
     }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    if ( ( aExp == 0x7FFF ) && (uint64_t) ( aSig<<1 ) ) aSign = 0;
-    shiftCount = 0x4037 - aExp;
-    if ( shiftCount <= 0 ) shiftCount = 1;
-    shift64RightJamming( aSig, shiftCount, &aSig );
-    return roundAndPackInt32(aSign, aSig, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point value `a' to the 32-bit two's complement integer format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic, except that the conversion is always rounded
-| toward zero.  If `a' is a NaN, the largest positive integer is returned.
-| Otherwise, if the conversion overflows, the largest integer with the same
-| sign as `a' is returned.
-*----------------------------------------------------------------------------*/
-
-int32_t floatx80_to_int32_round_to_zero(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig, savedASig;
-    int32_t z;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return 1 << 31;
+    zSig0 |= ( zSig1 != 0 );
+    switch (roundingMode) {
+    case float_round_nearest_even:
+    case float_round_ties_away:
+        break;
+    case float_round_to_zero:
+        roundIncrement = 0;
+        break;
+    case float_round_up:
+        roundIncrement = zSign ? 0 : roundMask;
+        break;
+    case float_round_down:
+        roundIncrement = zSign ? roundMask : 0;
+        break;
+    default:
+        abort();
     }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    if ( 0x401E < aExp ) {
-        if ( ( aExp == 0x7FFF ) && (uint64_t) ( aSig<<1 ) ) aSign = 0;
-        goto invalid;
-    }
-    else if ( aExp < 0x3FFF ) {
-        if (aExp || aSig) {
-            status->float_exception_flags |= float_flag_inexact;
+    roundBits = zSig0 & roundMask;
+    if ( 0x7FFD <= (uint32_t) ( zExp - 1 ) ) {
+        if (    ( 0x7FFE < zExp )
+             || ( ( zExp == 0x7FFE ) && ( zSig0 + roundIncrement < zSig0 ) )
+           ) {
+            goto overflow;
         }
-        return 0;
-    }
-    shiftCount = 0x403E - aExp;
-    savedASig = aSig;
-    aSig >>= shiftCount;
-    z = aSig;
-    if ( aSign ) z = - z;
-    if ( ( z < 0 ) ^ aSign ) {
- invalid:
-        float_raise(float_flag_invalid, status);
-        return aSign ? (int32_t) 0x80000000 : 0x7FFFFFFF;
-    }
-    if ( ( aSig<<shiftCount ) != savedASig ) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    return z;
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point value `a' to the 64-bit two's complement integer format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic---which means in particular that the conversion
-| is rounded according to the current rounding mode.  If `a' is a NaN,
-| the largest positive integer is returned.  Otherwise, if the conversion
-| overflows, the largest integer with the same sign as `a' is returned.
-*----------------------------------------------------------------------------*/
-
-int64_t floatx80_to_int64(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig, aSigExtra;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return 1ULL << 63;
-    }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    shiftCount = 0x403E - aExp;
-    if ( shiftCount <= 0 ) {
-        if ( shiftCount ) {
-            float_raise(float_flag_invalid, status);
-            if (!aSign || floatx80_is_any_nan(a)) {
-                return INT64_MAX;
+        if ( zExp <= 0 ) {
+            if (status->flush_to_zero) {
+                float_raise(float_flag_output_denormal, status);
+                return packFloatx80(zSign, 0, 0);
             }
-            return INT64_MIN;
-        }
-        aSigExtra = 0;
-    }
-    else {
-        shift64ExtraRightJamming( aSig, 0, shiftCount, &aSig, &aSigExtra );
-    }
-    return roundAndPackInt64(aSign, aSig, aSigExtra, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point value `a' to the 64-bit two's complement integer format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic, except that the conversion is always rounded
-| toward zero.  If `a' is a NaN, the largest positive integer is returned.
-| Otherwise, if the conversion overflows, the largest integer with the same
-| sign as `a' is returned.
-*----------------------------------------------------------------------------*/
-
-int64_t floatx80_to_int64_round_to_zero(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig;
-    int64_t z;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return 1ULL << 63;
-    }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    shiftCount = aExp - 0x403E;
-    if ( 0 <= shiftCount ) {
-        aSig &= UINT64_C(0x7FFFFFFFFFFFFFFF);
-        if ( ( a.high != 0xC03E ) || aSig ) {
-            float_raise(float_flag_invalid, status);
-            if ( ! aSign || ( ( aExp == 0x7FFF ) && aSig ) ) {
-                return INT64_MAX;
+            isTiny = status->tininess_before_rounding
+                  || (zExp < 0 )
+                  || (zSig0 <= zSig0 + roundIncrement);
+            shift64RightJamming( zSig0, 1 - zExp, &zSig0 );
+            zExp = 0;
+            roundBits = zSig0 & roundMask;
+            if (isTiny && roundBits) {
+                float_raise(float_flag_underflow, status);
             }
+            if (roundBits) {
+                float_raise(float_flag_inexact, status);
+            }
+            zSig0 += roundIncrement;
+            if ( (int64_t) zSig0 < 0 ) zExp = 1;
+            roundIncrement = roundMask + 1;
+            if ( roundNearestEven && ( roundBits<<1 == roundIncrement ) ) {
+                roundMask |= roundIncrement;
+            }
+            zSig0 &= ~ roundMask;
+            return packFloatx80( zSign, zExp, zSig0 );
         }
-        return INT64_MIN;
-    }
-    else if ( aExp < 0x3FFF ) {
-        if (aExp | aSig) {
-            status->float_exception_flags |= float_flag_inexact;
-        }
-        return 0;
-    }
-    z = aSig>>( - shiftCount );
-    if ( (uint64_t) ( aSig<<( shiftCount & 63 ) ) ) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    if ( aSign ) z = - z;
-    return z;
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point value `a' to the single-precision floating-point format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float32 floatx80_to_float32(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t aSig;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return float32_default_nan(status);
-    }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if ( (uint64_t) ( aSig<<1 ) ) {
-            return commonNaNToFloat32(floatx80ToCommonNaN(a, status), status);
-        }
-        return packFloat32( aSign, 0xFF, 0 );
-    }
-    shift64RightJamming( aSig, 33, &aSig );
-    if ( aExp || aSig ) aExp -= 0x3F81;
-    return roundAndPackFloat32(aSign, aExp, aSig, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point value `a' to the double-precision floating-point format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float64 floatx80_to_float64(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t aSig, zSig;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return float64_default_nan(status);
-    }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if ( (uint64_t) ( aSig<<1 ) ) {
-            return commonNaNToFloat64(floatx80ToCommonNaN(a, status), status);
-        }
-        return packFloat64( aSign, 0x7FF, 0 );
-    }
-    shift64RightJamming( aSig, 1, &zSig );
-    if ( aExp || aSig ) aExp -= 0x3C01;
-    return roundAndPackFloat64(aSign, aExp, zSig, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the extended double-precision floating-
-| point value `a' to the quadruple-precision floating-point format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 floatx80_to_float128(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int aExp;
-    uint64_t aSig, zSig0, zSig1;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return float128_default_nan(status);
     }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    if ( ( aExp == 0x7FFF ) && (uint64_t) ( aSig<<1 ) ) {
-        return commonNaNToFloat128(floatx80ToCommonNaN(a, status), status);
-    }
-    shift128Right( aSig<<1, 0, 16, &zSig0, &zSig1 );
-    return packFloat128( aSign, aExp, zSig0, zSig1 );
-
-}
-
-/*----------------------------------------------------------------------------
-| Rounds the extended double-precision floating-point value `a'
-| to the precision provided by floatx80_rounding_precision and returns the
-| result as an extended double-precision floating-point value.
-| The operation is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_round(floatx80 a, float_status *status)
-{
-    return roundAndPackFloatx80(status->floatx80_rounding_precision,
-                                extractFloatx80Sign(a),
-                                extractFloatx80Exp(a),
-                                extractFloatx80Frac(a), 0, status);
-}
-
-/*----------------------------------------------------------------------------
-| Rounds the extended double-precision floating-point value `a' to an integer,
-| and returns the result as an extended quadruple-precision floating-point
-| value.  The operation is performed according to the IEC/IEEE Standard for
-| Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_round_to_int(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t lastBitMask, roundBitsMask;
-    floatx80 z;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
+    if (roundBits) {
+        float_raise(float_flag_inexact, status);
     }
-    aExp = extractFloatx80Exp( a );
-    if ( 0x403E <= aExp ) {
-        if ( ( aExp == 0x7FFF ) && (uint64_t) ( extractFloatx80Frac( a )<<1 ) ) {
-            return propagateFloatx80NaN(a, a, status);
-        }
-        return a;
+    zSig0 += roundIncrement;
+    if ( zSig0 < roundIncrement ) {
+        ++zExp;
+        zSig0 = UINT64_C(0x8000000000000000);
     }
-    if ( aExp < 0x3FFF ) {
-        if (    ( aExp == 0 )
-             && ( (uint64_t) ( extractFloatx80Frac( a )<<1 ) == 0 ) ) {
-            return a;
-        }
-        status->float_exception_flags |= float_flag_inexact;
-        aSign = extractFloatx80Sign( a );
-        switch (status->float_rounding_mode) {
-         case float_round_nearest_even:
-            if ( ( aExp == 0x3FFE ) && (uint64_t) ( extractFloatx80Frac( a )<<1 )
-               ) {
-                return
-                    packFloatx80( aSign, 0x3FFF, UINT64_C(0x8000000000000000));
-            }
-            break;
-        case float_round_ties_away:
-            if (aExp == 0x3FFE) {
-                return packFloatx80(aSign, 0x3FFF, UINT64_C(0x8000000000000000));
-            }
-            break;
-         case float_round_down:
-            return
-                  aSign ?
-                      packFloatx80( 1, 0x3FFF, UINT64_C(0x8000000000000000))
-                : packFloatx80( 0, 0, 0 );
-         case float_round_up:
-            return
-                  aSign ? packFloatx80( 1, 0, 0 )
-                : packFloatx80( 0, 0x3FFF, UINT64_C(0x8000000000000000));
-        }
-        return packFloatx80( aSign, 0, 0 );
+    roundIncrement = roundMask + 1;
+    if ( roundNearestEven && ( roundBits<<1 == roundIncrement ) ) {
+        roundMask |= roundIncrement;
     }
-    lastBitMask = 1;
-    lastBitMask <<= 0x403E - aExp;
-    roundBitsMask = lastBitMask - 1;
-    z = a;
-    switch (status->float_rounding_mode) {
+    zSig0 &= ~ roundMask;
+    if ( zSig0 == 0 ) zExp = 0;
+    return packFloatx80( zSign, zExp, zSig0 );
+ precision80:
+    switch (roundingMode) {
     case float_round_nearest_even:
-        z.low += lastBitMask>>1;
-        if ((z.low & roundBitsMask) == 0) {
-            z.low &= ~lastBitMask;
-        }
-        break;
     case float_round_ties_away:
-        z.low += lastBitMask >> 1;
+        increment = ((int64_t)zSig1 < 0);
         break;
     case float_round_to_zero:
+        increment = 0;
         break;
     case float_round_up:
-        if (!extractFloatx80Sign(z)) {
-            z.low += roundBitsMask;
-        }
+        increment = !zSign && zSig1;
         break;
     case float_round_down:
-        if (extractFloatx80Sign(z)) {
-            z.low += roundBitsMask;
-        }
+        increment = zSign && zSig1;
         break;
     default:
         abort();
     }
-    z.low &= ~ roundBitsMask;
-    if ( z.low == 0 ) {
-        ++z.high;
-        z.low = UINT64_C(0x8000000000000000);
-    }
-    if (z.low != a.low) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    return z;
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of adding the absolute values of the extended double-
-| precision floating-point values `a' and `b'.  If `zSign' is 1, the sum is
-| negated before being returned.  `zSign' is ignored if the result is a NaN.
-| The addition is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-static floatx80 addFloatx80Sigs(floatx80 a, floatx80 b, flag zSign,
-                                float_status *status)
-{
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig, bSig, zSig0, zSig1;
-    int32_t expDiff;
-
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    bSig = extractFloatx80Frac( b );
-    bExp = extractFloatx80Exp( b );
-    expDiff = aExp - bExp;
-    if ( 0 < expDiff ) {
-        if ( aExp == 0x7FFF ) {
-            if ((uint64_t)(aSig << 1)) {
-                return propagateFloatx80NaN(a, b, status);
-            }
-            return a;
-        }
-        if ( bExp == 0 ) --expDiff;
-        shift64ExtraRightJamming( bSig, 0, expDiff, &bSig, &zSig1 );
-        zExp = aExp;
-    }
-    else if ( expDiff < 0 ) {
-        if ( bExp == 0x7FFF ) {
-            if ((uint64_t)(bSig << 1)) {
-                return propagateFloatx80NaN(a, b, status);
-            }
-            return packFloatx80(zSign,
-                                floatx80_infinity_high,
-                                floatx80_infinity_low);
-        }
-        if ( aExp == 0 ) ++expDiff;
-        shift64ExtraRightJamming( aSig, 0, - expDiff, &aSig, &zSig1 );
-        zExp = bExp;
-    }
-    else {
-        if ( aExp == 0x7FFF ) {
-            if ( (uint64_t) ( ( aSig | bSig )<<1 ) ) {
-                return propagateFloatx80NaN(a, b, status);
-            }
-            return a;
-        }
-        zSig1 = 0;
-        zSig0 = aSig + bSig;
-        if ( aExp == 0 ) {
-            if (zSig0 == 0) {
-                return packFloatx80(zSign, 0, 0);
-            }
-            normalizeFloatx80Subnormal( zSig0, &zExp, &zSig0 );
-            goto roundAndPack;
-        }
-        zExp = aExp;
-        goto shiftRight1;
-    }
-    zSig0 = aSig + bSig;
-    if ( (int64_t) zSig0 < 0 ) goto roundAndPack;
- shiftRight1:
-    shift64ExtraRightJamming( zSig0, zSig1, 1, &zSig0, &zSig1 );
-    zSig0 |= UINT64_C(0x8000000000000000);
-    ++zExp;
- roundAndPack:
-    return roundAndPackFloatx80(status->floatx80_rounding_precision,
-                                zSign, zExp, zSig0, zSig1, status);
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of subtracting the absolute values of the extended
-| double-precision floating-point values `a' and `b'.  If `zSign' is 1, the
-| difference is negated before being returned.  `zSign' is ignored if the
-| result is a NaN.  The subtraction is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-static floatx80 subFloatx80Sigs(floatx80 a, floatx80 b, flag zSign,
-                                float_status *status)
-{
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig, bSig, zSig0, zSig1;
-    int32_t expDiff;
-
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    bSig = extractFloatx80Frac( b );
-    bExp = extractFloatx80Exp( b );
-    expDiff = aExp - bExp;
-    if ( 0 < expDiff ) goto aExpBigger;
-    if ( expDiff < 0 ) goto bExpBigger;
-    if ( aExp == 0x7FFF ) {
-        if ( (uint64_t) ( ( aSig | bSig )<<1 ) ) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    if ( aExp == 0 ) {
-        aExp = 1;
-        bExp = 1;
-    }
-    zSig1 = 0;
-    if ( bSig < aSig ) goto aBigger;
-    if ( aSig < bSig ) goto bBigger;
-    return packFloatx80(status->float_rounding_mode == float_round_down, 0, 0);
- bExpBigger:
-    if ( bExp == 0x7FFF ) {
-        if ((uint64_t)(bSig << 1)) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        return packFloatx80(zSign ^ 1, floatx80_infinity_high,
-                            floatx80_infinity_low);
-    }
-    if ( aExp == 0 ) ++expDiff;
-    shift128RightJamming( aSig, 0, - expDiff, &aSig, &zSig1 );
- bBigger:
-    sub128( bSig, 0, aSig, zSig1, &zSig0, &zSig1 );
-    zExp = bExp;
-    zSign ^= 1;
-    goto normalizeRoundAndPack;
- aExpBigger:
-    if ( aExp == 0x7FFF ) {
-        if ((uint64_t)(aSig << 1)) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        return a;
-    }
-    if ( bExp == 0 ) --expDiff;
-    shift128RightJamming( bSig, 0, expDiff, &bSig, &zSig1 );
- aBigger:
-    sub128( aSig, 0, bSig, zSig1, &zSig0, &zSig1 );
-    zExp = aExp;
- normalizeRoundAndPack:
-    return normalizeRoundAndPackFloatx80(status->floatx80_rounding_precision,
-                                         zSign, zExp, zSig0, zSig1, status);
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of adding the extended double-precision floating-point
-| values `a' and `b'.  The operation is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_add(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    aSign = extractFloatx80Sign( a );
-    bSign = extractFloatx80Sign( b );
-    if ( aSign == bSign ) {
-        return addFloatx80Sigs(a, b, aSign, status);
-    }
-    else {
-        return subFloatx80Sigs(a, b, aSign, status);
-    }
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of subtracting the extended double-precision floating-
-| point values `a' and `b'.  The operation is performed according to the
-| IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_sub(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    aSign = extractFloatx80Sign( a );
-    bSign = extractFloatx80Sign( b );
-    if ( aSign == bSign ) {
-        return subFloatx80Sigs(a, b, aSign, status);
-    }
-    else {
-        return addFloatx80Sigs(a, b, aSign, status);
-    }
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of multiplying the extended double-precision floating-
-| point values `a' and `b'.  The operation is performed according to the
-| IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_mul(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign, zSign;
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig, bSig, zSig0, zSig1;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    bSig = extractFloatx80Frac( b );
-    bExp = extractFloatx80Exp( b );
-    bSign = extractFloatx80Sign( b );
-    zSign = aSign ^ bSign;
-    if ( aExp == 0x7FFF ) {
-        if (    (uint64_t) ( aSig<<1 )
-             || ( ( bExp == 0x7FFF ) && (uint64_t) ( bSig<<1 ) ) ) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        if ( ( bExp | bSig ) == 0 ) goto invalid;
-        return packFloatx80(zSign, floatx80_infinity_high,
-                                   floatx80_infinity_low);
-    }
-    if ( bExp == 0x7FFF ) {
-        if ((uint64_t)(bSig << 1)) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        if ( ( aExp | aSig ) == 0 ) {
- invalid:
-            float_raise(float_flag_invalid, status);
-            return floatx80_default_nan(status);
-        }
-        return packFloatx80(zSign, floatx80_infinity_high,
-                                   floatx80_infinity_low);
-    }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloatx80( zSign, 0, 0 );
-        normalizeFloatx80Subnormal( aSig, &aExp, &aSig );
-    }
-    if ( bExp == 0 ) {
-        if ( bSig == 0 ) return packFloatx80( zSign, 0, 0 );
-        normalizeFloatx80Subnormal( bSig, &bExp, &bSig );
-    }
-    zExp = aExp + bExp - 0x3FFE;
-    mul64To128( aSig, bSig, &zSig0, &zSig1 );
-    if ( 0 < (int64_t) zSig0 ) {
-        shortShift128Left( zSig0, zSig1, 1, &zSig0, &zSig1 );
-        --zExp;
-    }
-    return roundAndPackFloatx80(status->floatx80_rounding_precision,
-                                zSign, zExp, zSig0, zSig1, status);
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of dividing the extended double-precision floating-point
-| value `a' by the corresponding value `b'.  The operation is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_div(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign, zSign;
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig, bSig, zSig0, zSig1;
-    uint64_t rem0, rem1, rem2, term0, term1, term2;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    bSig = extractFloatx80Frac( b );
-    bExp = extractFloatx80Exp( b );
-    bSign = extractFloatx80Sign( b );
-    zSign = aSign ^ bSign;
-    if ( aExp == 0x7FFF ) {
-        if ((uint64_t)(aSig << 1)) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        if ( bExp == 0x7FFF ) {
-            if ((uint64_t)(bSig << 1)) {
-                return propagateFloatx80NaN(a, b, status);
-            }
-            goto invalid;
-        }
-        return packFloatx80(zSign, floatx80_infinity_high,
-                                   floatx80_infinity_low);
-    }
-    if ( bExp == 0x7FFF ) {
-        if ((uint64_t)(bSig << 1)) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        return packFloatx80( zSign, 0, 0 );
-    }
-    if ( bExp == 0 ) {
-        if ( bSig == 0 ) {
-            if ( ( aExp | aSig ) == 0 ) {
- invalid:
-                float_raise(float_flag_invalid, status);
-                return floatx80_default_nan(status);
-            }
-            float_raise(float_flag_divbyzero, status);
-            return packFloatx80(zSign, floatx80_infinity_high,
-                                       floatx80_infinity_low);
-        }
-        normalizeFloatx80Subnormal( bSig, &bExp, &bSig );
-    }
-    if ( aExp == 0 ) {
-        if ( aSig == 0 ) return packFloatx80( zSign, 0, 0 );
-        normalizeFloatx80Subnormal( aSig, &aExp, &aSig );
-    }
-    zExp = aExp - bExp + 0x3FFE;
-    rem1 = 0;
-    if ( bSig <= aSig ) {
-        shift128Right( aSig, 0, 1, &aSig, &rem1 );
-        ++zExp;
-    }
-    zSig0 = estimateDiv128To64( aSig, rem1, bSig );
-    mul64To128( bSig, zSig0, &term0, &term1 );
-    sub128( aSig, rem1, term0, term1, &rem0, &rem1 );
-    while ( (int64_t) rem0 < 0 ) {
-        --zSig0;
-        add128( rem0, rem1, 0, bSig, &rem0, &rem1 );
-    }
-    zSig1 = estimateDiv128To64( rem1, 0, bSig );
-    if ( (uint64_t) ( zSig1<<1 ) <= 8 ) {
-        mul64To128( bSig, zSig1, &term1, &term2 );
-        sub128( rem1, 0, term1, term2, &rem1, &rem2 );
-        while ( (int64_t) rem1 < 0 ) {
-            --zSig1;
-            add128( rem1, rem2, 0, bSig, &rem1, &rem2 );
-        }
-        zSig1 |= ( ( rem1 | rem2 ) != 0 );
-    }
-    return roundAndPackFloatx80(status->floatx80_rounding_precision,
-                                zSign, zExp, zSig0, zSig1, status);
-}
-
-/*----------------------------------------------------------------------------
-| Returns the remainder of the extended double-precision floating-point value
-| `a' with respect to the corresponding value `b'.  The operation is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_rem(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, zSign;
-    int32_t aExp, bExp, expDiff;
-    uint64_t aSig0, aSig1, bSig;
-    uint64_t q, term0, term1, alternateASig0, alternateASig1;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    aSig0 = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    bSig = extractFloatx80Frac( b );
-    bExp = extractFloatx80Exp( b );
-    if ( aExp == 0x7FFF ) {
-        if (    (uint64_t) ( aSig0<<1 )
-             || ( ( bExp == 0x7FFF ) && (uint64_t) ( bSig<<1 ) ) ) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        goto invalid;
-    }
-    if ( bExp == 0x7FFF ) {
-        if ((uint64_t)(bSig << 1)) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        return a;
-    }
-    if ( bExp == 0 ) {
-        if ( bSig == 0 ) {
- invalid:
-            float_raise(float_flag_invalid, status);
-            return floatx80_default_nan(status);
-        }
-        normalizeFloatx80Subnormal( bSig, &bExp, &bSig );
-    }
-    if ( aExp == 0 ) {
-        if ( (uint64_t) ( aSig0<<1 ) == 0 ) return a;
-        normalizeFloatx80Subnormal( aSig0, &aExp, &aSig0 );
-    }
-    bSig |= UINT64_C(0x8000000000000000);
-    zSign = aSign;
-    expDiff = aExp - bExp;
-    aSig1 = 0;
-    if ( expDiff < 0 ) {
-        if ( expDiff < -1 ) return a;
-        shift128Right( aSig0, 0, 1, &aSig0, &aSig1 );
-        expDiff = 0;
-    }
-    q = ( bSig <= aSig0 );
-    if ( q ) aSig0 -= bSig;
-    expDiff -= 64;
-    while ( 0 < expDiff ) {
-        q = estimateDiv128To64( aSig0, aSig1, bSig );
-        q = ( 2 < q ) ? q - 2 : 0;
-        mul64To128( bSig, q, &term0, &term1 );
-        sub128( aSig0, aSig1, term0, term1, &aSig0, &aSig1 );
-        shortShift128Left( aSig0, aSig1, 62, &aSig0, &aSig1 );
-        expDiff -= 62;
-    }
-    expDiff += 64;
-    if ( 0 < expDiff ) {
-        q = estimateDiv128To64( aSig0, aSig1, bSig );
-        q = ( 2 < q ) ? q - 2 : 0;
-        q >>= 64 - expDiff;
-        mul64To128( bSig, q<<( 64 - expDiff ), &term0, &term1 );
-        sub128( aSig0, aSig1, term0, term1, &aSig0, &aSig1 );
-        shortShift128Left( 0, bSig, 64 - expDiff, &term0, &term1 );
-        while ( le128( term0, term1, aSig0, aSig1 ) ) {
-            ++q;
-            sub128( aSig0, aSig1, term0, term1, &aSig0, &aSig1 );
-        }
-    }
-    else {
-        term1 = 0;
-        term0 = bSig;
-    }
-    sub128( term0, term1, aSig0, aSig1, &alternateASig0, &alternateASig1 );
-    if (    lt128( alternateASig0, alternateASig1, aSig0, aSig1 )
-         || (    eq128( alternateASig0, alternateASig1, aSig0, aSig1 )
-              && ( q & 1 ) )
-       ) {
-        aSig0 = alternateASig0;
-        aSig1 = alternateASig1;
-        zSign = ! zSign;
-    }
-    return
-        normalizeRoundAndPackFloatx80(
-            80, zSign, bExp + expDiff, aSig0, aSig1, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the square root of the extended double-precision floating-point
-| value `a'.  The operation is performed according to the IEC/IEEE Standard
-| for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 floatx80_sqrt(floatx80 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, zExp;
-    uint64_t aSig0, aSig1, zSig0, zSig1, doubleZSig0;
-    uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
-
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    aSig0 = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if ((uint64_t)(aSig0 << 1)) {
-            return propagateFloatx80NaN(a, a, status);
-        }
-        if ( ! aSign ) return a;
-        goto invalid;
-    }
-    if ( aSign ) {
-        if ( ( aExp | aSig0 ) == 0 ) return a;
- invalid:
-        float_raise(float_flag_invalid, status);
-        return floatx80_default_nan(status);
-    }
-    if ( aExp == 0 ) {
-        if ( aSig0 == 0 ) return packFloatx80( 0, 0, 0 );
-        normalizeFloatx80Subnormal( aSig0, &aExp, &aSig0 );
-    }
-    zExp = ( ( aExp - 0x3FFF )>>1 ) + 0x3FFF;
-    zSig0 = estimateSqrt32( aExp, aSig0>>32 );
-    shift128Right( aSig0, 0, 2 + ( aExp & 1 ), &aSig0, &aSig1 );
-    zSig0 = estimateDiv128To64( aSig0, aSig1, zSig0<<32 ) + ( zSig0<<30 );
-    doubleZSig0 = zSig0<<1;
-    mul64To128( zSig0, zSig0, &term0, &term1 );
-    sub128( aSig0, aSig1, term0, term1, &rem0, &rem1 );
-    while ( (int64_t) rem0 < 0 ) {
-        --zSig0;
-        doubleZSig0 -= 2;
-        add128( rem0, rem1, zSig0>>63, doubleZSig0 | 1, &rem0, &rem1 );
-    }
-    zSig1 = estimateDiv128To64( rem1, 0, doubleZSig0 );
-    if ( ( zSig1 & UINT64_C(0x3FFFFFFFFFFFFFFF) ) <= 5 ) {
-        if ( zSig1 == 0 ) zSig1 = 1;
-        mul64To128( doubleZSig0, zSig1, &term1, &term2 );
-        sub128( rem1, 0, term1, term2, &rem1, &rem2 );
-        mul64To128( zSig1, zSig1, &term2, &term3 );
-        sub192( rem1, rem2, 0, 0, term2, term3, &rem1, &rem2, &rem3 );
-        while ( (int64_t) rem1 < 0 ) {
-            --zSig1;
-            shortShift128Left( 0, zSig1, 1, &term2, &term3 );
-            term3 |= 1;
-            term2 |= doubleZSig0;
-            add192( rem1, rem2, rem3, 0, term2, term3, &rem1, &rem2, &rem3 );
-        }
-        zSig1 |= ( ( rem1 | rem2 | rem3 ) != 0 );
-    }
-    shortShift128Left( 0, zSig1, 1, &zSig0, &zSig1 );
-    zSig0 |= doubleZSig0;
-    return roundAndPackFloatx80(status->floatx80_rounding_precision,
-                                0, zExp, zSig0, zSig1, status);
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point value `a' is equal
-| to the corresponding value `b', and 0 otherwise.  The invalid exception is
-| raised if either operand is a NaN.  Otherwise, the comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int floatx80_eq(floatx80 a, floatx80 b, float_status *status)
-{
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)
-        || (extractFloatx80Exp(a) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(a) << 1))
-        || (extractFloatx80Exp(b) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(b) << 1))
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    return
-           ( a.low == b.low )
-        && (    ( a.high == b.high )
-             || (    ( a.low == 0 )
-                  && ( (uint16_t) ( ( a.high | b.high )<<1 ) == 0 ) )
-           );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point value `a' is
-| less than or equal to the corresponding value `b', and 0 otherwise.  The
-| invalid exception is raised if either operand is a NaN.  The comparison is
-| performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int floatx80_le(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)
-        || (extractFloatx80Exp(a) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(a) << 1))
-        || (extractFloatx80Exp(b) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(b) << 1))
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloatx80Sign( a );
-    bSign = extractFloatx80Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            || (    ( ( (uint16_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 == 0 );
-    }
-    return
-          aSign ? le128( b.high, b.low, a.high, a.low )
-        : le128( a.high, a.low, b.high, b.low );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point value `a' is
-| less than the corresponding value `b', and 0 otherwise.  The invalid
-| exception is raised if either operand is a NaN.  The comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int floatx80_lt(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)
-        || (extractFloatx80Exp(a) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(a) << 1))
-        || (extractFloatx80Exp(b) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(b) << 1))
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloatx80Sign( a );
-    bSign = extractFloatx80Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            && (    ( ( (uint16_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 != 0 );
-    }
-    return
-          aSign ? lt128( b.high, b.low, a.high, a.low )
-        : lt128( a.high, a.low, b.high, b.low );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point values `a' and `b'
-| cannot be compared, and 0 otherwise.  The invalid exception is raised if
-| either operand is a NaN.   The comparison is performed according to the
-| IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-int floatx80_unordered(floatx80 a, floatx80 b, float_status *status)
-{
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)
-        || (extractFloatx80Exp(a) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(a) << 1))
-        || (extractFloatx80Exp(b) == 0x7FFF
-            && (uint64_t) (extractFloatx80Frac(b) << 1))
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 1;
-    }
-    return 0;
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point value `a' is
-| equal to the corresponding value `b', and 0 otherwise.  Quiet NaNs do not
-| cause an exception.  The comparison is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int floatx80_eq_quiet(floatx80 a, floatx80 b, float_status *status)
-{
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( a )<<1 ) )
-         || (    ( extractFloatx80Exp( b ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
-       ) {
-        if (floatx80_is_signaling_nan(a, status)
-         || floatx80_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    return
-           ( a.low == b.low )
-        && (    ( a.high == b.high )
-             || (    ( a.low == 0 )
-                  && ( (uint16_t) ( ( a.high | b.high )<<1 ) == 0 ) )
-           );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point value `a' is less
-| than or equal to the corresponding value `b', and 0 otherwise.  Quiet NaNs
-| do not cause an exception.  Otherwise, the comparison is performed according
-| to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int floatx80_le_quiet(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( a )<<1 ) )
-         || (    ( extractFloatx80Exp( b ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
-       ) {
-        if (floatx80_is_signaling_nan(a, status)
-         || floatx80_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    aSign = extractFloatx80Sign( a );
-    bSign = extractFloatx80Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            || (    ( ( (uint16_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 == 0 );
-    }
-    return
-          aSign ? le128( b.high, b.low, a.high, a.low )
-        : le128( a.high, a.low, b.high, b.low );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point value `a' is less
-| than the corresponding value `b', and 0 otherwise.  Quiet NaNs do not cause
-| an exception.  Otherwise, the comparison is performed according to the
-| IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int floatx80_lt_quiet(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( a )<<1 ) )
-         || (    ( extractFloatx80Exp( b ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
-       ) {
-        if (floatx80_is_signaling_nan(a, status)
-         || floatx80_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    aSign = extractFloatx80Sign( a );
-    bSign = extractFloatx80Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            && (    ( ( (uint16_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 != 0 );
-    }
-    return
-          aSign ? lt128( b.high, b.low, a.high, a.low )
-        : lt128( a.high, a.low, b.high, b.low );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the extended double-precision floating-point values `a' and `b'
-| cannot be compared, and 0 otherwise.  Quiet NaNs do not cause an exception.
-| The comparison is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-int floatx80_unordered_quiet(floatx80 a, floatx80 b, float_status *status)
-{
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return 1;
-    }
-    if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( a )<<1 ) )
-         || (    ( extractFloatx80Exp( b ) == 0x7FFF )
-              && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
-       ) {
-        if (floatx80_is_signaling_nan(a, status)
-         || floatx80_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 1;
-    }
-    return 0;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the 32-bit two's complement integer format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic---which means in particular that the conversion is rounded
-| according to the current rounding mode.  If `a' is a NaN, the largest
-| positive integer is returned.  Otherwise, if the conversion overflows, the
-| largest integer with the same sign as `a' is returned.
-*----------------------------------------------------------------------------*/
-
-int32_t float128_to_int32(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig0, aSig1;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( ( aExp == 0x7FFF ) && ( aSig0 | aSig1 ) ) aSign = 0;
-    if ( aExp ) aSig0 |= UINT64_C(0x0001000000000000);
-    aSig0 |= ( aSig1 != 0 );
-    shiftCount = 0x4028 - aExp;
-    if ( 0 < shiftCount ) shift64RightJamming( aSig0, shiftCount, &aSig0 );
-    return roundAndPackInt32(aSign, aSig0, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the 32-bit two's complement integer format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic, except that the conversion is always rounded toward zero.  If
-| `a' is a NaN, the largest positive integer is returned.  Otherwise, if the
-| conversion overflows, the largest integer with the same sign as `a' is
-| returned.
-*----------------------------------------------------------------------------*/
-
-int32_t float128_to_int32_round_to_zero(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig0, aSig1, savedASig;
-    int32_t z;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    aSig0 |= ( aSig1 != 0 );
-    if ( 0x401E < aExp ) {
-        if ( ( aExp == 0x7FFF ) && aSig0 ) aSign = 0;
-        goto invalid;
-    }
-    else if ( aExp < 0x3FFF ) {
-        if (aExp || aSig0) {
-            status->float_exception_flags |= float_flag_inexact;
-        }
-        return 0;
-    }
-    aSig0 |= UINT64_C(0x0001000000000000);
-    shiftCount = 0x402F - aExp;
-    savedASig = aSig0;
-    aSig0 >>= shiftCount;
-    z = aSig0;
-    if ( aSign ) z = - z;
-    if ( ( z < 0 ) ^ aSign ) {
- invalid:
-        float_raise(float_flag_invalid, status);
-        return aSign ? INT32_MIN : INT32_MAX;
-    }
-    if ( ( aSig0<<shiftCount ) != savedASig ) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    return z;
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the 64-bit two's complement integer format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic---which means in particular that the conversion is rounded
-| according to the current rounding mode.  If `a' is a NaN, the largest
-| positive integer is returned.  Otherwise, if the conversion overflows, the
-| largest integer with the same sign as `a' is returned.
-*----------------------------------------------------------------------------*/
-
-int64_t float128_to_int64(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig0, aSig1;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( aExp ) aSig0 |= UINT64_C(0x0001000000000000);
-    shiftCount = 0x402F - aExp;
-    if ( shiftCount <= 0 ) {
-        if ( 0x403E < aExp ) {
-            float_raise(float_flag_invalid, status);
-            if (    ! aSign
-                 || (    ( aExp == 0x7FFF )
-                      && ( aSig1 || ( aSig0 != UINT64_C(0x0001000000000000) ) )
-                    )
-               ) {
-                return INT64_MAX;
-            }
-            return INT64_MIN;
-        }
-        shortShift128Left( aSig0, aSig1, - shiftCount, &aSig0, &aSig1 );
-    }
-    else {
-        shift64ExtraRightJamming( aSig0, aSig1, shiftCount, &aSig0, &aSig1 );
-    }
-    return roundAndPackInt64(aSign, aSig0, aSig1, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the 64-bit two's complement integer format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic, except that the conversion is always rounded toward zero.
-| If `a' is a NaN, the largest positive integer is returned.  Otherwise, if
-| the conversion overflows, the largest integer with the same sign as `a' is
-| returned.
-*----------------------------------------------------------------------------*/
-
-int64_t float128_to_int64_round_to_zero(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, shiftCount;
-    uint64_t aSig0, aSig1;
-    int64_t z;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( aExp ) aSig0 |= UINT64_C(0x0001000000000000);
-    shiftCount = aExp - 0x402F;
-    if ( 0 < shiftCount ) {
-        if ( 0x403E <= aExp ) {
-            aSig0 &= UINT64_C(0x0000FFFFFFFFFFFF);
-            if (    ( a.high == UINT64_C(0xC03E000000000000) )
-                 && ( aSig1 < UINT64_C(0x0002000000000000) ) ) {
-                if (aSig1) {
-                    status->float_exception_flags |= float_flag_inexact;
-                }
-            }
-            else {
-                float_raise(float_flag_invalid, status);
-                if ( ! aSign || ( ( aExp == 0x7FFF ) && ( aSig0 | aSig1 ) ) ) {
-                    return INT64_MAX;
-                }
-            }
-            return INT64_MIN;
-        }
-        z = ( aSig0<<shiftCount ) | ( aSig1>>( ( - shiftCount ) & 63 ) );
-        if ( (uint64_t) ( aSig1<<shiftCount ) ) {
-            status->float_exception_flags |= float_flag_inexact;
-        }
-    }
-    else {
-        if ( aExp < 0x3FFF ) {
-            if ( aExp | aSig0 | aSig1 ) {
-                status->float_exception_flags |= float_flag_inexact;
-            }
-            return 0;
-        }
-        z = aSig0>>( - shiftCount );
-        if (    aSig1
-             || ( shiftCount && (uint64_t) ( aSig0<<( shiftCount & 63 ) ) ) ) {
-            status->float_exception_flags |= float_flag_inexact;
-        }
-    }
-    if ( aSign ) z = - z;
-    return z;
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point value
-| `a' to the 64-bit unsigned integer format.  The conversion is
-| performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic---which means in particular that the conversion is rounded
-| according to the current rounding mode.  If `a' is a NaN, the largest
-| positive integer is returned.  If the conversion overflows, the
-| largest unsigned integer is returned.  If 'a' is negative, the value is
-| rounded and zero is returned; negative values that do not round to zero
-| will raise the inexact exception.
-*----------------------------------------------------------------------------*/
-
-uint64_t float128_to_uint64(float128 a, float_status *status)
-{
-    flag aSign;
-    int aExp;
-    int shiftCount;
-    uint64_t aSig0, aSig1;
-
-    aSig0 = extractFloat128Frac0(a);
-    aSig1 = extractFloat128Frac1(a);
-    aExp = extractFloat128Exp(a);
-    aSign = extractFloat128Sign(a);
-    if (aSign && (aExp > 0x3FFE)) {
-        float_raise(float_flag_invalid, status);
-        if (float128_is_any_nan(a)) {
-            return UINT64_MAX;
-        } else {
-            return 0;
-        }
-    }
-    if (aExp) {
-        aSig0 |= UINT64_C(0x0001000000000000);
-    }
-    shiftCount = 0x402F - aExp;
-    if (shiftCount <= 0) {
-        if (0x403E < aExp) {
-            float_raise(float_flag_invalid, status);
-            return UINT64_MAX;
-        }
-        shortShift128Left(aSig0, aSig1, -shiftCount, &aSig0, &aSig1);
-    } else {
-        shift64ExtraRightJamming(aSig0, aSig1, shiftCount, &aSig0, &aSig1);
-    }
-    return roundAndPackUint64(aSign, aSig0, aSig1, status);
-}
-
-uint64_t float128_to_uint64_round_to_zero(float128 a, float_status *status)
-{
-    uint64_t v;
-    signed char current_rounding_mode = status->float_rounding_mode;
-
-    set_float_rounding_mode(float_round_to_zero, status);
-    v = float128_to_uint64(a, status);
-    set_float_rounding_mode(current_rounding_mode, status);
-
-    return v;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the 32-bit unsigned integer format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic except that the conversion is always rounded toward zero.
-| If `a' is a NaN, the largest positive integer is returned.  Otherwise,
-| if the conversion overflows, the largest unsigned integer is returned.
-| If 'a' is negative, the value is rounded and zero is returned; negative
-| values that do not round to zero will raise the inexact exception.
-*----------------------------------------------------------------------------*/
-
-uint32_t float128_to_uint32_round_to_zero(float128 a, float_status *status)
-{
-    uint64_t v;
-    uint32_t res;
-    int old_exc_flags = get_float_exception_flags(status);
-
-    v = float128_to_uint64_round_to_zero(a, status);
-    if (v > 0xffffffff) {
-        res = 0xffffffff;
-    } else {
-        return v;
-    }
-    set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid, status);
-    return res;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point value
-| `a' to the 32-bit unsigned integer format.  The conversion is
-| performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic---which means in particular that the conversion is rounded
-| according to the current rounding mode.  If `a' is a NaN, the largest
-| positive integer is returned.  If the conversion overflows, the
-| largest unsigned integer is returned.  If 'a' is negative, the value is
-| rounded and zero is returned; negative values that do not round to zero
-| will raise the inexact exception.
-*----------------------------------------------------------------------------*/
-
-uint32_t float128_to_uint32(float128 a, float_status *status)
-{
-    uint64_t v;
-    uint32_t res;
-    int old_exc_flags = get_float_exception_flags(status);
-
-    v = float128_to_uint64(a, status);
-    if (v > 0xffffffff) {
-        res = 0xffffffff;
-    } else {
-        return v;
-    }
-    set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid, status);
-    return res;
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the single-precision floating-point format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float32 float128_to_float32(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t aSig0, aSig1;
-    uint32_t zSig;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 ) {
-            return commonNaNToFloat32(float128ToCommonNaN(a, status), status);
-        }
-        return packFloat32( aSign, 0xFF, 0 );
-    }
-    aSig0 |= ( aSig1 != 0 );
-    shift64RightJamming( aSig0, 18, &aSig0 );
-    zSig = aSig0;
-    if ( aExp || zSig ) {
-        zSig |= 0x40000000;
-        aExp -= 0x3F81;
-    }
-    return roundAndPackFloat32(aSign, aExp, zSig, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the double-precision floating-point format.  The conversion
-| is performed according to the IEC/IEEE Standard for Binary Floating-Point
-| Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float64 float128_to_float64(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t aSig0, aSig1;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 ) {
-            return commonNaNToFloat64(float128ToCommonNaN(a, status), status);
-        }
-        return packFloat64( aSign, 0x7FF, 0 );
-    }
-    shortShift128Left( aSig0, aSig1, 14, &aSig0, &aSig1 );
-    aSig0 |= ( aSig1 != 0 );
-    if ( aExp || aSig0 ) {
-        aSig0 |= UINT64_C(0x4000000000000000);
-        aExp -= 0x3C01;
-    }
-    return roundAndPackFloat64(aSign, aExp, aSig0, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of converting the quadruple-precision floating-point
-| value `a' to the extended double-precision floating-point format.  The
-| conversion is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-floatx80 float128_to_floatx80(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t aSig0, aSig1;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 ) {
-            return commonNaNToFloatx80(float128ToCommonNaN(a, status), status);
-        }
-        return packFloatx80(aSign, floatx80_infinity_high,
-                                   floatx80_infinity_low);
-    }
-    if ( aExp == 0 ) {
-        if ( ( aSig0 | aSig1 ) == 0 ) return packFloatx80( aSign, 0, 0 );
-        normalizeFloat128Subnormal( aSig0, aSig1, &aExp, &aSig0, &aSig1 );
-    }
-    else {
-        aSig0 |= UINT64_C(0x0001000000000000);
-    }
-    shortShift128Left( aSig0, aSig1, 15, &aSig0, &aSig1 );
-    return roundAndPackFloatx80(80, aSign, aExp, aSig0, aSig1, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Rounds the quadruple-precision floating-point value `a' to an integer, and
-| returns the result as a quadruple-precision floating-point value.  The
-| operation is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float128_round_to_int(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t lastBitMask, roundBitsMask;
-    float128 z;
-
-    aExp = extractFloat128Exp( a );
-    if ( 0x402F <= aExp ) {
-        if ( 0x406F <= aExp ) {
-            if (    ( aExp == 0x7FFF )
-                 && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) )
-               ) {
-                return propagateFloat128NaN(a, a, status);
-            }
-            return a;
-        }
-        lastBitMask = 1;
-        lastBitMask = ( lastBitMask<<( 0x406E - aExp ) )<<1;
-        roundBitsMask = lastBitMask - 1;
-        z = a;
-        switch (status->float_rounding_mode) {
-        case float_round_nearest_even:
-            if ( lastBitMask ) {
-                add128( z.high, z.low, 0, lastBitMask>>1, &z.high, &z.low );
-                if ( ( z.low & roundBitsMask ) == 0 ) z.low &= ~ lastBitMask;
-            }
-            else {
-                if ( (int64_t) z.low < 0 ) {
-                    ++z.high;
-                    if ( (uint64_t) ( z.low<<1 ) == 0 ) z.high &= ~1;
-                }
-            }
-            break;
-        case float_round_ties_away:
-            if (lastBitMask) {
-                add128(z.high, z.low, 0, lastBitMask >> 1, &z.high, &z.low);
-            } else {
-                if ((int64_t) z.low < 0) {
-                    ++z.high;
-                }
-            }
-            break;
-        case float_round_to_zero:
-            break;
-        case float_round_up:
-            if (!extractFloat128Sign(z)) {
-                add128(z.high, z.low, 0, roundBitsMask, &z.high, &z.low);
-            }
-            break;
-        case float_round_down:
-            if (extractFloat128Sign(z)) {
-                add128(z.high, z.low, 0, roundBitsMask, &z.high, &z.low);
-            }
-            break;
-        case float_round_to_odd:
-            /*
-             * Note that if lastBitMask == 0, the last bit is the lsb
-             * of high, and roundBitsMask == -1.
-             */
-            if ((lastBitMask ? z.low & lastBitMask : z.high & 1) == 0) {
-                add128(z.high, z.low, 0, roundBitsMask, &z.high, &z.low);
-            }
-            break;
-        default:
-            abort();
-        }
-        z.low &= ~ roundBitsMask;
-    }
-    else {
-        if ( aExp < 0x3FFF ) {
-            if ( ( ( (uint64_t) ( a.high<<1 ) ) | a.low ) == 0 ) return a;
-            status->float_exception_flags |= float_flag_inexact;
-            aSign = extractFloat128Sign( a );
-            switch (status->float_rounding_mode) {
-            case float_round_nearest_even:
-                if (    ( aExp == 0x3FFE )
-                     && (   extractFloat128Frac0( a )
-                          | extractFloat128Frac1( a ) )
-                   ) {
-                    return packFloat128( aSign, 0x3FFF, 0, 0 );
-                }
-                break;
-            case float_round_ties_away:
-                if (aExp == 0x3FFE) {
-                    return packFloat128(aSign, 0x3FFF, 0, 0);
-                }
-                break;
-            case float_round_down:
-                return
-                      aSign ? packFloat128( 1, 0x3FFF, 0, 0 )
-                    : packFloat128( 0, 0, 0, 0 );
-            case float_round_up:
-                return
-                      aSign ? packFloat128( 1, 0, 0, 0 )
-                    : packFloat128( 0, 0x3FFF, 0, 0 );
-
-            case float_round_to_odd:
-                return packFloat128(aSign, 0x3FFF, 0, 0);
-            }
-            return packFloat128( aSign, 0, 0, 0 );
-        }
-        lastBitMask = 1;
-        lastBitMask <<= 0x402F - aExp;
-        roundBitsMask = lastBitMask - 1;
-        z.low = 0;
-        z.high = a.high;
-        switch (status->float_rounding_mode) {
-        case float_round_nearest_even:
-            z.high += lastBitMask>>1;
-            if ( ( ( z.high & roundBitsMask ) | a.low ) == 0 ) {
-                z.high &= ~ lastBitMask;
-            }
-            break;
-        case float_round_ties_away:
-            z.high += lastBitMask>>1;
-            break;
-        case float_round_to_zero:
-            break;
-        case float_round_up:
-            if (!extractFloat128Sign(z)) {
-                z.high |= ( a.low != 0 );
-                z.high += roundBitsMask;
-            }
-            break;
-        case float_round_down:
-            if (extractFloat128Sign(z)) {
-                z.high |= (a.low != 0);
-                z.high += roundBitsMask;
-            }
-            break;
-        case float_round_to_odd:
-            if ((z.high & lastBitMask) == 0) {
-                z.high |= (a.low != 0);
-                z.high += roundBitsMask;
-            }
-            break;
-        default:
-            abort();
-        }
-        z.high &= ~ roundBitsMask;
-    }
-    if ( ( z.low != a.low ) || ( z.high != a.high ) ) {
-        status->float_exception_flags |= float_flag_inexact;
-    }
-    return z;
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of adding the absolute values of the quadruple-precision
-| floating-point values `a' and `b'.  If `zSign' is 1, the sum is negated
-| before being returned.  `zSign' is ignored if the result is a NaN.
-| The addition is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-static float128 addFloat128Sigs(float128 a, float128 b, flag zSign,
-                                float_status *status)
-{
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2;
-    int32_t expDiff;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    bSig1 = extractFloat128Frac1( b );
-    bSig0 = extractFloat128Frac0( b );
-    bExp = extractFloat128Exp( b );
-    expDiff = aExp - bExp;
-    if ( 0 < expDiff ) {
-        if ( aExp == 0x7FFF ) {
-            if (aSig0 | aSig1) {
-                return propagateFloat128NaN(a, b, status);
-            }
-            return a;
-        }
-        if ( bExp == 0 ) {
-            --expDiff;
-        }
-        else {
-            bSig0 |= UINT64_C(0x0001000000000000);
-        }
-        shift128ExtraRightJamming(
-            bSig0, bSig1, 0, expDiff, &bSig0, &bSig1, &zSig2 );
-        zExp = aExp;
-    }
-    else if ( expDiff < 0 ) {
-        if ( bExp == 0x7FFF ) {
-            if (bSig0 | bSig1) {
-                return propagateFloat128NaN(a, b, status);
-            }
-            return packFloat128( zSign, 0x7FFF, 0, 0 );
-        }
-        if ( aExp == 0 ) {
-            ++expDiff;
-        }
-        else {
-            aSig0 |= UINT64_C(0x0001000000000000);
-        }
-        shift128ExtraRightJamming(
-            aSig0, aSig1, 0, - expDiff, &aSig0, &aSig1, &zSig2 );
-        zExp = bExp;
-    }
-    else {
-        if ( aExp == 0x7FFF ) {
-            if ( aSig0 | aSig1 | bSig0 | bSig1 ) {
-                return propagateFloat128NaN(a, b, status);
-            }
-            return a;
-        }
-        add128( aSig0, aSig1, bSig0, bSig1, &zSig0, &zSig1 );
-        if ( aExp == 0 ) {
-            if (status->flush_to_zero) {
-                if (zSig0 | zSig1) {
-                    float_raise(float_flag_output_denormal, status);
-                }
-                return packFloat128(zSign, 0, 0, 0);
-            }
-            return packFloat128( zSign, 0, zSig0, zSig1 );
-        }
-        zSig2 = 0;
-        zSig0 |= UINT64_C(0x0002000000000000);
-        zExp = aExp;
-        goto shiftRight1;
-    }
-    aSig0 |= UINT64_C(0x0001000000000000);
-    add128( aSig0, aSig1, bSig0, bSig1, &zSig0, &zSig1 );
-    --zExp;
-    if ( zSig0 < UINT64_C(0x0002000000000000) ) goto roundAndPack;
-    ++zExp;
- shiftRight1:
-    shift128ExtraRightJamming(
-        zSig0, zSig1, zSig2, 1, &zSig0, &zSig1, &zSig2 );
- roundAndPack:
-    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of subtracting the absolute values of the quadruple-
-| precision floating-point values `a' and `b'.  If `zSign' is 1, the
-| difference is negated before being returned.  `zSign' is ignored if the
-| result is a NaN.  The subtraction is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-static float128 subFloat128Sigs(float128 a, float128 b, flag zSign,
-                                float_status *status)
-{
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1;
-    int32_t expDiff;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    bSig1 = extractFloat128Frac1( b );
-    bSig0 = extractFloat128Frac0( b );
-    bExp = extractFloat128Exp( b );
-    expDiff = aExp - bExp;
-    shortShift128Left( aSig0, aSig1, 14, &aSig0, &aSig1 );
-    shortShift128Left( bSig0, bSig1, 14, &bSig0, &bSig1 );
-    if ( 0 < expDiff ) goto aExpBigger;
-    if ( expDiff < 0 ) goto bExpBigger;
-    if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 | bSig0 | bSig1 ) {
-            return propagateFloat128NaN(a, b, status);
-        }
-        float_raise(float_flag_invalid, status);
-        return float128_default_nan(status);
-    }
-    if ( aExp == 0 ) {
-        aExp = 1;
-        bExp = 1;
-    }
-    if ( bSig0 < aSig0 ) goto aBigger;
-    if ( aSig0 < bSig0 ) goto bBigger;
-    if ( bSig1 < aSig1 ) goto aBigger;
-    if ( aSig1 < bSig1 ) goto bBigger;
-    return packFloat128(status->float_rounding_mode == float_round_down,
-                        0, 0, 0);
- bExpBigger:
-    if ( bExp == 0x7FFF ) {
-        if (bSig0 | bSig1) {
-            return propagateFloat128NaN(a, b, status);
-        }
-        return packFloat128( zSign ^ 1, 0x7FFF, 0, 0 );
-    }
-    if ( aExp == 0 ) {
-        ++expDiff;
-    }
-    else {
-        aSig0 |= UINT64_C(0x4000000000000000);
-    }
-    shift128RightJamming( aSig0, aSig1, - expDiff, &aSig0, &aSig1 );
-    bSig0 |= UINT64_C(0x4000000000000000);
- bBigger:
-    sub128( bSig0, bSig1, aSig0, aSig1, &zSig0, &zSig1 );
-    zExp = bExp;
-    zSign ^= 1;
-    goto normalizeRoundAndPack;
- aExpBigger:
-    if ( aExp == 0x7FFF ) {
-        if (aSig0 | aSig1) {
-            return propagateFloat128NaN(a, b, status);
-        }
-        return a;
-    }
-    if ( bExp == 0 ) {
-        --expDiff;
-    }
-    else {
-        bSig0 |= UINT64_C(0x4000000000000000);
-    }
-    shift128RightJamming( bSig0, bSig1, expDiff, &bSig0, &bSig1 );
-    aSig0 |= UINT64_C(0x4000000000000000);
- aBigger:
-    sub128( aSig0, aSig1, bSig0, bSig1, &zSig0, &zSig1 );
-    zExp = aExp;
- normalizeRoundAndPack:
-    --zExp;
-    return normalizeRoundAndPackFloat128(zSign, zExp - 14, zSig0, zSig1,
-                                         status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of adding the quadruple-precision floating-point values
-| `a' and `b'.  The operation is performed according to the IEC/IEEE Standard
-| for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float128_add(float128 a, float128 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    aSign = extractFloat128Sign( a );
-    bSign = extractFloat128Sign( b );
-    if ( aSign == bSign ) {
-        return addFloat128Sigs(a, b, aSign, status);
-    }
-    else {
-        return subFloat128Sigs(a, b, aSign, status);
-    }
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of subtracting the quadruple-precision floating-point
-| values `a' and `b'.  The operation is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float128_sub(float128 a, float128 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    aSign = extractFloat128Sign( a );
-    bSign = extractFloat128Sign( b );
-    if ( aSign == bSign ) {
-        return subFloat128Sigs(a, b, aSign, status);
-    }
-    else {
-        return addFloat128Sigs(a, b, aSign, status);
-    }
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of multiplying the quadruple-precision floating-point
-| values `a' and `b'.  The operation is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float128_mul(float128 a, float128 b, float_status *status)
-{
-    flag aSign, bSign, zSign;
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2, zSig3;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    bSig1 = extractFloat128Frac1( b );
-    bSig0 = extractFloat128Frac0( b );
-    bExp = extractFloat128Exp( b );
-    bSign = extractFloat128Sign( b );
-    zSign = aSign ^ bSign;
-    if ( aExp == 0x7FFF ) {
-        if (    ( aSig0 | aSig1 )
-             || ( ( bExp == 0x7FFF ) && ( bSig0 | bSig1 ) ) ) {
-            return propagateFloat128NaN(a, b, status);
-        }
-        if ( ( bExp | bSig0 | bSig1 ) == 0 ) goto invalid;
-        return packFloat128( zSign, 0x7FFF, 0, 0 );
-    }
-    if ( bExp == 0x7FFF ) {
-        if (bSig0 | bSig1) {
-            return propagateFloat128NaN(a, b, status);
-        }
-        if ( ( aExp | aSig0 | aSig1 ) == 0 ) {
- invalid:
-            float_raise(float_flag_invalid, status);
-            return float128_default_nan(status);
-        }
-        return packFloat128( zSign, 0x7FFF, 0, 0 );
-    }
-    if ( aExp == 0 ) {
-        if ( ( aSig0 | aSig1 ) == 0 ) return packFloat128( zSign, 0, 0, 0 );
-        normalizeFloat128Subnormal( aSig0, aSig1, &aExp, &aSig0, &aSig1 );
-    }
-    if ( bExp == 0 ) {
-        if ( ( bSig0 | bSig1 ) == 0 ) return packFloat128( zSign, 0, 0, 0 );
-        normalizeFloat128Subnormal( bSig0, bSig1, &bExp, &bSig0, &bSig1 );
-    }
-    zExp = aExp + bExp - 0x4000;
-    aSig0 |= UINT64_C(0x0001000000000000);
-    shortShift128Left( bSig0, bSig1, 16, &bSig0, &bSig1 );
-    mul128To256( aSig0, aSig1, bSig0, bSig1, &zSig0, &zSig1, &zSig2, &zSig3 );
-    add128( zSig0, zSig1, aSig0, aSig1, &zSig0, &zSig1 );
-    zSig2 |= ( zSig3 != 0 );
-    if (UINT64_C( 0x0002000000000000) <= zSig0 ) {
-        shift128ExtraRightJamming(
-            zSig0, zSig1, zSig2, 1, &zSig0, &zSig1, &zSig2 );
-        ++zExp;
-    }
-    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the result of dividing the quadruple-precision floating-point value
-| `a' by the corresponding value `b'.  The operation is performed according to
-| the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float128_div(float128 a, float128 b, float_status *status)
-{
-    flag aSign, bSign, zSign;
-    int32_t aExp, bExp, zExp;
-    uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2;
-    uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    bSig1 = extractFloat128Frac1( b );
-    bSig0 = extractFloat128Frac0( b );
-    bExp = extractFloat128Exp( b );
-    bSign = extractFloat128Sign( b );
-    zSign = aSign ^ bSign;
-    if ( aExp == 0x7FFF ) {
-        if (aSig0 | aSig1) {
-            return propagateFloat128NaN(a, b, status);
-        }
-        if ( bExp == 0x7FFF ) {
-            if (bSig0 | bSig1) {
-                return propagateFloat128NaN(a, b, status);
+    if ( 0x7FFD <= (uint32_t) ( zExp - 1 ) ) {
+        if (    ( 0x7FFE < zExp )
+             || (    ( zExp == 0x7FFE )
+                  && ( zSig0 == UINT64_C(0xFFFFFFFFFFFFFFFF) )
+                  && increment
+                )
+           ) {
+            roundMask = 0;
+ overflow:
+            float_raise(float_flag_overflow | float_flag_inexact, status);
+            if (    ( roundingMode == float_round_to_zero )
+                 || ( zSign && ( roundingMode == float_round_up ) )
+                 || ( ! zSign && ( roundingMode == float_round_down ) )
+               ) {
+                return packFloatx80( zSign, 0x7FFE, ~ roundMask );
             }
-            goto invalid;
-        }
-        return packFloat128( zSign, 0x7FFF, 0, 0 );
-    }
-    if ( bExp == 0x7FFF ) {
-        if (bSig0 | bSig1) {
-            return propagateFloat128NaN(a, b, status);
+            return packFloatx80(zSign,
+                                floatx80_infinity_high,
+                                floatx80_infinity_low);
         }
-        return packFloat128( zSign, 0, 0, 0 );
-    }
-    if ( bExp == 0 ) {
-        if ( ( bSig0 | bSig1 ) == 0 ) {
-            if ( ( aExp | aSig0 | aSig1 ) == 0 ) {
- invalid:
-                float_raise(float_flag_invalid, status);
-                return float128_default_nan(status);
+        if ( zExp <= 0 ) {
+            isTiny = status->tininess_before_rounding
+                  || (zExp < 0)
+                  || !increment
+                  || (zSig0 < UINT64_C(0xFFFFFFFFFFFFFFFF));
+            shift64ExtraRightJamming( zSig0, zSig1, 1 - zExp, &zSig0, &zSig1 );
+            zExp = 0;
+            if (isTiny && zSig1) {
+                float_raise(float_flag_underflow, status);
             }
-            float_raise(float_flag_divbyzero, status);
-            return packFloat128( zSign, 0x7FFF, 0, 0 );
-        }
-        normalizeFloat128Subnormal( bSig0, bSig1, &bExp, &bSig0, &bSig1 );
-    }
-    if ( aExp == 0 ) {
-        if ( ( aSig0 | aSig1 ) == 0 ) return packFloat128( zSign, 0, 0, 0 );
-        normalizeFloat128Subnormal( aSig0, aSig1, &aExp, &aSig0, &aSig1 );
-    }
-    zExp = aExp - bExp + 0x3FFD;
-    shortShift128Left(
-        aSig0 | UINT64_C(0x0001000000000000), aSig1, 15, &aSig0, &aSig1 );
-    shortShift128Left(
-        bSig0 | UINT64_C(0x0001000000000000), bSig1, 15, &bSig0, &bSig1 );
-    if ( le128( bSig0, bSig1, aSig0, aSig1 ) ) {
-        shift128Right( aSig0, aSig1, 1, &aSig0, &aSig1 );
-        ++zExp;
-    }
-    zSig0 = estimateDiv128To64( aSig0, aSig1, bSig0 );
-    mul128By64To192( bSig0, bSig1, zSig0, &term0, &term1, &term2 );
-    sub192( aSig0, aSig1, 0, term0, term1, term2, &rem0, &rem1, &rem2 );
-    while ( (int64_t) rem0 < 0 ) {
-        --zSig0;
-        add192( rem0, rem1, rem2, 0, bSig0, bSig1, &rem0, &rem1, &rem2 );
-    }
-    zSig1 = estimateDiv128To64( rem1, rem2, bSig0 );
-    if ( ( zSig1 & 0x3FFF ) <= 4 ) {
-        mul128By64To192( bSig0, bSig1, zSig1, &term1, &term2, &term3 );
-        sub192( rem1, rem2, 0, term1, term2, term3, &rem1, &rem2, &rem3 );
-        while ( (int64_t) rem1 < 0 ) {
-            --zSig1;
-            add192( rem1, rem2, rem3, 0, bSig0, bSig1, &rem1, &rem2, &rem3 );
-        }
-        zSig1 |= ( ( rem1 | rem2 | rem3 ) != 0 );
-    }
-    shift128ExtraRightJamming( zSig0, zSig1, 0, 15, &zSig0, &zSig1, &zSig2 );
-    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns the remainder of the quadruple-precision floating-point value `a'
-| with respect to the corresponding value `b'.  The operation is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float128_rem(float128 a, float128 b, float_status *status)
-{
-    flag aSign, zSign;
-    int32_t aExp, bExp, expDiff;
-    uint64_t aSig0, aSig1, bSig0, bSig1, q, term0, term1, term2;
-    uint64_t allZero, alternateASig0, alternateASig1, sigMean1;
-    int64_t sigMean0;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    bSig1 = extractFloat128Frac1( b );
-    bSig0 = extractFloat128Frac0( b );
-    bExp = extractFloat128Exp( b );
-    if ( aExp == 0x7FFF ) {
-        if (    ( aSig0 | aSig1 )
-             || ( ( bExp == 0x7FFF ) && ( bSig0 | bSig1 ) ) ) {
-            return propagateFloat128NaN(a, b, status);
+            if (zSig1) {
+                float_raise(float_flag_inexact, status);
+            }
+            switch (roundingMode) {
+            case float_round_nearest_even:
+            case float_round_ties_away:
+                increment = ((int64_t)zSig1 < 0);
+                break;
+            case float_round_to_zero:
+                increment = 0;
+                break;
+            case float_round_up:
+                increment = !zSign && zSig1;
+                break;
+            case float_round_down:
+                increment = zSign && zSig1;
+                break;
+            default:
+                abort();
+            }
+            if ( increment ) {
+                ++zSig0;
+                if (!(zSig1 << 1) && roundNearestEven) {
+                    zSig0 &= ~1;
+                }
+                if ( (int64_t) zSig0 < 0 ) zExp = 1;
+            }
+            return packFloatx80( zSign, zExp, zSig0 );
         }
-        goto invalid;
     }
-    if ( bExp == 0x7FFF ) {
-        if (bSig0 | bSig1) {
-            return propagateFloat128NaN(a, b, status);
-        }
-        return a;
+    if (zSig1) {
+        float_raise(float_flag_inexact, status);
     }
-    if ( bExp == 0 ) {
-        if ( ( bSig0 | bSig1 ) == 0 ) {
- invalid:
-            float_raise(float_flag_invalid, status);
-            return float128_default_nan(status);
-        }
-        normalizeFloat128Subnormal( bSig0, bSig1, &bExp, &bSig0, &bSig1 );
-    }
-    if ( aExp == 0 ) {
-        if ( ( aSig0 | aSig1 ) == 0 ) return a;
-        normalizeFloat128Subnormal( aSig0, aSig1, &aExp, &aSig0, &aSig1 );
-    }
-    expDiff = aExp - bExp;
-    if ( expDiff < -1 ) return a;
-    shortShift128Left(
-        aSig0 | UINT64_C(0x0001000000000000),
-        aSig1,
-        15 - ( expDiff < 0 ),
-        &aSig0,
-        &aSig1
-    );
-    shortShift128Left(
-        bSig0 | UINT64_C(0x0001000000000000), bSig1, 15, &bSig0, &bSig1 );
-    q = le128( bSig0, bSig1, aSig0, aSig1 );
-    if ( q ) sub128( aSig0, aSig1, bSig0, bSig1, &aSig0, &aSig1 );
-    expDiff -= 64;
-    while ( 0 < expDiff ) {
-        q = estimateDiv128To64( aSig0, aSig1, bSig0 );
-        q = ( 4 < q ) ? q - 4 : 0;
-        mul128By64To192( bSig0, bSig1, q, &term0, &term1, &term2 );
-        shortShift192Left( term0, term1, term2, 61, &term1, &term2, &allZero );
-        shortShift128Left( aSig0, aSig1, 61, &aSig0, &allZero );
-        sub128( aSig0, 0, term1, term2, &aSig0, &aSig1 );
-        expDiff -= 61;
-    }
-    if ( -64 < expDiff ) {
-        q = estimateDiv128To64( aSig0, aSig1, bSig0 );
-        q = ( 4 < q ) ? q - 4 : 0;
-        q >>= - expDiff;
-        shift128Right( bSig0, bSig1, 12, &bSig0, &bSig1 );
-        expDiff += 52;
-        if ( expDiff < 0 ) {
-            shift128Right( aSig0, aSig1, - expDiff, &aSig0, &aSig1 );
+    if ( increment ) {
+        ++zSig0;
+        if ( zSig0 == 0 ) {
+            ++zExp;
+            zSig0 = UINT64_C(0x8000000000000000);
         }
         else {
-            shortShift128Left( aSig0, aSig1, expDiff, &aSig0, &aSig1 );
+            if (!(zSig1 << 1) && roundNearestEven) {
+                zSig0 &= ~1;
+            }
         }
-        mul128By64To192( bSig0, bSig1, q, &term0, &term1, &term2 );
-        sub128( aSig0, aSig1, term1, term2, &aSig0, &aSig1 );
     }
     else {
-        shift128Right( aSig0, aSig1, 12, &aSig0, &aSig1 );
-        shift128Right( bSig0, bSig1, 12, &bSig0, &bSig1 );
-    }
-    do {
-        alternateASig0 = aSig0;
-        alternateASig1 = aSig1;
-        ++q;
-        sub128( aSig0, aSig1, bSig0, bSig1, &aSig0, &aSig1 );
-    } while ( 0 <= (int64_t) aSig0 );
-    add128(
-        aSig0, aSig1, alternateASig0, alternateASig1, (uint64_t *)&sigMean0, &sigMean1 );
-    if (    ( sigMean0 < 0 )
-         || ( ( ( sigMean0 | sigMean1 ) == 0 ) && ( q & 1 ) ) ) {
-        aSig0 = alternateASig0;
-        aSig1 = alternateASig1;
-    }
-    zSign = ( (int64_t) aSig0 < 0 );
-    if ( zSign ) sub128( 0, 0, aSig0, aSig1, &aSig0, &aSig1 );
-    return normalizeRoundAndPackFloat128(aSign ^ zSign, bExp - 4, aSig0, aSig1,
-                                         status);
-}
-
-/*----------------------------------------------------------------------------
-| Returns the square root of the quadruple-precision floating-point value `a'.
-| The operation is performed according to the IEC/IEEE Standard for Binary
-| Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-float128 float128_sqrt(float128 a, float_status *status)
-{
-    flag aSign;
-    int32_t aExp, zExp;
-    uint64_t aSig0, aSig1, zSig0, zSig1, zSig2, doubleZSig0;
-    uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if (aSig0 | aSig1) {
-            return propagateFloat128NaN(a, a, status);
-        }
-        if ( ! aSign ) return a;
-        goto invalid;
-    }
-    if ( aSign ) {
-        if ( ( aExp | aSig0 | aSig1 ) == 0 ) return a;
- invalid:
-        float_raise(float_flag_invalid, status);
-        return float128_default_nan(status);
-    }
-    if ( aExp == 0 ) {
-        if ( ( aSig0 | aSig1 ) == 0 ) return packFloat128( 0, 0, 0, 0 );
-        normalizeFloat128Subnormal( aSig0, aSig1, &aExp, &aSig0, &aSig1 );
-    }
-    zExp = ( ( aExp - 0x3FFF )>>1 ) + 0x3FFE;
-    aSig0 |= UINT64_C(0x0001000000000000);
-    zSig0 = estimateSqrt32( aExp, aSig0>>17 );
-    shortShift128Left( aSig0, aSig1, 13 - ( aExp & 1 ), &aSig0, &aSig1 );
-    zSig0 = estimateDiv128To64( aSig0, aSig1, zSig0<<32 ) + ( zSig0<<30 );
-    doubleZSig0 = zSig0<<1;
-    mul64To128( zSig0, zSig0, &term0, &term1 );
-    sub128( aSig0, aSig1, term0, term1, &rem0, &rem1 );
-    while ( (int64_t) rem0 < 0 ) {
-        --zSig0;
-        doubleZSig0 -= 2;
-        add128( rem0, rem1, zSig0>>63, doubleZSig0 | 1, &rem0, &rem1 );
-    }
-    zSig1 = estimateDiv128To64( rem1, 0, doubleZSig0 );
-    if ( ( zSig1 & 0x1FFF ) <= 5 ) {
-        if ( zSig1 == 0 ) zSig1 = 1;
-        mul64To128( doubleZSig0, zSig1, &term1, &term2 );
-        sub128( rem1, 0, term1, term2, &rem1, &rem2 );
-        mul64To128( zSig1, zSig1, &term2, &term3 );
-        sub192( rem1, rem2, 0, 0, term2, term3, &rem1, &rem2, &rem3 );
-        while ( (int64_t) rem1 < 0 ) {
-            --zSig1;
-            shortShift128Left( 0, zSig1, 1, &term2, &term3 );
-            term3 |= 1;
-            term2 |= doubleZSig0;
-            add192( rem1, rem2, rem3, 0, term2, term3, &rem1, &rem2, &rem3 );
-        }
-        zSig1 |= ( ( rem1 | rem2 | rem3 ) != 0 );
-    }
-    shift128ExtraRightJamming( zSig0, zSig1, 0, 14, &zSig0, &zSig1, &zSig2 );
-    return roundAndPackFloat128(0, zExp, zSig0, zSig1, zSig2, status);
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point value `a' is equal to
-| the corresponding value `b', and 0 otherwise.  The invalid exception is
-| raised if either operand is a NaN.  Otherwise, the comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float128_eq(float128 a, float128 b, float_status *status)
-{
-
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    return
-           ( a.low == b.low )
-        && (    ( a.high == b.high )
-             || (    ( a.low == 0 )
-                  && ( (uint64_t) ( ( a.high | b.high )<<1 ) == 0 ) )
-           );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point value `a' is less than
-| or equal to the corresponding value `b', and 0 otherwise.  The invalid
-| exception is raised if either operand is a NaN.  The comparison is performed
-| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float128_le(float128 a, float128 b, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloat128Sign( a );
-    bSign = extractFloat128Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            || (    ( ( (uint64_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 == 0 );
+        if ( zSig0 == 0 ) zExp = 0;
     }
-    return
-          aSign ? le128( b.high, b.low, a.high, a.low )
-        : le128( a.high, a.low, b.high, b.low );
+    return packFloatx80( zSign, zExp, zSig0 );
 
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point value `a' is less than
-| the corresponding value `b', and 0 otherwise.  The invalid exception is
-| raised if either operand is a NaN.  The comparison is performed according
-| to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+| Takes an abstract floating-point value having sign `zSign', exponent
+| `zExp', and significand formed by the concatenation of `zSig0' and `zSig1',
+| and returns the proper extended double-precision floating-point value
+| corresponding to the abstract input.  This routine is just like
+| `roundAndPackFloatx80' except that the input significand does not have to be
+| normalized.
 *----------------------------------------------------------------------------*/
 
-int float128_lt(float128 a, float128 b, float_status *status)
+floatx80 normalizeRoundAndPackFloatx80(FloatX80RoundPrec roundingPrecision,
+                                       bool zSign, int32_t zExp,
+                                       uint64_t zSig0, uint64_t zSig1,
+                                       float_status *status)
 {
-    flag aSign, bSign;
+    int8_t shiftCount;
 
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 0;
-    }
-    aSign = extractFloat128Sign( a );
-    bSign = extractFloat128Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            && (    ( ( (uint64_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 != 0 );
+    if ( zSig0 == 0 ) {
+        zSig0 = zSig1;
+        zSig1 = 0;
+        zExp -= 64;
     }
-    return
-          aSign ? lt128( b.high, b.low, a.high, a.low )
-        : lt128( a.high, a.low, b.high, b.low );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point values `a' and `b' cannot
-| be compared, and 0 otherwise.  The invalid exception is raised if either
-| operand is a NaN. The comparison is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
+    shiftCount = clz64(zSig0);
+    shortShift128Left( zSig0, zSig1, shiftCount, &zSig0, &zSig1 );
+    zExp -= shiftCount;
+    return roundAndPackFloatx80(roundingPrecision, zSign, zExp,
+                                zSig0, zSig1, status);
 
-int float128_unordered(float128 a, float128 b, float_status *status)
-{
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        float_raise(float_flag_invalid, status);
-        return 1;
-    }
-    return 0;
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point value `a' is equal to
-| the corresponding value `b', and 0 otherwise.  Quiet NaNs do not cause an
-| exception.  The comparison is performed according to the IEC/IEEE Standard
-| for Binary Floating-Point Arithmetic.
+| Returns the binary exponential of the single-precision floating-point value
+| `a'. The operation is performed according to the IEC/IEEE Standard for
+| Binary Floating-Point Arithmetic.
+|
+| Uses the following identities:
+|
+| 1. -------------------------------------------------------------------------
+|      x    x*ln(2)
+|     2  = e
+|
+| 2. -------------------------------------------------------------------------
+|                      2     3     4     5           n
+|      x        x     x     x     x     x           x
+|     e  = 1 + --- + --- + --- + --- + --- + ... + --- + ...
+|               1!    2!    3!    4!    5!          n!
 *----------------------------------------------------------------------------*/
 
-int float128_eq_quiet(float128 a, float128 b, float_status *status)
+static const float64 float32_exp2_coefficients[15] =
 {
+    const_float64( 0x3ff0000000000000ll ), /*  1 */
+    const_float64( 0x3fe0000000000000ll ), /*  2 */
+    const_float64( 0x3fc5555555555555ll ), /*  3 */
+    const_float64( 0x3fa5555555555555ll ), /*  4 */
+    const_float64( 0x3f81111111111111ll ), /*  5 */
+    const_float64( 0x3f56c16c16c16c17ll ), /*  6 */
+    const_float64( 0x3f2a01a01a01a01all ), /*  7 */
+    const_float64( 0x3efa01a01a01a01all ), /*  8 */
+    const_float64( 0x3ec71de3a556c734ll ), /*  9 */
+    const_float64( 0x3e927e4fb7789f5cll ), /* 10 */
+    const_float64( 0x3e5ae64567f544e4ll ), /* 11 */
+    const_float64( 0x3e21eed8eff8d898ll ), /* 12 */
+    const_float64( 0x3de6124613a86d09ll ), /* 13 */
+    const_float64( 0x3da93974a8c07c9dll ), /* 14 */
+    const_float64( 0x3d6ae7f3e733b81fll ), /* 15 */
+};
 
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        if (float128_is_signaling_nan(a, status)
-         || float128_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    return
-           ( a.low == b.low )
-        && (    ( a.high == b.high )
-             || (    ( a.low == 0 )
-                  && ( (uint64_t) ( ( a.high | b.high )<<1 ) == 0 ) )
-           );
-
-}
-
-/*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point value `a' is less than
-| or equal to the corresponding value `b', and 0 otherwise.  Quiet NaNs do not
-| cause an exception.  Otherwise, the comparison is performed according to the
-| IEC/IEEE Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
-
-int float128_le_quiet(float128 a, float128 b, float_status *status)
+float32 float32_exp2(float32 a, float_status *status)
 {
-    flag aSign, bSign;
+    FloatParts64 xp, xnp, tp, rp;
+    int i;
 
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        if (float128_is_signaling_nan(a, status)
-         || float128_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
+    float32_unpack_canonical(&xp, a, status);
+    if (unlikely(xp.cls != float_class_normal)) {
+        switch (xp.cls) {
+        case float_class_snan:
+        case float_class_qnan:
+            parts_return_nan(&xp, status);
+            return float32_round_pack_canonical(&xp, status);
+        case float_class_inf:
+            return xp.sign ? float32_zero : a;
+        case float_class_zero:
+            return float32_one;
+        default:
+            break;
         }
-        return 0;
-    }
-    aSign = extractFloat128Sign( a );
-    bSign = extractFloat128Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            || (    ( ( (uint64_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 == 0 );
+        g_assert_not_reached();
     }
-    return
-          aSign ? le128( b.high, b.low, a.high, a.low )
-        : le128( a.high, a.low, b.high, b.low );
-
-}
 
-/*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point value `a' is less than
-| the corresponding value `b', and 0 otherwise.  Quiet NaNs do not cause an
-| exception.  Otherwise, the comparison is performed according to the IEC/IEEE
-| Standard for Binary Floating-Point Arithmetic.
-*----------------------------------------------------------------------------*/
+    float_raise(float_flag_inexact, status);
 
-int float128_lt_quiet(float128 a, float128 b, float_status *status)
-{
-    flag aSign, bSign;
+    float64_unpack_canonical(&tp, float64_ln2, status);
+    xp = *parts_mul(&xp, &tp, status);
+    xnp = xp;
 
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        if (float128_is_signaling_nan(a, status)
-         || float128_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 0;
-    }
-    aSign = extractFloat128Sign( a );
-    bSign = extractFloat128Sign( b );
-    if ( aSign != bSign ) {
-        return
-               aSign
-            && (    ( ( (uint64_t) ( ( a.high | b.high )<<1 ) ) | a.low | b.low )
-                 != 0 );
+    float64_unpack_canonical(&rp, float64_one, status);
+    for (i = 0 ; i < 15 ; i++) {
+        float64_unpack_canonical(&tp, float32_exp2_coefficients[i], status);
+        rp = *parts_muladd(&tp, &xnp, &rp, 0, status);
+        xnp = *parts_mul(&xnp, &xp, status);
     }
-    return
-          aSign ? lt128( b.high, b.low, a.high, a.low )
-        : lt128( a.high, a.low, b.high, b.low );
 
+    return float32_round_pack_canonical(&rp, status);
 }
 
 /*----------------------------------------------------------------------------
-| Returns 1 if the quadruple-precision floating-point values `a' and `b' cannot
-| be compared, and 0 otherwise.  Quiet NaNs do not cause an exception.  The
-| comparison is performed according to the IEC/IEEE Standard for Binary
+| Rounds the extended double-precision floating-point value `a'
+| to the precision provided by floatx80_rounding_precision and returns the
+| result as an extended double-precision floating-point value.
+| The operation is performed according to the IEC/IEEE Standard for Binary
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_unordered_quiet(float128 a, float128 b, float_status *status)
-{
-    if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
-              && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
-         || (    ( extractFloat128Exp( b ) == 0x7FFF )
-              && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
-       ) {
-        if (float128_is_signaling_nan(a, status)
-         || float128_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return 1;
-    }
-    return 0;
-}
-
-static inline int floatx80_compare_internal(floatx80 a, floatx80 b,
-                                            int is_quiet, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (floatx80_invalid_encoding(a) || floatx80_invalid_encoding(b)) {
-        float_raise(float_flag_invalid, status);
-        return float_relation_unordered;
-    }
-    if (( ( extractFloatx80Exp( a ) == 0x7fff ) &&
-          ( extractFloatx80Frac( a )<<1 ) ) ||
-        ( ( extractFloatx80Exp( b ) == 0x7fff ) &&
-          ( extractFloatx80Frac( b )<<1 ) )) {
-        if (!is_quiet ||
-            floatx80_is_signaling_nan(a, status) ||
-            floatx80_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return float_relation_unordered;
-    }
-    aSign = extractFloatx80Sign( a );
-    bSign = extractFloatx80Sign( b );
-    if ( aSign != bSign ) {
-
-        if ( ( ( (uint16_t) ( ( a.high | b.high ) << 1 ) ) == 0) &&
-             ( ( a.low | b.low ) == 0 ) ) {
-            /* zero case */
-            return float_relation_equal;
-        } else {
-            return 1 - (2 * aSign);
-        }
-    } else {
-        if (a.low == b.low && a.high == b.high) {
-            return float_relation_equal;
-        } else {
-            return 1 - 2 * (aSign ^ ( lt128( a.high, a.low, b.high, b.low ) ));
-        }
-    }
-}
-
-int floatx80_compare(floatx80 a, floatx80 b, float_status *status)
-{
-    return floatx80_compare_internal(a, b, 0, status);
-}
-
-int floatx80_compare_quiet(floatx80 a, floatx80 b, float_status *status)
-{
-    return floatx80_compare_internal(a, b, 1, status);
-}
-
-static inline int float128_compare_internal(float128 a, float128 b,
-                                            int is_quiet, float_status *status)
-{
-    flag aSign, bSign;
-
-    if (( ( extractFloat128Exp( a ) == 0x7fff ) &&
-          ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) ) ||
-        ( ( extractFloat128Exp( b ) == 0x7fff ) &&
-          ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )) {
-        if (!is_quiet ||
-            float128_is_signaling_nan(a, status) ||
-            float128_is_signaling_nan(b, status)) {
-            float_raise(float_flag_invalid, status);
-        }
-        return float_relation_unordered;
-    }
-    aSign = extractFloat128Sign( a );
-    bSign = extractFloat128Sign( b );
-    if ( aSign != bSign ) {
-        if ( ( ( ( a.high | b.high )<<1 ) | a.low | b.low ) == 0 ) {
-            /* zero case */
-            return float_relation_equal;
-        } else {
-            return 1 - (2 * aSign);
-        }
-    } else {
-        if (a.low == b.low && a.high == b.high) {
-            return float_relation_equal;
-        } else {
-            return 1 - 2 * (aSign ^ ( lt128( a.high, a.low, b.high, b.low ) ));
-        }
-    }
-}
-
-int float128_compare(float128 a, float128 b, float_status *status)
-{
-    return float128_compare_internal(a, b, 0, status);
-}
-
-int float128_compare_quiet(float128 a, float128 b, float_status *status)
-{
-    return float128_compare_internal(a, b, 1, status);
-}
-
-floatx80 floatx80_scalbn(floatx80 a, int n, float_status *status)
+floatx80 floatx80_round(floatx80 a, float_status *status)
 {
-    flag aSign;
-    int32_t aExp;
-    uint64_t aSig;
+    FloatParts128 p;
 
-    if (floatx80_invalid_encoding(a)) {
-        float_raise(float_flag_invalid, status);
+    if (!floatx80_unpack_canonical(&p, a, status)) {
         return floatx80_default_nan(status);
     }
-    aSig = extractFloatx80Frac( a );
-    aExp = extractFloatx80Exp( a );
-    aSign = extractFloatx80Sign( a );
-
-    if ( aExp == 0x7FFF ) {
-        if ( aSig<<1 ) {
-            return propagateFloatx80NaN(a, a, status);
-        }
-        return a;
-    }
-
-    if (aExp == 0) {
-        if (aSig == 0) {
-            return a;
-        }
-        aExp++;
-    }
-
-    if (n > 0x10000) {
-        n = 0x10000;
-    } else if (n < -0x10000) {
-        n = -0x10000;
-    }
-
-    aExp += n;
-    return normalizeRoundAndPackFloatx80(status->floatx80_rounding_precision,
-                                         aSign, aExp, aSig, 0, status);
-}
-
-float128 float128_scalbn(float128 a, int n, float_status *status)
-{
-    flag aSign;
-    int32_t aExp;
-    uint64_t aSig0, aSig1;
-
-    aSig1 = extractFloat128Frac1( a );
-    aSig0 = extractFloat128Frac0( a );
-    aExp = extractFloat128Exp( a );
-    aSign = extractFloat128Sign( a );
-    if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 ) {
-            return propagateFloat128NaN(a, a, status);
-        }
-        return a;
-    }
-    if (aExp != 0) {
-        aSig0 |= UINT64_C(0x0001000000000000);
-    } else if (aSig0 == 0 && aSig1 == 0) {
-        return a;
-    } else {
-        aExp++;
-    }
-
-    if (n > 0x10000) {
-        n = 0x10000;
-    } else if (n < -0x10000) {
-        n = -0x10000;
-    }
-
-    aExp += n - 1;
-    return normalizeRoundAndPackFloat128( aSign, aExp, aSig0, aSig1
-                                         , status);
-
+    return floatx80_round_pack_canonical(&p, status);
 }
 
 void softfloat_init(void)
diff --git a/qemu/hw/core/cpu.c b/qemu/hw/core/cpu.c
index 0bd16dc92e..e46dcd5c06 100644
--- a/qemu/hw/core/cpu.c
+++ b/qemu/hw/core/cpu.c
@@ -71,6 +71,62 @@ static bool cpu_common_exec_interrupt(CPUState *cpu, int int_req)
     return false;
 }
 
+static void cpu_legacy_debug_excp_handler(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    cc->debug_excp_handler(cpu);
+}
+
+static void cpu_legacy_exec_enter(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    cc->cpu_exec_enter(cpu);
+}
+
+static void cpu_legacy_exec_exit(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    cc->cpu_exec_exit(cpu);
+}
+
+static bool cpu_legacy_exec_interrupt(CPUState *cpu, int interrupt_request)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    return cc->cpu_exec_interrupt(cpu, interrupt_request);
+}
+
+static bool cpu_legacy_tlb_fill(CPUState *cpu, vaddr address, int size,
+                                MMUAccessType access_type, int mmu_idx,
+                                bool probe, uintptr_t retaddr)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    return cc->tlb_fill(cpu, address, size, access_type, mmu_idx, probe,
+                        retaddr);
+}
+
+static void cpu_legacy_unaligned_access(CPUState *cpu, vaddr addr,
+                                        MMUAccessType access_type,
+                                        int mmu_idx, uintptr_t retaddr)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    cc->do_unaligned_access(cpu, addr, access_type, mmu_idx, retaddr);
+}
+
+static const struct TCGCPUOps cpu_legacy_tcg_ops = {
+    .cpu_exec_enter = cpu_legacy_exec_enter,
+    .cpu_exec_exit = cpu_legacy_exec_exit,
+    .debug_excp_handler = cpu_legacy_debug_excp_handler,
+    .cpu_exec_interrupt = cpu_legacy_exec_interrupt,
+    .tlb_fill = cpu_legacy_tlb_fill,
+    .do_unaligned_access = cpu_legacy_unaligned_access,
+};
+
 void cpu_reset(CPUState *cpu)
 {
     CPUClass *klass = CPU_GET_CLASS(cpu);
@@ -118,6 +174,7 @@ void cpu_class_init(struct uc_struct *uc, CPUClass *k)
     k->cpu_exec_enter = cpu_common_noop;
     k->cpu_exec_exit = cpu_common_noop;
     k->cpu_exec_interrupt = cpu_common_exec_interrupt;
+    k->tcg_ops = &cpu_legacy_tcg_ops;
     /* instead of dc->reset. */
     k->reset = cpu_common_reset;
 
diff --git a/qemu/include/crypto/aes.h b/qemu/include/crypto/aes.h
index 12fb321b89..ba297d6a73 100644
--- a/qemu/include/crypto/aes.h
+++ b/qemu/include/crypto/aes.h
@@ -16,7 +16,6 @@ typedef struct aes_key_st AES_KEY;
 #define AES_set_decrypt_key QEMU_AES_set_decrypt_key
 #define AES_encrypt QEMU_AES_encrypt
 #define AES_decrypt QEMU_AES_decrypt
-#define AES_cbc_encrypt QEMU_AES_cbc_encrypt
 
 int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
 	AES_KEY *key);
@@ -27,9 +26,6 @@ void AES_encrypt(const unsigned char *in, unsigned char *out,
 	const AES_KEY *key);
 void AES_decrypt(const unsigned char *in, unsigned char *out,
 	const AES_KEY *key);
-void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
-		     const unsigned long length, const AES_KEY *key,
-		     unsigned char *ivec, const int enc);
 
 extern const uint8_t AES_sbox[256];
 extern const uint8_t AES_isbox[256];
diff --git a/qemu/include/crypto/sm4.h b/qemu/include/crypto/sm4.h
new file mode 100644
index 0000000000..9bd3ebc62e
--- /dev/null
+++ b/qemu/include/crypto/sm4.h
@@ -0,0 +1,6 @@
+#ifndef QEMU_SM4_H
+#define QEMU_SM4_H
+
+extern const uint8_t sm4_sbox[256];
+
+#endif
diff --git a/qemu/include/exec/cputlb.h b/qemu/include/exec/cputlb.h
index 0a7757114d..3436ddaea3 100644
--- a/qemu/include/exec/cputlb.h
+++ b/qemu/include/exec/cputlb.h
@@ -6,7 +6,7 @@
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
+ * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
diff --git a/qemu/include/exec/helper-head.h b/qemu/include/exec/helper-head.h
index 9d7add43bb..e1d14edeab 100644
--- a/qemu/include/exec/helper-head.h
+++ b/qemu/include/exec/helper-head.h
@@ -46,20 +46,23 @@
 #define dh_ctype_ptr void *
 #define dh_ctype_cptr const void *
 #define dh_ctype_void void
-#define dh_ctype_noreturn void QEMU_NORETURN
+#define dh_ctype_noreturn G_NORETURN void
 #define dh_ctype(t) dh_ctype_##t
 
 #ifdef NEED_CPU_H
 # ifdef TARGET_LONG_BITS
 #  if TARGET_LONG_BITS == 32
 #   define dh_alias_tl i32
+#   define dh_typecode_tl dh_typecode_i32
 #  else
 #   define dh_alias_tl i64
+#   define dh_typecode_tl dh_typecode_i64
 #  endif
 # endif
-# define dh_alias_env ptr
 # define dh_ctype_tl target_ulong
+# define dh_alias_env ptr
 # define dh_ctype_env CPUArchState *
+# define dh_typecode_env dh_typecode_ptr
 #endif
 
 /* We can't use glue() here because it falls foul of C preprocessor
@@ -85,6 +88,20 @@
 #define dh_retvar_ptr tcgv_ptr_temp(tcg_ctx, retval)
 #define dh_retvar(t) glue(dh_retvar_, dh_alias(t))
 
+#define dh_typecode_void 0
+#define dh_typecode_noreturn 0
+#define dh_typecode_i32 2
+#define dh_typecode_s32 3
+#define dh_typecode_i64 4
+#define dh_typecode_s64 5
+#define dh_typecode_ptr 6
+#define dh_typecode_int dh_typecode_s32
+#define dh_typecode_f16 dh_typecode_i32
+#define dh_typecode_f32 dh_typecode_i32
+#define dh_typecode_f64 dh_typecode_i64
+#define dh_typecode_cptr dh_typecode_ptr
+#define dh_typecode(t) dh_typecode_##t
+
 #define dh_is_64bit_void 0
 #define dh_is_64bit_noreturn 0
 #define dh_is_64bit_i32 0
@@ -104,30 +121,23 @@
 #define dh_is_signed_f64 0
 #define dh_is_signed_tl  0
 #define dh_is_signed_int 1
-/* ??? This is highly specific to the host cpu.  There are even special
-   extension instructions that may be required, e.g. ia64's addp4.  But
-   for now we don't support any 64-bit targets with 32-bit pointers.  */
 #define dh_is_signed_ptr 0
 #define dh_is_signed_cptr dh_is_signed_ptr
+#ifdef NEED_CPU_H
 #define dh_is_signed_env dh_is_signed_ptr
+#endif
 #define dh_is_signed(t) dh_is_signed_##t
 
 #define dh_callflag_i32  0
-#define dh_callflag_s32  0
-#define dh_callflag_int  0
 #define dh_callflag_i64  0
-#define dh_callflag_s64  0
-#define dh_callflag_f16  0
-#define dh_callflag_f32  0
-#define dh_callflag_f64  0
 #define dh_callflag_ptr  0
-#define dh_callflag_cptr dh_callflag_ptr
 #define dh_callflag_void 0
 #define dh_callflag_noreturn TCG_CALL_NO_RETURN
 #define dh_callflag(t) glue(dh_callflag_, dh_alias(t))
 
+#define dh_typemask(t, n)  (dh_typecode(t) << (n * 3))
 #define dh_sizemask(t, n) \
-  ((dh_is_64bit(t) << (n*2)) | (dh_is_signed(t) << (n*2+1)))
+  ((dh_is_64bit(t) << (n * 2)) | (dh_is_signed(t) << (n * 2 + 1)))
 
 #define dh_arg(t, n) \
   glue(glue(tcgv_, dh_alias(t)), _temp)(tcg_ctx, glue(arg, n))
diff --git a/qemu/include/exec/hwaddr.h b/qemu/include/exec/hwaddr.h
index a71c93cc81..8f16d179a8 100644
--- a/qemu/include/exec/hwaddr.h
+++ b/qemu/include/exec/hwaddr.h
@@ -18,4 +18,9 @@ typedef uint64_t hwaddr;
 #define HWADDR_PRIx PRIx64
 #define HWADDR_PRIX PRIX64
 
+typedef struct MemMapEntry {
+    hwaddr base;
+    hwaddr size;
+} MemMapEntry;
+
 #endif
diff --git a/qemu/include/exec/ioport.h b/qemu/include/exec/ioport.h
index cdd6b2dfbd..0f023def09 100644
--- a/qemu/include/exec/ioport.h
+++ b/qemu/include/exec/ioport.h
@@ -6,7 +6,7 @@
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
+ * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
diff --git a/qemu/include/exec/memattrs.h b/qemu/include/exec/memattrs.h
index 95f2d20d55..9fb98bc1ef 100644
--- a/qemu/include/exec/memattrs.h
+++ b/qemu/include/exec/memattrs.h
@@ -35,6 +35,14 @@ typedef struct MemTxAttrs {
     unsigned int secure:1;
     /* Memory access is usermode (unprivileged) */
     unsigned int user:1;
+    /*
+     * Bus interconnect and peripherals can access anything (memories,
+     * devices) by default. By setting the 'memory' bit, bus transaction
+     * are restricted to "normal" memories (per the AMBA documentation)
+     * versus devices. Access to devices will be logged and rejected
+     * (see MEMTX_ACCESS_ERROR).
+     */
+    unsigned int memory:1;
     /* Requester ID (for MSI for example) */
     unsigned int requester_id:16;
     /* Invert endianness for this page */
@@ -66,6 +74,7 @@ typedef struct MemTxAttrs {
 #define MEMTX_OK 0
 #define MEMTX_ERROR             (1U << 0) /* device returned an error */
 #define MEMTX_DECODE_ERROR      (1U << 1) /* nothing at that address */
+#define MEMTX_ACCESS_ERROR      (1U << 2) /* access denied */
 typedef uint32_t MemTxResult;
 
 #endif
diff --git a/qemu/include/exec/memop.h b/qemu/include/exec/memop.h
index 529d07b02d..bcaae1532a 100644
--- a/qemu/include/exec/memop.h
+++ b/qemu/include/exec/memop.h
@@ -19,12 +19,16 @@ typedef enum MemOp {
     MO_16    = 1,
     MO_32    = 2,
     MO_64    = 3,
-    MO_SIZE  = 3,   /* Mask for the above.  */
+    MO_128   = 4,
+    MO_256   = 5,
+    MO_512   = 6,
+    MO_1024  = 7,
+    MO_SIZE  = 0x07,   /* Mask for the above.  */
 
-    MO_SIGN  = 4,   /* Sign-extended, otherwise zero-extended.  */
+    MO_SIGN  = 0x08,   /* Sign-extended, otherwise zero-extended.  */
 
-    MO_BSWAP = 8,   /* Host reverse endian.  */
-#ifdef HOST_WORDS_BIGENDIAN
+    MO_BSWAP = 0x10,   /* Host reverse endian.  */
+#if HOST_BIG_ENDIAN
     MO_LE    = MO_BSWAP,
     MO_BE    = 0,
 #else
@@ -32,11 +36,19 @@ typedef enum MemOp {
     MO_BE    = MO_BSWAP,
 #endif
 #ifdef NEED_CPU_H
-#ifdef TARGET_WORDS_BIGENDIAN
+#if defined(TARGET_BIG_ENDIAN)
+# define MEMOP_TARGET_BIG_ENDIAN TARGET_BIG_ENDIAN
+#elif defined(TARGET_WORDS_BIGENDIAN)
+# define MEMOP_TARGET_BIG_ENDIAN 1
+#else
+# define MEMOP_TARGET_BIG_ENDIAN 0
+#endif
+#if MEMOP_TARGET_BIG_ENDIAN
     MO_TE    = MO_BE,
 #else
     MO_TE    = MO_LE,
 #endif
+#undef MEMOP_TARGET_BIG_ENDIAN
 #endif
 
     /*
@@ -59,8 +71,8 @@ typedef enum MemOp {
      * - an alignment to a specified size, which may be more or less than
      *   the access size (MO_ALIGN_x where 'x' is a size in bytes);
      */
-    MO_ASHIFT = 4,
-    MO_AMASK = 7 << MO_ASHIFT,
+    MO_ASHIFT = 5,
+    MO_AMASK = 0x7 << MO_ASHIFT,
 #ifdef NEED_CPU_H
 #ifdef TARGET_ALIGNED_ONLY
     MO_ALIGN = 0,
@@ -81,29 +93,41 @@ typedef enum MemOp {
     MO_UB    = MO_8,
     MO_UW    = MO_16,
     MO_UL    = MO_32,
+    MO_UQ    = MO_64,
+    MO_UO    = MO_128,
     MO_SB    = MO_SIGN | MO_8,
     MO_SW    = MO_SIGN | MO_16,
     MO_SL    = MO_SIGN | MO_32,
-    MO_Q     = MO_64,
+    MO_SQ    = MO_SIGN | MO_64,
+    MO_SO    = MO_SIGN | MO_128,
 
     MO_LEUW  = MO_LE | MO_UW,
     MO_LEUL  = MO_LE | MO_UL,
+    MO_LEUQ  = MO_LE | MO_UQ,
     MO_LESW  = MO_LE | MO_SW,
     MO_LESL  = MO_LE | MO_SL,
-    MO_LEQ   = MO_LE | MO_Q,
+    MO_LESQ  = MO_LE | MO_SQ,
 
     MO_BEUW  = MO_BE | MO_UW,
     MO_BEUL  = MO_BE | MO_UL,
+    MO_BEUQ  = MO_BE | MO_UQ,
     MO_BESW  = MO_BE | MO_SW,
     MO_BESL  = MO_BE | MO_SL,
-    MO_BEQ   = MO_BE | MO_Q,
+    MO_BESQ  = MO_BE | MO_SQ,
+
+    MO_Q     = MO_UQ,
+    MO_LEQ   = MO_LEUQ,
+    MO_BEQ   = MO_BEUQ,
 
 #ifdef NEED_CPU_H
     MO_TEUW  = MO_TE | MO_UW,
     MO_TEUL  = MO_TE | MO_UL,
+    MO_TEUQ  = MO_TE | MO_UQ,
+    MO_TEUO  = MO_TE | MO_UO,
     MO_TESW  = MO_TE | MO_SW,
     MO_TESL  = MO_TE | MO_SL,
-    MO_TEQ   = MO_TE | MO_Q,
+    MO_TESQ  = MO_TE | MO_SQ,
+    MO_TEQ   = MO_TEUQ,
 #endif
 
     MO_SSIZE = MO_SIZE | MO_SIGN,
diff --git a/qemu/include/exec/poison.h b/qemu/include/exec/poison.h
index 7b9ac361dc..99fe4f3007 100644
--- a/qemu/include/exec/poison.h
+++ b/qemu/include/exec/poison.h
@@ -3,7 +3,13 @@
 
 #ifndef HW_POISON_H
 #define HW_POISON_H
+
 #ifdef __GNUC__
+#if defined(__has_include)
+#if __has_include("config-poison.h")
+#include "config-poison.h"
+#endif
+#endif
 
 #pragma GCC poison TARGET_I386
 #pragma GCC poison TARGET_X86_64
@@ -11,8 +17,9 @@
 #pragma GCC poison TARGET_ALPHA
 #pragma GCC poison TARGET_ARM
 #pragma GCC poison TARGET_CRIS
+#pragma GCC poison TARGET_HEXAGON
 #pragma GCC poison TARGET_HPPA
-#pragma GCC poison TARGET_LM32
+#pragma GCC poison TARGET_LOONGARCH64
 #pragma GCC poison TARGET_M68K
 #pragma GCC poison TARGET_MICROBLAZE
 #pragma GCC poison TARGET_MIPS
@@ -20,7 +27,6 @@
 #pragma GCC poison TARGET_ABI_MIPSO32
 #pragma GCC poison TARGET_MIPS64
 #pragma GCC poison TARGET_ABI_MIPSN64
-#pragma GCC poison TARGET_MOXIE
 #pragma GCC poison TARGET_NIOS2
 #pragma GCC poison TARGET_OPENRISC
 #pragma GCC poison TARGET_PPC
@@ -31,16 +37,14 @@
 #pragma GCC poison TARGET_SH4
 #pragma GCC poison TARGET_SPARC
 #pragma GCC poison TARGET_SPARC64
-#pragma GCC poison TARGET_TILEGX
 #pragma GCC poison TARGET_TRICORE
-#pragma GCC poison TARGET_UNICORE32
 #pragma GCC poison TARGET_XTENSA
 
 #pragma GCC poison TARGET_ALIGNED_ONLY
 #pragma GCC poison TARGET_HAS_BFLT
 #pragma GCC poison TARGET_NAME
 #pragma GCC poison TARGET_SUPPORTS_MTTCG
-#pragma GCC poison TARGET_WORDS_BIGENDIAN
+#pragma GCC poison TARGET_BIG_ENDIAN
 #pragma GCC poison BSWAP_NEEDED
 
 #pragma GCC poison TARGET_LONG_BITS
@@ -53,8 +57,6 @@
 #pragma GCC poison TARGET_PAGE_BITS
 #pragma GCC poison TARGET_PAGE_ALIGN
 
-#pragma GCC poison CPUArchState
-
 #pragma GCC poison CPU_INTERRUPT_HARD
 #pragma GCC poison CPU_INTERRUPT_EXITTB
 #pragma GCC poison CPU_INTERRUPT_HALT
@@ -69,17 +71,15 @@
 #pragma GCC poison CPU_INTERRUPT_TGT_INT_2
 
 #pragma GCC poison CONFIG_ALPHA_DIS
-#pragma GCC poison CONFIG_ARM_A64_DIS
-#pragma GCC poison CONFIG_ARM_DIS
 #pragma GCC poison CONFIG_CRIS_DIS
 #pragma GCC poison CONFIG_HPPA_DIS
 #pragma GCC poison CONFIG_I386_DIS
-#pragma GCC poison CONFIG_LM32_DIS
+#pragma GCC poison CONFIG_HEXAGON_DIS
+#pragma GCC poison CONFIG_LOONGARCH_DIS
 #pragma GCC poison CONFIG_M68K_DIS
 #pragma GCC poison CONFIG_MICROBLAZE_DIS
 #pragma GCC poison CONFIG_MIPS_DIS
 #pragma GCC poison CONFIG_NANOMIPS_DIS
-#pragma GCC poison CONFIG_MOXIE_DIS
 #pragma GCC poison CONFIG_NIOS2_DIS
 #pragma GCC poison CONFIG_PPC_DIS
 #pragma GCC poison CONFIG_RISCV_DIS
@@ -88,9 +88,13 @@
 #pragma GCC poison CONFIG_SPARC_DIS
 #pragma GCC poison CONFIG_XTENSA_DIS
 
+#pragma GCC poison CONFIG_HAX
+#pragma GCC poison CONFIG_HVF
 #pragma GCC poison CONFIG_LINUX_USER
 #pragma GCC poison CONFIG_KVM
 #pragma GCC poison CONFIG_SOFTMMU
+#pragma GCC poison CONFIG_WHPX
+#pragma GCC poison CONFIG_XEN
 
 #endif
 #endif
diff --git a/qemu/include/exec/translator.h b/qemu/include/exec/translator.h
index a86cd75977..075a79cb71 100644
--- a/qemu/include/exec/translator.h
+++ b/qemu/include/exec/translator.h
@@ -73,6 +73,7 @@ typedef struct DisasContextBase {
     int num_insns;
     int max_insns;
     bool singlestep_enabled;
+    target_ulong target_page_mask;
 } DisasContextBase;
 
 /**
@@ -143,6 +144,17 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
 
 void translator_loop_temp_check(DisasContextBase *db);
 
+static inline bool translator_use_goto_tb(DisasContextBase *db,
+                                          target_ulong dest)
+{
+    return ((db->pc_first ^ dest) & db->target_page_mask) == 0;
+}
+
+static inline bool is_same_page(const DisasContextBase *db, target_ulong addr)
+{
+    return ((addr ^ db->pc_first) & db->target_page_mask) == 0;
+}
+
 /*
  * Translator Load Functions
  *
diff --git a/qemu/include/fpu/softfloat-helpers.h b/qemu/include/fpu/softfloat-helpers.h
index e0baf24c8f..94cbe073ec 100644
--- a/qemu/include/fpu/softfloat-helpers.h
+++ b/qemu/include/fpu/softfloat-helpers.h
@@ -48,17 +48,18 @@ this code that are retained.
 ===============================================================================
 */
 
-#ifndef _SOFTFLOAT_HELPERS_H_
-#define _SOFTFLOAT_HELPERS_H_
+#ifndef SOFTFLOAT_HELPERS_H
+#define SOFTFLOAT_HELPERS_H
 
 #include "fpu/softfloat-types.h"
 
-static inline void set_float_detect_tininess(int val, float_status *status)
+static inline void set_float_detect_tininess(bool val, float_status *status)
 {
-    status->float_detect_tininess = val;
+    status->tininess_before_rounding = val;
 }
 
-static inline void set_float_rounding_mode(int val, float_status *status)
+static inline void set_float_rounding_mode(FloatRoundMode val,
+                                           float_status *status)
 {
     status->float_rounding_mode = val;
 }
@@ -68,38 +69,48 @@ static inline void set_float_exception_flags(int val, float_status *status)
     status->float_exception_flags = val;
 }
 
-static inline void set_floatx80_rounding_precision(int val,
+static inline void set_floatx80_rounding_precision(FloatX80RoundPrec val,
                                                    float_status *status)
 {
     status->floatx80_rounding_precision = val;
 }
 
-static inline void set_flush_to_zero(flag val, float_status *status)
+static inline void set_flush_to_zero(bool val, float_status *status)
 {
     status->flush_to_zero = val;
 }
 
-static inline void set_flush_inputs_to_zero(flag val, float_status *status)
+static inline void set_flush_inputs_to_zero(bool val, float_status *status)
 {
     status->flush_inputs_to_zero = val;
 }
 
-static inline void set_default_nan_mode(flag val, float_status *status)
+static inline void set_default_nan_mode(bool val, float_status *status)
 {
     status->default_nan_mode = val;
 }
 
-static inline void set_snan_bit_is_one(flag val, float_status *status)
+static inline void set_snan_bit_is_one(bool val, float_status *status)
 {
     status->snan_bit_is_one = val;
 }
 
-static inline int get_float_detect_tininess(float_status *status)
+static inline void set_use_first_nan(bool val, float_status *status)
 {
-    return status->float_detect_tininess;
+    status->use_first_nan = val;
 }
 
-static inline int get_float_rounding_mode(float_status *status)
+static inline void set_no_signaling_nans(bool val, float_status *status)
+{
+    status->no_signaling_nans = val;
+}
+
+static inline bool get_float_detect_tininess(float_status *status)
+{
+    return status->tininess_before_rounding;
+}
+
+static inline FloatRoundMode get_float_rounding_mode(float_status *status)
 {
     return status->float_rounding_mode;
 }
@@ -109,24 +120,25 @@ static inline int get_float_exception_flags(float_status *status)
     return status->float_exception_flags;
 }
 
-static inline int get_floatx80_rounding_precision(float_status *status)
+static inline FloatX80RoundPrec
+get_floatx80_rounding_precision(float_status *status)
 {
     return status->floatx80_rounding_precision;
 }
 
-static inline flag get_flush_to_zero(float_status *status)
+static inline bool get_flush_to_zero(float_status *status)
 {
     return status->flush_to_zero;
 }
 
-static inline flag get_flush_inputs_to_zero(float_status *status)
+static inline bool get_flush_inputs_to_zero(float_status *status)
 {
     return status->flush_inputs_to_zero;
 }
 
-static inline flag get_default_nan_mode(float_status *status)
+static inline bool get_default_nan_mode(float_status *status)
 {
     return status->default_nan_mode;
 }
 
-#endif /* _SOFTFLOAT_HELPERS_H_ */
+#endif /* SOFTFLOAT_HELPERS_H */
diff --git a/qemu/include/fpu/softfloat-macros.h b/qemu/include/fpu/softfloat-macros.h
index afae4f7404..d1564bc7ee 100644
--- a/qemu/include/fpu/softfloat-macros.h
+++ b/qemu/include/fpu/softfloat-macros.h
@@ -8,7 +8,6 @@
  * so some portions are provided under:
  *  the SoftFloat-2a license
  *  the BSD license
- *  GPL-v2-or-later
  *
  * Any future contributions to this file after December 1st 2014 will be
  * taken to be licensed under the Softfloat-2a license unless specifically
@@ -75,14 +74,47 @@ this code that are retained.
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-/* Portions of this work are licensed under the terms of the GNU GPL,
- * version 2 or later. See the COPYING file in the top-level directory.
- */
-
 #ifndef FPU_SOFTFLOAT_MACROS_H
 #define FPU_SOFTFLOAT_MACROS_H
 
 #include "fpu/softfloat-types.h"
+#include "qemu/host-utils.h"
+
+/**
+ * shl_double: double-word merging left shift
+ * @l: left or most-significant word
+ * @r: right or least-significant word
+ * @c: shift count
+ *
+ * Shift @l left by @c bits, shifting in bits from @r.
+ */
+static inline uint64_t shl_double(uint64_t l, uint64_t r, int c)
+{
+#if defined(__x86_64__) && !defined(_MSC_VER)
+    asm("shld %b2, %1, %0" : "+r"(l) : "r"(r), "ci"(c));
+    return l;
+#else
+    return c ? (l << c) | (r >> (64 - c)) : l;
+#endif
+}
+
+/**
+ * shr_double: double-word merging right shift
+ * @l: left or most-significant word
+ * @r: right or least-significant word
+ * @c: shift count
+ *
+ * Shift @r right by @c bits, shifting in bits from @l.
+ */
+static inline uint64_t shr_double(uint64_t l, uint64_t r, int c)
+{
+#if defined(__x86_64__) && !defined(_MSC_VER)
+    asm("shrd %b2, %1, %0" : "+r"(r) : "r"(l), "ci"(c));
+    return r;
+#else
+    return c ? (r >> c) | (l << (64 - c)) : r;
+#endif
+}
 
 /*----------------------------------------------------------------------------
 | Shifts `a' right by the number of bits given in `count'.  If any nonzero
@@ -403,16 +435,12 @@ static inline void
 | are stored at the locations pointed to by `z0Ptr' and `z1Ptr'.
 *----------------------------------------------------------------------------*/
 
-static inline void
- add128(
-     uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1, uint64_t *z0Ptr, uint64_t *z1Ptr )
+static inline void add128(uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1,
+                          uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
-    uint64_t z1;
-
-    z1 = a1 + b1;
-    *z1Ptr = z1;
-    *z0Ptr = a0 + b0 + ( z1 < a1 );
-
+    bool c = 0;
+    *z1Ptr = uadd64_carry(a1, b1, &c);
+    *z0Ptr = uadd64_carry(a0, b0, &c);
 }
 
 /*----------------------------------------------------------------------------
@@ -423,34 +451,14 @@ static inline void
 | `z1Ptr', and `z2Ptr'.
 *----------------------------------------------------------------------------*/
 
-static inline void
- add192(
-     uint64_t a0,
-     uint64_t a1,
-     uint64_t a2,
-     uint64_t b0,
-     uint64_t b1,
-     uint64_t b2,
-     uint64_t *z0Ptr,
-     uint64_t *z1Ptr,
-     uint64_t *z2Ptr
- )
+static inline void add192(uint64_t a0, uint64_t a1, uint64_t a2,
+                          uint64_t b0, uint64_t b1, uint64_t b2,
+                          uint64_t *z0Ptr, uint64_t *z1Ptr, uint64_t *z2Ptr)
 {
-    uint64_t z0, z1, z2;
-    int8_t carry0, carry1;
-
-    z2 = a2 + b2;
-    carry1 = ( z2 < a2 );
-    z1 = a1 + b1;
-    carry0 = ( z1 < a1 );
-    z0 = a0 + b0;
-    z1 += carry1;
-    z0 += ( z1 < carry1 );
-    z0 += carry0;
-    *z2Ptr = z2;
-    *z1Ptr = z1;
-    *z0Ptr = z0;
-
+    bool c = 0;
+    *z2Ptr = uadd64_carry(a2, b2, &c);
+    *z1Ptr = uadd64_carry(a1, b1, &c);
+    *z0Ptr = uadd64_carry(a0, b0, &c);
 }
 
 /*----------------------------------------------------------------------------
@@ -461,14 +469,12 @@ static inline void
 | `z1Ptr'.
 *----------------------------------------------------------------------------*/
 
-static inline void
- sub128(
-     uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1, uint64_t *z0Ptr, uint64_t *z1Ptr )
+static inline void sub128(uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1,
+                          uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
-
-    *z1Ptr = a1 - b1;
-    *z0Ptr = a0 - b0 - ( a1 < b1 );
-
+    bool c = 0;
+    *z1Ptr = usub64_borrow(a1, b1, &c);
+    *z0Ptr = usub64_borrow(a0, b0, &c);
 }
 
 /*----------------------------------------------------------------------------
@@ -479,34 +485,14 @@ static inline void
 | pointed to by `z0Ptr', `z1Ptr', and `z2Ptr'.
 *----------------------------------------------------------------------------*/
 
-static inline void
- sub192(
-     uint64_t a0,
-     uint64_t a1,
-     uint64_t a2,
-     uint64_t b0,
-     uint64_t b1,
-     uint64_t b2,
-     uint64_t *z0Ptr,
-     uint64_t *z1Ptr,
-     uint64_t *z2Ptr
- )
+static inline void sub192(uint64_t a0, uint64_t a1, uint64_t a2,
+                          uint64_t b0, uint64_t b1, uint64_t b2,
+                          uint64_t *z0Ptr, uint64_t *z1Ptr, uint64_t *z2Ptr)
 {
-    uint64_t z0, z1, z2;
-    int8_t borrow0, borrow1;
-
-    z2 = a2 - b2;
-    borrow1 = ( a2 < b2 );
-    z1 = a1 - b1;
-    borrow0 = ( a1 < b1 );
-    z0 = a0 - b0;
-    z0 -= ( z1 < borrow1 );
-    z1 -= borrow1;
-    z0 -= borrow0;
-    *z2Ptr = z2;
-    *z1Ptr = z1;
-    *z0Ptr = z0;
-
+    bool c = 0;
+    *z2Ptr = usub64_borrow(a2, b2, &c);
+    *z1Ptr = usub64_borrow(a1, b1, &c);
+    *z0Ptr = usub64_borrow(a0, b0, &c);
 }
 
 /*----------------------------------------------------------------------------
@@ -515,27 +501,10 @@ static inline void
 | `z0Ptr' and `z1Ptr'.
 *----------------------------------------------------------------------------*/
 
-static inline void mul64To128( uint64_t a, uint64_t b, uint64_t *z0Ptr, uint64_t *z1Ptr )
+static inline void
+mul64To128(uint64_t a, uint64_t b, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
-    uint32_t aHigh, aLow, bHigh, bLow;
-    uint64_t z0, zMiddleA, zMiddleB, z1;
-
-    aLow = a;
-    aHigh = a>>32;
-    bLow = b;
-    bHigh = b>>32;
-    z1 = ( (uint64_t) aLow ) * bLow;
-    zMiddleA = ( (uint64_t) aLow ) * bHigh;
-    zMiddleB = ( (uint64_t) aHigh ) * bLow;
-    z0 = ( (uint64_t) aHigh ) * bHigh;
-    zMiddleA += zMiddleB;
-    z0 += ( ( (uint64_t) ( zMiddleA < zMiddleB ) )<<32 ) + ( zMiddleA>>32 );
-    zMiddleA <<= 32;
-    z1 += zMiddleA;
-    z0 += ( z1 < zMiddleA );
-    *z1Ptr = z1;
-    *z0Ptr = z0;
-
+    mulu64(z1Ptr, z0Ptr, a, b);
 }
 
 /*----------------------------------------------------------------------------
@@ -546,24 +515,14 @@ static inline void mul64To128( uint64_t a, uint64_t b, uint64_t *z0Ptr, uint64_t
 *----------------------------------------------------------------------------*/
 
 static inline void
- mul128By64To192(
-     uint64_t a0,
-     uint64_t a1,
-     uint64_t b,
-     uint64_t *z0Ptr,
-     uint64_t *z1Ptr,
-     uint64_t *z2Ptr
- )
+mul128By64To192(uint64_t a0, uint64_t a1, uint64_t b,
+                uint64_t *z0Ptr, uint64_t *z1Ptr, uint64_t *z2Ptr)
 {
-    uint64_t z0, z1, z2, more1;
-
-    mul64To128( a1, b, &z1, &z2 );
-    mul64To128( a0, b, &z0, &more1 );
-    add128( z0, more1, 0, z1, &z0, &z1 );
-    *z2Ptr = z2;
-    *z1Ptr = z1;
-    *z0Ptr = z0;
+    uint64_t z0, z1, m1;
 
+    mul64To128(a1, b, &m1, z2Ptr);
+    mul64To128(a0, b, &z0, &z1);
+    add128(z0, z1, 0, m1, z0Ptr, z1Ptr);
 }
 
 /*----------------------------------------------------------------------------
@@ -573,34 +532,21 @@ static inline void
 | the locations pointed to by `z0Ptr', `z1Ptr', `z2Ptr', and `z3Ptr'.
 *----------------------------------------------------------------------------*/
 
-static inline void
- mul128To256(
-     uint64_t a0,
-     uint64_t a1,
-     uint64_t b0,
-     uint64_t b1,
-     uint64_t *z0Ptr,
-     uint64_t *z1Ptr,
-     uint64_t *z2Ptr,
-     uint64_t *z3Ptr
- )
+static inline void mul128To256(uint64_t a0, uint64_t a1,
+                               uint64_t b0, uint64_t b1,
+                               uint64_t *z0Ptr, uint64_t *z1Ptr,
+                               uint64_t *z2Ptr, uint64_t *z3Ptr)
 {
-    uint64_t z0, z1, z2, z3;
-    uint64_t more1, more2;
-
-    mul64To128( a1, b1, &z2, &z3 );
-    mul64To128( a1, b0, &z1, &more2 );
-    add128( z1, more2, 0, z2, &z1, &z2 );
-    mul64To128( a0, b0, &z0, &more1 );
-    add128( z0, more1, 0, z1, &z0, &z1 );
-    mul64To128( a0, b1, &more1, &more2 );
-    add128( more1, more2, 0, z2, &more1, &z2 );
-    add128( z0, z1, 0, more1, &z0, &z1 );
-    *z3Ptr = z3;
-    *z2Ptr = z2;
-    *z1Ptr = z1;
-    *z0Ptr = z0;
+    uint64_t z0, z1, z2;
+    uint64_t m0, m1, m2, n1, n2;
+
+    mul64To128(a1, b0, &m1, &m2);
+    mul64To128(a0, b1, &n1, &n2);
+    mul64To128(a1, b1, &z2, z3Ptr);
+    mul64To128(a0, b0, &z0, &z1);
 
+    add192( 0, m1, m2,  0, n1, n2, &m0, &m1, &m2);
+    add192(m0, m1, m2, z0, z1, z2, z0Ptr, z1Ptr, z2Ptr);
 }
 
 /*----------------------------------------------------------------------------
@@ -634,83 +580,6 @@ static inline uint64_t estimateDiv128To64(uint64_t a0, uint64_t a1, uint64_t b)
 
 }
 
-/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd
- * (https://gmplib.org/repo/gmp/file/tip/longlong.h)
- *
- * Licensed under the GPLv2/LGPLv3
- */
-static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1,
-                                  uint64_t n0, uint64_t d)
-{
-#if defined(__x86_64__) && !defined(_MSC_VER)
-    uint64_t q;
-    asm ("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d));
-    return q;
-#elif defined(__s390x__) && !defined(__clang__)
-    /* Need to use a TImode type to get an even register pair for DLGR.  */
-    unsigned __int128 n = (unsigned __int128)n1 << 64 | n0;
-    asm("dlgr %0, %1" : "+r"(n) : "r"(d));
-    *r = n >> 64;
-    return n;
-#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7)
-    /* From Power ISA 2.06, programming note for divdeu.  */
-    uint64_t q1, q2, Q, r1, r2, R;
-    asm("divdeu %0,%2,%4; divdu %1,%3,%4"
-        : "=&r"(q1), "=r"(q2)
-        : "r"(n1), "r"(n0), "r"(d));
-    r1 = -(q1 * d);         /* low part of (n1<<64) - (q1 * d) */
-    r2 = n0 - (q2 * d);
-    Q = q1 + q2;
-    R = r1 + r2;
-    if (R >= d || R < r2) { /* overflow implies R > d */
-        Q += 1;
-        R -= d;
-    }
-    *r = R;
-    return Q;
-#else
-    uint64_t d0, d1, q0, q1, r1, r0, m;
-
-    d0 = (uint32_t)d;
-    d1 = d >> 32;
-
-    r1 = n1 % d1;
-    q1 = n1 / d1;
-    m = q1 * d0;
-    r1 = (r1 << 32) | (n0 >> 32);
-    if (r1 < m) {
-        q1 -= 1;
-        r1 += d;
-        if (r1 >= d) {
-            if (r1 < m) {
-                q1 -= 1;
-                r1 += d;
-            }
-        }
-    }
-    r1 -= m;
-
-    r0 = r1 % d1;
-    q0 = r1 / d1;
-    m = q0 * d0;
-    r0 = (r0 << 32) | (uint32_t)n0;
-    if (r0 < m) {
-        q0 -= 1;
-        r0 += d;
-        if (r0 >= d) {
-            if (r0 < m) {
-                q0 -= 1;
-                r0 += d;
-            }
-        }
-    }
-    r0 -= m;
-
-    *r = r0;
-    return (q1 << 32) | q0;
-#endif
-}
-
 /*----------------------------------------------------------------------------
 | Returns an approximation to the square root of the 32-bit significand given
 | by `a'.  Considered as an integer, `a' must be at least 2^31.  If bit 0 of
@@ -756,11 +625,9 @@ static inline uint32_t estimateSqrt32(int aExp, uint32_t a)
 | Otherwise, returns 0.
 *----------------------------------------------------------------------------*/
 
-static inline flag eq128( uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1 )
+static inline bool eq128(uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1)
 {
-
-    return ( a0 == b0 ) && ( a1 == b1 );
-
+    return a0 == b0 && a1 == b1;
 }
 
 /*----------------------------------------------------------------------------
@@ -769,11 +636,9 @@ static inline flag eq128( uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1 )
 | Otherwise, returns 0.
 *----------------------------------------------------------------------------*/
 
-static inline flag le128( uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1 )
+static inline bool le128(uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1)
 {
-
-    return ( a0 < b0 ) || ( ( a0 == b0 ) && ( a1 <= b1 ) );
-
+    return a0 < b0 || (a0 == b0 && a1 <= b1);
 }
 
 /*----------------------------------------------------------------------------
@@ -782,11 +647,9 @@ static inline flag le128( uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1 )
 | returns 0.
 *----------------------------------------------------------------------------*/
 
-static inline flag lt128( uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1 )
+static inline bool lt128(uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1)
 {
-
-    return ( a0 < b0 ) || ( ( a0 == b0 ) && ( a1 < b1 ) );
-
+    return a0 < b0 || (a0 == b0 && a1 < b1);
 }
 
 /*----------------------------------------------------------------------------
@@ -795,11 +658,43 @@ static inline flag lt128( uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1 )
 | Otherwise, returns 0.
 *----------------------------------------------------------------------------*/
 
-static inline flag ne128( uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1 )
+static inline bool ne128(uint64_t a0, uint64_t a1, uint64_t b0, uint64_t b1)
 {
+    return a0 != b0 || a1 != b1;
+}
+
+/*
+ * Similarly, comparisons of 192-bit values.
+ */
+
+static inline bool eq192(uint64_t a0, uint64_t a1, uint64_t a2,
+                         uint64_t b0, uint64_t b1, uint64_t b2)
+{
+    return ((a0 ^ b0) | (a1 ^ b1) | (a2 ^ b2)) == 0;
+}
 
-    return ( a0 != b0 ) || ( a1 != b1 );
+static inline bool le192(uint64_t a0, uint64_t a1, uint64_t a2,
+                         uint64_t b0, uint64_t b1, uint64_t b2)
+{
+    if (a0 != b0) {
+        return a0 < b0;
+    }
+    if (a1 != b1) {
+        return a1 < b1;
+    }
+    return a2 <= b2;
+}
 
+static inline bool lt192(uint64_t a0, uint64_t a1, uint64_t a2,
+                         uint64_t b0, uint64_t b1, uint64_t b2)
+{
+    if (a0 != b0) {
+        return a0 < b0;
+    }
+    if (a1 != b1) {
+        return a1 < b1;
+    }
+    return a2 < b2;
 }
 
 #endif
diff --git a/qemu/include/fpu/softfloat-types.h b/qemu/include/fpu/softfloat-types.h
index 565dced559..cd95f78c87 100644
--- a/qemu/include/fpu/softfloat-types.h
+++ b/qemu/include/fpu/softfloat-types.h
@@ -80,14 +80,6 @@ this code that are retained.
 #ifndef SOFTFLOAT_TYPES_H
 #define SOFTFLOAT_TYPES_H
 
-#include <stdint.h>
-
-/* This 'flag' type must be able to hold at least 0 and 1. It should
- * probably be replaced with 'bool' but the uses would need to be audited
- * to check that they weren't accidentally relying on it being a larger type.
- */
-typedef uint8_t flag;
-
 /*
  * Software IEC/IEEE floating-point types.
  */
@@ -111,7 +103,7 @@ typedef struct {
 #define make_floatx80(exp, mant) ((floatx80) { mant, exp })
 #define make_floatx80_init(exp, mant) { .low = mant, .high = exp }
 typedef struct {
-#ifdef HOST_WORDS_BIGENDIAN
+#if HOST_BIG_ENDIAN
     uint64_t high, low;
 #else
     uint64_t low, high;
@@ -120,43 +112,64 @@ typedef struct {
 #define make_float128(high_, low_) ((float128) { .high = high_, .low = low_ })
 #define make_float128_init(high_, low_) { .high = high_, .low = low_ }
 
+/*
+ * Software neural-network floating-point types.
+ */
+typedef uint16_t bfloat16;
+typedef bool flag;
+
 /*
  * Software IEC/IEEE floating-point underflow tininess-detection mode.
  */
 
-enum {
-    float_tininess_after_rounding  = 0,
-    float_tininess_before_rounding = 1
-};
+#define float_tininess_after_rounding  false
+#define float_tininess_before_rounding true
 
 /*
  *Software IEC/IEEE floating-point rounding mode.
  */
 
-enum {
+typedef enum __attribute__((__packed__)) {
     float_round_nearest_even = 0,
     float_round_down         = 1,
     float_round_up           = 2,
     float_round_to_zero      = 3,
     float_round_ties_away    = 4,
-    /* Not an IEEE rounding mode: round to the closest odd mantissa value */
+    /* Not an IEEE rounding mode: round to closest odd, overflow to max */
     float_round_to_odd       = 5,
-};
+    /* Not an IEEE rounding mode: round to closest odd, overflow to inf */
+    float_round_to_odd_inf   = 6,
+} FloatRoundMode;
 
 /*
  * Software IEC/IEEE floating-point exception flags.
  */
 
 enum {
-    float_flag_invalid   =  1,
-    float_flag_divbyzero =  4,
-    float_flag_overflow  =  8,
-    float_flag_underflow = 16,
-    float_flag_inexact   = 32,
-    float_flag_input_denormal = 64,
-    float_flag_output_denormal = 128
+    float_flag_invalid         = 0x0001,
+    float_flag_divbyzero       = 0x0002,
+    float_flag_overflow        = 0x0004,
+    float_flag_underflow       = 0x0008,
+    float_flag_inexact         = 0x0010,
+    float_flag_input_denormal  = 0x0020,
+    float_flag_output_denormal = 0x0040,
+    float_flag_invalid_isi     = 0x0080,  /* inf - inf */
+    float_flag_invalid_imz     = 0x0100,  /* inf * 0 */
+    float_flag_invalid_idi     = 0x0200,  /* inf / inf */
+    float_flag_invalid_zdz     = 0x0400,  /* 0 / 0 */
+    float_flag_invalid_sqrt    = 0x0800,  /* sqrt(-x) */
+    float_flag_invalid_cvti    = 0x1000,  /* non-nan to integer */
+    float_flag_invalid_snan    = 0x2000,  /* any operand was snan */
 };
 
+/*
+ * Rounding precision for floatx80.
+ */
+typedef enum __attribute__((__packed__)) {
+    floatx80_precision_x,
+    floatx80_precision_d,
+    floatx80_precision_s,
+} FloatX80RoundPrec;
 
 /*
  * Floating Point Status. Individual architectures may maintain
@@ -166,17 +179,27 @@ enum {
  */
 
 typedef struct float_status {
-    signed char float_detect_tininess;
-    signed char float_rounding_mode;
-    uint8_t     float_exception_flags;
-    signed char floatx80_rounding_precision;
+    uint16_t float_exception_flags;
+    FloatRoundMode float_rounding_mode;
+    FloatX80RoundPrec floatx80_rounding_precision;
+    bool tininess_before_rounding;
     /* should denormalised results go to zero and set the inexact flag? */
-    flag flush_to_zero;
+    bool flush_to_zero;
     /* should denormalised inputs go to zero and set the input_denormal flag? */
-    flag flush_inputs_to_zero;
-    flag default_nan_mode;
-    /* not always used -- see snan_bit_is_one() in softfloat-specialize.h */
-    flag snan_bit_is_one;
+    bool flush_inputs_to_zero;
+    bool default_nan_mode;
+    /*
+     * The flags below are not used on all specializations and may
+     * constant fold away (see snan_bit_is_one()/no_signalling_nans() in
+     * softfloat-specialize.inc.c)
+     */
+    bool snan_bit_is_one;
+    bool use_first_nan;
+    bool no_signaling_nans;
+    /* should overflowed results subtract re_bias to its exponent? */
+    bool rebias_overflow;
+    /* should underflowed results add re_bias to its exponent? */
+    bool rebias_underflow;
 } float_status;
 
 #endif /* SOFTFLOAT_TYPES_H */
diff --git a/qemu/include/fpu/softfloat.h b/qemu/include/fpu/softfloat.h
index ecb8ba0114..3dcf20e3a2 100644
--- a/qemu/include/fpu/softfloat.h
+++ b/qemu/include/fpu/softfloat.h
@@ -85,21 +85,26 @@ this code that are retained.
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE floating-point ordering relations
 *----------------------------------------------------------------------------*/
-enum {
+
+typedef enum {
     float_relation_less      = -1,
     float_relation_equal     =  0,
     float_relation_greater   =  1,
     float_relation_unordered =  2
-};
+} FloatRelation;
 
 #include "fpu/softfloat-types.h"
 #include "fpu/softfloat-helpers.h"
+#include "qemu/int128.h"
 
 /*----------------------------------------------------------------------------
 | Routine to raise any or all of the software IEC/IEEE floating-point
 | exception flags.
 *----------------------------------------------------------------------------*/
-void float_raise(uint8_t flags, float_status *status);
+static inline void float_raise(uint16_t flags, float_status *status)
+{
+    status->float_exception_flags |= flags;
+}
 
 /*----------------------------------------------------------------------------
 | If `a' is denormal and we are in flush-to-zero mode then set the
@@ -108,6 +113,7 @@ void float_raise(uint8_t flags, float_status *status);
 float16 float16_squash_input_denormal(float16 a, float_status *status);
 float32 float32_squash_input_denormal(float32 a, float_status *status);
 float64 float64_squash_input_denormal(float64 a, float_status *status);
+bfloat16 bfloat16_squash_input_denormal(bfloat16 a, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Options to indicate which negations to perform in float*_muladd()
@@ -135,9 +141,11 @@ float16 uint16_to_float16_scalbn(uint16_t a, int, float_status *status);
 float16 uint32_to_float16_scalbn(uint32_t a, int, float_status *status);
 float16 uint64_to_float16_scalbn(uint64_t a, int, float_status *status);
 
+float16 int8_to_float16(int8_t a, float_status *status);
 float16 int16_to_float16(int16_t a, float_status *status);
 float16 int32_to_float16(int32_t a, float_status *status);
 float16 int64_to_float16(int64_t a, float_status *status);
+float16 uint8_to_float16(uint8_t a, float_status *status);
 float16 uint16_to_float16(uint16_t a, float_status *status);
 float16 uint32_to_float16(uint32_t a, float_status *status);
 float16 uint64_to_float16(uint64_t a, float_status *status);
@@ -175,7 +183,9 @@ floatx80 int64_to_floatx80(int64_t, float_status *status);
 
 float128 int32_to_float128(int32_t, float_status *status);
 float128 int64_to_float128(int64_t, float_status *status);
+float128 int128_to_float128(Int128, float_status *status);
 float128 uint64_to_float128(uint64_t, float_status *status);
+float128 uint128_to_float128(Int128, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software half-precision conversion routines.
@@ -186,10 +196,13 @@ float32 float16_to_float32(float16, bool ieee, float_status *status);
 float16 float64_to_float16(float64 a, bool ieee, float_status *status);
 float64 float16_to_float64(float16 a, bool ieee, float_status *status);
 
-int16_t float16_to_int16_scalbn(float16, int, int, float_status *status);
-int32_t float16_to_int32_scalbn(float16, int, int, float_status *status);
-int64_t float16_to_int64_scalbn(float16, int, int, float_status *status);
+int8_t  float16_to_int8_scalbn(float16, FloatRoundMode, int,
+                               float_status *status);
+int16_t float16_to_int16_scalbn(float16, FloatRoundMode, int, float_status *);
+int32_t float16_to_int32_scalbn(float16, FloatRoundMode, int, float_status *);
+int64_t float16_to_int64_scalbn(float16, FloatRoundMode, int, float_status *);
 
+int8_t  float16_to_int8(float16, float_status *status);
 int16_t float16_to_int16(float16, float_status *status);
 int32_t float16_to_int32(float16, float_status *status);
 int64_t float16_to_int64(float16, float_status *status);
@@ -198,10 +211,16 @@ int16_t float16_to_int16_round_to_zero(float16, float_status *status);
 int32_t float16_to_int32_round_to_zero(float16, float_status *status);
 int64_t float16_to_int64_round_to_zero(float16, float_status *status);
 
-uint16_t float16_to_uint16_scalbn(float16 a, int, int, float_status *status);
-uint32_t float16_to_uint32_scalbn(float16 a, int, int, float_status *status);
-uint64_t float16_to_uint64_scalbn(float16 a, int, int, float_status *status);
+uint8_t float16_to_uint8_scalbn(float16 a, FloatRoundMode,
+                                int, float_status *status);
+uint16_t float16_to_uint16_scalbn(float16 a, FloatRoundMode,
+                                  int, float_status *status);
+uint32_t float16_to_uint32_scalbn(float16 a, FloatRoundMode,
+                                  int, float_status *status);
+uint64_t float16_to_uint64_scalbn(float16 a, FloatRoundMode,
+                                  int, float_status *status);
 
+uint8_t  float16_to_uint8(float16 a, float_status *status);
 uint16_t float16_to_uint16(float16 a, float_status *status);
 uint32_t float16_to_uint32(float16 a, float_status *status);
 uint64_t float16_to_uint64(float16 a, float_status *status);
@@ -227,39 +246,46 @@ float16 float16_minnum(float16, float16, float_status *status);
 float16 float16_maxnum(float16, float16, float_status *status);
 float16 float16_minnummag(float16, float16, float_status *status);
 float16 float16_maxnummag(float16, float16, float_status *status);
+float16 float16_minimum_number(float16, float16, float_status *status);
+float16 float16_maximum_number(float16, float16, float_status *status);
 float16 float16_sqrt(float16, float_status *status);
-int float16_compare(float16, float16, float_status *status);
-int float16_compare_quiet(float16, float16, float_status *status);
+FloatRelation float16_compare(float16, float16, float_status *status);
+FloatRelation float16_compare_quiet(float16, float16, float_status *status);
 
-int float16_is_quiet_nan(float16, float_status *status);
-int float16_is_signaling_nan(float16, float_status *status);
+bool float16_is_quiet_nan(float16, float_status *status);
+bool float16_is_signaling_nan(float16, float_status *status);
 float16 float16_silence_nan(float16, float_status *status);
 
-static inline int float16_is_any_nan(float16 a)
+static inline bool float16_is_any_nan(float16 a)
 {
     return ((float16_val(a) & ~0x8000) > 0x7c00);
 }
 
-static inline int float16_is_neg(float16 a)
+static inline bool float16_is_neg(float16 a)
 {
     return float16_val(a) >> 15;
 }
 
-static inline int float16_is_infinity(float16 a)
+static inline bool float16_is_infinity(float16 a)
 {
     return (float16_val(a) & 0x7fff) == 0x7c00;
 }
 
-static inline int float16_is_zero(float16 a)
+static inline bool float16_is_zero(float16 a)
 {
     return (float16_val(a) & 0x7fff) == 0;
 }
 
-static inline int float16_is_zero_or_denormal(float16 a)
+static inline bool float16_is_zero_or_denormal(float16 a)
 {
     return (float16_val(a) & 0x7c00) == 0;
 }
 
+static inline bool float16_is_normal(float16 a)
+{
+    return (((float16_val(a) >> 10) + 1) & 0x1f) >= 2;
+}
+
 static inline float16 float16_abs(float16 a)
 {
     /* Note that abs does *not* handle NaN specially, nor does
@@ -281,6 +307,47 @@ static inline float16 float16_set_sign(float16 a, int sign)
     return make_float16((float16_val(a) & 0x7fff) | (sign << 15));
 }
 
+static inline bool float16_eq(float16 a, float16 b, float_status *s)
+{
+    return float16_compare(a, b, s) == float_relation_equal;
+}
+
+static inline bool float16_le(float16 a, float16 b, float_status *s)
+{
+    return float16_compare(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float16_lt(float16 a, float16 b, float_status *s)
+{
+    return float16_compare(a, b, s) < float_relation_equal;
+}
+
+static inline bool float16_unordered(float16 a, float16 b, float_status *s)
+{
+    return float16_compare(a, b, s) == float_relation_unordered;
+}
+
+static inline bool float16_eq_quiet(float16 a, float16 b, float_status *s)
+{
+    return float16_compare_quiet(a, b, s) == float_relation_equal;
+}
+
+static inline bool float16_le_quiet(float16 a, float16 b, float_status *s)
+{
+    return float16_compare_quiet(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float16_lt_quiet(float16 a, float16 b, float_status *s)
+{
+    return float16_compare_quiet(a, b, s) < float_relation_equal;
+}
+
+static inline bool float16_unordered_quiet(float16 a, float16 b,
+                                           float_status *s)
+{
+    return float16_compare_quiet(a, b, s) == float_relation_unordered;
+}
+
 #define float16_zero make_float16(0)
 #define float16_half make_float16(0x3800)
 #define float16_one make_float16(0x3c00)
@@ -289,6 +356,188 @@ static inline float16 float16_set_sign(float16 a, int sign)
 #define float16_three make_float16(0x4200)
 #define float16_infinity make_float16(0x7c00)
 
+/*----------------------------------------------------------------------------
+| Software bfloat16 conversion routines.
+*----------------------------------------------------------------------------*/
+
+bfloat16 bfloat16_round_to_int(bfloat16, float_status *status);
+bfloat16 float32_to_bfloat16(float32, float_status *status);
+float32 bfloat16_to_float32(bfloat16, float_status *status);
+bfloat16 float64_to_bfloat16(float64 a, float_status *status);
+float64 bfloat16_to_float64(bfloat16 a, float_status *status);
+
+int16_t bfloat16_to_int16_scalbn(bfloat16, FloatRoundMode,
+                                 int, float_status *status);
+int32_t bfloat16_to_int32_scalbn(bfloat16, FloatRoundMode,
+                                 int, float_status *status);
+int64_t bfloat16_to_int64_scalbn(bfloat16, FloatRoundMode,
+                                 int, float_status *status);
+
+int16_t bfloat16_to_int16(bfloat16, float_status *status);
+int32_t bfloat16_to_int32(bfloat16, float_status *status);
+int64_t bfloat16_to_int64(bfloat16, float_status *status);
+
+int16_t bfloat16_to_int16_round_to_zero(bfloat16, float_status *status);
+int32_t bfloat16_to_int32_round_to_zero(bfloat16, float_status *status);
+int64_t bfloat16_to_int64_round_to_zero(bfloat16, float_status *status);
+
+uint16_t bfloat16_to_uint16_scalbn(bfloat16 a, FloatRoundMode,
+                                   int, float_status *status);
+uint32_t bfloat16_to_uint32_scalbn(bfloat16 a, FloatRoundMode,
+                                   int, float_status *status);
+uint64_t bfloat16_to_uint64_scalbn(bfloat16 a, FloatRoundMode,
+                                   int, float_status *status);
+
+uint16_t bfloat16_to_uint16(bfloat16 a, float_status *status);
+uint32_t bfloat16_to_uint32(bfloat16 a, float_status *status);
+uint64_t bfloat16_to_uint64(bfloat16 a, float_status *status);
+
+uint16_t bfloat16_to_uint16_round_to_zero(bfloat16 a, float_status *status);
+uint32_t bfloat16_to_uint32_round_to_zero(bfloat16 a, float_status *status);
+uint64_t bfloat16_to_uint64_round_to_zero(bfloat16 a, float_status *status);
+
+bfloat16 int16_to_bfloat16_scalbn(int16_t a, int, float_status *status);
+bfloat16 int32_to_bfloat16_scalbn(int32_t a, int, float_status *status);
+bfloat16 int64_to_bfloat16_scalbn(int64_t a, int, float_status *status);
+bfloat16 uint16_to_bfloat16_scalbn(uint16_t a, int, float_status *status);
+bfloat16 uint32_to_bfloat16_scalbn(uint32_t a, int, float_status *status);
+bfloat16 uint64_to_bfloat16_scalbn(uint64_t a, int, float_status *status);
+
+bfloat16 int16_to_bfloat16(int16_t a, float_status *status);
+bfloat16 int32_to_bfloat16(int32_t a, float_status *status);
+bfloat16 int64_to_bfloat16(int64_t a, float_status *status);
+bfloat16 uint16_to_bfloat16(uint16_t a, float_status *status);
+bfloat16 uint32_to_bfloat16(uint32_t a, float_status *status);
+bfloat16 uint64_to_bfloat16(uint64_t a, float_status *status);
+
+/*----------------------------------------------------------------------------
+| Software bfloat16 operations.
+*----------------------------------------------------------------------------*/
+
+bfloat16 bfloat16_add(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_sub(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_mul(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_div(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_muladd(bfloat16, bfloat16, bfloat16, int,
+                         float_status *status);
+float16 bfloat16_scalbn(bfloat16, int, float_status *status);
+bfloat16 bfloat16_min(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_max(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_minnum(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_maxnum(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_minnummag(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_maxnummag(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_minimum_number(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_maximum_number(bfloat16, bfloat16, float_status *status);
+bfloat16 bfloat16_sqrt(bfloat16, float_status *status);
+FloatRelation bfloat16_compare(bfloat16, bfloat16, float_status *status);
+FloatRelation bfloat16_compare_quiet(bfloat16, bfloat16, float_status *status);
+
+bool bfloat16_is_quiet_nan(bfloat16, float_status *status);
+bool bfloat16_is_signaling_nan(bfloat16, float_status *status);
+bfloat16 bfloat16_silence_nan(bfloat16, float_status *status);
+bfloat16 bfloat16_default_nan(float_status *status);
+
+static inline bool bfloat16_is_any_nan(bfloat16 a)
+{
+    return ((a & ~0x8000) > 0x7F80);
+}
+
+static inline bool bfloat16_is_neg(bfloat16 a)
+{
+    return a >> 15;
+}
+
+static inline bool bfloat16_is_infinity(bfloat16 a)
+{
+    return (a & 0x7fff) == 0x7F80;
+}
+
+static inline bool bfloat16_is_zero(bfloat16 a)
+{
+    return (a & 0x7fff) == 0;
+}
+
+static inline bool bfloat16_is_zero_or_denormal(bfloat16 a)
+{
+    return (a & 0x7F80) == 0;
+}
+
+static inline bool bfloat16_is_normal(bfloat16 a)
+{
+    return (((a >> 7) + 1) & 0xff) >= 2;
+}
+
+static inline bfloat16 bfloat16_abs(bfloat16 a)
+{
+    /* Note that abs does *not* handle NaN specially, nor does
+     * it flush denormal inputs to zero.
+     */
+    return a & 0x7fff;
+}
+
+static inline bfloat16 bfloat16_chs(bfloat16 a)
+{
+    /* Note that chs does *not* handle NaN specially, nor does
+     * it flush denormal inputs to zero.
+     */
+    return a ^ 0x8000;
+}
+
+static inline bfloat16 bfloat16_set_sign(bfloat16 a, int sign)
+{
+    return (a & 0x7fff) | (sign << 15);
+}
+
+static inline bool bfloat16_eq(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_compare(a, b, s) == float_relation_equal;
+}
+
+static inline bool bfloat16_le(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_compare(a, b, s) <= float_relation_equal;
+}
+
+static inline bool bfloat16_lt(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_compare(a, b, s) < float_relation_equal;
+}
+
+static inline bool bfloat16_unordered(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_compare(a, b, s) == float_relation_unordered;
+}
+
+static inline bool bfloat16_eq_quiet(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_compare_quiet(a, b, s) == float_relation_equal;
+}
+
+static inline bool bfloat16_le_quiet(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_compare_quiet(a, b, s) <= float_relation_equal;
+}
+
+static inline bool bfloat16_lt_quiet(bfloat16 a, bfloat16 b, float_status *s)
+{
+    return bfloat16_compare_quiet(a, b, s) < float_relation_equal;
+}
+
+static inline bool bfloat16_unordered_quiet(bfloat16 a, bfloat16 b,
+                                           float_status *s)
+{
+    return bfloat16_compare_quiet(a, b, s) == float_relation_unordered;
+}
+
+#define bfloat16_zero 0
+#define bfloat16_half 0x3f00
+#define bfloat16_one 0x3f80
+#define bfloat16_one_point_five 0x3fc0
+#define bfloat16_two 0x4000
+#define bfloat16_three 0x4040
+#define bfloat16_infinity 0x7f80
+
 /*----------------------------------------------------------------------------
 | The pattern for a default generated half-precision NaN.
 *----------------------------------------------------------------------------*/
@@ -298,9 +547,9 @@ float16 float16_default_nan(float_status *status);
 | Software IEC/IEEE single-precision conversion routines.
 *----------------------------------------------------------------------------*/
 
-int16_t float32_to_int16_scalbn(float32, int, int, float_status *status);
-int32_t float32_to_int32_scalbn(float32, int, int, float_status *status);
-int64_t float32_to_int64_scalbn(float32, int, int, float_status *status);
+int16_t float32_to_int16_scalbn(float32, FloatRoundMode, int, float_status *);
+int32_t float32_to_int32_scalbn(float32, FloatRoundMode, int, float_status *);
+int64_t float32_to_int64_scalbn(float32, FloatRoundMode, int, float_status *);
 
 int16_t float32_to_int16(float32, float_status *status);
 int32_t float32_to_int32(float32, float_status *status);
@@ -310,9 +559,9 @@ int16_t float32_to_int16_round_to_zero(float32, float_status *status);
 int32_t float32_to_int32_round_to_zero(float32, float_status *status);
 int64_t float32_to_int64_round_to_zero(float32, float_status *status);
 
-uint16_t float32_to_uint16_scalbn(float32, int, int, float_status *status);
-uint32_t float32_to_uint32_scalbn(float32, int, int, float_status *status);
-uint64_t float32_to_uint64_scalbn(float32, int, int, float_status *status);
+uint16_t float32_to_uint16_scalbn(float32, FloatRoundMode, int, float_status *);
+uint32_t float32_to_uint32_scalbn(float32, FloatRoundMode, int, float_status *);
+uint64_t float32_to_uint64_scalbn(float32, FloatRoundMode, int, float_status *);
 
 uint16_t float32_to_uint16(float32, float_status *status);
 uint32_t float32_to_uint32(float32, float_status *status);
@@ -339,24 +588,18 @@ float32 float32_muladd(float32, float32, float32, int, float_status *status);
 float32 float32_sqrt(float32, float_status *status);
 float32 float32_exp2(float32, float_status *status);
 float32 float32_log2(float32, float_status *status);
-int float32_eq(float32, float32, float_status *status);
-int float32_le(float32, float32, float_status *status);
-int float32_lt(float32, float32, float_status *status);
-int float32_unordered(float32, float32, float_status *status);
-int float32_eq_quiet(float32, float32, float_status *status);
-int float32_le_quiet(float32, float32, float_status *status);
-int float32_lt_quiet(float32, float32, float_status *status);
-int float32_unordered_quiet(float32, float32, float_status *status);
-int float32_compare(float32, float32, float_status *status);
-int float32_compare_quiet(float32, float32, float_status *status);
+FloatRelation float32_compare(float32, float32, float_status *status);
+FloatRelation float32_compare_quiet(float32, float32, float_status *status);
 float32 float32_min(float32, float32, float_status *status);
 float32 float32_max(float32, float32, float_status *status);
 float32 float32_minnum(float32, float32, float_status *status);
 float32 float32_maxnum(float32, float32, float_status *status);
 float32 float32_minnummag(float32, float32, float_status *status);
 float32 float32_maxnummag(float32, float32, float_status *status);
-int float32_is_quiet_nan(float32, float_status *status);
-int float32_is_signaling_nan(float32, float_status *status);
+float32 float32_minimum_number(float32, float32, float_status *status);
+float32 float32_maximum_number(float32, float32, float_status *status);
+bool float32_is_quiet_nan(float32, float_status *status);
+bool float32_is_signaling_nan(float32, float_status *status);
 float32 float32_silence_nan(float32, float_status *status);
 float32 float32_scalbn(float32, int, float_status *status);
 
@@ -376,27 +619,27 @@ static inline float32 float32_chs(float32 a)
     return make_float32(float32_val(a) ^ 0x80000000);
 }
 
-static inline int float32_is_infinity(float32 a)
+static inline bool float32_is_infinity(float32 a)
 {
     return (float32_val(a) & 0x7fffffff) == 0x7f800000;
 }
 
-static inline int float32_is_neg(float32 a)
+static inline bool float32_is_neg(float32 a)
 {
     return float32_val(a) >> 31;
 }
 
-static inline int float32_is_zero(float32 a)
+static inline bool float32_is_zero(float32 a)
 {
     return (float32_val(a) & 0x7fffffff) == 0;
 }
 
-static inline int float32_is_any_nan(float32 a)
+static inline bool float32_is_any_nan(float32 a)
 {
     return ((float32_val(a) & ~(1 << 31)) > 0x7f800000UL);
 }
 
-static inline int float32_is_zero_or_denormal(float32 a)
+static inline bool float32_is_zero_or_denormal(float32 a)
 {
     return (float32_val(a) & 0x7f800000) == 0;
 }
@@ -421,6 +664,47 @@ static inline float32 float32_set_sign(float32 a, int sign)
     return make_float32((float32_val(a) & 0x7fffffff) | (sign << 31));
 }
 
+static inline bool float32_eq(float32 a, float32 b, float_status *s)
+{
+    return float32_compare(a, b, s) == float_relation_equal;
+}
+
+static inline bool float32_le(float32 a, float32 b, float_status *s)
+{
+    return float32_compare(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float32_lt(float32 a, float32 b, float_status *s)
+{
+    return float32_compare(a, b, s) < float_relation_equal;
+}
+
+static inline bool float32_unordered(float32 a, float32 b, float_status *s)
+{
+    return float32_compare(a, b, s) == float_relation_unordered;
+}
+
+static inline bool float32_eq_quiet(float32 a, float32 b, float_status *s)
+{
+    return float32_compare_quiet(a, b, s) == float_relation_equal;
+}
+
+static inline bool float32_le_quiet(float32 a, float32 b, float_status *s)
+{
+    return float32_compare_quiet(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float32_lt_quiet(float32 a, float32 b, float_status *s)
+{
+    return float32_compare_quiet(a, b, s) < float_relation_equal;
+}
+
+static inline bool float32_unordered_quiet(float32 a, float32 b,
+                                           float_status *s)
+{
+    return float32_compare_quiet(a, b, s) == float_relation_unordered;
+}
+
 #define float32_zero make_float32(0)
 #define float32_half make_float32(0x3f000000)
 #define float32_one make_float32(0x3f800000)
@@ -440,7 +724,7 @@ static inline float32 float32_set_sign(float32 a, int sign)
 | significand.
 *----------------------------------------------------------------------------*/
 
-static inline float32 packFloat32(flag zSign, int zExp, uint32_t zSig)
+static inline float32 packFloat32(bool zSign, int zExp, uint32_t zSig)
 {
     return make_float32(
           (((uint32_t)zSign) << 31) + (((uint32_t)zExp) << 23) + zSig);
@@ -455,9 +739,9 @@ float32 float32_default_nan(float_status *status);
 | Software IEC/IEEE double-precision conversion routines.
 *----------------------------------------------------------------------------*/
 
-int16_t float64_to_int16_scalbn(float64, int, int, float_status *status);
-int32_t float64_to_int32_scalbn(float64, int, int, float_status *status);
-int64_t float64_to_int64_scalbn(float64, int, int, float_status *status);
+int16_t float64_to_int16_scalbn(float64, FloatRoundMode, int, float_status *);
+int32_t float64_to_int32_scalbn(float64, FloatRoundMode, int, float_status *);
+int64_t float64_to_int64_scalbn(float64, FloatRoundMode, int, float_status *);
 
 int16_t float64_to_int16(float64, float_status *status);
 int32_t float64_to_int32(float64, float_status *status);
@@ -467,9 +751,9 @@ int16_t float64_to_int16_round_to_zero(float64, float_status *status);
 int32_t float64_to_int32_round_to_zero(float64, float_status *status);
 int64_t float64_to_int64_round_to_zero(float64, float_status *status);
 
-uint16_t float64_to_uint16_scalbn(float64, int, int, float_status *status);
-uint32_t float64_to_uint32_scalbn(float64, int, int, float_status *status);
-uint64_t float64_to_uint64_scalbn(float64, int, int, float_status *status);
+uint16_t float64_to_uint16_scalbn(float64, FloatRoundMode, int, float_status *);
+uint32_t float64_to_uint32_scalbn(float64, FloatRoundMode, int, float_status *);
+uint64_t float64_to_uint64_scalbn(float64, FloatRoundMode, int, float_status *);
 
 uint16_t float64_to_uint16(float64, float_status *status);
 uint32_t float64_to_uint32(float64, float_status *status);
@@ -495,24 +779,18 @@ float64 float64_rem(float64, float64, float_status *status);
 float64 float64_muladd(float64, float64, float64, int, float_status *status);
 float64 float64_sqrt(float64, float_status *status);
 float64 float64_log2(float64, float_status *status);
-int float64_eq(float64, float64, float_status *status);
-int float64_le(float64, float64, float_status *status);
-int float64_lt(float64, float64, float_status *status);
-int float64_unordered(float64, float64, float_status *status);
-int float64_eq_quiet(float64, float64, float_status *status);
-int float64_le_quiet(float64, float64, float_status *status);
-int float64_lt_quiet(float64, float64, float_status *status);
-int float64_unordered_quiet(float64, float64, float_status *status);
-int float64_compare(float64, float64, float_status *status);
-int float64_compare_quiet(float64, float64, float_status *status);
+FloatRelation float64_compare(float64, float64, float_status *status);
+FloatRelation float64_compare_quiet(float64, float64, float_status *status);
 float64 float64_min(float64, float64, float_status *status);
 float64 float64_max(float64, float64, float_status *status);
 float64 float64_minnum(float64, float64, float_status *status);
 float64 float64_maxnum(float64, float64, float_status *status);
 float64 float64_minnummag(float64, float64, float_status *status);
 float64 float64_maxnummag(float64, float64, float_status *status);
-int float64_is_quiet_nan(float64 a, float_status *status);
-int float64_is_signaling_nan(float64, float_status *status);
+float64 float64_minimum_number(float64, float64, float_status *status);
+float64 float64_maximum_number(float64, float64, float_status *status);
+bool float64_is_quiet_nan(float64 a, float_status *status);
+bool float64_is_signaling_nan(float64, float_status *status);
 float64 float64_silence_nan(float64, float_status *status);
 float64 float64_scalbn(float64, int, float_status *status);
 
@@ -532,27 +810,27 @@ static inline float64 float64_chs(float64 a)
     return make_float64(float64_val(a) ^ 0x8000000000000000LL);
 }
 
-static inline int float64_is_infinity(float64 a)
+static inline bool float64_is_infinity(float64 a)
 {
     return (float64_val(a) & 0x7fffffffffffffffLL ) == 0x7ff0000000000000LL;
 }
 
-static inline int float64_is_neg(float64 a)
+static inline bool float64_is_neg(float64 a)
 {
     return float64_val(a) >> 63;
 }
 
-static inline int float64_is_zero(float64 a)
+static inline bool float64_is_zero(float64 a)
 {
     return (float64_val(a) & 0x7fffffffffffffffLL) == 0;
 }
 
-static inline int float64_is_any_nan(float64 a)
+static inline bool float64_is_any_nan(float64 a)
 {
     return ((float64_val(a) & ~(1ULL << 63)) > 0x7ff0000000000000ULL);
 }
 
-static inline int float64_is_zero_or_denormal(float64 a)
+static inline bool float64_is_zero_or_denormal(float64 a)
 {
     return (float64_val(a) & 0x7ff0000000000000LL) == 0;
 }
@@ -578,6 +856,47 @@ static inline float64 float64_set_sign(float64 a, int sign)
                         | ((int64_t)sign << 63));
 }
 
+static inline bool float64_eq(float64 a, float64 b, float_status *s)
+{
+    return float64_compare(a, b, s) == float_relation_equal;
+}
+
+static inline bool float64_le(float64 a, float64 b, float_status *s)
+{
+    return float64_compare(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float64_lt(float64 a, float64 b, float_status *s)
+{
+    return float64_compare(a, b, s) < float_relation_equal;
+}
+
+static inline bool float64_unordered(float64 a, float64 b, float_status *s)
+{
+    return float64_compare(a, b, s) == float_relation_unordered;
+}
+
+static inline bool float64_eq_quiet(float64 a, float64 b, float_status *s)
+{
+    return float64_compare_quiet(a, b, s) == float_relation_equal;
+}
+
+static inline bool float64_le_quiet(float64 a, float64 b, float_status *s)
+{
+    return float64_compare_quiet(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float64_lt_quiet(float64 a, float64 b, float_status *s)
+{
+    return float64_compare_quiet(a, b, s) < float_relation_equal;
+}
+
+static inline bool float64_unordered_quiet(float64 a, float64 b,
+                                           float_status *s)
+{
+    return float64_compare_quiet(a, b, s) == float_relation_unordered;
+}
+
 #define float64_zero make_float64(0)
 #define float64_half make_float64(0x3fe0000000000000LL)
 #define float64_one make_float64(0x3ff0000000000000LL)
@@ -592,6 +911,18 @@ static inline float64 float64_set_sign(float64 a, int sign)
 *----------------------------------------------------------------------------*/
 float64 float64_default_nan(float_status *status);
 
+/*----------------------------------------------------------------------------
+| Software IEC/IEEE double-precision operations, rounding to single precision,
+| returning a result in double precision, with only one rounding step.
+*----------------------------------------------------------------------------*/
+
+float64 float64r32_add(float64, float64, float_status *status);
+float64 float64r32_sub(float64, float64, float_status *status);
+float64 float64r32_mul(float64, float64, float_status *status);
+float64 float64r32_div(float64, float64, float_status *status);
+float64 float64r32_muladd(float64, float64, float64, int, float_status *status);
+float64 float64r32_sqrt(float64, float_status *status);
+
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE extended double-precision conversion routines.
 *----------------------------------------------------------------------------*/
@@ -617,18 +948,13 @@ floatx80 floatx80_add(floatx80, floatx80, float_status *status);
 floatx80 floatx80_sub(floatx80, floatx80, float_status *status);
 floatx80 floatx80_mul(floatx80, floatx80, float_status *status);
 floatx80 floatx80_div(floatx80, floatx80, float_status *status);
+floatx80 floatx80_modrem(floatx80, floatx80, bool, uint64_t *,
+                         float_status *status);
+floatx80 floatx80_mod(floatx80, floatx80, float_status *status);
 floatx80 floatx80_rem(floatx80, floatx80, float_status *status);
 floatx80 floatx80_sqrt(floatx80, float_status *status);
-int floatx80_eq(floatx80, floatx80, float_status *status);
-int floatx80_le(floatx80, floatx80, float_status *status);
-int floatx80_lt(floatx80, floatx80, float_status *status);
-int floatx80_unordered(floatx80, floatx80, float_status *status);
-int floatx80_eq_quiet(floatx80, floatx80, float_status *status);
-int floatx80_le_quiet(floatx80, floatx80, float_status *status);
-int floatx80_lt_quiet(floatx80, floatx80, float_status *status);
-int floatx80_unordered_quiet(floatx80, floatx80, float_status *status);
-int floatx80_compare(floatx80, floatx80, float_status *status);
-int floatx80_compare_quiet(floatx80, floatx80, float_status *status);
+FloatRelation floatx80_compare(floatx80, floatx80, float_status *status);
+FloatRelation floatx80_compare_quiet(floatx80, floatx80, float_status *status);
 int floatx80_is_quiet_nan(floatx80, float_status *status);
 int floatx80_is_signaling_nan(floatx80, float_status *status);
 floatx80 floatx80_silence_nan(floatx80, float_status *status);
@@ -646,7 +972,7 @@ static inline floatx80 floatx80_chs(floatx80 a)
     return a;
 }
 
-static inline int floatx80_is_infinity(floatx80 a)
+static inline bool floatx80_is_infinity(floatx80 a)
 {
 #if defined(TARGET_M68K)
     return (a.high & 0x7fff) == floatx80_infinity.high && !(a.low << 1);
@@ -656,26 +982,67 @@ static inline int floatx80_is_infinity(floatx80 a)
 #endif
 }
 
-static inline int floatx80_is_neg(floatx80 a)
+static inline bool floatx80_is_neg(floatx80 a)
 {
     return a.high >> 15;
 }
 
-static inline int floatx80_is_zero(floatx80 a)
+static inline bool floatx80_is_zero(floatx80 a)
 {
     return (a.high & 0x7fff) == 0 && a.low == 0;
 }
 
-static inline int floatx80_is_zero_or_denormal(floatx80 a)
+static inline bool floatx80_is_zero_or_denormal(floatx80 a)
 {
     return (a.high & 0x7fff) == 0;
 }
 
-static inline int floatx80_is_any_nan(floatx80 a)
+static inline bool floatx80_is_any_nan(floatx80 a)
 {
     return ((a.high & 0x7fff) == 0x7fff) && (a.low<<1);
 }
 
+static inline bool floatx80_eq(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_compare(a, b, s) == float_relation_equal;
+}
+
+static inline bool floatx80_le(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_compare(a, b, s) <= float_relation_equal;
+}
+
+static inline bool floatx80_lt(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_compare(a, b, s) < float_relation_equal;
+}
+
+static inline bool floatx80_unordered(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_compare(a, b, s) == float_relation_unordered;
+}
+
+static inline bool floatx80_eq_quiet(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_compare_quiet(a, b, s) == float_relation_equal;
+}
+
+static inline bool floatx80_le_quiet(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_compare_quiet(a, b, s) <= float_relation_equal;
+}
+
+static inline bool floatx80_lt_quiet(floatx80 a, floatx80 b, float_status *s)
+{
+    return floatx80_compare_quiet(a, b, s) < float_relation_equal;
+}
+
+static inline bool floatx80_unordered_quiet(floatx80 a, floatx80 b,
+                                           float_status *s)
+{
+    return floatx80_compare_quiet(a, b, s) == float_relation_unordered;
+}
+
 /*----------------------------------------------------------------------------
 | Return whether the given value is an invalid floatx80 encoding.
 | Invalid floatx80 encodings arise when the integer bit is not set, but
@@ -688,10 +1055,35 @@ static inline int floatx80_is_any_nan(floatx80 a)
 *----------------------------------------------------------------------------*/
 static inline bool floatx80_invalid_encoding(floatx80 a)
 {
+#if defined(TARGET_M68K)
+    /*-------------------------------------------------------------------------
+    | With m68k, the explicit integer bit can be zero in the case of:
+    | - zeros                (exp == 0, mantissa == 0)
+    | - denormalized numbers (exp == 0, mantissa != 0)
+    | - unnormalized numbers (exp != 0, exp < 0x7FFF)
+    | - infinities           (exp == 0x7FFF, mantissa == 0)
+    | - not-a-numbers        (exp == 0x7FFF, mantissa != 0)
+    |
+    | For infinities and NaNs, the explicit integer bit can be either one or
+    | zero.
+    |
+    | The IEEE 754 standard does not define a zero integer bit. Such a number
+    | is an unnormalized number. Hardware does not directly support
+    | denormalized and unnormalized numbers, but implicitly supports them by
+    | trapping them as unimplemented data types, allowing efficient conversion
+    | in software.
+    |
+    | See "M68000 FAMILY PROGRAMMER’S REFERENCE MANUAL",
+    |     "1.6 FLOATING-POINT DATA TYPES"
+    *------------------------------------------------------------------------*/
+    return false;
+#else
     return (a.low & (1ULL << 63)) == 0 && (a.high & 0x7FFF) != 0;
+#endif
 }
 
 #define floatx80_zero make_floatx80(0x0000, 0x0000000000000000LL)
+#define floatx80_zero_init make_floatx80_init(0x0000, 0x0000000000000000LL)
 #define floatx80_one make_floatx80(0x3fff, 0x8000000000000000LL)
 #define floatx80_ln2 make_floatx80(0x3ffe, 0xb17217f7d1cf79acLL)
 #define floatx80_pi make_floatx80(0x4000, 0xc90fdaa22168c235LL)
@@ -722,7 +1114,7 @@ static inline int32_t extractFloatx80Exp(floatx80 a)
 | `a'.
 *----------------------------------------------------------------------------*/
 
-static inline flag extractFloatx80Sign(floatx80 a)
+static inline bool extractFloatx80Sign(floatx80 a)
 {
     return a.high >> 15;
 }
@@ -732,7 +1124,7 @@ static inline flag extractFloatx80Sign(floatx80 a)
 | extended double-precision floating-point value, returning the result.
 *----------------------------------------------------------------------------*/
 
-static inline floatx80 packFloatx80(flag zSign, int32_t zExp, uint64_t zSig)
+static inline floatx80 packFloatx80(bool zSign, int32_t zExp, uint64_t zSig)
 {
     floatx80 z;
 
@@ -783,7 +1175,7 @@ floatx80 propagateFloatx80NaN(floatx80 a, floatx80 b, float_status *status);
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 roundAndPackFloatx80(int8_t roundingPrecision, flag zSign,
+floatx80 roundAndPackFloatx80(FloatX80RoundPrec roundingPrecision, bool zSign,
                               int32_t zExp, uint64_t zSig0, uint64_t zSig1,
                               float_status *status);
 
@@ -796,8 +1188,8 @@ floatx80 roundAndPackFloatx80(int8_t roundingPrecision, flag zSign,
 | normalized.
 *----------------------------------------------------------------------------*/
 
-floatx80 normalizeRoundAndPackFloatx80(int8_t roundingPrecision,
-                                       flag zSign, int32_t zExp,
+floatx80 normalizeRoundAndPackFloatx80(FloatX80RoundPrec roundingPrecision,
+                                       bool zSign, int32_t zExp,
                                        uint64_t zSig0, uint64_t zSig1,
                                        float_status *status);
 
@@ -812,9 +1204,13 @@ floatx80 floatx80_default_nan(float_status *status);
 int32_t float128_to_int32(float128, float_status *status);
 int32_t float128_to_int32_round_to_zero(float128, float_status *status);
 int64_t float128_to_int64(float128, float_status *status);
+Int128 float128_to_int128(float128, float_status *status);
 int64_t float128_to_int64_round_to_zero(float128, float_status *status);
+Int128 float128_to_int128_round_to_zero(float128, float_status *status);
 uint64_t float128_to_uint64(float128, float_status *status);
+Int128 float128_to_uint128(float128, float_status *status);
 uint64_t float128_to_uint64_round_to_zero(float128, float_status *status);
+Int128 float128_to_uint128_round_to_zero(float128, float_status *status);
 uint32_t float128_to_uint32(float128, float_status *status);
 uint32_t float128_to_uint32_round_to_zero(float128, float_status *status);
 float32 float128_to_float32(float128, float_status *status);
@@ -828,21 +1224,23 @@ float128 float128_round_to_int(float128, float_status *status);
 float128 float128_add(float128, float128, float_status *status);
 float128 float128_sub(float128, float128, float_status *status);
 float128 float128_mul(float128, float128, float_status *status);
+float128 float128_muladd(float128, float128, float128, int,
+                         float_status *status);
 float128 float128_div(float128, float128, float_status *status);
 float128 float128_rem(float128, float128, float_status *status);
 float128 float128_sqrt(float128, float_status *status);
-int float128_eq(float128, float128, float_status *status);
-int float128_le(float128, float128, float_status *status);
-int float128_lt(float128, float128, float_status *status);
-int float128_unordered(float128, float128, float_status *status);
-int float128_eq_quiet(float128, float128, float_status *status);
-int float128_le_quiet(float128, float128, float_status *status);
-int float128_lt_quiet(float128, float128, float_status *status);
-int float128_unordered_quiet(float128, float128, float_status *status);
-int float128_compare(float128, float128, float_status *status);
-int float128_compare_quiet(float128, float128, float_status *status);
-int float128_is_quiet_nan(float128, float_status *status);
-int float128_is_signaling_nan(float128, float_status *status);
+FloatRelation float128_compare(float128, float128, float_status *status);
+FloatRelation float128_compare_quiet(float128, float128, float_status *status);
+float128 float128_min(float128, float128, float_status *status);
+float128 float128_max(float128, float128, float_status *status);
+float128 float128_minnum(float128, float128, float_status *status);
+float128 float128_maxnum(float128, float128, float_status *status);
+float128 float128_minnummag(float128, float128, float_status *status);
+float128 float128_maxnummag(float128, float128, float_status *status);
+float128 float128_minimum_number(float128, float128, float_status *status);
+float128 float128_maximum_number(float128, float128, float_status *status);
+bool float128_is_quiet_nan(float128, float_status *status);
+bool float128_is_signaling_nan(float128, float_status *status);
 float128 float128_silence_nan(float128, float_status *status);
 float128 float128_scalbn(float128, int, float_status *status);
 
@@ -858,22 +1256,22 @@ static inline float128 float128_chs(float128 a)
     return a;
 }
 
-static inline int float128_is_infinity(float128 a)
+static inline bool float128_is_infinity(float128 a)
 {
     return (a.high & 0x7fffffffffffffffLL) == 0x7fff000000000000LL && a.low == 0;
 }
 
-static inline int float128_is_neg(float128 a)
+static inline bool float128_is_neg(float128 a)
 {
     return a.high >> 63;
 }
 
-static inline int float128_is_zero(float128 a)
+static inline bool float128_is_zero(float128 a)
 {
     return (a.high & 0x7fffffffffffffffLL) == 0 && a.low == 0;
 }
 
-static inline int float128_is_zero_or_denormal(float128 a)
+static inline bool float128_is_zero_or_denormal(float128 a)
 {
     return (a.high & 0x7fff000000000000LL) == 0;
 }
@@ -888,12 +1286,53 @@ static inline bool float128_is_denormal(float128 a)
     return float128_is_zero_or_denormal(a) && !float128_is_zero(a);
 }
 
-static inline int float128_is_any_nan(float128 a)
+static inline bool float128_is_any_nan(float128 a)
 {
     return ((a.high >> 48) & 0x7fff) == 0x7fff &&
         ((a.low != 0) || ((a.high & 0xffffffffffffLL) != 0));
 }
 
+static inline bool float128_eq(float128 a, float128 b, float_status *s)
+{
+    return float128_compare(a, b, s) == float_relation_equal;
+}
+
+static inline bool float128_le(float128 a, float128 b, float_status *s)
+{
+    return float128_compare(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float128_lt(float128 a, float128 b, float_status *s)
+{
+    return float128_compare(a, b, s) < float_relation_equal;
+}
+
+static inline bool float128_unordered(float128 a, float128 b, float_status *s)
+{
+    return float128_compare(a, b, s) == float_relation_unordered;
+}
+
+static inline bool float128_eq_quiet(float128 a, float128 b, float_status *s)
+{
+    return float128_compare_quiet(a, b, s) == float_relation_equal;
+}
+
+static inline bool float128_le_quiet(float128 a, float128 b, float_status *s)
+{
+    return float128_compare_quiet(a, b, s) <= float_relation_equal;
+}
+
+static inline bool float128_lt_quiet(float128 a, float128 b, float_status *s)
+{
+    return float128_compare_quiet(a, b, s) < float_relation_equal;
+}
+
+static inline bool float128_unordered_quiet(float128 a, float128 b,
+                                           float_status *s)
+{
+    return float128_compare_quiet(a, b, s) == float_relation_unordered;
+}
+
 #define float128_zero make_float128(0, 0)
 
 /*----------------------------------------------------------------------------
diff --git a/qemu/include/hw/core/cpu.h b/qemu/include/hw/core/cpu.h
index e01ff76c85..2e7a4db569 100644
--- a/qemu/include/hw/core/cpu.h
+++ b/qemu/include/hw/core/cpu.h
@@ -49,6 +49,37 @@ typedef struct CPUWatchpoint CPUWatchpoint;
 
 struct TranslationBlock;
 
+struct TCGCPUOps {
+    void (*initialize)(void);
+    void (*synchronize_from_tb)(CPUState *cpu,
+                                const struct TranslationBlock *tb);
+    void (*restore_state_to_opc)(CPUState *cpu,
+                                 const struct TranslationBlock *tb,
+                                 const uint64_t *data);
+    void (*cpu_exec_enter)(CPUState *cpu);
+    void (*cpu_exec_exit)(CPUState *cpu);
+    void (*debug_excp_handler)(CPUState *cpu);
+    void (*do_interrupt)(CPUState *cpu);
+    bool (*cpu_exec_interrupt)(CPUState *cpu, int interrupt_request);
+    bool (*tlb_fill)(CPUState *cpu, vaddr address, int size,
+                     MMUAccessType access_type, int mmu_idx,
+                     bool probe, uintptr_t retaddr);
+    void (*do_transaction_failed)(CPUState *cpu, hwaddr physaddr,
+                                  vaddr addr, unsigned size,
+                                  MMUAccessType access_type, int mmu_idx,
+                                  MemTxAttrs attrs,
+                                  MemTxResult response,
+                                  uintptr_t retaddr);
+    void (*do_unaligned_access)(CPUState *cpu, vaddr addr,
+                                MMUAccessType access_type, int mmu_idx,
+                                uintptr_t retaddr);
+    vaddr (*adjust_watchpoint_address)(CPUState *cpu, vaddr addr, int len);
+    bool (*debug_check_watchpoint)(CPUState *cpu, CPUWatchpoint *wp);
+    bool (*debug_check_breakpoint)(CPUState *cpu);
+    bool (*io_recompile_replay_branch)(CPUState *cpu,
+                                       const struct TranslationBlock *tb);
+};
+
 /**
  * CPUClass:
  * @class_by_name: Callback to map -cpu command line model name to an
@@ -131,6 +162,7 @@ typedef struct CPUClass {
     void (*cpu_exec_exit)(CPUState *cpu);
     bool (*cpu_exec_interrupt)(CPUState *cpu, int interrupt_request);
 
+    const struct TCGCPUOps *tcg_ops;
     vaddr (*adjust_watchpoint_address)(CPUState *cpu, vaddr addr, int len);
     void (*tcg_initialize)(struct uc_struct *uc);
 } CPUClass;
diff --git a/qemu/include/hw/core/tcg-cpu-ops.h b/qemu/include/hw/core/tcg-cpu-ops.h
new file mode 100644
index 0000000000..2e6b0fd42f
--- /dev/null
+++ b/qemu/include/hw/core/tcg-cpu-ops.h
@@ -0,0 +1,103 @@
+/*
+ * TCG CPU-specific operations
+ *
+ * Copyright 2021 SUSE LLC
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef TCG_CPU_OPS_H
+#define TCG_CPU_OPS_H
+
+#include "hw/core/cpu.h"
+#include "exec/exec-all.h"
+
+static inline void cpu_tcg_synchronize_from_tb(CPUState *cpu,
+                                               struct TranslationBlock *tb)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->tcg_ops && cc->tcg_ops->synchronize_from_tb) {
+        cc->tcg_ops->synchronize_from_tb(cpu, tb);
+    } else if (cc->synchronize_from_tb) {
+        cc->synchronize_from_tb(cpu, tb);
+    } else {
+        assert(cc->set_pc);
+        cc->set_pc(cpu, tb->pc);
+    }
+}
+
+static inline void cpu_tcg_debug_excp_handler(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->tcg_ops && cc->tcg_ops->debug_excp_handler) {
+        cc->tcg_ops->debug_excp_handler(cpu);
+    } else {
+        cc->debug_excp_handler(cpu);
+    }
+}
+
+static inline void cpu_tcg_exec_enter(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->tcg_ops && cc->tcg_ops->cpu_exec_enter) {
+        cc->tcg_ops->cpu_exec_enter(cpu);
+    } else {
+        cc->cpu_exec_enter(cpu);
+    }
+}
+
+static inline void cpu_tcg_exec_exit(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->tcg_ops && cc->tcg_ops->cpu_exec_exit) {
+        cc->tcg_ops->cpu_exec_exit(cpu);
+    } else {
+        cc->cpu_exec_exit(cpu);
+    }
+}
+
+static inline bool cpu_tcg_exec_interrupt(CPUState *cpu,
+                                          int interrupt_request)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->tcg_ops && cc->tcg_ops->cpu_exec_interrupt) {
+        return cc->tcg_ops->cpu_exec_interrupt(cpu, interrupt_request);
+    }
+    return cc->cpu_exec_interrupt(cpu, interrupt_request);
+}
+
+static inline bool cpu_tcg_tlb_fill(CPUState *cpu, vaddr address, int size,
+                                    MMUAccessType access_type, int mmu_idx,
+                                    bool probe, uintptr_t retaddr)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->tcg_ops && cc->tcg_ops->tlb_fill) {
+        return cc->tcg_ops->tlb_fill(cpu, address, size, access_type,
+                                     mmu_idx, probe, retaddr);
+    }
+    return cc->tlb_fill(cpu, address, size, access_type, mmu_idx,
+                        probe, retaddr);
+}
+
+static inline void cpu_tcg_unaligned_access(CPUState *cpu, vaddr addr,
+                                            MMUAccessType access_type,
+                                            int mmu_idx, uintptr_t retaddr)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->tcg_ops && cc->tcg_ops->do_unaligned_access) {
+        cc->tcg_ops->do_unaligned_access(cpu, addr, access_type,
+                                         mmu_idx, retaddr);
+    } else {
+        cc->do_unaligned_access(cpu, addr, access_type, mmu_idx, retaddr);
+    }
+}
+
+#endif /* TCG_CPU_OPS_H */
diff --git a/qemu/include/hw/mips/cpudevs.h b/qemu/include/hw/mips/cpudevs.h
index 291f59281a..f7c9728fa9 100644
--- a/qemu/include/hw/mips/cpudevs.h
+++ b/qemu/include/hw/mips/cpudevs.h
@@ -5,13 +5,6 @@
 
 /* Definitions for MIPS CPU internal devices.  */
 
-/* addr.c */
-uint64_t cpu_mips_kseg0_to_phys(void *opaque, uint64_t addr);
-uint64_t cpu_mips_phys_to_kseg0(void *opaque, uint64_t addr);
-uint64_t cpu_mips_kvm_um_phys_to_kseg0(void *opaque, uint64_t addr);
-bool mips_um_ksegs_enabled(void);
-void mips_um_ksegs_enable(void);
-
 /* mips_int.c */
 void cpu_mips_irq_init_cpu(MIPSCPU *cpu);
 
diff --git a/qemu/include/libdecnumber/dconfig.h b/qemu/include/libdecnumber/dconfig.h
index 0f7dccef1f..2bc0ba7f14 100644
--- a/qemu/include/libdecnumber/dconfig.h
+++ b/qemu/include/libdecnumber/dconfig.h
@@ -28,7 +28,7 @@
    02110-1301, USA.  */
 
 
-#if defined(HOST_WORDS_BIGENDIAN)
+#if HOST_BIG_ENDIAN
 #define WORDS_BIGENDIAN 1
 #else
 #define WORDS_BIGENDIAN 0
diff --git a/qemu/include/libdecnumber/decNumber.h b/qemu/include/libdecnumber/decNumber.h
index aa115fed07..41bc2a0d36 100644
--- a/qemu/include/libdecnumber/decNumber.h
+++ b/qemu/include/libdecnumber/decNumber.h
@@ -116,12 +116,16 @@
   decNumber * decNumberFromUInt32(decNumber *, uint32_t);
   decNumber *decNumberFromInt64(decNumber *, int64_t);
   decNumber *decNumberFromUInt64(decNumber *, uint64_t);
+  decNumber *decNumberFromInt128(decNumber *, uint64_t, int64_t);
+  decNumber *decNumberFromUInt128(decNumber *, uint64_t, uint64_t);
   decNumber * decNumberFromString(decNumber *, const char *, decContext *);
   char	    * decNumberToString(const decNumber *, char *);
   char	    * decNumberToEngString(const decNumber *, char *);
   uint32_t    decNumberToUInt32(const decNumber *, decContext *);
   int32_t     decNumberToInt32(const decNumber *, decContext *);
   int64_t     decNumberIntegralToInt64(const decNumber *dn, decContext *set);
+  void        decNumberIntegralToInt128(const decNumber *dn, decContext *set,
+        uint64_t *plow, uint64_t *phigh);
   uint8_t   * decNumberGetBCD(const decNumber *, uint8_t *);
   decNumber * decNumberSetBCD(decNumber *, const uint8_t *, uint32_t);
 
diff --git a/qemu/include/libdecnumber/decNumberLocal.h b/qemu/include/libdecnumber/decNumberLocal.h
index 4d53c077f2..6198ca8593 100644
--- a/qemu/include/libdecnumber/decNumberLocal.h
+++ b/qemu/include/libdecnumber/decNumberLocal.h
@@ -98,7 +98,7 @@
 
   /* Shared lookup tables					      */
   extern const uByte  DECSTICKYTAB[10]; /* re-round digits if sticky  */
-  extern const uLong  DECPOWERS[19];    /* powers of ten table        */
+  extern const uLong  DECPOWERS[20];    /* powers of ten table        */
   /* The following are included from decDPD.h			      */
   extern const uShort DPD2BIN[1024];	/* DPD -> 0-999		      */
   extern const uShort BIN2DPD[1000];	/* 0-999 -> DPD		      */
diff --git a/qemu/include/qemu/atomic128.h b/qemu/include/qemu/atomic128.h
index 4183863d11..b18bcc3751 100644
--- a/qemu/include/qemu/atomic128.h
+++ b/qemu/include/qemu/atomic128.h
@@ -6,7 +6,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
- * See docs/devel/atomics.txt for discussion about the guarantees each
+ * See docs/devel/atomics.rst for discussion about the guarantees each
  * atomic primitive is meant to provide.
  */
 
@@ -44,7 +44,7 @@
 #if defined(CONFIG_ATOMIC128)
 static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
 {
-    return atomic_cmpxchg__nocheck(ptr, cmp, new);
+    return qatomic_cmpxchg__nocheck(ptr, cmp, new);
 }
 # define HAVE_CMPXCHG128 1
 #elif defined(CONFIG_CMPXCHG128)
@@ -99,12 +99,12 @@ Int128 QEMU_ERROR("unsupported atomic")
 #if defined(CONFIG_ATOMIC128)
 static inline Int128 atomic16_read(Int128 *ptr)
 {
-    return atomic_read__nocheck(ptr);
+    return qatomic_read__nocheck(ptr);
 }
 
 static inline void atomic16_set(Int128 *ptr, Int128 val)
 {
-    atomic_set__nocheck(ptr, val);
+    qatomic_set__nocheck(ptr, val);
 }
 
 # define HAVE_ATOMIC128 1
diff --git a/qemu/include/qemu/bitmap.h b/qemu/include/qemu/bitmap.h
index 267f04f275..937162eb4b 100644
--- a/qemu/include/qemu/bitmap.h
+++ b/qemu/include/qemu/bitmap.h
@@ -69,6 +69,14 @@
 #define DECLARE_BITMAP(name,bits)                  \
         unsigned long name[BITS_TO_LONGS(bits)]
 
+/*
+ * This is for use with the bit32 versions of set_bit() etc;
+ * we don't currently support the full range of bitmap operations
+ * on bitmaps backed by an array of uint32_t.
+ */
+#define DECLARE_BITMAP32(name, bits)            \
+        uint32_t name[BITS_TO_U32S(bits)]
+
 #define small_nbits(nbits)                      \
         ((nbits) <= BITS_PER_LONG)
 
@@ -253,6 +261,7 @@ void qemu_bitmap_set(unsigned long *map, long i, long len);
 void bitmap_set_atomic(unsigned long *map, long i, long len);
 void qemu_bitmap_clear(unsigned long *map, long start, long nr);
 bool bitmap_test_and_clear_atomic(unsigned long *map, long start, long nr);
+bool bitmap_test_and_clear(unsigned long *map, long start, long nr);
 void bitmap_copy_and_clear_atomic(unsigned long *dst, unsigned long *src,
                                   long nr);
 unsigned long bitmap_find_next_zero_area(unsigned long *map,
diff --git a/qemu/include/qemu/compiler.h b/qemu/include/qemu/compiler.h
index 971aa12721..c471e5c13e 100644
--- a/qemu/include/qemu/compiler.h
+++ b/qemu/include/qemu/compiler.h
@@ -16,11 +16,30 @@
 #define tostring(s)	#s
 #endif
 
+#ifndef HOST_BIG_ENDIAN
+# if defined(HOST_WORDS_BIGENDIAN)
+#  define HOST_BIG_ENDIAN 1
+# else
+#  define HOST_BIG_ENDIAN 0
+# endif
+#endif
+
+#ifndef HOST_LONG_BITS
+# if UINTPTR_MAX == UINT64_MAX
+#  define HOST_LONG_BITS 64
+# else
+#  define HOST_LONG_BITS 32
+# endif
+#endif
+
 #ifdef _MSC_VER
 // MSVC support
 
 #define inline		__inline
 #define __func__	__FUNCTION__
+#ifndef __attribute__
+#define __attribute__(x)
+#endif
 
 #include <math.h>
 #include <float.h>
@@ -84,11 +103,35 @@ static union MSVC_FLOAT_HACK __NAN = {{0x00, 0x00, 0xC0, 0x7F}};
 #define likely(x) (x)
 #define unlikely(x) (x)
 
+#ifndef __has_warning
+#define __has_warning(x) 0
+#endif
+
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+#ifndef __has_attribute
+#define __has_attribute(x) 0
+#endif
+
 #define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member)))
 
 #define QEMU_FLATTEN
 #define QEMU_ALWAYS_INLINE  __declspec(inline)
 
+#ifndef G_GNUC_WARN_UNUSED_RESULT
+#define G_GNUC_WARN_UNUSED_RESULT
+#endif
+
+#ifndef G_NORETURN
+#define G_NORETURN __declspec(noreturn)
+#endif
+
 #else  // Unix compilers
 
 #ifndef NAN
@@ -112,11 +155,19 @@ static union MSVC_FLOAT_HACK __NAN = {{0x00, 0x00, 0xC0, 0x7F}};
 
 #define QEMU_NORETURN __attribute__ ((__noreturn__))
 
+#ifndef G_NORETURN
+#define G_NORETURN __attribute__((noreturn))
+#endif
+
 #define QEMU_UNUSED_VAR __attribute__((unused))
 #define QEMU_UNUSED_FUNC __attribute__((unused))
 
 #define QEMU_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
 
+#ifndef G_GNUC_WARN_UNUSED_RESULT
+#define G_GNUC_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#endif
+
 #define QEMU_SENTINEL __attribute__((sentinel))
 
 #if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__))
diff --git a/qemu/include/qemu/cpuid.h b/qemu/include/qemu/cpuid.h
index 9c09dd1bf6..c10d60fa6d 100644
--- a/qemu/include/qemu/cpuid.h
+++ b/qemu/include/qemu/cpuid.h
@@ -52,12 +52,26 @@
 #ifndef bit_AVX2
 #define bit_AVX2        (1 << 5)
 #endif
-#ifndef bit_AVX512F
-#define bit_AVX512F        (1 << 16)
-#endif
 #ifndef bit_BMI2
 #define bit_BMI2        (1 << 8)
 #endif
+#ifndef bit_AVX512F
+#define bit_AVX512F     (1 << 16)
+#endif
+#ifndef bit_AVX512DQ
+#define bit_AVX512DQ    (1 << 17)
+#endif
+#ifndef bit_AVX512BW
+#define bit_AVX512BW    (1 << 30)
+#endif
+#ifndef bit_AVX512VL
+#define bit_AVX512VL    (1u << 31)
+#endif
+
+/* Leaf 7, %ecx */
+#ifndef bit_AVX512VBMI2
+#define bit_AVX512VBMI2 (1 << 6)
+#endif
 
 /* Leaf 0x80000001, %ecx */
 #ifndef bit_LZCNT
diff --git a/qemu/include/qemu/host-utils.h b/qemu/include/qemu/host-utils.h
index 0c5b30ff67..06385707ed 100644
--- a/qemu/include/qemu/host-utils.h
+++ b/qemu/include/qemu/host-utils.h
@@ -23,12 +23,21 @@
  * THE SOFTWARE.
  */
 
+/* Portions of this work are licensed under the terms of the GNU GPL,
+ * version 2 or later. See the COPYING file in the top-level directory.
+ */
+
 #ifndef HOST_UTILS_H
 #define HOST_UTILS_H
 
+#include "qemu/compiler.h"
 #include "qemu/bswap.h"
 #include "qemu/int128.h"
 
+#if defined(_MSC_VER) && defined(_M_X64)
+#include <intrin.h>
+#endif
+
 #ifdef CONFIG_INT128
 static inline void mulu64(uint64_t *plow, uint64_t *phigh,
                           uint64_t a, uint64_t b)
@@ -74,43 +83,45 @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
     #endif
 }
 
-static inline int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor)
+static inline uint64_t muldiv64_round_up(uint64_t a, uint32_t b, uint32_t c)
 {
-    if (divisor == 0) {
-        return 1;
-    } else {
-        __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow;
-        __uint128_t result = dividend / divisor;
-        *plow = result;
-        *phigh = dividend % divisor;
-        return result > UINT64_MAX;
-    }
+    return ((__int128_t)a * b + c - 1) / c;
 }
 
-static inline int divs128(int64_t *plow, int64_t *phigh, int64_t divisor)
+static inline uint64_t divu128(uint64_t *plow, uint64_t *phigh,
+                               uint64_t divisor)
 {
-    if (divisor == 0) {
-        return 1;
-    } else {
-        __int128_t dividend = ((__int128_t)*phigh << 64) | *plow;
-        __int128_t result = dividend / divisor;
-        *plow = result;
-        *phigh = dividend % divisor;
-        return result != *plow;
-    }
+    __uint128_t dividend = ((__uint128_t)*phigh << 64) | *plow;
+    __uint128_t result = dividend / divisor;
+
+    *plow = result;
+    *phigh = result >> 64;
+    return dividend % divisor;
+}
+
+static inline int64_t divs128(uint64_t *plow, int64_t *phigh,
+                              int64_t divisor)
+{
+    __int128_t dividend = ((__int128_t)*phigh << 64) | *plow;
+    __int128_t result = dividend / divisor;
+
+    *plow = result;
+    *phigh = result >> 64;
+    return dividend % divisor;
 }
 #else
-void muls64(uint64_t *phigh, uint64_t *plow, int64_t a, int64_t b);
-void mulu64(uint64_t *phigh, uint64_t *plow, uint64_t a, uint64_t b);
-int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor);
-int divs128(int64_t *plow, int64_t *phigh, int64_t divisor);
+void muls64(uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b);
+void mulu64(uint64_t *plow, uint64_t *phigh, uint64_t a, uint64_t b);
+uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor);
+int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor);
 
-static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+static inline uint64_t muldiv64_rounding(uint64_t a, uint32_t b, uint32_t c,
+                                  bool round_up)
 {
     union {
         uint64_t ll;
         struct {
-#ifdef HOST_WORDS_BIGENDIAN
+#if HOST_BIG_ENDIAN
             uint32_t high, low;
 #else
             uint32_t low, high;
@@ -121,12 +132,25 @@ static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
 
     u.ll = a;
     rl = (uint64_t)u.l.low * (uint64_t)b;
+    if (round_up) {
+        rl += c - 1;
+    }
     rh = (uint64_t)u.l.high * (uint64_t)b;
     rh += (rl >> 32);
     res.l.high = rh / c;
     res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
     return res.ll;
 }
+
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    return muldiv64_rounding(a, b, c, false);
+}
+
+static inline uint64_t muldiv64_round_up(uint64_t a, uint32_t b, uint32_t c)
+{
+    return muldiv64_rounding(a, b, c, true);
+}
 #endif
 
 /**
@@ -419,6 +443,9 @@ static inline int ctpop64(uint64_t val)
  */
 static inline uint8_t revbit8(uint8_t x)
 {
+#if __has_builtin(__builtin_bitreverse8)
+    return __builtin_bitreverse8(x);
+#else
     /* Assign the correct nibble position.  */
     x = ((x & 0xf0) >> 4)
       | ((x & 0x0f) << 4);
@@ -428,6 +455,7 @@ static inline uint8_t revbit8(uint8_t x)
       | ((x & 0x22) << 1)
       | ((x & 0x11) << 3);
     return x;
+#endif
 }
 
 /**
@@ -436,6 +464,9 @@ static inline uint8_t revbit8(uint8_t x)
  */
 static inline uint16_t revbit16(uint16_t x)
 {
+#if __has_builtin(__builtin_bitreverse16)
+    return __builtin_bitreverse16(x);
+#else
     /* Assign the correct byte position.  */
     x = bswap16(x);
     /* Assign the correct nibble position.  */
@@ -447,6 +478,7 @@ static inline uint16_t revbit16(uint16_t x)
       | ((x & 0x2222) << 1)
       | ((x & 0x1111) << 3);
     return x;
+#endif
 }
 
 /**
@@ -455,6 +487,9 @@ static inline uint16_t revbit16(uint16_t x)
  */
 static inline uint32_t revbit32(uint32_t x)
 {
+#if __has_builtin(__builtin_bitreverse32)
+    return __builtin_bitreverse32(x);
+#else
     /* Assign the correct byte position.  */
     x = bswap32(x);
     /* Assign the correct nibble position.  */
@@ -466,6 +501,7 @@ static inline uint32_t revbit32(uint32_t x)
       | ((x & 0x22222222u) << 1)
       | ((x & 0x11111111u) << 3);
     return x;
+#endif
 }
 
 /**
@@ -474,6 +510,9 @@ static inline uint32_t revbit32(uint32_t x)
  */
 static inline uint64_t revbit64(uint64_t x)
 {
+#if __has_builtin(__builtin_bitreverse64)
+    return __builtin_bitreverse64(x);
+#else
     /* Assign the correct byte position.  */
     x = bswap64(x);
     /* Assign the correct nibble position.  */
@@ -485,6 +524,315 @@ static inline uint64_t revbit64(uint64_t x)
       | ((x & 0x2222222222222222ull) << 1)
       | ((x & 0x1111111111111111ull) << 3);
     return x;
+#endif
+}
+
+/**
+ * Return the absolute value of a 64-bit integer as an unsigned 64-bit value
+ */
+static inline uint64_t uabs64(int64_t v)
+{
+    return v < 0 ? -v : v;
+}
+
+/**
+ * sadd32_overflow - addition with overflow indication
+ * @x, @y: addends
+ * @ret: Output for sum
+ *
+ * Computes *@ret = @x + @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool sadd32_overflow(int32_t x, int32_t y, int32_t *ret)
+{
+#ifdef _MSC_VER
+    uint32_t ux = x;
+    uint32_t uy = y;
+    uint32_t ur = ux + uy;
+
+    *ret = (int32_t)ur;
+    return ((~(ux ^ uy) & (ux ^ ur)) >> 31) != 0;
+#else
+    return __builtin_add_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * sadd64_overflow - addition with overflow indication
+ * @x, @y: addends
+ * @ret: Output for sum
+ *
+ * Computes *@ret = @x + @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool sadd64_overflow(int64_t x, int64_t y, int64_t *ret)
+{
+#ifdef _MSC_VER
+    uint64_t ux = x;
+    uint64_t uy = y;
+    uint64_t ur = ux + uy;
+
+    *ret = (int64_t)ur;
+    return ((~(ux ^ uy) & (ux ^ ur)) >> 63) != 0;
+#else
+    return __builtin_add_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * uadd32_overflow - addition with overflow indication
+ * @x, @y: addends
+ * @ret: Output for sum
+ *
+ * Computes *@ret = @x + @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool uadd32_overflow(uint32_t x, uint32_t y, uint32_t *ret)
+{
+#ifdef _MSC_VER
+    *ret = x + y;
+    return *ret < x;
+#else
+    return __builtin_add_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * uadd64_overflow - addition with overflow indication
+ * @x, @y: addends
+ * @ret: Output for sum
+ *
+ * Computes *@ret = @x + @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool uadd64_overflow(uint64_t x, uint64_t y, uint64_t *ret)
+{
+#ifdef _MSC_VER
+    *ret = x + y;
+    return *ret < x;
+#else
+    return __builtin_add_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * ssub32_overflow - subtraction with overflow indication
+ * @x: Minuend
+ * @y: Subtrahend
+ * @ret: Output for difference
+ *
+ * Computes *@ret = @x - @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool ssub32_overflow(int32_t x, int32_t y, int32_t *ret)
+{
+#ifdef _MSC_VER
+    uint32_t ux = x;
+    uint32_t uy = y;
+    uint32_t ur = ux - uy;
+
+    *ret = (int32_t)ur;
+    return (((ux ^ uy) & (ux ^ ur)) >> 31) != 0;
+#else
+    return __builtin_sub_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * ssub64_overflow - subtraction with overflow indication
+ * @x: Minuend
+ * @y: Subtrahend
+ * @ret: Output for sum
+ *
+ * Computes *@ret = @x - @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool ssub64_overflow(int64_t x, int64_t y, int64_t *ret)
+{
+#ifdef _MSC_VER
+    uint64_t ux = x;
+    uint64_t uy = y;
+    uint64_t ur = ux - uy;
+
+    *ret = (int64_t)ur;
+    return (((ux ^ uy) & (ux ^ ur)) >> 63) != 0;
+#else
+    return __builtin_sub_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * usub32_overflow - subtraction with overflow indication
+ * @x: Minuend
+ * @y: Subtrahend
+ * @ret: Output for sum
+ *
+ * Computes *@ret = @x - @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool usub32_overflow(uint32_t x, uint32_t y, uint32_t *ret)
+{
+#ifdef _MSC_VER
+    *ret = x - y;
+    return x < y;
+#else
+    return __builtin_sub_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * usub64_overflow - subtraction with overflow indication
+ * @x: Minuend
+ * @y: Subtrahend
+ * @ret: Output for sum
+ *
+ * Computes *@ret = @x - @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool usub64_overflow(uint64_t x, uint64_t y, uint64_t *ret)
+{
+#ifdef _MSC_VER
+    *ret = x - y;
+    return x < y;
+#else
+    return __builtin_sub_overflow(x, y, ret);
+#endif
+}
+
+/**
+ * smul32_overflow - multiplication with overflow indication
+ * @x, @y: Input multipliers
+ * @ret: Output for product
+ *
+ * Computes *@ret = @x * @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool smul32_overflow(int32_t x, int32_t y, int32_t *ret)
+{
+    return __builtin_mul_overflow(x, y, ret);
+}
+
+/**
+ * smul64_overflow - multiplication with overflow indication
+ * @x, @y: Input multipliers
+ * @ret: Output for product
+ *
+ * Computes *@ret = @x * @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool smul64_overflow(int64_t x, int64_t y, int64_t *ret)
+{
+    return __builtin_mul_overflow(x, y, ret);
+}
+
+/**
+ * umul32_overflow - multiplication with overflow indication
+ * @x, @y: Input multipliers
+ * @ret: Output for product
+ *
+ * Computes *@ret = @x * @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool umul32_overflow(uint32_t x, uint32_t y, uint32_t *ret)
+{
+    return __builtin_mul_overflow(x, y, ret);
+}
+
+/**
+ * umul64_overflow - multiplication with overflow indication
+ * @x, @y: Input multipliers
+ * @ret: Output for product
+ *
+ * Computes *@ret = @x * @y, and returns true if and only if that
+ * value has been truncated.
+ */
+static inline bool umul64_overflow(uint64_t x, uint64_t y, uint64_t *ret)
+{
+    return __builtin_mul_overflow(x, y, ret);
+}
+
+/*
+ * Unsigned 128x64 multiplication.
+ * Returns true if the result got truncated to 128 bits.
+ * Otherwise, returns false and the multiplication result via plow and phigh.
+ */
+static inline bool mulu128(uint64_t *plow, uint64_t *phigh, uint64_t factor)
+{
+#if defined(CONFIG_INT128)
+    bool res;
+    __uint128_t r;
+    __uint128_t f = ((__uint128_t)*phigh << 64) | *plow;
+    res = __builtin_mul_overflow(f, factor, &r);
+
+    *plow = r;
+    *phigh = r >> 64;
+
+    return res;
+#else
+    uint64_t dhi = *phigh;
+    uint64_t dlo = *plow;
+    uint64_t ahi;
+    uint64_t blo, bhi;
+
+    if (dhi == 0) {
+        mulu64(plow, phigh, dlo, factor);
+        return false;
+    }
+
+    mulu64(plow, &ahi, dlo, factor);
+    mulu64(&blo, &bhi, dhi, factor);
+
+    return uadd64_overflow(ahi, blo, phigh) || bhi != 0;
+#endif
+}
+
+/**
+ * uadd64_carry - addition with carry-in and carry-out
+ * @x, @y: addends
+ * @pcarry: in-out carry value
+ *
+ * Computes @x + @y + *@pcarry, placing the carry-out back
+ * into *@pcarry and returning the 64-bit sum.
+ */
+static inline uint64_t uadd64_carry(uint64_t x, uint64_t y, bool *pcarry)
+{
+#if __has_builtin(__builtin_addcll)
+    unsigned long long c = *pcarry;
+    x = __builtin_addcll(x, y, c, &c);
+    *pcarry = c & 1;
+    return x;
+#else
+    bool c = *pcarry;
+    /* This is clang's internal expansion of __builtin_addc. */
+    c = uadd64_overflow(x, c, &x);
+    c |= uadd64_overflow(x, y, &x);
+    *pcarry = c;
+    return x;
+#endif
+}
+
+/**
+ * usub64_borrow - subtraction with borrow-in and borrow-out
+ * @x, @y: addends
+ * @pborrow: in-out borrow value
+ *
+ * Computes @x - @y - *@pborrow, placing the borrow-out back
+ * into *@pborrow and returning the 64-bit sum.
+ */
+static inline uint64_t usub64_borrow(uint64_t x, uint64_t y, bool *pborrow)
+{
+#if __has_builtin(__builtin_subcll) && !defined(BUILTIN_SUBCLL_BROKEN)
+    unsigned long long b = *pborrow;
+    x = __builtin_subcll(x, y, b, &b);
+    *pborrow = b & 1;
+    return x;
+#else
+    bool b = *pborrow;
+    b = usub64_overflow(x, b, &x);
+    b |= usub64_overflow(x, y, &x);
+    *pborrow = b;
+    return x;
+#endif
 }
 
 /* Host type specific sizes of these routines.  */
@@ -584,4 +932,85 @@ void urshift(uint64_t *plow, uint64_t *phigh, int32_t shift);
  */
 void ulshift(uint64_t *plow, uint64_t *phigh, int32_t shift, bool *overflow);
 
+/* From the GNU Multi Precision Library - longlong.h __udiv_qrnnd
+ * (https://gmplib.org/repo/gmp/file/tip/longlong.h)
+ *
+ * Licensed under the GPLv2/LGPLv3
+ */
+static inline uint64_t udiv_qrnnd(uint64_t *r, uint64_t n1,
+                                  uint64_t n0, uint64_t d)
+{
+#if defined(_MSC_VER) && defined(_M_X64)
+    return _udiv128(n1, n0, d, r);
+#elif defined(__x86_64__)
+    uint64_t q;
+    asm("divq %4" : "=a"(q), "=d"(*r) : "0"(n0), "1"(n1), "rm"(d));
+    return q;
+#elif defined(__s390x__) && !defined(__clang__)
+    /* Need to use a TImode type to get an even register pair for DLGR.  */
+    unsigned __int128 n = (unsigned __int128)n1 << 64 | n0;
+    asm("dlgr %0, %1" : "+r"(n) : "r"(d));
+    *r = n >> 64;
+    return n;
+#elif defined(_ARCH_PPC64) && defined(_ARCH_PWR7)
+    /* From Power ISA 2.06, programming note for divdeu.  */
+    uint64_t q1, q2, Q, r1, r2, R;
+    asm("divdeu %0,%2,%4; divdu %1,%3,%4"
+        : "=&r"(q1), "=r"(q2)
+        : "r"(n1), "r"(n0), "r"(d));
+    r1 = -(q1 * d);         /* low part of (n1<<64) - (q1 * d) */
+    r2 = n0 - (q2 * d);
+    Q = q1 + q2;
+    R = r1 + r2;
+    if (R >= d || R < r2) { /* overflow implies R > d */
+        Q += 1;
+        R -= d;
+    }
+    *r = R;
+    return Q;
+#else
+    uint64_t d0, d1, q0, q1, r1, r0, m;
+
+    d0 = (uint32_t)d;
+    d1 = d >> 32;
+
+    r1 = n1 % d1;
+    q1 = n1 / d1;
+    m = q1 * d0;
+    r1 = (r1 << 32) | (n0 >> 32);
+    if (r1 < m) {
+        q1 -= 1;
+        r1 += d;
+        if (r1 >= d) {
+            if (r1 < m) {
+                q1 -= 1;
+                r1 += d;
+            }
+        }
+    }
+    r1 -= m;
+
+    r0 = r1 % d1;
+    q0 = r1 / d1;
+    m = q0 * d0;
+    r0 = (r0 << 32) | (uint32_t)n0;
+    if (r0 < m) {
+        q0 -= 1;
+        r0 += d;
+        if (r0 >= d) {
+            if (r0 < m) {
+                q0 -= 1;
+                r0 += d;
+            }
+        }
+    }
+    r0 -= m;
+
+    *r = r0;
+    return (q1 << 32) | q0;
+#endif
+}
+
+Int128 divu256(Int128 *plow, Int128 *phigh, Int128 divisor);
+Int128 divs256(Int128 *plow, Int128 *phigh, Int128 divisor);
 #endif
diff --git a/qemu/include/qemu/int128.h b/qemu/include/qemu/int128.h
index f3908f7eb3..d6c3cb63d9 100644
--- a/qemu/include/qemu/int128.h
+++ b/qemu/include/qemu/int128.h
@@ -2,8 +2,43 @@
 #define INT128_H
 
 #include "qemu/bswap.h"
-#ifdef CONFIG_INT128
 
+static inline int int128_clz64(uint64_t val)
+{
+#ifndef _MSC_VER
+    return val ? __builtin_clzll(val) : 64;
+#else
+    int cnt = 0;
+
+    if (!(val >> 32)) {
+        cnt += 32;
+    } else {
+        val >>= 32;
+    }
+    if (!(val & 0xFFFF0000U)) {
+        cnt += 16;
+        val <<= 16;
+    }
+    if (!(val & 0xFF000000U)) {
+        cnt += 8;
+        val <<= 8;
+    }
+    if (!(val & 0xF0000000U)) {
+        cnt += 4;
+        val <<= 4;
+    }
+    if (!(val & 0xC0000000U)) {
+        cnt += 2;
+        val <<= 2;
+    }
+    if (!(val & 0x80000000U)) {
+        cnt++;
+    }
+    return cnt;
+#endif
+}
+
+#ifdef CONFIG_INT128
 typedef __int128_t Int128;
 
 static inline Int128 int128_make64(uint64_t a)
@@ -11,6 +46,11 @@ static inline Int128 int128_make64(uint64_t a)
     return a;
 }
 
+static inline Int128 int128_makes64(int64_t a)
+{
+    return a;
+}
+
 static inline Int128 int128_make128(uint64_t lo, uint64_t hi)
 {
     return (__uint128_t)hi << 64 | lo;
@@ -53,16 +93,41 @@ static inline Int128 int128_exts64(int64_t a)
     return a;
 }
 
+static inline Int128 int128_not(Int128 a)
+{
+    return ~a;
+}
+
 static inline Int128 int128_and(Int128 a, Int128 b)
 {
     return a & b;
 }
 
+static inline Int128 int128_or(Int128 a, Int128 b)
+{
+    return a | b;
+}
+
+static inline Int128 int128_xor(Int128 a, Int128 b)
+{
+    return a ^ b;
+}
+
 static inline Int128 int128_rshift(Int128 a, int n)
 {
     return a >> n;
 }
 
+static inline Int128 int128_urshift(Int128 a, int n)
+{
+    return (__uint128_t)a >> n;
+}
+
+static inline Int128 int128_lshift(Int128 a, int n)
+{
+    return a << n;
+}
+
 static inline Int128 int128_add(Int128 a, Int128 b)
 {
     return a + b;
@@ -98,11 +163,21 @@ static inline bool int128_ge(Int128 a, Int128 b)
     return a >= b;
 }
 
+static inline bool int128_uge(Int128 a, Int128 b)
+{
+    return ((__uint128_t)a) >= ((__uint128_t)b);
+}
+
 static inline bool int128_lt(Int128 a, Int128 b)
 {
     return a < b;
 }
 
+static inline bool int128_ult(Int128 a, Int128 b)
+{
+    return (__uint128_t)a < (__uint128_t)b;
+}
+
 static inline bool int128_le(Int128 a, Int128 b)
 {
     return a <= b;
@@ -140,29 +215,69 @@ static inline void int128_subfrom(Int128 *a, Int128 b)
 
 static inline Int128 bswap128(Int128 a)
 {
+#if __has_builtin(__builtin_bswap128)
+    return __builtin_bswap128(a);
+#else
     return int128_make128(bswap64(int128_gethi(a)), bswap64(int128_getlo(a)));
+#endif
+}
+
+static inline int clz128(Int128 a)
+{
+    if (a >> 64) {
+        return int128_clz64(a >> 64);
+    } else {
+        return a ? int128_clz64((uint64_t)a) + 64 : 128;
+    }
+}
+
+static inline Int128 int128_divu(Int128 a, Int128 b)
+{
+    return (__uint128_t)a / (__uint128_t)b;
+}
+
+static inline Int128 int128_remu(Int128 a, Int128 b)
+{
+    return (__uint128_t)a % (__uint128_t)b;
+}
+
+static inline Int128 int128_divs(Int128 a, Int128 b)
+{
+    return a / b;
+}
+
+static inline Int128 int128_rems(Int128 a, Int128 b)
+{
+    return a % b;
 }
 
 #else /* !CONFIG_INT128 */
 
 typedef struct Int128 Int128;
-#if !defined(__clang__)
-typedef Int128 __int128_t;
-#endif
 
 struct Int128 {
+#if HOST_BIG_ENDIAN
+    int64_t hi;
+    uint64_t lo;
+#else
     uint64_t lo;
     int64_t hi;
+#endif
 };
 
 static inline Int128 int128_make64(uint64_t a)
 {
-    return (Int128) { a, 0 };
+    return (Int128) { .lo = a, .hi = 0 };
+}
+
+static inline Int128 int128_makes64(int64_t a)
+{
+    return (Int128) { .lo = a, .hi = a >> 63 };
 }
 
 static inline Int128 int128_make128(uint64_t lo, uint64_t hi)
 {
-    return (Int128) { lo, hi };
+    return (Int128) { .lo = lo, .hi = hi };
 }
 
 static inline uint64_t int128_get64(Int128 a)
@@ -193,22 +308,38 @@ static inline Int128 int128_one(void)
 
 static inline Int128 int128_2_64(void)
 {
-    return (Int128) { 0, 1 };
+    return int128_make128(0, 1);
 }
 
 static inline Int128 int128_exts64(int64_t a)
 {
-    return (Int128) { .lo = a, .hi = (a < 0) ? -1 : 0 };
+    return int128_make128(a, (a < 0) ? -1 : 0);
+}
+
+static inline Int128 int128_not(Int128 a)
+{
+    return int128_make128(~a.lo, ~a.hi);
 }
 
 static inline Int128 int128_and(Int128 a, Int128 b)
 {
-    return (Int128) { a.lo & b.lo, a.hi & b.hi };
+    return int128_make128(a.lo & b.lo, a.hi & b.hi);
+}
+
+static inline Int128 int128_or(Int128 a, Int128 b)
+{
+    return int128_make128(a.lo | b.lo, a.hi | b.hi);
+}
+
+static inline Int128 int128_xor(Int128 a, Int128 b)
+{
+    return int128_make128(a.lo ^ b.lo, a.hi ^ b.hi);
 }
 
 static inline Int128 int128_rshift(Int128 a, int n)
 {
     int64_t h;
+
     if (!n) {
         return a;
     }
@@ -220,27 +351,44 @@ static inline Int128 int128_rshift(Int128 a, int n)
     }
 }
 
+static inline Int128 int128_urshift(Int128 a, int n)
+{
+    uint64_t h = a.hi;
+
+    if (!n) {
+        return a;
+    }
+    h = h >> (n & 63);
+    if (n >= 64) {
+        return int128_make64(h);
+    } else {
+        return int128_make128((a.lo >> n) | ((uint64_t)a.hi << (64 - n)), h);
+    }
+}
+
+static inline Int128 int128_lshift(Int128 a, int n)
+{
+    uint64_t l = a.lo << (n & 63);
+
+    if (n >= 64) {
+        return int128_make128(0, l);
+    } else if (n > 0) {
+        return int128_make128(l, (a.hi << n) | (a.lo >> (64 - n)));
+    }
+    return a;
+}
+
 static inline Int128 int128_add(Int128 a, Int128 b)
 {
     uint64_t lo = a.lo + b.lo;
 
-    /* a.lo <= a.lo + b.lo < a.lo + k (k is the base, 2^64).  Hence,
-     * a.lo + b.lo >= k implies 0 <= lo = a.lo + b.lo - k < a.lo.
-     * Similarly, a.lo + b.lo < k implies a.lo <= lo = a.lo + b.lo < k.
-     *
-     * So the carry is lo < a.lo.
-     */
     return int128_make128(lo, (uint64_t)a.hi + b.hi + (lo < a.lo));
 }
 
 static inline Int128 int128_neg(Int128 a)
 {
-#ifdef _MSC_VER
-    uint64_t lo = a.lo;
-    lo = 0 - lo;
-#else
-    uint64_t lo = (uint64_t)(-a.lo);
-#endif
+    uint64_t lo = -a.lo;
+
     return int128_make128(lo, ~(uint64_t)a.hi + !lo);
 }
 
@@ -269,11 +417,21 @@ static inline bool int128_ge(Int128 a, Int128 b)
     return a.hi > b.hi || (a.hi == b.hi && a.lo >= b.lo);
 }
 
+static inline bool int128_uge(Int128 a, Int128 b)
+{
+    return (uint64_t)a.hi > (uint64_t)b.hi || (a.hi == b.hi && a.lo >= b.lo);
+}
+
 static inline bool int128_lt(Int128 a, Int128 b)
 {
     return !int128_ge(a, b);
 }
 
+static inline bool int128_ult(Int128 a, Int128 b)
+{
+    return !int128_uge(a, b);
+}
+
 static inline bool int128_le(Int128 a, Int128 b)
 {
     return int128_ge(b, a);
@@ -311,8 +469,32 @@ static inline void int128_subfrom(Int128 *a, Int128 b)
 
 static inline Int128 bswap128(Int128 a)
 {
-    return int128_make128(bswap64(int128_gethi(a)), bswap64(int128_getlo(a)));
+    return int128_make128(bswap64(a.hi), bswap64(a.lo));
+}
+
+static inline int clz128(Int128 a)
+{
+    if (a.hi) {
+        return int128_clz64(a.hi);
+    } else {
+        return a.lo ? int128_clz64(a.lo) + 64 : 128;
+    }
 }
 
+Int128 int128_divu(Int128, Int128);
+Int128 int128_remu(Int128, Int128);
+Int128 int128_divs(Int128, Int128);
+Int128 int128_rems(Int128, Int128);
+
 #endif /* CONFIG_INT128 */
+
+static inline void bswap128s(Int128 *s)
+{
+    *s = bswap128(*s);
+}
+
+#define UINT128_MAX int128_make128(~0ULL, ~0ULL)
+#define INT128_MAX int128_make128(UINT64_MAX, INT64_MAX)
+#define INT128_MIN int128_make128(0, INT64_MIN)
+
 #endif /* INT128_H */
diff --git a/qemu/include/qemu/log.h b/qemu/include/qemu/log.h
index 10c298cbf6..6c3dc265c9 100644
--- a/qemu/include/qemu/log.h
+++ b/qemu/include/qemu/log.h
@@ -1,6 +1,8 @@
 #ifndef QEMU_LOG_H
 #define QEMU_LOG_H
 
+#include <stdio.h>
+
 #define CPU_LOG_TB_OUT_ASM (1 << 0)
 #define CPU_LOG_TB_IN_ASM  (1 << 1)
 #define CPU_LOG_TB_OP      (1 << 2)
@@ -234,4 +236,23 @@ static inline bool is_log_level_active(uint32_t level)
 #define qemu_log_mask(mask, fmt, ...) \
     LOG_MESSAGE(mask, fmt, ## __VA_ARGS__)
 
+static inline bool qemu_loglevel_mask(int mask)
+{
+    return is_log_level_active(mask);
+}
+
+static inline FILE *qemu_log_trylock(void)
+{
+#ifdef UNICORN_LOGGING
+    return stdout;
+#else
+    return NULL;
+#endif
+}
+
+static inline void qemu_log_unlock(FILE *fd)
+{
+    (void)fd;
+}
+
 #endif /* QEMU_LOG_H */
diff --git a/qemu/include/qemu/queue.h b/qemu/include/qemu/queue.h
index f0108346ec..15e1bb00d5 100644
--- a/qemu/include/qemu/queue.h
+++ b/qemu/include/qemu/queue.h
@@ -218,12 +218,12 @@ struct {                                                                \
         typeof(elm) save_sle_next;                                           \
         do {                                                                 \
             save_sle_next = (elm)->field.sle_next = (head)->slh_first;       \
-        } while (atomic_cmpxchg(&(head)->slh_first, save_sle_next, (elm)) != \
+        } while (qatomic_cmpxchg(&(head)->slh_first, save_sle_next, (elm)) !=\
                  save_sle_next);                                             \
 } while (/*CONSTCOND*/0)
 
 #define QSLIST_MOVE_ATOMIC(dest, src) do {                               \
-        (dest)->slh_first = atomic_xchg(&(src)->slh_first, NULL);        \
+        (dest)->slh_first = qatomic_xchg(&(src)->slh_first, NULL);       \
 } while (/*CONSTCOND*/0)
 
 #define QSLIST_REMOVE_HEAD(head, field) do {                             \
@@ -376,7 +376,8 @@ struct {                                                                \
 /*
  * Simple queue access methods.
  */
-#define QSIMPLEQ_EMPTY_ATOMIC(head) (atomic_read(&((head)->sqh_first)) == NULL)
+#define QSIMPLEQ_EMPTY_ATOMIC(head) \
+    (qatomic_read(&((head)->sqh_first)) == NULL)
 #define QSIMPLEQ_EMPTY(head)        ((head)->sqh_first == NULL)
 #define QSIMPLEQ_FIRST(head)        ((head)->sqh_first)
 #define QSIMPLEQ_NEXT(elm, field)   ((elm)->field.sqe_next)
diff --git a/qemu/include/qemu/range.h b/qemu/include/qemu/range.h
index f62b363e0d..7e2b1cc447 100644
--- a/qemu/include/qemu/range.h
+++ b/qemu/include/qemu/range.h
@@ -114,8 +114,8 @@ static inline uint64_t range_upb(Range *range)
  * @size may be 0. If the range would overflow, returns -ERANGE, otherwise
  * 0.
  */
-static inline int QEMU_WARN_UNUSED_RESULT range_init(Range *range, uint64_t lob,
-                                                     uint64_t size)
+G_GNUC_WARN_UNUSED_RESULT
+static inline int range_init(Range *range, uint64_t lob, uint64_t size)
 {
     if (lob + size < lob) {
         return -ERANGE;
diff --git a/qemu/include/qemu/xxhash.h b/qemu/include/qemu/xxhash.h
index f26fa94729..abf6e643a4 100644
--- a/qemu/include/qemu/xxhash.h
+++ b/qemu/include/qemu/xxhash.h
@@ -126,4 +126,102 @@ static inline uint32_t qemu_xxhash6(uint64_t ab, uint64_t cd, uint32_t e,
     return qemu_xxhash7(ab, cd, e, f, 0);
 }
 
+/*
+ * Component parts of the XXH64 algorithm from
+ * https://github.com/Cyan4973/xxHash/blob/v0.8.0/xxhash.h
+ *
+ * The complete algorithm looks like
+ *
+ *  i = 0;
+ *  if (len >= 32) {
+ *      v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+ *      v2 = seed + XXH_PRIME64_2;
+ *      v3 = seed + 0;
+ *      v4 = seed - XXH_PRIME64_1;
+ *      do {
+ *          v1 = XXH64_round(v1, get64bits(input + i));
+ *          v2 = XXH64_round(v2, get64bits(input + i + 8));
+ *          v3 = XXH64_round(v3, get64bits(input + i + 16));
+ *          v4 = XXH64_round(v4, get64bits(input + i + 24));
+ *      } while ((i += 32) <= len);
+ *      h64 = XXH64_mergerounds(v1, v2, v3, v4);
+ *  } else {
+ *      h64 = seed + XXH_PRIME64_5;
+ *  }
+ *  h64 += len;
+ *
+ *  for (; i + 8 <= len; i += 8) {
+ *      h64 ^= XXH64_round(0, get64bits(input + i));
+ *      h64 = rol64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4;
+ *  }
+ *  for (; i + 4 <= len; i += 4) {
+ *      h64 ^= get32bits(input + i) * PRIME64_1;
+ *      h64 = rol64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
+ *  }
+ *  for (; i < len; i += 1) {
+ *      h64 ^= get8bits(input + i) * XXH_PRIME64_5;
+ *      h64 = rol64(h64, 11) * XXH_PRIME64_1;
+ *  }
+ *
+ *  return XXH64_avalanche(h64)
+ *
+ * Exposing the pieces instead allows for simplified usage when
+ * the length is a known constant and the inputs are in registers.
+ */
+#define XXH_PRIME64_1   0x9E3779B185EBCA87ULL
+#define XXH_PRIME64_2   0xC2B2AE3D27D4EB4FULL
+#define XXH_PRIME64_3   0x165667B19E3779F9ULL
+#define XXH_PRIME64_4   0x85EBCA77C2B2AE63ULL
+#define XXH_PRIME64_5   0x27D4EB2F165667C5ULL
+
+static inline uint64_t XXH64_round(uint64_t acc, uint64_t input)
+{
+    return rol64(acc + input * XXH_PRIME64_2, 31) * XXH_PRIME64_1;
+}
+
+static inline uint64_t XXH64_mergeround(uint64_t acc, uint64_t val)
+{
+    return (acc ^ XXH64_round(0, val)) * XXH_PRIME64_1 + XXH_PRIME64_4;
+}
+
+static inline uint64_t XXH64_mergerounds(uint64_t v1, uint64_t v2,
+                                         uint64_t v3, uint64_t v4)
+{
+    uint64_t h64;
+
+    h64 = rol64(v1, 1) + rol64(v2, 7) + rol64(v3, 12) + rol64(v4, 18);
+    h64 = XXH64_mergeround(h64, v1);
+    h64 = XXH64_mergeround(h64, v2);
+    h64 = XXH64_mergeround(h64, v3);
+    h64 = XXH64_mergeround(h64, v4);
+
+    return h64;
+}
+
+static inline uint64_t XXH64_avalanche(uint64_t h64)
+{
+    h64 ^= h64 >> 33;
+    h64 *= XXH_PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= XXH_PRIME64_3;
+    h64 ^= h64 >> 32;
+    return h64;
+}
+
+static inline uint64_t qemu_xxhash64_4(uint64_t a, uint64_t b,
+                                       uint64_t c, uint64_t d)
+{
+    uint64_t v1 = QEMU_XXHASH_SEED + XXH_PRIME64_1 + XXH_PRIME64_2;
+    uint64_t v2 = QEMU_XXHASH_SEED + XXH_PRIME64_2;
+    uint64_t v3 = QEMU_XXHASH_SEED + 0;
+    uint64_t v4 = QEMU_XXHASH_SEED - XXH_PRIME64_1;
+
+    v1 = XXH64_round(v1, a);
+    v2 = XXH64_round(v2, b);
+    v3 = XXH64_round(v3, c);
+    v4 = XXH64_round(v4, d);
+
+    return XXH64_avalanche(XXH64_mergerounds(v1, v2, v3, v4));
+}
+
 #endif /* QEMU_XXHASH_H */
diff --git a/qemu/include/sysemu/memory_mapping.h b/qemu/include/sysemu/memory_mapping.h
index 4fec27364e..e2fa8f4b22 100644
--- a/qemu/include/sysemu/memory_mapping.h
+++ b/qemu/include/sysemu/memory_mapping.h
@@ -15,13 +15,12 @@
 #define MEMORY_MAPPING_H
 
 #include "qemu/queue.h"
-#include "exec/cpu-defs.h"
-#include "exec/memory.h"
+#include "exec/cpu-common.h"
 
     /* The physical and virtual address in the memory mapping are contiguous. */
 typedef struct MemoryMapping {
     hwaddr phys_addr;
-    target_ulong virt_addr;
+    vaddr virt_addr;
     ram_addr_t length;
     QTAILQ_ENTRY(MemoryMapping) next;
 } MemoryMapping;
diff --git a/qemu/include/tcg/tcg-gvec-desc.h b/qemu/include/tcg/tcg-gvec-desc.h
index 0224ac3e78..704bd86454 100644
--- a/qemu/include/tcg/tcg-gvec-desc.h
+++ b/qemu/include/tcg/tcg-gvec-desc.h
@@ -20,29 +20,41 @@
 #ifndef TCG_TCG_GVEC_DESC_H
 #define TCG_TCG_GVEC_DESC_H
 
-/* ??? These bit widths are set for ARM SVE, maxing out at 256 byte vectors. */
-#define SIMD_OPRSZ_SHIFT   0
-#define SIMD_OPRSZ_BITS    5
+/*
+ * This configuration allows MAXSZ to represent 2048 bytes, and
+ * OPRSZ to match MAXSZ, or represent the smaller values 8, 16, or 32.
+ *
+ * Encode this with:
+ *   0, 1, 3 -> 8, 16, 32
+ *   2       -> maxsz
+ *
+ * This steals the input that would otherwise map to 24 to match maxsz.
+ */
+#define SIMD_MAXSZ_SHIFT   0
+#define SIMD_MAXSZ_BITS    8
 
-#define SIMD_MAXSZ_SHIFT   (SIMD_OPRSZ_SHIFT + SIMD_OPRSZ_BITS)
-#define SIMD_MAXSZ_BITS    5
+#define SIMD_OPRSZ_SHIFT   (SIMD_MAXSZ_SHIFT + SIMD_MAXSZ_BITS)
+#define SIMD_OPRSZ_BITS    2
 
-#define SIMD_DATA_SHIFT    (SIMD_MAXSZ_SHIFT + SIMD_MAXSZ_BITS)
+#define SIMD_DATA_SHIFT    (SIMD_OPRSZ_SHIFT + SIMD_OPRSZ_BITS)
 #define SIMD_DATA_BITS     (32 - SIMD_DATA_SHIFT)
 
 /* Create a descriptor from components.  */
 uint32_t simd_desc(uint32_t oprsz, uint32_t maxsz, int32_t data);
 
-/* Extract the operation size from a descriptor.  */
-static inline intptr_t simd_oprsz(uint32_t desc)
+/* Extract the max vector size from a descriptor.  */
+static inline intptr_t simd_maxsz(uint32_t desc)
 {
-    return (extract32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS) + 1) * 8;
+    return extract32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS) * 8 + 8;
 }
 
-/* Extract the max vector size from a descriptor.  */
-static inline intptr_t simd_maxsz(uint32_t desc)
+/* Extract the operation size from a descriptor.  */
+static inline intptr_t simd_oprsz(uint32_t desc)
 {
-    return (extract32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS) + 1) * 8;
+    uint32_t f = extract32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS);
+    intptr_t o = f * 8 + 8;
+    intptr_t m = simd_maxsz(desc);
+    return f == 2 ? m : o;
 }
 
 /* Extract the operation-specific data from a descriptor.  */
diff --git a/qemu/include/tcg/tcg-op-gvec.h b/qemu/include/tcg/tcg-op-gvec.h
index dd414fc768..22c72e4f1f 100644
--- a/qemu/include/tcg/tcg-op-gvec.h
+++ b/qemu/include/tcg/tcg-op-gvec.h
@@ -322,6 +322,8 @@ void tcg_gen_gvec_dup8i(TCGContext *tcg_ctx, uint32_t dofs, uint32_t s, uint32_t
 void tcg_gen_gvec_dup16i(TCGContext *tcg_ctx, uint32_t dofs, uint32_t s, uint32_t m, uint16_t x);
 void tcg_gen_gvec_dup32i(TCGContext *tcg_ctx, uint32_t dofs, uint32_t s, uint32_t m, uint32_t x);
 void tcg_gen_gvec_dup64i(TCGContext *tcg_ctx, uint32_t dofs, uint32_t s, uint32_t m, uint64_t x);
+void tcg_gen_gvec_dup_imm(TCGContext *tcg_ctx, unsigned vece, uint32_t dofs,
+                          uint32_t s, uint32_t m, uint64_t x);
 
 void tcg_gen_gvec_shli(TCGContext *tcg_ctx, unsigned vece, uint32_t dofs, uint32_t aofs,
                        int64_t shift, uint32_t oprsz, uint32_t maxsz);
diff --git a/qemu/include/tcg/tcg-op.h b/qemu/include/tcg/tcg-op.h
index 93026d1d51..f3e10ac069 100644
--- a/qemu/include/tcg/tcg-op.h
+++ b/qemu/include/tcg/tcg-op.h
@@ -978,7 +978,7 @@ static inline void tcg_gen_qemu_ld32s(TCGContext *tcg_ctx, TCGv ret, TCGv addr,
 
 static inline void tcg_gen_qemu_ld64(TCGContext *tcg_ctx, TCGv_i64 ret, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_ld_i64(tcg_ctx, ret, addr, mem_index, MO_TEQ);
+    tcg_gen_qemu_ld_i64(tcg_ctx, ret, addr, mem_index, MO_TEUQ);
 }
 
 static inline void tcg_gen_qemu_st8(TCGContext *tcg_ctx, TCGv arg, TCGv addr, int mem_index)
@@ -998,7 +998,7 @@ static inline void tcg_gen_qemu_st32(TCGContext *tcg_ctx, TCGv arg, TCGv addr, i
 
 static inline void tcg_gen_qemu_st64(TCGContext *tcg_ctx, TCGv_i64 arg, TCGv addr, int mem_index)
 {
-    tcg_gen_qemu_st_i64(tcg_ctx, arg, addr, mem_index, MO_TEQ);
+    tcg_gen_qemu_st_i64(tcg_ctx, arg, addr, mem_index, MO_TEUQ);
 }
 
 void tcg_gen_atomic_cmpxchg_i32(TCGContext *tcg_ctx, TCGv_i32, TCGv, TCGv_i32, TCGv_i32,
diff --git a/qemu/include/tcg/tcg.h b/qemu/include/tcg/tcg.h
index bbeb4b9afd..b4dfd34a02 100644
--- a/qemu/include/tcg/tcg.h
+++ b/qemu/include/tcg/tcg.h
@@ -708,6 +708,7 @@ struct TCGContext {
 
     /* qemu/target/i386/translate.c: global register indexes */
     TCGv cpu_cc_dst, cpu_cc_src, cpu_cc_src2;
+    TCGv cpu_eip;
     TCGv_i32 cpu_cc_op;
     TCGv cpu_regs[56]; // 16 GRP for x64
     /* only x86 need cpu_seg_base[]. */
@@ -1171,6 +1172,7 @@ TCGOp *tcg_emit_op(TCGContext *tcg_ctx, TCGOpcode opc);
 void tcg_op_remove(TCGContext *s, TCGOp *op);
 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc);
 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc);
+void tcg_remove_ops_after(TCGContext *tcg_ctx, TCGOp *op);
 
 void tcg_optimize(TCGContext *s);
 
@@ -1182,6 +1184,13 @@ TCGv_vec tcg_const_zeros_vec(TCGContext *tcg_ctx, TCGType);
 TCGv_vec tcg_const_ones_vec(TCGContext *tcg_ctx, TCGType);
 TCGv_vec tcg_const_zeros_vec_matching(TCGContext *tcg_ctx, TCGv_vec);
 TCGv_vec tcg_const_ones_vec_matching(TCGContext *tcg_ctx, TCGv_vec);
+TCGv_vec tcg_constant_vec_matching(TCGContext *tcg_ctx, TCGv_vec match,
+                                   unsigned vece, int64_t val);
+
+#define tcg_constant_i32(tcg_ctx, val) tcg_const_i32(tcg_ctx, val)
+#define tcg_constant_i64(tcg_ctx, val) tcg_const_i64(tcg_ctx, val)
+#define tcg_constant_tl(tcg_ctx, val) tcg_const_tl(tcg_ctx, val)
+#define tcg_constant_ptr(tcg_ctx, val) tcg_const_ptr(tcg_ctx, val)
 
 #if UINTPTR_MAX == UINT32_MAX
 # define tcg_const_ptr(tcg_ctx, x)        ((TCGv_ptr)tcg_const_i32(tcg_ctx, (intptr_t)(x)))
diff --git a/qemu/libdecnumber/decContext.c b/qemu/libdecnumber/decContext.c
index 7d97a65ac5..1956edf0a7 100644
--- a/qemu/libdecnumber/decContext.c
+++ b/qemu/libdecnumber/decContext.c
@@ -53,12 +53,13 @@ static	const  Flag *mfctop=(Flag *)&mfcone; /* -> top byte */
 const uByte DECSTICKYTAB[10]={1,1,2,3,4,6,6,7,8,9}; /* used if sticky */
 
 /* ------------------------------------------------------------------ */
-/* Powers of ten (powers[n]==10**n, 0<=n<=9)			      */
+/* Powers of ten (powers[n]==10**n, 0<=n<=19)                         */
 /* ------------------------------------------------------------------ */
-const uLong DECPOWERS[19] = {1, 10, 100, 1000, 10000, 100000, 1000000,
+const uLong DECPOWERS[20] = {1, 10, 100, 1000, 10000, 100000, 1000000,
   10000000, 100000000, 1000000000, 10000000000ULL, 100000000000ULL,
   1000000000000ULL, 10000000000000ULL, 100000000000000ULL, 1000000000000000ULL,
-  10000000000000000ULL, 100000000000000000ULL, 1000000000000000000ULL, };
+  10000000000000000ULL, 100000000000000000ULL, 1000000000000000000ULL,
+  10000000000000000000ULL,};
 
 /* ------------------------------------------------------------------ */
 /* decContextClearStatus -- clear bits in current status	      */
diff --git a/qemu/libdecnumber/decNumber.c b/qemu/libdecnumber/decNumber.c
index 1e84e9b8d3..02d0faa659 100644
--- a/qemu/libdecnumber/decNumber.c
+++ b/qemu/libdecnumber/decNumber.c
@@ -167,6 +167,7 @@
 /* ------------------------------------------------------------------ */
 
 #include "qemu/osdep.h"
+#include "qemu/host-utils.h"
 #include "libdecnumber/dconfig.h"
 #include "libdecnumber/decNumber.h"
 #include "libdecnumber/decNumberLocal.h"
@@ -263,6 +264,7 @@ static decNumber * decTrim(decNumber *, decContext *, Flag, Int *);
 static Int	   decUnitAddSub(const Unit *, Int, const Unit *, Int, Int,
 			      Unit *, Int);
 static Int	   decUnitCompare(const Unit *, Int, const Unit *, Int, Int);
+static bool        mulUInt128ByPowOf10(uLong *, uLong *, uInt);
 
 #if !DECSUBSET
 /* decFinish == decFinalize when no subset arithmetic needed */
@@ -466,6 +468,41 @@ decNumber *decNumberFromUInt64(decNumber *dn, uint64_t uin)
     return dn;
 } /* decNumberFromUInt64 */
 
+decNumber *decNumberFromInt128(decNumber *dn, uint64_t lo, int64_t hi)
+{
+    uint64_t unsig_hi = hi;
+    if (hi < 0) {
+        if (lo == 0) {
+            unsig_hi = -unsig_hi;
+        } else {
+            unsig_hi = ~unsig_hi;
+            lo = -lo;
+        }
+    }
+
+    decNumberFromUInt128(dn, lo, unsig_hi);
+    if (hi < 0) {
+        dn->bits = DECNEG;        /* sign needed */
+    }
+    return dn;
+} /* decNumberFromInt128 */
+
+decNumber *decNumberFromUInt128(decNumber *dn, uint64_t lo, uint64_t hi)
+{
+    uint64_t rem;
+    Unit *up;                             /* work pointer */
+    decNumberZero(dn);                    /* clean */
+    if (lo == 0 && hi == 0) {
+        return dn;                /* [or decGetDigits bad call] */
+    }
+    for (up = dn->lsu; hi > 0 || lo > 0; up++) {
+        rem = divu128(&lo, &hi, DECDPUNMAX + 1);
+        *up = (Unit)rem;
+    }
+    dn->digits = decGetDigits(dn->lsu, up - dn->lsu);
+    return dn;
+} /* decNumberFromUInt128 */
+
 /* ------------------------------------------------------------------ */
 /* to-int64 -- conversion to int64                                    */
 /*                                                                    */
@@ -510,6 +547,68 @@ int64_t decNumberIntegralToInt64(const decNumber *dn, decContext *set)
     return 0;
 } /* decNumberIntegralToInt64 */
 
+/* ------------------------------------------------------------------ */
+/* decNumberIntegralToInt128 -- conversion to int128                  */
+/*                                                                    */
+/*  dn is the decNumber to convert.  dn is assumed to have been       */
+/*    rounded to a floating point integer value.                      */
+/*  set is the context for reporting errors                           */
+/*  returns the converted decNumber via plow and phigh                */
+/*                                                                    */
+/* Invalid is set if the decNumber is a NaN, Infinite or is out of    */
+/* range for a signed 128 bit integer.                                */
+/* ------------------------------------------------------------------ */
+
+void decNumberIntegralToInt128(const decNumber *dn, decContext *set,
+        uint64_t *plow, uint64_t *phigh)
+{
+    int d;        /* work */
+    const Unit *up;   /* .. */
+    uint64_t lo = 0, hi = 0;
+
+    if (decNumberIsSpecial(dn) || (dn->exponent < 0) ||
+       (dn->digits + dn->exponent > 39)) {
+        goto Invalid;
+    }
+
+    up = dn->lsu;     /* -> lsu */
+
+    for (d = (dn->digits - 1) / DECDPUN; d >= 0; d--) {
+        if (mulu128(&lo, &hi, DECDPUNMAX + 1)) {
+            /* overflow */
+            goto Invalid;
+        }
+        if (uadd64_overflow(lo, up[d], &lo)) {
+            if (uadd64_overflow(hi, 1, &hi)) {
+                /* overflow */
+                goto Invalid;
+            }
+        }
+    }
+
+    if (mulUInt128ByPowOf10(&lo, &hi, dn->exponent)) {
+        /* overflow */
+        goto Invalid;
+    }
+
+    if (decNumberIsNegative(dn)) {
+        if (lo == 0) {
+            *phigh = -hi;
+            *plow = 0;
+        } else {
+            *phigh = ~hi;
+            *plow = -lo;
+        }
+    } else {
+        *plow = lo;
+        *phigh = hi;
+    }
+
+    return;
+
+Invalid:
+    decContextSetStatus(set, DEC_Invalid_operation);
+} /* decNumberIntegralToInt128 */
 
 /* ------------------------------------------------------------------ */
 /* to-scientific-string -- conversion to numeric string		      */
@@ -5630,7 +5729,7 @@ static const uShort LNnn[90] = {
 /*    would certainly save at least one if it were made ten times     */
 /*    bigger, too (for truncated fractions 0.100 through 0.999).      */
 /*    However, for most practical evaluations, at least four or five  */
-/*    iterations will be neede -- so this would only speed up by      */
+/*    iterations will be needed -- so this would only speed up by      */
 /*    20-25% and that probably does not justify increasing the table  */
 /*    size.							      */
 /*								      */
@@ -7853,6 +7952,38 @@ static Int decGetDigits(Unit *uar, Int len) {
   return digits;
   } /* decGetDigits */
 
+/* ------------------------------------------------------------------ */
+/* mulUInt128ByPowOf10 -- multiply a 128-bit unsigned integer by a    */
+/* power of 10.                                                       */
+/*                                                                    */
+/*   The 128-bit factor composed of plow and phigh is multiplied      */
+/*   by 10^exp.                                                       */
+/*                                                                    */
+/*   plow   pointer to the low 64 bits of the first factor            */
+/*   phigh  pointer to the high 64 bits of the first factor           */
+/*   exp    the exponent of the power of 10 of the second factor      */
+/*                                                                    */
+/* If the result fits in 128 bits, returns false and the              */
+/* multiplication result through plow and phigh.                      */
+/* Otherwise, returns true.                                           */
+/* ------------------------------------------------------------------ */
+static bool mulUInt128ByPowOf10(uLong *plow, uLong *phigh, uInt pow10)
+{
+    while (pow10 >= ARRAY_SIZE(powers)) {
+        if (mulu128(plow, phigh, powers[ARRAY_SIZE(powers) - 1])) {
+            /* Overflow */
+            return true;
+        }
+        pow10 -= ARRAY_SIZE(powers) - 1;
+    }
+
+    if (pow10 > 0) {
+        return mulu128(plow, phigh, powers[pow10]);
+    } else {
+        return false;
+    }
+}
+
 #if DECTRACE | DECCHECK
 /* ------------------------------------------------------------------ */
 /* decNumberShow -- display a number [debug aid]		      */
diff --git a/qemu/libdecnumber/dpd/decimal64.c b/qemu/libdecnumber/dpd/decimal64.c
index 4816176410..290dbe8177 100644
--- a/qemu/libdecnumber/dpd/decimal64.c
+++ b/qemu/libdecnumber/dpd/decimal64.c
@@ -617,7 +617,6 @@ static const uInt multies[]={131073, 26215, 5243, 1049, 210};
 #endif
 void decDigitsToDPD(const decNumber *dn, uInt *targ, Int shift) {
   Int  cut;		      /* work */
-  Int  n;		      /* output bunch counter */
   Int  digits=dn->digits;     /* digit countdown */
   uInt dpd;		      /* densely packed decimal value */
   uInt bin;		      /* binary value 0-999 */
@@ -676,7 +675,7 @@ void decDigitsToDPD(const decNumber *dn, uInt *targ, Int shift) {
     bin=0;			   /* [keep compiler quiet] */
   #endif
 
-  for(n=0; digits>0; n++) {	   /* each output bunch */
+  while (digits > 0) {             /* each output bunch */
     #if DECDPUN==3		   /* fast path, 3-at-a-time */
       bin=*inu;			   /* 3 digits ready for convert */
       digits-=3;		   /* [may go negative] */
diff --git a/qemu/m68k.h b/qemu/m68k.h
index 67f94364d2..76072a5bfb 100644
--- a/qemu/m68k.h
+++ b/qemu/m68k.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_m68k
 #define float32_squash_input_denormal float32_squash_input_denormal_m68k
 #define float64_squash_input_denormal float64_squash_input_denormal_m68k
+#define bfloat16_add bfloat16_add_m68k
+#define bfloat16_compare bfloat16_compare_m68k
+#define bfloat16_compare_quiet bfloat16_compare_quiet_m68k
+#define bfloat16_default_nan bfloat16_default_nan_m68k
+#define bfloat16_div bfloat16_div_m68k
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_m68k
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_m68k
+#define bfloat16_max bfloat16_max_m68k
+#define bfloat16_maximum_number bfloat16_maximum_number_m68k
+#define bfloat16_maxnum bfloat16_maxnum_m68k
+#define bfloat16_maxnummag bfloat16_maxnummag_m68k
+#define bfloat16_min bfloat16_min_m68k
+#define bfloat16_minimum_number bfloat16_minimum_number_m68k
+#define bfloat16_minnum bfloat16_minnum_m68k
+#define bfloat16_minnummag bfloat16_minnummag_m68k
+#define bfloat16_mul bfloat16_mul_m68k
+#define bfloat16_muladd bfloat16_muladd_m68k
+#define bfloat16_round_to_int bfloat16_round_to_int_m68k
+#define bfloat16_scalbn bfloat16_scalbn_m68k
+#define bfloat16_silence_nan bfloat16_silence_nan_m68k
+#define bfloat16_sqrt bfloat16_sqrt_m68k
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_m68k
+#define bfloat16_sub bfloat16_sub_m68k
+#define bfloat16_to_float32 bfloat16_to_float32_m68k
+#define bfloat16_to_float64 bfloat16_to_float64_m68k
+#define bfloat16_to_int16 bfloat16_to_int16_m68k
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_m68k
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_m68k
+#define bfloat16_to_int32 bfloat16_to_int32_m68k
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_m68k
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_m68k
+#define bfloat16_to_int64 bfloat16_to_int64_m68k
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_m68k
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_m68k
+#define bfloat16_to_uint16 bfloat16_to_uint16_m68k
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_m68k
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_m68k
+#define bfloat16_to_uint32 bfloat16_to_uint32_m68k
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_m68k
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_m68k
+#define bfloat16_to_uint64 bfloat16_to_uint64_m68k
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_m68k
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_m68k
+#define float128_maximum_number float128_maximum_number_m68k
+#define float128_max float128_max_m68k
+#define float128_maxnum float128_maxnum_m68k
+#define float128_maxnummag float128_maxnummag_m68k
+#define float128_min float128_min_m68k
+#define float128_minimum_number float128_minimum_number_m68k
+#define float128_minnum float128_minnum_m68k
+#define float128_minnummag float128_minnummag_m68k
+#define float128_muladd float128_muladd_m68k
+#define float128_to_int128 float128_to_int128_m68k
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_m68k
+#define float128_to_uint128 float128_to_uint128_m68k
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_m68k
+#define float16_maximum_number float16_maximum_number_m68k
+#define float16_minimum_number float16_minimum_number_m68k
+#define float16_to_int8 float16_to_int8_m68k
+#define float16_to_int8_scalbn float16_to_int8_scalbn_m68k
+#define float16_to_uint8 float16_to_uint8_m68k
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_m68k
+#define float32_maximum_number float32_maximum_number_m68k
+#define float32_minimum_number float32_minimum_number_m68k
+#define float32_to_bfloat16 float32_to_bfloat16_m68k
+#define float64_maximum_number float64_maximum_number_m68k
+#define float64_minimum_number float64_minimum_number_m68k
+#define float64_to_bfloat16 float64_to_bfloat16_m68k
+#define float64r32_add float64r32_add_m68k
+#define float64r32_div float64r32_div_m68k
+#define float64r32_mul float64r32_mul_m68k
+#define float64r32_muladd float64r32_muladd_m68k
+#define float64r32_sqrt float64r32_sqrt_m68k
+#define float64r32_sub float64r32_sub_m68k
+#define floatx80_mod floatx80_mod_m68k
+#define floatx80_modrem floatx80_modrem_m68k
+#define int128_to_float128 int128_to_float128_m68k
+#define int16_to_bfloat16 int16_to_bfloat16_m68k
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_m68k
+#define int32_to_bfloat16 int32_to_bfloat16_m68k
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_m68k
+#define int64_to_bfloat16 int64_to_bfloat16_m68k
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_m68k
+#define int8_to_float16 int8_to_float16_m68k
+#define uint128_to_float128 uint128_to_float128_m68k
+#define uint16_to_bfloat16 uint16_to_bfloat16_m68k
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_m68k
+#define uint32_to_bfloat16 uint32_to_bfloat16_m68k
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_m68k
+#define uint64_to_bfloat16 uint64_to_bfloat16_m68k
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_m68k
+#define uint8_to_float16 uint8_to_float16_m68k
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_m68k
 #define roundAndPackFloatx80 roundAndPackFloatx80_m68k
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_m68k
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_m68k
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_m68k
 #define helper_exit_atomic helper_exit_atomic_m68k
+#define helper_memset helper_memset_m68k
+#define helper_emu_stop helper_emu_stop_m68k
+#define tcg_remove_ops_after tcg_remove_ops_after_m68k
+#define tcg_constant_vec_matching tcg_constant_vec_matching_m68k
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_m68k
 #define helper_gvec_add8 helper_gvec_add8_m68k
 #define helper_gvec_add16 helper_gvec_add16_m68k
 #define helper_gvec_add32 helper_gvec_add32_m68k
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_m68k
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_m68k
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_m68k
+#define gen_helper_mve_vctp gen_helper_mve_vctp_m68k
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_m68k
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_m68k
+#define gen_helper_mve_vdup gen_helper_mve_vdup_m68k
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_m68k
+#define gen_helper_mve_vandi gen_helper_mve_vandi_m68k
+#define gen_helper_mve_vorri gen_helper_mve_vorri_m68k
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_m68k
+#define gen_helper_mve_viduph gen_helper_mve_viduph_m68k
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_m68k
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_m68k
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_m68k
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_m68k
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_m68k
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_m68k
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_m68k
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_m68k
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_m68k
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_m68k
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_m68k
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_m68k
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_m68k
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_m68k
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_m68k
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_m68k
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_m68k
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_m68k
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_m68k
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_m68k
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_m68k
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_m68k
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_m68k
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_m68k
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_m68k
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_m68k
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_m68k
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_m68k
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_m68k
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_m68k
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_m68k
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_m68k
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_m68k
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_m68k
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_m68k
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_m68k
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_m68k
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_m68k
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_m68k
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_m68k
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_m68k
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_m68k
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_m68k
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_m68k
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_m68k
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_m68k
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_m68k
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_m68k
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_m68k
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_m68k
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_m68k
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_m68k
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_m68k
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_m68k
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_m68k
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_m68k
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_m68k
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_m68k
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_m68k
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_m68k
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_m68k
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_m68k
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_m68k
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_m68k
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_m68k
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_m68k
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_m68k
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_m68k
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_m68k
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_m68k
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_m68k
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_m68k
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_m68k
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_m68k
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_m68k
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_m68k
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_m68k
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_m68k
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_m68k
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_m68k
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_m68k
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_m68k
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_m68k
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_m68k
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_m68k
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_m68k
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_m68k
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_m68k
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_m68k
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_m68k
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_m68k
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_m68k
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_m68k
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_m68k
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_m68k
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_m68k
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_m68k
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_m68k
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_m68k
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_m68k
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_m68k
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_m68k
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_m68k
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_m68k
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_m68k
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_m68k
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_m68k
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_m68k
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_m68k
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_m68k
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_m68k
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_m68k
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_m68k
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_m68k
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_m68k
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_m68k
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_m68k
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_m68k
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_m68k
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_m68k
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_m68k
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_m68k
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_m68k
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_m68k
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_m68k
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_m68k
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_m68k
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_m68k
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_m68k
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_m68k
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_m68k
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_m68k
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_m68k
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_m68k
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_m68k
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_m68k
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_m68k
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_m68k
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_m68k
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_m68k
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_m68k
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_m68k
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_m68k
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_m68k
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_m68k
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_m68k
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_m68k
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_m68k
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_m68k
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_m68k
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_m68k
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_m68k
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_m68k
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_m68k
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_m68k
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_m68k
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_m68k
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_m68k
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_m68k
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_m68k
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_m68k
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_m68k
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_m68k
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_m68k
+#define gen_helper_mve_vand gen_helper_mve_vand_m68k
+#define gen_helper_mve_vbic gen_helper_mve_vbic_m68k
+#define gen_helper_mve_vorr gen_helper_mve_vorr_m68k
+#define gen_helper_mve_vorn gen_helper_mve_vorn_m68k
+#define gen_helper_mve_veor gen_helper_mve_veor_m68k
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_m68k
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_m68k
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_m68k
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_m68k
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_m68k
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_m68k
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_m68k
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_m68k
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_m68k
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_m68k
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_m68k
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_m68k
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_m68k
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_m68k
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_m68k
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_m68k
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_m68k
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_m68k
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_m68k
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_m68k
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_m68k
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_m68k
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_m68k
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_m68k
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_m68k
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_m68k
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_m68k
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_m68k
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_m68k
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_m68k
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_m68k
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_m68k
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_m68k
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_m68k
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_m68k
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_m68k
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_m68k
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_m68k
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_m68k
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_m68k
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_m68k
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_m68k
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_m68k
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_m68k
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_m68k
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_m68k
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_m68k
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_m68k
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_m68k
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_m68k
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_m68k
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_m68k
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_m68k
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_m68k
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_m68k
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_m68k
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_m68k
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_m68k
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_m68k
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_m68k
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_m68k
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_m68k
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_m68k
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_m68k
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_m68k
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_m68k
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_m68k
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_m68k
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_m68k
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_m68k
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_m68k
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_m68k
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_m68k
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_m68k
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_m68k
+#define gen_helper_mve_vminub gen_helper_mve_vminub_m68k
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_m68k
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_m68k
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_m68k
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_m68k
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_m68k
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_m68k
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_m68k
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_m68k
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_m68k
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_m68k
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_m68k
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_m68k
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_m68k
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_m68k
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_m68k
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_m68k
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_m68k
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_m68k
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_m68k
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_m68k
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_m68k
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_m68k
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_m68k
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_m68k
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_m68k
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_m68k
+#define gen_helper_mve_vadc gen_helper_mve_vadc_m68k
+#define gen_helper_mve_vadci gen_helper_mve_vadci_m68k
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_m68k
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_m68k
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_m68k
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_m68k
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_m68k
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_m68k
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_m68k
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_m68k
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_m68k
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_m68k
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_m68k
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_m68k
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_m68k
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_m68k
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_m68k
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_m68k
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_m68k
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_m68k
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_m68k
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_m68k
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_m68k
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_m68k
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_m68k
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_m68k
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_m68k
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_m68k
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_m68k
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_m68k
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_m68k
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_m68k
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_m68k
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_m68k
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_m68k
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_m68k
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_m68k
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_m68k
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_m68k
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_m68k
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_m68k
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_m68k
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_m68k
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_m68k
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_m68k
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_m68k
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_m68k
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_m68k
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_m68k
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_m68k
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_m68k
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_m68k
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_m68k
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_m68k
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_m68k
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_m68k
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_m68k
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_m68k
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_m68k
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_m68k
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_m68k
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_m68k
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_m68k
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_m68k
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_m68k
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_m68k
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_m68k
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_m68k
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_m68k
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_m68k
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_m68k
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_m68k
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_m68k
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_m68k
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_m68k
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_m68k
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_m68k
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_m68k
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_m68k
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_m68k
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_m68k
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_m68k
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_m68k
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_m68k
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_m68k
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_m68k
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_m68k
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_m68k
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_m68k
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_m68k
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_m68k
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_m68k
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_m68k
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_m68k
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_m68k
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_m68k
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_m68k
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_m68k
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_m68k
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_m68k
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_m68k
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_m68k
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_m68k
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_m68k
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_m68k
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_m68k
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_m68k
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_m68k
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_m68k
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_m68k
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_m68k
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_m68k
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_m68k
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_m68k
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_m68k
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_m68k
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_m68k
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_m68k
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_m68k
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_m68k
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_m68k
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_m68k
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_m68k
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_m68k
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_m68k
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_m68k
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_m68k
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_m68k
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_m68k
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_m68k
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_m68k
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_m68k
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_m68k
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_m68k
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_m68k
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_m68k
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_m68k
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_m68k
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_m68k
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_m68k
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_m68k
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_m68k
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_m68k
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_m68k
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_m68k
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_m68k
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_m68k
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_m68k
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_m68k
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_m68k
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_m68k
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_m68k
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_m68k
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_m68k
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_m68k
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_m68k
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_m68k
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_m68k
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_m68k
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_m68k
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_m68k
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_m68k
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_m68k
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_m68k
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_m68k
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_m68k
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_m68k
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_m68k
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_m68k
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_m68k
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_m68k
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_m68k
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_m68k
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_m68k
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_m68k
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_m68k
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_m68k
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_m68k
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_m68k
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_m68k
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_m68k
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_m68k
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_m68k
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_m68k
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_m68k
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_m68k
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_m68k
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_m68k
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_m68k
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_m68k
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_m68k
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_m68k
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_m68k
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_m68k
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_m68k
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_m68k
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_m68k
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_m68k
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_m68k
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_m68k
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_m68k
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_m68k
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_m68k
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_m68k
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_m68k
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_m68k
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_m68k
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_m68k
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_m68k
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_m68k
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_m68k
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_m68k
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_m68k
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_m68k
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_m68k
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_m68k
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_m68k
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_m68k
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_m68k
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_m68k
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_m68k
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_m68k
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_m68k
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_m68k
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_m68k
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_m68k
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_m68k
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_m68k
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_m68k
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_m68k
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_m68k
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_m68k
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_m68k
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_m68k
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_m68k
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_m68k
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_m68k
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_m68k
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_m68k
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_m68k
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_m68k
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_m68k
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_m68k
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_m68k
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_m68k
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_m68k
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_m68k
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_m68k
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_m68k
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_m68k
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_m68k
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_m68k
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_m68k
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_m68k
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_m68k
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_m68k
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_m68k
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_m68k
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_m68k
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_m68k
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_m68k
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_m68k
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_m68k
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_m68k
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_m68k
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_m68k
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_m68k
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_m68k
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_m68k
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_m68k
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_m68k
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_m68k
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_m68k
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_m68k
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_m68k
+#define gen_helper_mve_ushll gen_helper_mve_ushll_m68k
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_m68k
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_m68k
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_m68k
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_m68k
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_m68k
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_m68k
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_m68k
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_m68k
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_m68k
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_m68k
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_m68k
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_m68k
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_m68k
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_m68k
+#define gen_helper_mve_vslib gen_helper_mve_vslib_m68k
+#define gen_helper_mve_vslih gen_helper_mve_vslih_m68k
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_m68k
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_m68k
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_m68k
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_m68k
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_m68k
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_m68k
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_m68k
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_m68k
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_m68k
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_m68k
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_m68k
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_m68k
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_m68k
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_m68k
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_m68k
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_m68k
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_m68k
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_m68k
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_m68k
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_m68k
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_m68k
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_m68k
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_m68k
+#define gen_helper_mve_vminab gen_helper_mve_vminab_m68k
+#define gen_helper_mve_vminah gen_helper_mve_vminah_m68k
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_m68k
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_m68k
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_m68k
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_m68k
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_m68k
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_m68k
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_m68k
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_m68k
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_m68k
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_m68k
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_m68k
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_m68k
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_m68k
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_m68k
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_m68k
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_m68k
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_m68k
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_m68k
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_m68k
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_m68k
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_m68k
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_m68k
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_m68k
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_m68k
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_m68k
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_m68k
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_m68k
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_m68k
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_m68k
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_m68k
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_m68k
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_m68k
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_m68k
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_m68k
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_m68k
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_m68k
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_m68k
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_m68k
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_m68k
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_m68k
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_m68k
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_m68k
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_m68k
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_m68k
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_m68k
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_m68k
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_m68k
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_m68k
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_m68k
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_m68k
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_m68k
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_m68k
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_m68k
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_m68k
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_m68k
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_m68k
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_m68k
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_m68k
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_m68k
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_m68k
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_m68k
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_m68k
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_m68k
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_m68k
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_m68k
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_m68k
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_m68k
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_m68k
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_m68k
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_m68k
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_m68k
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_m68k
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_m68k
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_m68k
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_m68k
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_m68k
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_m68k
 #define gen_helper_cpsr_read gen_helper_cpsr_read_m68k
 #define gen_helper_cpsr_write gen_helper_cpsr_write_m68k
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_m68k
diff --git a/qemu/mips.h b/qemu/mips.h
index 9b0fa37cd0..91440cb504 100644
--- a/qemu/mips.h
+++ b/qemu/mips.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_mips
 #define float32_squash_input_denormal float32_squash_input_denormal_mips
 #define float64_squash_input_denormal float64_squash_input_denormal_mips
+#define bfloat16_add bfloat16_add_mips
+#define bfloat16_compare bfloat16_compare_mips
+#define bfloat16_compare_quiet bfloat16_compare_quiet_mips
+#define bfloat16_default_nan bfloat16_default_nan_mips
+#define bfloat16_div bfloat16_div_mips
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_mips
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_mips
+#define bfloat16_max bfloat16_max_mips
+#define bfloat16_maximum_number bfloat16_maximum_number_mips
+#define bfloat16_maxnum bfloat16_maxnum_mips
+#define bfloat16_maxnummag bfloat16_maxnummag_mips
+#define bfloat16_min bfloat16_min_mips
+#define bfloat16_minimum_number bfloat16_minimum_number_mips
+#define bfloat16_minnum bfloat16_minnum_mips
+#define bfloat16_minnummag bfloat16_minnummag_mips
+#define bfloat16_mul bfloat16_mul_mips
+#define bfloat16_muladd bfloat16_muladd_mips
+#define bfloat16_round_to_int bfloat16_round_to_int_mips
+#define bfloat16_scalbn bfloat16_scalbn_mips
+#define bfloat16_silence_nan bfloat16_silence_nan_mips
+#define bfloat16_sqrt bfloat16_sqrt_mips
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_mips
+#define bfloat16_sub bfloat16_sub_mips
+#define bfloat16_to_float32 bfloat16_to_float32_mips
+#define bfloat16_to_float64 bfloat16_to_float64_mips
+#define bfloat16_to_int16 bfloat16_to_int16_mips
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_mips
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_mips
+#define bfloat16_to_int32 bfloat16_to_int32_mips
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_mips
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_mips
+#define bfloat16_to_int64 bfloat16_to_int64_mips
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_mips
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_mips
+#define bfloat16_to_uint16 bfloat16_to_uint16_mips
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_mips
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_mips
+#define bfloat16_to_uint32 bfloat16_to_uint32_mips
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_mips
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_mips
+#define bfloat16_to_uint64 bfloat16_to_uint64_mips
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_mips
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_mips
+#define float128_maximum_number float128_maximum_number_mips
+#define float128_max float128_max_mips
+#define float128_maxnum float128_maxnum_mips
+#define float128_maxnummag float128_maxnummag_mips
+#define float128_min float128_min_mips
+#define float128_minimum_number float128_minimum_number_mips
+#define float128_minnum float128_minnum_mips
+#define float128_minnummag float128_minnummag_mips
+#define float128_muladd float128_muladd_mips
+#define float128_to_int128 float128_to_int128_mips
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_mips
+#define float128_to_uint128 float128_to_uint128_mips
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_mips
+#define float16_maximum_number float16_maximum_number_mips
+#define float16_minimum_number float16_minimum_number_mips
+#define float16_to_int8 float16_to_int8_mips
+#define float16_to_int8_scalbn float16_to_int8_scalbn_mips
+#define float16_to_uint8 float16_to_uint8_mips
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_mips
+#define float32_maximum_number float32_maximum_number_mips
+#define float32_minimum_number float32_minimum_number_mips
+#define float32_to_bfloat16 float32_to_bfloat16_mips
+#define float64_maximum_number float64_maximum_number_mips
+#define float64_minimum_number float64_minimum_number_mips
+#define float64_to_bfloat16 float64_to_bfloat16_mips
+#define float64r32_add float64r32_add_mips
+#define float64r32_div float64r32_div_mips
+#define float64r32_mul float64r32_mul_mips
+#define float64r32_muladd float64r32_muladd_mips
+#define float64r32_sqrt float64r32_sqrt_mips
+#define float64r32_sub float64r32_sub_mips
+#define floatx80_mod floatx80_mod_mips
+#define floatx80_modrem floatx80_modrem_mips
+#define int128_to_float128 int128_to_float128_mips
+#define int16_to_bfloat16 int16_to_bfloat16_mips
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_mips
+#define int32_to_bfloat16 int32_to_bfloat16_mips
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_mips
+#define int64_to_bfloat16 int64_to_bfloat16_mips
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_mips
+#define int8_to_float16 int8_to_float16_mips
+#define uint128_to_float128 uint128_to_float128_mips
+#define uint16_to_bfloat16 uint16_to_bfloat16_mips
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_mips
+#define uint32_to_bfloat16 uint32_to_bfloat16_mips
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_mips
+#define uint64_to_bfloat16 uint64_to_bfloat16_mips
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_mips
+#define uint8_to_float16 uint8_to_float16_mips
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_mips
 #define roundAndPackFloatx80 roundAndPackFloatx80_mips
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_mips
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_mips
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_mips
 #define helper_exit_atomic helper_exit_atomic_mips
+#define helper_memset helper_memset_mips
+#define helper_emu_stop helper_emu_stop_mips
+#define tcg_remove_ops_after tcg_remove_ops_after_mips
+#define tcg_constant_vec_matching tcg_constant_vec_matching_mips
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_mips
 #define helper_gvec_add8 helper_gvec_add8_mips
 #define helper_gvec_add16 helper_gvec_add16_mips
 #define helper_gvec_add32 helper_gvec_add32_mips
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_mips
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_mips
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_mips
+#define gen_helper_mve_vctp gen_helper_mve_vctp_mips
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_mips
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_mips
+#define gen_helper_mve_vdup gen_helper_mve_vdup_mips
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_mips
+#define gen_helper_mve_vandi gen_helper_mve_vandi_mips
+#define gen_helper_mve_vorri gen_helper_mve_vorri_mips
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_mips
+#define gen_helper_mve_viduph gen_helper_mve_viduph_mips
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_mips
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_mips
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_mips
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_mips
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_mips
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_mips
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_mips
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_mips
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_mips
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_mips
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_mips
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_mips
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_mips
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_mips
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_mips
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_mips
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_mips
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_mips
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_mips
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_mips
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_mips
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_mips
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_mips
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_mips
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_mips
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_mips
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_mips
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_mips
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_mips
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_mips
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_mips
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_mips
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_mips
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_mips
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_mips
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_mips
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_mips
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_mips
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_mips
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_mips
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_mips
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_mips
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_mips
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_mips
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_mips
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_mips
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_mips
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_mips
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_mips
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_mips
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_mips
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_mips
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_mips
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_mips
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_mips
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_mips
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_mips
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_mips
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_mips
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_mips
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_mips
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_mips
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_mips
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_mips
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_mips
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_mips
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_mips
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_mips
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_mips
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_mips
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_mips
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_mips
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_mips
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_mips
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_mips
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_mips
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_mips
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_mips
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_mips
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_mips
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_mips
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_mips
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_mips
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_mips
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_mips
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_mips
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_mips
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_mips
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_mips
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_mips
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_mips
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_mips
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_mips
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_mips
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_mips
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_mips
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_mips
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_mips
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_mips
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_mips
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_mips
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_mips
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_mips
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_mips
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_mips
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_mips
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_mips
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_mips
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_mips
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_mips
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_mips
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_mips
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_mips
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_mips
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_mips
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_mips
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_mips
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_mips
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_mips
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_mips
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_mips
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_mips
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_mips
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_mips
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_mips
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_mips
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_mips
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_mips
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_mips
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_mips
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_mips
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_mips
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_mips
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_mips
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_mips
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_mips
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_mips
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_mips
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_mips
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_mips
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_mips
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_mips
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_mips
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_mips
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_mips
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_mips
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_mips
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_mips
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_mips
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_mips
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_mips
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_mips
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_mips
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_mips
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_mips
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_mips
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_mips
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_mips
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_mips
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_mips
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_mips
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_mips
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_mips
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_mips
+#define gen_helper_mve_vand gen_helper_mve_vand_mips
+#define gen_helper_mve_vbic gen_helper_mve_vbic_mips
+#define gen_helper_mve_vorr gen_helper_mve_vorr_mips
+#define gen_helper_mve_vorn gen_helper_mve_vorn_mips
+#define gen_helper_mve_veor gen_helper_mve_veor_mips
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_mips
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_mips
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_mips
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_mips
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_mips
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_mips
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_mips
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_mips
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_mips
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_mips
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_mips
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_mips
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_mips
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_mips
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_mips
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_mips
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_mips
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_mips
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_mips
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_mips
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_mips
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_mips
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_mips
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_mips
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_mips
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_mips
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_mips
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_mips
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_mips
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_mips
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_mips
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_mips
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_mips
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_mips
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_mips
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_mips
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_mips
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_mips
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_mips
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_mips
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_mips
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_mips
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_mips
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_mips
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_mips
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_mips
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_mips
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_mips
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_mips
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_mips
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_mips
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_mips
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_mips
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_mips
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_mips
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_mips
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_mips
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_mips
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_mips
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_mips
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_mips
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_mips
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_mips
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_mips
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_mips
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_mips
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_mips
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_mips
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_mips
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_mips
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_mips
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_mips
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_mips
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_mips
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_mips
+#define gen_helper_mve_vminub gen_helper_mve_vminub_mips
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_mips
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_mips
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_mips
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_mips
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_mips
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_mips
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_mips
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_mips
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_mips
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_mips
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_mips
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_mips
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_mips
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_mips
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_mips
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_mips
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_mips
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_mips
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_mips
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_mips
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_mips
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_mips
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_mips
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_mips
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_mips
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_mips
+#define gen_helper_mve_vadc gen_helper_mve_vadc_mips
+#define gen_helper_mve_vadci gen_helper_mve_vadci_mips
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_mips
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_mips
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_mips
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_mips
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_mips
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_mips
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_mips
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_mips
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_mips
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_mips
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_mips
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_mips
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_mips
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_mips
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_mips
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_mips
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_mips
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_mips
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_mips
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_mips
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_mips
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_mips
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_mips
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_mips
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_mips
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_mips
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_mips
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_mips
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_mips
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_mips
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_mips
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_mips
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_mips
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_mips
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_mips
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_mips
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_mips
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_mips
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_mips
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_mips
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_mips
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_mips
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_mips
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_mips
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_mips
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_mips
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_mips
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_mips
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_mips
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_mips
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_mips
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_mips
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_mips
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_mips
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_mips
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_mips
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_mips
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_mips
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_mips
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_mips
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_mips
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_mips
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_mips
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_mips
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_mips
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_mips
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_mips
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_mips
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_mips
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_mips
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_mips
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_mips
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_mips
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_mips
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_mips
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_mips
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_mips
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_mips
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_mips
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_mips
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_mips
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_mips
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_mips
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_mips
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_mips
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_mips
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_mips
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_mips
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_mips
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_mips
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_mips
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_mips
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_mips
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_mips
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_mips
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_mips
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_mips
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_mips
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_mips
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_mips
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_mips
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_mips
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_mips
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_mips
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_mips
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_mips
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_mips
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_mips
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_mips
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_mips
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_mips
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_mips
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_mips
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_mips
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_mips
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_mips
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_mips
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_mips
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_mips
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_mips
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_mips
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_mips
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_mips
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_mips
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_mips
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_mips
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_mips
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_mips
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_mips
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_mips
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_mips
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_mips
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_mips
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_mips
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_mips
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_mips
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_mips
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_mips
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_mips
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_mips
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_mips
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_mips
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_mips
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_mips
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_mips
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_mips
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_mips
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_mips
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_mips
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_mips
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_mips
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_mips
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_mips
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_mips
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_mips
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_mips
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_mips
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_mips
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_mips
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_mips
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_mips
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_mips
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_mips
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_mips
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_mips
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_mips
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_mips
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_mips
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_mips
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_mips
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_mips
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_mips
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_mips
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_mips
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_mips
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_mips
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_mips
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_mips
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_mips
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_mips
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_mips
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_mips
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_mips
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_mips
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_mips
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_mips
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_mips
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_mips
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_mips
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_mips
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_mips
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_mips
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_mips
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_mips
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_mips
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_mips
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_mips
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_mips
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_mips
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_mips
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_mips
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_mips
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_mips
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_mips
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_mips
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_mips
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_mips
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_mips
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_mips
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_mips
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_mips
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_mips
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_mips
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_mips
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_mips
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_mips
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_mips
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_mips
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_mips
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_mips
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_mips
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_mips
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_mips
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_mips
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_mips
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_mips
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_mips
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_mips
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_mips
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_mips
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_mips
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_mips
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_mips
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_mips
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_mips
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_mips
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_mips
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_mips
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_mips
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_mips
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_mips
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_mips
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_mips
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_mips
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_mips
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_mips
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_mips
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_mips
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_mips
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_mips
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_mips
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_mips
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_mips
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_mips
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_mips
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_mips
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_mips
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_mips
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_mips
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_mips
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_mips
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_mips
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_mips
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_mips
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_mips
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_mips
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_mips
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_mips
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_mips
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_mips
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_mips
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_mips
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_mips
+#define gen_helper_mve_ushll gen_helper_mve_ushll_mips
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_mips
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_mips
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_mips
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_mips
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_mips
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_mips
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_mips
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_mips
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_mips
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_mips
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_mips
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_mips
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_mips
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_mips
+#define gen_helper_mve_vslib gen_helper_mve_vslib_mips
+#define gen_helper_mve_vslih gen_helper_mve_vslih_mips
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_mips
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_mips
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_mips
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_mips
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_mips
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_mips
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_mips
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_mips
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_mips
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_mips
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_mips
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_mips
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_mips
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_mips
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_mips
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_mips
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_mips
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_mips
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_mips
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_mips
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_mips
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_mips
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_mips
+#define gen_helper_mve_vminab gen_helper_mve_vminab_mips
+#define gen_helper_mve_vminah gen_helper_mve_vminah_mips
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_mips
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_mips
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_mips
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_mips
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_mips
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_mips
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_mips
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_mips
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_mips
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_mips
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_mips
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_mips
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_mips
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_mips
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_mips
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_mips
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_mips
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_mips
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_mips
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_mips
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_mips
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_mips
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_mips
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_mips
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_mips
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_mips
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_mips
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_mips
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_mips
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_mips
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_mips
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_mips
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_mips
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_mips
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_mips
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_mips
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_mips
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_mips
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_mips
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_mips
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_mips
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_mips
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_mips
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_mips
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_mips
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_mips
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_mips
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_mips
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_mips
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_mips
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_mips
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_mips
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_mips
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_mips
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_mips
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_mips
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_mips
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_mips
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_mips
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_mips
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_mips
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_mips
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_mips
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_mips
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_mips
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_mips
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_mips
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_mips
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_mips
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_mips
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_mips
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_mips
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_mips
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_mips
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_mips
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_mips
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_mips
 #define gen_helper_cpsr_read gen_helper_cpsr_read_mips
 #define gen_helper_cpsr_write gen_helper_cpsr_write_mips
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_mips
@@ -1440,7 +2211,10 @@
 #define helper_dvp helper_dvp_mips
 #define helper_evp helper_evp_mips
 #define cpu_mips_get_random cpu_mips_get_random_mips
+#define cpu_mips_get_count cpu_mips_get_count_mips
 #define cpu_mips_init cpu_mips_init_mips
+#define cpu_mips_store_count cpu_mips_store_count_mips
+#define cpu_mips_store_compare cpu_mips_store_compare_mips
 #define helper_absq_s_ph helper_absq_s_ph_mips
 #define helper_absq_s_qb helper_absq_s_qb_mips
 #define helper_absq_s_w helper_absq_s_w_mips
diff --git a/qemu/mips64.h b/qemu/mips64.h
index ec030ff261..24422eef29 100644
--- a/qemu/mips64.h
+++ b/qemu/mips64.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_mips64
 #define float32_squash_input_denormal float32_squash_input_denormal_mips64
 #define float64_squash_input_denormal float64_squash_input_denormal_mips64
+#define bfloat16_add bfloat16_add_mips64
+#define bfloat16_compare bfloat16_compare_mips64
+#define bfloat16_compare_quiet bfloat16_compare_quiet_mips64
+#define bfloat16_default_nan bfloat16_default_nan_mips64
+#define bfloat16_div bfloat16_div_mips64
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_mips64
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_mips64
+#define bfloat16_max bfloat16_max_mips64
+#define bfloat16_maximum_number bfloat16_maximum_number_mips64
+#define bfloat16_maxnum bfloat16_maxnum_mips64
+#define bfloat16_maxnummag bfloat16_maxnummag_mips64
+#define bfloat16_min bfloat16_min_mips64
+#define bfloat16_minimum_number bfloat16_minimum_number_mips64
+#define bfloat16_minnum bfloat16_minnum_mips64
+#define bfloat16_minnummag bfloat16_minnummag_mips64
+#define bfloat16_mul bfloat16_mul_mips64
+#define bfloat16_muladd bfloat16_muladd_mips64
+#define bfloat16_round_to_int bfloat16_round_to_int_mips64
+#define bfloat16_scalbn bfloat16_scalbn_mips64
+#define bfloat16_silence_nan bfloat16_silence_nan_mips64
+#define bfloat16_sqrt bfloat16_sqrt_mips64
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_mips64
+#define bfloat16_sub bfloat16_sub_mips64
+#define bfloat16_to_float32 bfloat16_to_float32_mips64
+#define bfloat16_to_float64 bfloat16_to_float64_mips64
+#define bfloat16_to_int16 bfloat16_to_int16_mips64
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_mips64
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_mips64
+#define bfloat16_to_int32 bfloat16_to_int32_mips64
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_mips64
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_mips64
+#define bfloat16_to_int64 bfloat16_to_int64_mips64
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_mips64
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_mips64
+#define bfloat16_to_uint16 bfloat16_to_uint16_mips64
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_mips64
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_mips64
+#define bfloat16_to_uint32 bfloat16_to_uint32_mips64
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_mips64
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_mips64
+#define bfloat16_to_uint64 bfloat16_to_uint64_mips64
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_mips64
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_mips64
+#define float128_maximum_number float128_maximum_number_mips64
+#define float128_max float128_max_mips64
+#define float128_maxnum float128_maxnum_mips64
+#define float128_maxnummag float128_maxnummag_mips64
+#define float128_min float128_min_mips64
+#define float128_minimum_number float128_minimum_number_mips64
+#define float128_minnum float128_minnum_mips64
+#define float128_minnummag float128_minnummag_mips64
+#define float128_muladd float128_muladd_mips64
+#define float128_to_int128 float128_to_int128_mips64
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_mips64
+#define float128_to_uint128 float128_to_uint128_mips64
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_mips64
+#define float16_maximum_number float16_maximum_number_mips64
+#define float16_minimum_number float16_minimum_number_mips64
+#define float16_to_int8 float16_to_int8_mips64
+#define float16_to_int8_scalbn float16_to_int8_scalbn_mips64
+#define float16_to_uint8 float16_to_uint8_mips64
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_mips64
+#define float32_maximum_number float32_maximum_number_mips64
+#define float32_minimum_number float32_minimum_number_mips64
+#define float32_to_bfloat16 float32_to_bfloat16_mips64
+#define float64_maximum_number float64_maximum_number_mips64
+#define float64_minimum_number float64_minimum_number_mips64
+#define float64_to_bfloat16 float64_to_bfloat16_mips64
+#define float64r32_add float64r32_add_mips64
+#define float64r32_div float64r32_div_mips64
+#define float64r32_mul float64r32_mul_mips64
+#define float64r32_muladd float64r32_muladd_mips64
+#define float64r32_sqrt float64r32_sqrt_mips64
+#define float64r32_sub float64r32_sub_mips64
+#define floatx80_mod floatx80_mod_mips64
+#define floatx80_modrem floatx80_modrem_mips64
+#define int128_to_float128 int128_to_float128_mips64
+#define int16_to_bfloat16 int16_to_bfloat16_mips64
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_mips64
+#define int32_to_bfloat16 int32_to_bfloat16_mips64
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_mips64
+#define int64_to_bfloat16 int64_to_bfloat16_mips64
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_mips64
+#define int8_to_float16 int8_to_float16_mips64
+#define uint128_to_float128 uint128_to_float128_mips64
+#define uint16_to_bfloat16 uint16_to_bfloat16_mips64
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_mips64
+#define uint32_to_bfloat16 uint32_to_bfloat16_mips64
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_mips64
+#define uint64_to_bfloat16 uint64_to_bfloat16_mips64
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_mips64
+#define uint8_to_float16 uint8_to_float16_mips64
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_mips64
 #define roundAndPackFloatx80 roundAndPackFloatx80_mips64
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_mips64
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_mips64
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_mips64
 #define helper_exit_atomic helper_exit_atomic_mips64
+#define helper_memset helper_memset_mips64
+#define helper_emu_stop helper_emu_stop_mips64
+#define tcg_remove_ops_after tcg_remove_ops_after_mips64
+#define tcg_constant_vec_matching tcg_constant_vec_matching_mips64
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_mips64
 #define helper_gvec_add8 helper_gvec_add8_mips64
 #define helper_gvec_add16 helper_gvec_add16_mips64
 #define helper_gvec_add32 helper_gvec_add32_mips64
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_mips64
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_mips64
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_mips64
+#define gen_helper_mve_vctp gen_helper_mve_vctp_mips64
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_mips64
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_mips64
+#define gen_helper_mve_vdup gen_helper_mve_vdup_mips64
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_mips64
+#define gen_helper_mve_vandi gen_helper_mve_vandi_mips64
+#define gen_helper_mve_vorri gen_helper_mve_vorri_mips64
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_mips64
+#define gen_helper_mve_viduph gen_helper_mve_viduph_mips64
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_mips64
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_mips64
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_mips64
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_mips64
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_mips64
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_mips64
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_mips64
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_mips64
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_mips64
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_mips64
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_mips64
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_mips64
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_mips64
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_mips64
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_mips64
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_mips64
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_mips64
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_mips64
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_mips64
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_mips64
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_mips64
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_mips64
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_mips64
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_mips64
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_mips64
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_mips64
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_mips64
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_mips64
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_mips64
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_mips64
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_mips64
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_mips64
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_mips64
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_mips64
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_mips64
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_mips64
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_mips64
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_mips64
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_mips64
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_mips64
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_mips64
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_mips64
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_mips64
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_mips64
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_mips64
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_mips64
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_mips64
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_mips64
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_mips64
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_mips64
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_mips64
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_mips64
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_mips64
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_mips64
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_mips64
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_mips64
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_mips64
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_mips64
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_mips64
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_mips64
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_mips64
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_mips64
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_mips64
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_mips64
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_mips64
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_mips64
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_mips64
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_mips64
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_mips64
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_mips64
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_mips64
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_mips64
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_mips64
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_mips64
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_mips64
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_mips64
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_mips64
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_mips64
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_mips64
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_mips64
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_mips64
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_mips64
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_mips64
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_mips64
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_mips64
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_mips64
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_mips64
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_mips64
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_mips64
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_mips64
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_mips64
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_mips64
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_mips64
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_mips64
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_mips64
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_mips64
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_mips64
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_mips64
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_mips64
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_mips64
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_mips64
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_mips64
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_mips64
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_mips64
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_mips64
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_mips64
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_mips64
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_mips64
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_mips64
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_mips64
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_mips64
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_mips64
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_mips64
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_mips64
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_mips64
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_mips64
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_mips64
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_mips64
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_mips64
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_mips64
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_mips64
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_mips64
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_mips64
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_mips64
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_mips64
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_mips64
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_mips64
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_mips64
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_mips64
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_mips64
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_mips64
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_mips64
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_mips64
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_mips64
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_mips64
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_mips64
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_mips64
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_mips64
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_mips64
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_mips64
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_mips64
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_mips64
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_mips64
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_mips64
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_mips64
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_mips64
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_mips64
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_mips64
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_mips64
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_mips64
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_mips64
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_mips64
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_mips64
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_mips64
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_mips64
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_mips64
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_mips64
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_mips64
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_mips64
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_mips64
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_mips64
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_mips64
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_mips64
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_mips64
+#define gen_helper_mve_vand gen_helper_mve_vand_mips64
+#define gen_helper_mve_vbic gen_helper_mve_vbic_mips64
+#define gen_helper_mve_vorr gen_helper_mve_vorr_mips64
+#define gen_helper_mve_vorn gen_helper_mve_vorn_mips64
+#define gen_helper_mve_veor gen_helper_mve_veor_mips64
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_mips64
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_mips64
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_mips64
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_mips64
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_mips64
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_mips64
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_mips64
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_mips64
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_mips64
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_mips64
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_mips64
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_mips64
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_mips64
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_mips64
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_mips64
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_mips64
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_mips64
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_mips64
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_mips64
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_mips64
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_mips64
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_mips64
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_mips64
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_mips64
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_mips64
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_mips64
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_mips64
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_mips64
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_mips64
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_mips64
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_mips64
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_mips64
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_mips64
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_mips64
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_mips64
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_mips64
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_mips64
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_mips64
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_mips64
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_mips64
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_mips64
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_mips64
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_mips64
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_mips64
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_mips64
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_mips64
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_mips64
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_mips64
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_mips64
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_mips64
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_mips64
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_mips64
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_mips64
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_mips64
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_mips64
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_mips64
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_mips64
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_mips64
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_mips64
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_mips64
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_mips64
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_mips64
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_mips64
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_mips64
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_mips64
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_mips64
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_mips64
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_mips64
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_mips64
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_mips64
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_mips64
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_mips64
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_mips64
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_mips64
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_mips64
+#define gen_helper_mve_vminub gen_helper_mve_vminub_mips64
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_mips64
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_mips64
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_mips64
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_mips64
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_mips64
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_mips64
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_mips64
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_mips64
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_mips64
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_mips64
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_mips64
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_mips64
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_mips64
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_mips64
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_mips64
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_mips64
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_mips64
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_mips64
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_mips64
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_mips64
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_mips64
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_mips64
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_mips64
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_mips64
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_mips64
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_mips64
+#define gen_helper_mve_vadc gen_helper_mve_vadc_mips64
+#define gen_helper_mve_vadci gen_helper_mve_vadci_mips64
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_mips64
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_mips64
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_mips64
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_mips64
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_mips64
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_mips64
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_mips64
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_mips64
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_mips64
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_mips64
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_mips64
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_mips64
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_mips64
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_mips64
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_mips64
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_mips64
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_mips64
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_mips64
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_mips64
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_mips64
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_mips64
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_mips64
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_mips64
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_mips64
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_mips64
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_mips64
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_mips64
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_mips64
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_mips64
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_mips64
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_mips64
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_mips64
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_mips64
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_mips64
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_mips64
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_mips64
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_mips64
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_mips64
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_mips64
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_mips64
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_mips64
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_mips64
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_mips64
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_mips64
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_mips64
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_mips64
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_mips64
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_mips64
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_mips64
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_mips64
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_mips64
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_mips64
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_mips64
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_mips64
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_mips64
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_mips64
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_mips64
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_mips64
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_mips64
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_mips64
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_mips64
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_mips64
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_mips64
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_mips64
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_mips64
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_mips64
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_mips64
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_mips64
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_mips64
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_mips64
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_mips64
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_mips64
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_mips64
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_mips64
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_mips64
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_mips64
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_mips64
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_mips64
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_mips64
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_mips64
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_mips64
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_mips64
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_mips64
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_mips64
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_mips64
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_mips64
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_mips64
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_mips64
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_mips64
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_mips64
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_mips64
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_mips64
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_mips64
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_mips64
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_mips64
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_mips64
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_mips64
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_mips64
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_mips64
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_mips64
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_mips64
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_mips64
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_mips64
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_mips64
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_mips64
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_mips64
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_mips64
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_mips64
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_mips64
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_mips64
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_mips64
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_mips64
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_mips64
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_mips64
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_mips64
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_mips64
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_mips64
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_mips64
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_mips64
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_mips64
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_mips64
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_mips64
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_mips64
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_mips64
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_mips64
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_mips64
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_mips64
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_mips64
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_mips64
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_mips64
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_mips64
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_mips64
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_mips64
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_mips64
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_mips64
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_mips64
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_mips64
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_mips64
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_mips64
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_mips64
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_mips64
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_mips64
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_mips64
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_mips64
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_mips64
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_mips64
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_mips64
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_mips64
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_mips64
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_mips64
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_mips64
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_mips64
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_mips64
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_mips64
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_mips64
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_mips64
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_mips64
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_mips64
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_mips64
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_mips64
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_mips64
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_mips64
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_mips64
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_mips64
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_mips64
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_mips64
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_mips64
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_mips64
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_mips64
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_mips64
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_mips64
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_mips64
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_mips64
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_mips64
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_mips64
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_mips64
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_mips64
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_mips64
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_mips64
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_mips64
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_mips64
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_mips64
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_mips64
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_mips64
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_mips64
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_mips64
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_mips64
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_mips64
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_mips64
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_mips64
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_mips64
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_mips64
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_mips64
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_mips64
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_mips64
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_mips64
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_mips64
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_mips64
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_mips64
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_mips64
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_mips64
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_mips64
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_mips64
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_mips64
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_mips64
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_mips64
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_mips64
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_mips64
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_mips64
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_mips64
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_mips64
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_mips64
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_mips64
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_mips64
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_mips64
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_mips64
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_mips64
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_mips64
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_mips64
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_mips64
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_mips64
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_mips64
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_mips64
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_mips64
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_mips64
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_mips64
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_mips64
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_mips64
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_mips64
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_mips64
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_mips64
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_mips64
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_mips64
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_mips64
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_mips64
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_mips64
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_mips64
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_mips64
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_mips64
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_mips64
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_mips64
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_mips64
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_mips64
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_mips64
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_mips64
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_mips64
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_mips64
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_mips64
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_mips64
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_mips64
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_mips64
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_mips64
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_mips64
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_mips64
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_mips64
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_mips64
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_mips64
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_mips64
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_mips64
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_mips64
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_mips64
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_mips64
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_mips64
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_mips64
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_mips64
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_mips64
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_mips64
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_mips64
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_mips64
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_mips64
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_mips64
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_mips64
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_mips64
+#define gen_helper_mve_ushll gen_helper_mve_ushll_mips64
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_mips64
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_mips64
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_mips64
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_mips64
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_mips64
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_mips64
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_mips64
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_mips64
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_mips64
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_mips64
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_mips64
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_mips64
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_mips64
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_mips64
+#define gen_helper_mve_vslib gen_helper_mve_vslib_mips64
+#define gen_helper_mve_vslih gen_helper_mve_vslih_mips64
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_mips64
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_mips64
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_mips64
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_mips64
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_mips64
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_mips64
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_mips64
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_mips64
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_mips64
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_mips64
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_mips64
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_mips64
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_mips64
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_mips64
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_mips64
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_mips64
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_mips64
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_mips64
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_mips64
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_mips64
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_mips64
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_mips64
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_mips64
+#define gen_helper_mve_vminab gen_helper_mve_vminab_mips64
+#define gen_helper_mve_vminah gen_helper_mve_vminah_mips64
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_mips64
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_mips64
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_mips64
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_mips64
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_mips64
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_mips64
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_mips64
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_mips64
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_mips64
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_mips64
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_mips64
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_mips64
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_mips64
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_mips64
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_mips64
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_mips64
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_mips64
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_mips64
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_mips64
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_mips64
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_mips64
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_mips64
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_mips64
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_mips64
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_mips64
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_mips64
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_mips64
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_mips64
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_mips64
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_mips64
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_mips64
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_mips64
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_mips64
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_mips64
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_mips64
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_mips64
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_mips64
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_mips64
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_mips64
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_mips64
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_mips64
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_mips64
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_mips64
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_mips64
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_mips64
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_mips64
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_mips64
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_mips64
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_mips64
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_mips64
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_mips64
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_mips64
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_mips64
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_mips64
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_mips64
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_mips64
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_mips64
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_mips64
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_mips64
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_mips64
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_mips64
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_mips64
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_mips64
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_mips64
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_mips64
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_mips64
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_mips64
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_mips64
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_mips64
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_mips64
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_mips64
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_mips64
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_mips64
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_mips64
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_mips64
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_mips64
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_mips64
 #define gen_helper_cpsr_read gen_helper_cpsr_read_mips64
 #define gen_helper_cpsr_write gen_helper_cpsr_write_mips64
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_mips64
@@ -1440,7 +2211,10 @@
 #define helper_dvp helper_dvp_mips64
 #define helper_evp helper_evp_mips64
 #define cpu_mips_get_random cpu_mips_get_random_mips64
+#define cpu_mips_get_count cpu_mips_get_count_mips64
 #define cpu_mips_init cpu_mips_init_mips64
+#define cpu_mips_store_count cpu_mips_store_count_mips64
+#define cpu_mips_store_compare cpu_mips_store_compare_mips64
 #define helper_absq_s_ph helper_absq_s_ph_mips64
 #define helper_absq_s_qb helper_absq_s_qb_mips64
 #define helper_absq_s_w helper_absq_s_w_mips64
diff --git a/qemu/mips64el.h b/qemu/mips64el.h
index ea8c92739c..0e6e85b557 100644
--- a/qemu/mips64el.h
+++ b/qemu/mips64el.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_mips64el
 #define float32_squash_input_denormal float32_squash_input_denormal_mips64el
 #define float64_squash_input_denormal float64_squash_input_denormal_mips64el
+#define bfloat16_add bfloat16_add_mips64el
+#define bfloat16_compare bfloat16_compare_mips64el
+#define bfloat16_compare_quiet bfloat16_compare_quiet_mips64el
+#define bfloat16_default_nan bfloat16_default_nan_mips64el
+#define bfloat16_div bfloat16_div_mips64el
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_mips64el
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_mips64el
+#define bfloat16_max bfloat16_max_mips64el
+#define bfloat16_maximum_number bfloat16_maximum_number_mips64el
+#define bfloat16_maxnum bfloat16_maxnum_mips64el
+#define bfloat16_maxnummag bfloat16_maxnummag_mips64el
+#define bfloat16_min bfloat16_min_mips64el
+#define bfloat16_minimum_number bfloat16_minimum_number_mips64el
+#define bfloat16_minnum bfloat16_minnum_mips64el
+#define bfloat16_minnummag bfloat16_minnummag_mips64el
+#define bfloat16_mul bfloat16_mul_mips64el
+#define bfloat16_muladd bfloat16_muladd_mips64el
+#define bfloat16_round_to_int bfloat16_round_to_int_mips64el
+#define bfloat16_scalbn bfloat16_scalbn_mips64el
+#define bfloat16_silence_nan bfloat16_silence_nan_mips64el
+#define bfloat16_sqrt bfloat16_sqrt_mips64el
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_mips64el
+#define bfloat16_sub bfloat16_sub_mips64el
+#define bfloat16_to_float32 bfloat16_to_float32_mips64el
+#define bfloat16_to_float64 bfloat16_to_float64_mips64el
+#define bfloat16_to_int16 bfloat16_to_int16_mips64el
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_mips64el
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_mips64el
+#define bfloat16_to_int32 bfloat16_to_int32_mips64el
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_mips64el
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_mips64el
+#define bfloat16_to_int64 bfloat16_to_int64_mips64el
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_mips64el
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_mips64el
+#define bfloat16_to_uint16 bfloat16_to_uint16_mips64el
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_mips64el
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_mips64el
+#define bfloat16_to_uint32 bfloat16_to_uint32_mips64el
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_mips64el
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_mips64el
+#define bfloat16_to_uint64 bfloat16_to_uint64_mips64el
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_mips64el
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_mips64el
+#define float128_maximum_number float128_maximum_number_mips64el
+#define float128_max float128_max_mips64el
+#define float128_maxnum float128_maxnum_mips64el
+#define float128_maxnummag float128_maxnummag_mips64el
+#define float128_min float128_min_mips64el
+#define float128_minimum_number float128_minimum_number_mips64el
+#define float128_minnum float128_minnum_mips64el
+#define float128_minnummag float128_minnummag_mips64el
+#define float128_muladd float128_muladd_mips64el
+#define float128_to_int128 float128_to_int128_mips64el
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_mips64el
+#define float128_to_uint128 float128_to_uint128_mips64el
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_mips64el
+#define float16_maximum_number float16_maximum_number_mips64el
+#define float16_minimum_number float16_minimum_number_mips64el
+#define float16_to_int8 float16_to_int8_mips64el
+#define float16_to_int8_scalbn float16_to_int8_scalbn_mips64el
+#define float16_to_uint8 float16_to_uint8_mips64el
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_mips64el
+#define float32_maximum_number float32_maximum_number_mips64el
+#define float32_minimum_number float32_minimum_number_mips64el
+#define float32_to_bfloat16 float32_to_bfloat16_mips64el
+#define float64_maximum_number float64_maximum_number_mips64el
+#define float64_minimum_number float64_minimum_number_mips64el
+#define float64_to_bfloat16 float64_to_bfloat16_mips64el
+#define float64r32_add float64r32_add_mips64el
+#define float64r32_div float64r32_div_mips64el
+#define float64r32_mul float64r32_mul_mips64el
+#define float64r32_muladd float64r32_muladd_mips64el
+#define float64r32_sqrt float64r32_sqrt_mips64el
+#define float64r32_sub float64r32_sub_mips64el
+#define floatx80_mod floatx80_mod_mips64el
+#define floatx80_modrem floatx80_modrem_mips64el
+#define int128_to_float128 int128_to_float128_mips64el
+#define int16_to_bfloat16 int16_to_bfloat16_mips64el
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_mips64el
+#define int32_to_bfloat16 int32_to_bfloat16_mips64el
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_mips64el
+#define int64_to_bfloat16 int64_to_bfloat16_mips64el
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_mips64el
+#define int8_to_float16 int8_to_float16_mips64el
+#define uint128_to_float128 uint128_to_float128_mips64el
+#define uint16_to_bfloat16 uint16_to_bfloat16_mips64el
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_mips64el
+#define uint32_to_bfloat16 uint32_to_bfloat16_mips64el
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_mips64el
+#define uint64_to_bfloat16 uint64_to_bfloat16_mips64el
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_mips64el
+#define uint8_to_float16 uint8_to_float16_mips64el
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_mips64el
 #define roundAndPackFloatx80 roundAndPackFloatx80_mips64el
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_mips64el
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_mips64el
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_mips64el
 #define helper_exit_atomic helper_exit_atomic_mips64el
+#define helper_memset helper_memset_mips64el
+#define helper_emu_stop helper_emu_stop_mips64el
+#define tcg_remove_ops_after tcg_remove_ops_after_mips64el
+#define tcg_constant_vec_matching tcg_constant_vec_matching_mips64el
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_mips64el
 #define helper_gvec_add8 helper_gvec_add8_mips64el
 #define helper_gvec_add16 helper_gvec_add16_mips64el
 #define helper_gvec_add32 helper_gvec_add32_mips64el
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_mips64el
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_mips64el
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_mips64el
+#define gen_helper_mve_vctp gen_helper_mve_vctp_mips64el
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_mips64el
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_mips64el
+#define gen_helper_mve_vdup gen_helper_mve_vdup_mips64el
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_mips64el
+#define gen_helper_mve_vandi gen_helper_mve_vandi_mips64el
+#define gen_helper_mve_vorri gen_helper_mve_vorri_mips64el
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_mips64el
+#define gen_helper_mve_viduph gen_helper_mve_viduph_mips64el
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_mips64el
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_mips64el
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_mips64el
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_mips64el
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_mips64el
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_mips64el
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_mips64el
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_mips64el
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_mips64el
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_mips64el
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_mips64el
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_mips64el
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_mips64el
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_mips64el
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_mips64el
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_mips64el
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_mips64el
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_mips64el
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_mips64el
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_mips64el
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_mips64el
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_mips64el
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_mips64el
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_mips64el
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_mips64el
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_mips64el
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_mips64el
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_mips64el
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_mips64el
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_mips64el
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_mips64el
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_mips64el
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_mips64el
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_mips64el
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_mips64el
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_mips64el
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_mips64el
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_mips64el
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_mips64el
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_mips64el
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_mips64el
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_mips64el
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_mips64el
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_mips64el
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_mips64el
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_mips64el
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_mips64el
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_mips64el
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_mips64el
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_mips64el
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_mips64el
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_mips64el
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_mips64el
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_mips64el
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_mips64el
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_mips64el
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_mips64el
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_mips64el
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_mips64el
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_mips64el
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_mips64el
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_mips64el
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_mips64el
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_mips64el
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_mips64el
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_mips64el
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_mips64el
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_mips64el
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_mips64el
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_mips64el
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_mips64el
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_mips64el
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_mips64el
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_mips64el
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_mips64el
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_mips64el
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_mips64el
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_mips64el
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_mips64el
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_mips64el
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_mips64el
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_mips64el
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_mips64el
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_mips64el
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_mips64el
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_mips64el
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_mips64el
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_mips64el
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_mips64el
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_mips64el
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_mips64el
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_mips64el
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_mips64el
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_mips64el
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_mips64el
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_mips64el
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_mips64el
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_mips64el
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_mips64el
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_mips64el
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_mips64el
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_mips64el
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_mips64el
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_mips64el
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_mips64el
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_mips64el
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_mips64el
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_mips64el
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_mips64el
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_mips64el
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_mips64el
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_mips64el
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_mips64el
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_mips64el
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_mips64el
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_mips64el
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_mips64el
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_mips64el
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_mips64el
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_mips64el
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_mips64el
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_mips64el
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_mips64el
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_mips64el
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_mips64el
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_mips64el
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_mips64el
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_mips64el
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_mips64el
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_mips64el
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_mips64el
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_mips64el
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_mips64el
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_mips64el
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_mips64el
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_mips64el
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_mips64el
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_mips64el
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_mips64el
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_mips64el
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_mips64el
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_mips64el
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_mips64el
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_mips64el
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_mips64el
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_mips64el
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_mips64el
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_mips64el
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_mips64el
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_mips64el
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_mips64el
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_mips64el
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_mips64el
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_mips64el
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_mips64el
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_mips64el
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_mips64el
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_mips64el
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_mips64el
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_mips64el
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_mips64el
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_mips64el
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_mips64el
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_mips64el
+#define gen_helper_mve_vand gen_helper_mve_vand_mips64el
+#define gen_helper_mve_vbic gen_helper_mve_vbic_mips64el
+#define gen_helper_mve_vorr gen_helper_mve_vorr_mips64el
+#define gen_helper_mve_vorn gen_helper_mve_vorn_mips64el
+#define gen_helper_mve_veor gen_helper_mve_veor_mips64el
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_mips64el
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_mips64el
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_mips64el
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_mips64el
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_mips64el
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_mips64el
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_mips64el
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_mips64el
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_mips64el
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_mips64el
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_mips64el
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_mips64el
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_mips64el
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_mips64el
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_mips64el
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_mips64el
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_mips64el
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_mips64el
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_mips64el
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_mips64el
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_mips64el
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_mips64el
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_mips64el
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_mips64el
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_mips64el
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_mips64el
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_mips64el
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_mips64el
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_mips64el
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_mips64el
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_mips64el
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_mips64el
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_mips64el
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_mips64el
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_mips64el
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_mips64el
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_mips64el
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_mips64el
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_mips64el
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_mips64el
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_mips64el
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_mips64el
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_mips64el
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_mips64el
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_mips64el
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_mips64el
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_mips64el
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_mips64el
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_mips64el
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_mips64el
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_mips64el
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_mips64el
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_mips64el
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_mips64el
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_mips64el
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_mips64el
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_mips64el
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_mips64el
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_mips64el
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_mips64el
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_mips64el
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_mips64el
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_mips64el
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_mips64el
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_mips64el
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_mips64el
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_mips64el
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_mips64el
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_mips64el
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_mips64el
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_mips64el
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_mips64el
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_mips64el
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_mips64el
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_mips64el
+#define gen_helper_mve_vminub gen_helper_mve_vminub_mips64el
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_mips64el
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_mips64el
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_mips64el
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_mips64el
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_mips64el
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_mips64el
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_mips64el
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_mips64el
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_mips64el
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_mips64el
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_mips64el
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_mips64el
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_mips64el
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_mips64el
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_mips64el
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_mips64el
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_mips64el
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_mips64el
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_mips64el
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_mips64el
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_mips64el
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_mips64el
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_mips64el
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_mips64el
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_mips64el
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_mips64el
+#define gen_helper_mve_vadc gen_helper_mve_vadc_mips64el
+#define gen_helper_mve_vadci gen_helper_mve_vadci_mips64el
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_mips64el
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_mips64el
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_mips64el
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_mips64el
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_mips64el
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_mips64el
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_mips64el
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_mips64el
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_mips64el
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_mips64el
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_mips64el
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_mips64el
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_mips64el
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_mips64el
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_mips64el
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_mips64el
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_mips64el
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_mips64el
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_mips64el
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_mips64el
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_mips64el
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_mips64el
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_mips64el
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_mips64el
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_mips64el
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_mips64el
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_mips64el
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_mips64el
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_mips64el
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_mips64el
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_mips64el
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_mips64el
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_mips64el
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_mips64el
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_mips64el
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_mips64el
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_mips64el
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_mips64el
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_mips64el
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_mips64el
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_mips64el
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_mips64el
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_mips64el
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_mips64el
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_mips64el
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_mips64el
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_mips64el
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_mips64el
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_mips64el
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_mips64el
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_mips64el
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_mips64el
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_mips64el
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_mips64el
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_mips64el
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_mips64el
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_mips64el
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_mips64el
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_mips64el
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_mips64el
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_mips64el
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_mips64el
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_mips64el
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_mips64el
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_mips64el
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_mips64el
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_mips64el
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_mips64el
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_mips64el
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_mips64el
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_mips64el
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_mips64el
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_mips64el
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_mips64el
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_mips64el
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_mips64el
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_mips64el
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_mips64el
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_mips64el
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_mips64el
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_mips64el
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_mips64el
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_mips64el
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_mips64el
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_mips64el
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_mips64el
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_mips64el
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_mips64el
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_mips64el
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_mips64el
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_mips64el
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_mips64el
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_mips64el
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_mips64el
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_mips64el
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_mips64el
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_mips64el
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_mips64el
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_mips64el
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_mips64el
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_mips64el
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_mips64el
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_mips64el
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_mips64el
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_mips64el
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_mips64el
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_mips64el
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_mips64el
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_mips64el
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_mips64el
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_mips64el
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_mips64el
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_mips64el
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_mips64el
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_mips64el
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_mips64el
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_mips64el
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_mips64el
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_mips64el
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_mips64el
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_mips64el
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_mips64el
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_mips64el
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_mips64el
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_mips64el
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_mips64el
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_mips64el
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_mips64el
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_mips64el
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_mips64el
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_mips64el
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_mips64el
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_mips64el
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_mips64el
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_mips64el
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_mips64el
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_mips64el
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_mips64el
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_mips64el
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_mips64el
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_mips64el
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_mips64el
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_mips64el
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_mips64el
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_mips64el
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_mips64el
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_mips64el
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_mips64el
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_mips64el
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_mips64el
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_mips64el
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_mips64el
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_mips64el
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_mips64el
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_mips64el
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_mips64el
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_mips64el
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_mips64el
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_mips64el
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_mips64el
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_mips64el
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_mips64el
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_mips64el
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_mips64el
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_mips64el
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_mips64el
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_mips64el
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_mips64el
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_mips64el
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_mips64el
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_mips64el
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_mips64el
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_mips64el
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_mips64el
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_mips64el
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_mips64el
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_mips64el
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_mips64el
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_mips64el
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_mips64el
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_mips64el
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_mips64el
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_mips64el
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_mips64el
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_mips64el
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_mips64el
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_mips64el
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_mips64el
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_mips64el
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_mips64el
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_mips64el
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_mips64el
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_mips64el
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_mips64el
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_mips64el
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_mips64el
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_mips64el
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_mips64el
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_mips64el
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_mips64el
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_mips64el
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_mips64el
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_mips64el
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_mips64el
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_mips64el
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_mips64el
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_mips64el
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_mips64el
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_mips64el
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_mips64el
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_mips64el
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_mips64el
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_mips64el
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_mips64el
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_mips64el
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_mips64el
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_mips64el
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_mips64el
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_mips64el
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_mips64el
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_mips64el
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_mips64el
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_mips64el
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_mips64el
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_mips64el
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_mips64el
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_mips64el
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_mips64el
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_mips64el
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_mips64el
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_mips64el
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_mips64el
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_mips64el
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_mips64el
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_mips64el
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_mips64el
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_mips64el
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_mips64el
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_mips64el
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_mips64el
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_mips64el
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_mips64el
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_mips64el
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_mips64el
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_mips64el
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_mips64el
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_mips64el
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_mips64el
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_mips64el
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_mips64el
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_mips64el
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_mips64el
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_mips64el
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_mips64el
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_mips64el
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_mips64el
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_mips64el
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_mips64el
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_mips64el
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_mips64el
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_mips64el
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_mips64el
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_mips64el
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_mips64el
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_mips64el
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_mips64el
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_mips64el
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_mips64el
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_mips64el
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_mips64el
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_mips64el
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_mips64el
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_mips64el
+#define gen_helper_mve_ushll gen_helper_mve_ushll_mips64el
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_mips64el
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_mips64el
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_mips64el
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_mips64el
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_mips64el
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_mips64el
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_mips64el
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_mips64el
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_mips64el
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_mips64el
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_mips64el
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_mips64el
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_mips64el
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_mips64el
+#define gen_helper_mve_vslib gen_helper_mve_vslib_mips64el
+#define gen_helper_mve_vslih gen_helper_mve_vslih_mips64el
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_mips64el
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_mips64el
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_mips64el
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_mips64el
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_mips64el
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_mips64el
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_mips64el
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_mips64el
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_mips64el
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_mips64el
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_mips64el
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_mips64el
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_mips64el
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_mips64el
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_mips64el
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_mips64el
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_mips64el
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_mips64el
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_mips64el
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_mips64el
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_mips64el
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_mips64el
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_mips64el
+#define gen_helper_mve_vminab gen_helper_mve_vminab_mips64el
+#define gen_helper_mve_vminah gen_helper_mve_vminah_mips64el
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_mips64el
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_mips64el
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_mips64el
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_mips64el
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_mips64el
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_mips64el
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_mips64el
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_mips64el
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_mips64el
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_mips64el
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_mips64el
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_mips64el
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_mips64el
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_mips64el
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_mips64el
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_mips64el
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_mips64el
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_mips64el
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_mips64el
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_mips64el
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_mips64el
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_mips64el
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_mips64el
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_mips64el
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_mips64el
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_mips64el
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_mips64el
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_mips64el
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_mips64el
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_mips64el
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_mips64el
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_mips64el
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_mips64el
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_mips64el
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_mips64el
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_mips64el
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_mips64el
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_mips64el
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_mips64el
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_mips64el
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_mips64el
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_mips64el
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_mips64el
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_mips64el
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_mips64el
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_mips64el
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_mips64el
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_mips64el
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_mips64el
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_mips64el
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_mips64el
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_mips64el
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_mips64el
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_mips64el
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_mips64el
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_mips64el
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_mips64el
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_mips64el
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_mips64el
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_mips64el
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_mips64el
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_mips64el
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_mips64el
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_mips64el
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_mips64el
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_mips64el
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_mips64el
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_mips64el
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_mips64el
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_mips64el
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_mips64el
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_mips64el
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_mips64el
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_mips64el
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_mips64el
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_mips64el
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_mips64el
 #define gen_helper_cpsr_read gen_helper_cpsr_read_mips64el
 #define gen_helper_cpsr_write gen_helper_cpsr_write_mips64el
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_mips64el
@@ -1440,7 +2211,10 @@
 #define helper_dvp helper_dvp_mips64el
 #define helper_evp helper_evp_mips64el
 #define cpu_mips_get_random cpu_mips_get_random_mips64el
+#define cpu_mips_get_count cpu_mips_get_count_mips64el
 #define cpu_mips_init cpu_mips_init_mips64el
+#define cpu_mips_store_count cpu_mips_store_count_mips64el
+#define cpu_mips_store_compare cpu_mips_store_compare_mips64el
 #define helper_absq_s_ph helper_absq_s_ph_mips64el
 #define helper_absq_s_qb helper_absq_s_qb_mips64el
 #define helper_absq_s_w helper_absq_s_w_mips64el
diff --git a/qemu/mipsel.h b/qemu/mipsel.h
index 1b30cbc229..c7495fed90 100644
--- a/qemu/mipsel.h
+++ b/qemu/mipsel.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_mipsel
 #define float32_squash_input_denormal float32_squash_input_denormal_mipsel
 #define float64_squash_input_denormal float64_squash_input_denormal_mipsel
+#define bfloat16_add bfloat16_add_mipsel
+#define bfloat16_compare bfloat16_compare_mipsel
+#define bfloat16_compare_quiet bfloat16_compare_quiet_mipsel
+#define bfloat16_default_nan bfloat16_default_nan_mipsel
+#define bfloat16_div bfloat16_div_mipsel
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_mipsel
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_mipsel
+#define bfloat16_max bfloat16_max_mipsel
+#define bfloat16_maximum_number bfloat16_maximum_number_mipsel
+#define bfloat16_maxnum bfloat16_maxnum_mipsel
+#define bfloat16_maxnummag bfloat16_maxnummag_mipsel
+#define bfloat16_min bfloat16_min_mipsel
+#define bfloat16_minimum_number bfloat16_minimum_number_mipsel
+#define bfloat16_minnum bfloat16_minnum_mipsel
+#define bfloat16_minnummag bfloat16_minnummag_mipsel
+#define bfloat16_mul bfloat16_mul_mipsel
+#define bfloat16_muladd bfloat16_muladd_mipsel
+#define bfloat16_round_to_int bfloat16_round_to_int_mipsel
+#define bfloat16_scalbn bfloat16_scalbn_mipsel
+#define bfloat16_silence_nan bfloat16_silence_nan_mipsel
+#define bfloat16_sqrt bfloat16_sqrt_mipsel
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_mipsel
+#define bfloat16_sub bfloat16_sub_mipsel
+#define bfloat16_to_float32 bfloat16_to_float32_mipsel
+#define bfloat16_to_float64 bfloat16_to_float64_mipsel
+#define bfloat16_to_int16 bfloat16_to_int16_mipsel
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_mipsel
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_mipsel
+#define bfloat16_to_int32 bfloat16_to_int32_mipsel
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_mipsel
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_mipsel
+#define bfloat16_to_int64 bfloat16_to_int64_mipsel
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_mipsel
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_mipsel
+#define bfloat16_to_uint16 bfloat16_to_uint16_mipsel
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_mipsel
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_mipsel
+#define bfloat16_to_uint32 bfloat16_to_uint32_mipsel
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_mipsel
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_mipsel
+#define bfloat16_to_uint64 bfloat16_to_uint64_mipsel
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_mipsel
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_mipsel
+#define float128_maximum_number float128_maximum_number_mipsel
+#define float128_max float128_max_mipsel
+#define float128_maxnum float128_maxnum_mipsel
+#define float128_maxnummag float128_maxnummag_mipsel
+#define float128_min float128_min_mipsel
+#define float128_minimum_number float128_minimum_number_mipsel
+#define float128_minnum float128_minnum_mipsel
+#define float128_minnummag float128_minnummag_mipsel
+#define float128_muladd float128_muladd_mipsel
+#define float128_to_int128 float128_to_int128_mipsel
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_mipsel
+#define float128_to_uint128 float128_to_uint128_mipsel
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_mipsel
+#define float16_maximum_number float16_maximum_number_mipsel
+#define float16_minimum_number float16_minimum_number_mipsel
+#define float16_to_int8 float16_to_int8_mipsel
+#define float16_to_int8_scalbn float16_to_int8_scalbn_mipsel
+#define float16_to_uint8 float16_to_uint8_mipsel
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_mipsel
+#define float32_maximum_number float32_maximum_number_mipsel
+#define float32_minimum_number float32_minimum_number_mipsel
+#define float32_to_bfloat16 float32_to_bfloat16_mipsel
+#define float64_maximum_number float64_maximum_number_mipsel
+#define float64_minimum_number float64_minimum_number_mipsel
+#define float64_to_bfloat16 float64_to_bfloat16_mipsel
+#define float64r32_add float64r32_add_mipsel
+#define float64r32_div float64r32_div_mipsel
+#define float64r32_mul float64r32_mul_mipsel
+#define float64r32_muladd float64r32_muladd_mipsel
+#define float64r32_sqrt float64r32_sqrt_mipsel
+#define float64r32_sub float64r32_sub_mipsel
+#define floatx80_mod floatx80_mod_mipsel
+#define floatx80_modrem floatx80_modrem_mipsel
+#define int128_to_float128 int128_to_float128_mipsel
+#define int16_to_bfloat16 int16_to_bfloat16_mipsel
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_mipsel
+#define int32_to_bfloat16 int32_to_bfloat16_mipsel
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_mipsel
+#define int64_to_bfloat16 int64_to_bfloat16_mipsel
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_mipsel
+#define int8_to_float16 int8_to_float16_mipsel
+#define uint128_to_float128 uint128_to_float128_mipsel
+#define uint16_to_bfloat16 uint16_to_bfloat16_mipsel
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_mipsel
+#define uint32_to_bfloat16 uint32_to_bfloat16_mipsel
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_mipsel
+#define uint64_to_bfloat16 uint64_to_bfloat16_mipsel
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_mipsel
+#define uint8_to_float16 uint8_to_float16_mipsel
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_mipsel
 #define roundAndPackFloatx80 roundAndPackFloatx80_mipsel
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_mipsel
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_mipsel
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_mipsel
 #define helper_exit_atomic helper_exit_atomic_mipsel
+#define helper_memset helper_memset_mipsel
+#define helper_emu_stop helper_emu_stop_mipsel
+#define tcg_remove_ops_after tcg_remove_ops_after_mipsel
+#define tcg_constant_vec_matching tcg_constant_vec_matching_mipsel
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_mipsel
 #define helper_gvec_add8 helper_gvec_add8_mipsel
 #define helper_gvec_add16 helper_gvec_add16_mipsel
 #define helper_gvec_add32 helper_gvec_add32_mipsel
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_mipsel
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_mipsel
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_mipsel
+#define gen_helper_mve_vctp gen_helper_mve_vctp_mipsel
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_mipsel
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_mipsel
+#define gen_helper_mve_vdup gen_helper_mve_vdup_mipsel
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_mipsel
+#define gen_helper_mve_vandi gen_helper_mve_vandi_mipsel
+#define gen_helper_mve_vorri gen_helper_mve_vorri_mipsel
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_mipsel
+#define gen_helper_mve_viduph gen_helper_mve_viduph_mipsel
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_mipsel
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_mipsel
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_mipsel
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_mipsel
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_mipsel
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_mipsel
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_mipsel
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_mipsel
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_mipsel
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_mipsel
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_mipsel
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_mipsel
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_mipsel
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_mipsel
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_mipsel
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_mipsel
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_mipsel
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_mipsel
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_mipsel
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_mipsel
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_mipsel
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_mipsel
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_mipsel
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_mipsel
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_mipsel
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_mipsel
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_mipsel
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_mipsel
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_mipsel
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_mipsel
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_mipsel
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_mipsel
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_mipsel
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_mipsel
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_mipsel
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_mipsel
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_mipsel
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_mipsel
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_mipsel
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_mipsel
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_mipsel
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_mipsel
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_mipsel
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_mipsel
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_mipsel
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_mipsel
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_mipsel
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_mipsel
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_mipsel
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_mipsel
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_mipsel
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_mipsel
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_mipsel
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_mipsel
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_mipsel
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_mipsel
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_mipsel
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_mipsel
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_mipsel
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_mipsel
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_mipsel
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_mipsel
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_mipsel
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_mipsel
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_mipsel
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_mipsel
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_mipsel
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_mipsel
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_mipsel
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_mipsel
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_mipsel
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_mipsel
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_mipsel
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_mipsel
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_mipsel
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_mipsel
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_mipsel
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_mipsel
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_mipsel
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_mipsel
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_mipsel
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_mipsel
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_mipsel
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_mipsel
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_mipsel
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_mipsel
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_mipsel
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_mipsel
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_mipsel
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_mipsel
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_mipsel
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_mipsel
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_mipsel
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_mipsel
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_mipsel
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_mipsel
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_mipsel
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_mipsel
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_mipsel
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_mipsel
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_mipsel
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_mipsel
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_mipsel
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_mipsel
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_mipsel
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_mipsel
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_mipsel
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_mipsel
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_mipsel
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_mipsel
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_mipsel
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_mipsel
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_mipsel
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_mipsel
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_mipsel
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_mipsel
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_mipsel
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_mipsel
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_mipsel
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_mipsel
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_mipsel
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_mipsel
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_mipsel
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_mipsel
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_mipsel
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_mipsel
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_mipsel
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_mipsel
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_mipsel
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_mipsel
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_mipsel
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_mipsel
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_mipsel
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_mipsel
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_mipsel
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_mipsel
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_mipsel
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_mipsel
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_mipsel
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_mipsel
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_mipsel
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_mipsel
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_mipsel
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_mipsel
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_mipsel
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_mipsel
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_mipsel
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_mipsel
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_mipsel
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_mipsel
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_mipsel
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_mipsel
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_mipsel
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_mipsel
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_mipsel
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_mipsel
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_mipsel
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_mipsel
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_mipsel
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_mipsel
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_mipsel
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_mipsel
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_mipsel
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_mipsel
+#define gen_helper_mve_vand gen_helper_mve_vand_mipsel
+#define gen_helper_mve_vbic gen_helper_mve_vbic_mipsel
+#define gen_helper_mve_vorr gen_helper_mve_vorr_mipsel
+#define gen_helper_mve_vorn gen_helper_mve_vorn_mipsel
+#define gen_helper_mve_veor gen_helper_mve_veor_mipsel
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_mipsel
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_mipsel
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_mipsel
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_mipsel
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_mipsel
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_mipsel
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_mipsel
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_mipsel
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_mipsel
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_mipsel
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_mipsel
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_mipsel
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_mipsel
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_mipsel
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_mipsel
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_mipsel
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_mipsel
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_mipsel
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_mipsel
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_mipsel
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_mipsel
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_mipsel
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_mipsel
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_mipsel
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_mipsel
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_mipsel
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_mipsel
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_mipsel
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_mipsel
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_mipsel
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_mipsel
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_mipsel
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_mipsel
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_mipsel
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_mipsel
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_mipsel
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_mipsel
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_mipsel
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_mipsel
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_mipsel
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_mipsel
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_mipsel
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_mipsel
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_mipsel
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_mipsel
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_mipsel
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_mipsel
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_mipsel
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_mipsel
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_mipsel
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_mipsel
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_mipsel
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_mipsel
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_mipsel
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_mipsel
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_mipsel
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_mipsel
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_mipsel
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_mipsel
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_mipsel
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_mipsel
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_mipsel
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_mipsel
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_mipsel
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_mipsel
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_mipsel
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_mipsel
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_mipsel
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_mipsel
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_mipsel
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_mipsel
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_mipsel
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_mipsel
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_mipsel
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_mipsel
+#define gen_helper_mve_vminub gen_helper_mve_vminub_mipsel
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_mipsel
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_mipsel
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_mipsel
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_mipsel
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_mipsel
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_mipsel
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_mipsel
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_mipsel
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_mipsel
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_mipsel
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_mipsel
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_mipsel
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_mipsel
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_mipsel
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_mipsel
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_mipsel
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_mipsel
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_mipsel
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_mipsel
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_mipsel
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_mipsel
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_mipsel
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_mipsel
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_mipsel
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_mipsel
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_mipsel
+#define gen_helper_mve_vadc gen_helper_mve_vadc_mipsel
+#define gen_helper_mve_vadci gen_helper_mve_vadci_mipsel
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_mipsel
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_mipsel
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_mipsel
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_mipsel
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_mipsel
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_mipsel
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_mipsel
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_mipsel
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_mipsel
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_mipsel
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_mipsel
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_mipsel
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_mipsel
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_mipsel
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_mipsel
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_mipsel
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_mipsel
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_mipsel
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_mipsel
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_mipsel
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_mipsel
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_mipsel
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_mipsel
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_mipsel
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_mipsel
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_mipsel
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_mipsel
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_mipsel
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_mipsel
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_mipsel
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_mipsel
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_mipsel
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_mipsel
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_mipsel
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_mipsel
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_mipsel
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_mipsel
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_mipsel
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_mipsel
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_mipsel
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_mipsel
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_mipsel
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_mipsel
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_mipsel
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_mipsel
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_mipsel
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_mipsel
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_mipsel
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_mipsel
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_mipsel
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_mipsel
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_mipsel
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_mipsel
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_mipsel
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_mipsel
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_mipsel
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_mipsel
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_mipsel
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_mipsel
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_mipsel
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_mipsel
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_mipsel
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_mipsel
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_mipsel
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_mipsel
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_mipsel
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_mipsel
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_mipsel
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_mipsel
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_mipsel
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_mipsel
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_mipsel
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_mipsel
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_mipsel
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_mipsel
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_mipsel
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_mipsel
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_mipsel
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_mipsel
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_mipsel
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_mipsel
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_mipsel
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_mipsel
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_mipsel
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_mipsel
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_mipsel
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_mipsel
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_mipsel
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_mipsel
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_mipsel
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_mipsel
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_mipsel
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_mipsel
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_mipsel
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_mipsel
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_mipsel
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_mipsel
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_mipsel
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_mipsel
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_mipsel
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_mipsel
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_mipsel
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_mipsel
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_mipsel
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_mipsel
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_mipsel
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_mipsel
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_mipsel
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_mipsel
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_mipsel
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_mipsel
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_mipsel
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_mipsel
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_mipsel
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_mipsel
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_mipsel
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_mipsel
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_mipsel
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_mipsel
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_mipsel
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_mipsel
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_mipsel
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_mipsel
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_mipsel
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_mipsel
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_mipsel
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_mipsel
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_mipsel
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_mipsel
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_mipsel
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_mipsel
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_mipsel
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_mipsel
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_mipsel
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_mipsel
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_mipsel
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_mipsel
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_mipsel
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_mipsel
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_mipsel
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_mipsel
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_mipsel
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_mipsel
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_mipsel
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_mipsel
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_mipsel
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_mipsel
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_mipsel
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_mipsel
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_mipsel
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_mipsel
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_mipsel
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_mipsel
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_mipsel
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_mipsel
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_mipsel
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_mipsel
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_mipsel
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_mipsel
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_mipsel
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_mipsel
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_mipsel
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_mipsel
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_mipsel
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_mipsel
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_mipsel
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_mipsel
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_mipsel
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_mipsel
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_mipsel
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_mipsel
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_mipsel
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_mipsel
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_mipsel
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_mipsel
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_mipsel
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_mipsel
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_mipsel
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_mipsel
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_mipsel
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_mipsel
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_mipsel
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_mipsel
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_mipsel
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_mipsel
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_mipsel
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_mipsel
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_mipsel
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_mipsel
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_mipsel
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_mipsel
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_mipsel
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_mipsel
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_mipsel
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_mipsel
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_mipsel
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_mipsel
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_mipsel
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_mipsel
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_mipsel
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_mipsel
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_mipsel
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_mipsel
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_mipsel
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_mipsel
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_mipsel
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_mipsel
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_mipsel
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_mipsel
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_mipsel
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_mipsel
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_mipsel
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_mipsel
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_mipsel
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_mipsel
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_mipsel
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_mipsel
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_mipsel
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_mipsel
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_mipsel
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_mipsel
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_mipsel
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_mipsel
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_mipsel
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_mipsel
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_mipsel
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_mipsel
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_mipsel
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_mipsel
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_mipsel
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_mipsel
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_mipsel
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_mipsel
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_mipsel
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_mipsel
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_mipsel
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_mipsel
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_mipsel
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_mipsel
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_mipsel
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_mipsel
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_mipsel
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_mipsel
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_mipsel
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_mipsel
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_mipsel
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_mipsel
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_mipsel
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_mipsel
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_mipsel
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_mipsel
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_mipsel
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_mipsel
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_mipsel
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_mipsel
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_mipsel
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_mipsel
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_mipsel
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_mipsel
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_mipsel
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_mipsel
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_mipsel
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_mipsel
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_mipsel
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_mipsel
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_mipsel
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_mipsel
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_mipsel
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_mipsel
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_mipsel
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_mipsel
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_mipsel
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_mipsel
+#define gen_helper_mve_ushll gen_helper_mve_ushll_mipsel
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_mipsel
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_mipsel
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_mipsel
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_mipsel
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_mipsel
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_mipsel
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_mipsel
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_mipsel
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_mipsel
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_mipsel
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_mipsel
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_mipsel
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_mipsel
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_mipsel
+#define gen_helper_mve_vslib gen_helper_mve_vslib_mipsel
+#define gen_helper_mve_vslih gen_helper_mve_vslih_mipsel
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_mipsel
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_mipsel
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_mipsel
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_mipsel
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_mipsel
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_mipsel
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_mipsel
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_mipsel
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_mipsel
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_mipsel
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_mipsel
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_mipsel
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_mipsel
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_mipsel
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_mipsel
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_mipsel
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_mipsel
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_mipsel
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_mipsel
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_mipsel
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_mipsel
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_mipsel
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_mipsel
+#define gen_helper_mve_vminab gen_helper_mve_vminab_mipsel
+#define gen_helper_mve_vminah gen_helper_mve_vminah_mipsel
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_mipsel
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_mipsel
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_mipsel
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_mipsel
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_mipsel
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_mipsel
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_mipsel
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_mipsel
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_mipsel
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_mipsel
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_mipsel
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_mipsel
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_mipsel
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_mipsel
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_mipsel
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_mipsel
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_mipsel
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_mipsel
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_mipsel
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_mipsel
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_mipsel
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_mipsel
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_mipsel
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_mipsel
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_mipsel
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_mipsel
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_mipsel
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_mipsel
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_mipsel
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_mipsel
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_mipsel
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_mipsel
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_mipsel
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_mipsel
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_mipsel
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_mipsel
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_mipsel
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_mipsel
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_mipsel
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_mipsel
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_mipsel
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_mipsel
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_mipsel
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_mipsel
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_mipsel
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_mipsel
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_mipsel
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_mipsel
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_mipsel
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_mipsel
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_mipsel
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_mipsel
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_mipsel
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_mipsel
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_mipsel
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_mipsel
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_mipsel
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_mipsel
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_mipsel
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_mipsel
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_mipsel
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_mipsel
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_mipsel
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_mipsel
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_mipsel
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_mipsel
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_mipsel
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_mipsel
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_mipsel
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_mipsel
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_mipsel
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_mipsel
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_mipsel
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_mipsel
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_mipsel
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_mipsel
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_mipsel
 #define gen_helper_cpsr_read gen_helper_cpsr_read_mipsel
 #define gen_helper_cpsr_write gen_helper_cpsr_write_mipsel
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_mipsel
@@ -1440,7 +2211,10 @@
 #define helper_dvp helper_dvp_mipsel
 #define helper_evp helper_evp_mipsel
 #define cpu_mips_get_random cpu_mips_get_random_mipsel
+#define cpu_mips_get_count cpu_mips_get_count_mipsel
 #define cpu_mips_init cpu_mips_init_mipsel
+#define cpu_mips_store_count cpu_mips_store_count_mipsel
+#define cpu_mips_store_compare cpu_mips_store_compare_mipsel
 #define helper_absq_s_ph helper_absq_s_ph_mipsel
 #define helper_absq_s_qb helper_absq_s_qb_mipsel
 #define helper_absq_s_w helper_absq_s_w_mipsel
diff --git a/qemu/ppc.h b/qemu/ppc.h
index 9648fd2fe5..bc6683ca62 100644
--- a/qemu/ppc.h
+++ b/qemu/ppc.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_ppc
 #define float32_squash_input_denormal float32_squash_input_denormal_ppc
 #define float64_squash_input_denormal float64_squash_input_denormal_ppc
+#define bfloat16_add bfloat16_add_ppc
+#define bfloat16_compare bfloat16_compare_ppc
+#define bfloat16_compare_quiet bfloat16_compare_quiet_ppc
+#define bfloat16_default_nan bfloat16_default_nan_ppc
+#define bfloat16_div bfloat16_div_ppc
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_ppc
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_ppc
+#define bfloat16_max bfloat16_max_ppc
+#define bfloat16_maximum_number bfloat16_maximum_number_ppc
+#define bfloat16_maxnum bfloat16_maxnum_ppc
+#define bfloat16_maxnummag bfloat16_maxnummag_ppc
+#define bfloat16_min bfloat16_min_ppc
+#define bfloat16_minimum_number bfloat16_minimum_number_ppc
+#define bfloat16_minnum bfloat16_minnum_ppc
+#define bfloat16_minnummag bfloat16_minnummag_ppc
+#define bfloat16_mul bfloat16_mul_ppc
+#define bfloat16_muladd bfloat16_muladd_ppc
+#define bfloat16_round_to_int bfloat16_round_to_int_ppc
+#define bfloat16_scalbn bfloat16_scalbn_ppc
+#define bfloat16_silence_nan bfloat16_silence_nan_ppc
+#define bfloat16_sqrt bfloat16_sqrt_ppc
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_ppc
+#define bfloat16_sub bfloat16_sub_ppc
+#define bfloat16_to_float32 bfloat16_to_float32_ppc
+#define bfloat16_to_float64 bfloat16_to_float64_ppc
+#define bfloat16_to_int16 bfloat16_to_int16_ppc
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_ppc
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_ppc
+#define bfloat16_to_int32 bfloat16_to_int32_ppc
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_ppc
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_ppc
+#define bfloat16_to_int64 bfloat16_to_int64_ppc
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_ppc
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_ppc
+#define bfloat16_to_uint16 bfloat16_to_uint16_ppc
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_ppc
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_ppc
+#define bfloat16_to_uint32 bfloat16_to_uint32_ppc
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_ppc
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_ppc
+#define bfloat16_to_uint64 bfloat16_to_uint64_ppc
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_ppc
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_ppc
+#define float128_maximum_number float128_maximum_number_ppc
+#define float128_max float128_max_ppc
+#define float128_maxnum float128_maxnum_ppc
+#define float128_maxnummag float128_maxnummag_ppc
+#define float128_min float128_min_ppc
+#define float128_minimum_number float128_minimum_number_ppc
+#define float128_minnum float128_minnum_ppc
+#define float128_minnummag float128_minnummag_ppc
+#define float128_muladd float128_muladd_ppc
+#define float128_to_int128 float128_to_int128_ppc
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_ppc
+#define float128_to_uint128 float128_to_uint128_ppc
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_ppc
+#define float16_maximum_number float16_maximum_number_ppc
+#define float16_minimum_number float16_minimum_number_ppc
+#define float16_to_int8 float16_to_int8_ppc
+#define float16_to_int8_scalbn float16_to_int8_scalbn_ppc
+#define float16_to_uint8 float16_to_uint8_ppc
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_ppc
+#define float32_maximum_number float32_maximum_number_ppc
+#define float32_minimum_number float32_minimum_number_ppc
+#define float32_to_bfloat16 float32_to_bfloat16_ppc
+#define float64_maximum_number float64_maximum_number_ppc
+#define float64_minimum_number float64_minimum_number_ppc
+#define float64_to_bfloat16 float64_to_bfloat16_ppc
+#define float64r32_add float64r32_add_ppc
+#define float64r32_div float64r32_div_ppc
+#define float64r32_mul float64r32_mul_ppc
+#define float64r32_muladd float64r32_muladd_ppc
+#define float64r32_sqrt float64r32_sqrt_ppc
+#define float64r32_sub float64r32_sub_ppc
+#define floatx80_mod floatx80_mod_ppc
+#define floatx80_modrem floatx80_modrem_ppc
+#define int128_to_float128 int128_to_float128_ppc
+#define int16_to_bfloat16 int16_to_bfloat16_ppc
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_ppc
+#define int32_to_bfloat16 int32_to_bfloat16_ppc
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_ppc
+#define int64_to_bfloat16 int64_to_bfloat16_ppc
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_ppc
+#define int8_to_float16 int8_to_float16_ppc
+#define uint128_to_float128 uint128_to_float128_ppc
+#define uint16_to_bfloat16 uint16_to_bfloat16_ppc
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_ppc
+#define uint32_to_bfloat16 uint32_to_bfloat16_ppc
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_ppc
+#define uint64_to_bfloat16 uint64_to_bfloat16_ppc
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_ppc
+#define uint8_to_float16 uint8_to_float16_ppc
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_ppc
 #define roundAndPackFloatx80 roundAndPackFloatx80_ppc
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_ppc
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_ppc
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_ppc
 #define helper_exit_atomic helper_exit_atomic_ppc
+#define helper_memset helper_memset_ppc
+#define helper_emu_stop helper_emu_stop_ppc
+#define tcg_remove_ops_after tcg_remove_ops_after_ppc
+#define tcg_constant_vec_matching tcg_constant_vec_matching_ppc
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_ppc
 #define helper_gvec_add8 helper_gvec_add8_ppc
 #define helper_gvec_add16 helper_gvec_add16_ppc
 #define helper_gvec_add32 helper_gvec_add32_ppc
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_ppc
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_ppc
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_ppc
+#define gen_helper_mve_vctp gen_helper_mve_vctp_ppc
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_ppc
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_ppc
+#define gen_helper_mve_vdup gen_helper_mve_vdup_ppc
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_ppc
+#define gen_helper_mve_vandi gen_helper_mve_vandi_ppc
+#define gen_helper_mve_vorri gen_helper_mve_vorri_ppc
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_ppc
+#define gen_helper_mve_viduph gen_helper_mve_viduph_ppc
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_ppc
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_ppc
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_ppc
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_ppc
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_ppc
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_ppc
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_ppc
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_ppc
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_ppc
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_ppc
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_ppc
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_ppc
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_ppc
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_ppc
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_ppc
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_ppc
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_ppc
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_ppc
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_ppc
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_ppc
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_ppc
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_ppc
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_ppc
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_ppc
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_ppc
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_ppc
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_ppc
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_ppc
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_ppc
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_ppc
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_ppc
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_ppc
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_ppc
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_ppc
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_ppc
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_ppc
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_ppc
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_ppc
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_ppc
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_ppc
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_ppc
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_ppc
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_ppc
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_ppc
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_ppc
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_ppc
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_ppc
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_ppc
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_ppc
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_ppc
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_ppc
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_ppc
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_ppc
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_ppc
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_ppc
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_ppc
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_ppc
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_ppc
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_ppc
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_ppc
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_ppc
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_ppc
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_ppc
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_ppc
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_ppc
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_ppc
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_ppc
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_ppc
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_ppc
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_ppc
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_ppc
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_ppc
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_ppc
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_ppc
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_ppc
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_ppc
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_ppc
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_ppc
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_ppc
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_ppc
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_ppc
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_ppc
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_ppc
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_ppc
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_ppc
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_ppc
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_ppc
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_ppc
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_ppc
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_ppc
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_ppc
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_ppc
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_ppc
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_ppc
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_ppc
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_ppc
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_ppc
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_ppc
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_ppc
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_ppc
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_ppc
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_ppc
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_ppc
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_ppc
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_ppc
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_ppc
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_ppc
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_ppc
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_ppc
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_ppc
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_ppc
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_ppc
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_ppc
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_ppc
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_ppc
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_ppc
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_ppc
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_ppc
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_ppc
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_ppc
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_ppc
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_ppc
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_ppc
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_ppc
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_ppc
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_ppc
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_ppc
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_ppc
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_ppc
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_ppc
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_ppc
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_ppc
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_ppc
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_ppc
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_ppc
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_ppc
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_ppc
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_ppc
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_ppc
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_ppc
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_ppc
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_ppc
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_ppc
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_ppc
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_ppc
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_ppc
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_ppc
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_ppc
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_ppc
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_ppc
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_ppc
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_ppc
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_ppc
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_ppc
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_ppc
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_ppc
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_ppc
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_ppc
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_ppc
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_ppc
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_ppc
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_ppc
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_ppc
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_ppc
+#define gen_helper_mve_vand gen_helper_mve_vand_ppc
+#define gen_helper_mve_vbic gen_helper_mve_vbic_ppc
+#define gen_helper_mve_vorr gen_helper_mve_vorr_ppc
+#define gen_helper_mve_vorn gen_helper_mve_vorn_ppc
+#define gen_helper_mve_veor gen_helper_mve_veor_ppc
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_ppc
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_ppc
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_ppc
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_ppc
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_ppc
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_ppc
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_ppc
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_ppc
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_ppc
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_ppc
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_ppc
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_ppc
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_ppc
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_ppc
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_ppc
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_ppc
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_ppc
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_ppc
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_ppc
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_ppc
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_ppc
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_ppc
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_ppc
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_ppc
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_ppc
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_ppc
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_ppc
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_ppc
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_ppc
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_ppc
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_ppc
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_ppc
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_ppc
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_ppc
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_ppc
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_ppc
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_ppc
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_ppc
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_ppc
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_ppc
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_ppc
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_ppc
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_ppc
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_ppc
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_ppc
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_ppc
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_ppc
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_ppc
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_ppc
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_ppc
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_ppc
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_ppc
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_ppc
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_ppc
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_ppc
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_ppc
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_ppc
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_ppc
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_ppc
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_ppc
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_ppc
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_ppc
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_ppc
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_ppc
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_ppc
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_ppc
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_ppc
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_ppc
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_ppc
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_ppc
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_ppc
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_ppc
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_ppc
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_ppc
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_ppc
+#define gen_helper_mve_vminub gen_helper_mve_vminub_ppc
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_ppc
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_ppc
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_ppc
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_ppc
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_ppc
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_ppc
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_ppc
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_ppc
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_ppc
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_ppc
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_ppc
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_ppc
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_ppc
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_ppc
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_ppc
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_ppc
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_ppc
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_ppc
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_ppc
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_ppc
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_ppc
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_ppc
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_ppc
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_ppc
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_ppc
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_ppc
+#define gen_helper_mve_vadc gen_helper_mve_vadc_ppc
+#define gen_helper_mve_vadci gen_helper_mve_vadci_ppc
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_ppc
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_ppc
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_ppc
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_ppc
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_ppc
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_ppc
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_ppc
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_ppc
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_ppc
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_ppc
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_ppc
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_ppc
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_ppc
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_ppc
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_ppc
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_ppc
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_ppc
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_ppc
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_ppc
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_ppc
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_ppc
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_ppc
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_ppc
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_ppc
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_ppc
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_ppc
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_ppc
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_ppc
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_ppc
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_ppc
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_ppc
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_ppc
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_ppc
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_ppc
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_ppc
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_ppc
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_ppc
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_ppc
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_ppc
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_ppc
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_ppc
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_ppc
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_ppc
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_ppc
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_ppc
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_ppc
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_ppc
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_ppc
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_ppc
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_ppc
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_ppc
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_ppc
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_ppc
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_ppc
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_ppc
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_ppc
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_ppc
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_ppc
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_ppc
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_ppc
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_ppc
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_ppc
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_ppc
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_ppc
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_ppc
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_ppc
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_ppc
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_ppc
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_ppc
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_ppc
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_ppc
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_ppc
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_ppc
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_ppc
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_ppc
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_ppc
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_ppc
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_ppc
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_ppc
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_ppc
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_ppc
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_ppc
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_ppc
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_ppc
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_ppc
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_ppc
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_ppc
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_ppc
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_ppc
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_ppc
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_ppc
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_ppc
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_ppc
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_ppc
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_ppc
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_ppc
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_ppc
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_ppc
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_ppc
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_ppc
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_ppc
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_ppc
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_ppc
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_ppc
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_ppc
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_ppc
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_ppc
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_ppc
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_ppc
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_ppc
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_ppc
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_ppc
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_ppc
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_ppc
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_ppc
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_ppc
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_ppc
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_ppc
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_ppc
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_ppc
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_ppc
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_ppc
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_ppc
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_ppc
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_ppc
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_ppc
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_ppc
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_ppc
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_ppc
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_ppc
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_ppc
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_ppc
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_ppc
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_ppc
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_ppc
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_ppc
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_ppc
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_ppc
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_ppc
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_ppc
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_ppc
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_ppc
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_ppc
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_ppc
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_ppc
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_ppc
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_ppc
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_ppc
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_ppc
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_ppc
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_ppc
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_ppc
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_ppc
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_ppc
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_ppc
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_ppc
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_ppc
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_ppc
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_ppc
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_ppc
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_ppc
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_ppc
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_ppc
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_ppc
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_ppc
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_ppc
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_ppc
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_ppc
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_ppc
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_ppc
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_ppc
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_ppc
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_ppc
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_ppc
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_ppc
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_ppc
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_ppc
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_ppc
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_ppc
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_ppc
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_ppc
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_ppc
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_ppc
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_ppc
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_ppc
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_ppc
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_ppc
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_ppc
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_ppc
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_ppc
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_ppc
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_ppc
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_ppc
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_ppc
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_ppc
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_ppc
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_ppc
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_ppc
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_ppc
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_ppc
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_ppc
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_ppc
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_ppc
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_ppc
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_ppc
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_ppc
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_ppc
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_ppc
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_ppc
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_ppc
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_ppc
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_ppc
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_ppc
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_ppc
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_ppc
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_ppc
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_ppc
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_ppc
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_ppc
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_ppc
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_ppc
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_ppc
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_ppc
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_ppc
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_ppc
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_ppc
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_ppc
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_ppc
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_ppc
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_ppc
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_ppc
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_ppc
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_ppc
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_ppc
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_ppc
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_ppc
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_ppc
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_ppc
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_ppc
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_ppc
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_ppc
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_ppc
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_ppc
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_ppc
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_ppc
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_ppc
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_ppc
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_ppc
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_ppc
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_ppc
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_ppc
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_ppc
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_ppc
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_ppc
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_ppc
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_ppc
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_ppc
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_ppc
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_ppc
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_ppc
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_ppc
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_ppc
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_ppc
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_ppc
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_ppc
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_ppc
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_ppc
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_ppc
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_ppc
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_ppc
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_ppc
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_ppc
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_ppc
+#define gen_helper_mve_ushll gen_helper_mve_ushll_ppc
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_ppc
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_ppc
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_ppc
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_ppc
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_ppc
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_ppc
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_ppc
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_ppc
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_ppc
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_ppc
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_ppc
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_ppc
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_ppc
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_ppc
+#define gen_helper_mve_vslib gen_helper_mve_vslib_ppc
+#define gen_helper_mve_vslih gen_helper_mve_vslih_ppc
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_ppc
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_ppc
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_ppc
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_ppc
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_ppc
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_ppc
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_ppc
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_ppc
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_ppc
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_ppc
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_ppc
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_ppc
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_ppc
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_ppc
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_ppc
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_ppc
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_ppc
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_ppc
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_ppc
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_ppc
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_ppc
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_ppc
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_ppc
+#define gen_helper_mve_vminab gen_helper_mve_vminab_ppc
+#define gen_helper_mve_vminah gen_helper_mve_vminah_ppc
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_ppc
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_ppc
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_ppc
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_ppc
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_ppc
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_ppc
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_ppc
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_ppc
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_ppc
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_ppc
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_ppc
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_ppc
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_ppc
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_ppc
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_ppc
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_ppc
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_ppc
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_ppc
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_ppc
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_ppc
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_ppc
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_ppc
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_ppc
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_ppc
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_ppc
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_ppc
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_ppc
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_ppc
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_ppc
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_ppc
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_ppc
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_ppc
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_ppc
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_ppc
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_ppc
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_ppc
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_ppc
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_ppc
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_ppc
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_ppc
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_ppc
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_ppc
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_ppc
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_ppc
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_ppc
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_ppc
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_ppc
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_ppc
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_ppc
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_ppc
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_ppc
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_ppc
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_ppc
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_ppc
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_ppc
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_ppc
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_ppc
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_ppc
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_ppc
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_ppc
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_ppc
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_ppc
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_ppc
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_ppc
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_ppc
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_ppc
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_ppc
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_ppc
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_ppc
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_ppc
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_ppc
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_ppc
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_ppc
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_ppc
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_ppc
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_ppc
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_ppc
 #define gen_helper_cpsr_read gen_helper_cpsr_read_ppc
 #define gen_helper_cpsr_write gen_helper_cpsr_write_ppc
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_ppc
@@ -1332,6 +2103,18 @@
 #define helper_vprtybd helper_vprtybd_ppc
 #define helper_vprtybq helper_vprtybq_ppc
 #define helper_vmuluwm helper_vmuluwm_ppc
+#define helper_VDIVSQ helper_VDIVSQ_ppc
+#define helper_VDIVUQ helper_VDIVUQ_ppc
+#define helper_VDIVESD helper_VDIVESD_ppc
+#define helper_VDIVEUD helper_VDIVEUD_ppc
+#define helper_VDIVESQ helper_VDIVESQ_ppc
+#define helper_VDIVEUQ helper_VDIVEUQ_ppc
+#define helper_VMODSQ helper_VMODSQ_ppc
+#define helper_VMODUQ helper_VMODUQ_ppc
+#define helper_VSTRIBL helper_VSTRIBL_ppc
+#define helper_VSTRIBR helper_VSTRIBR_ppc
+#define helper_VSTRIHL helper_VSTRIHL_ppc
+#define helper_VSTRIHR helper_VSTRIHR_ppc
 #define helper_vaddfp helper_vaddfp_ppc
 #define helper_vsubfp helper_vsubfp_ppc
 #define helper_vminfp helper_vminfp_ppc
@@ -1479,6 +2262,10 @@
 #define helper_vextubrx helper_vextubrx_ppc
 #define helper_vextuhrx helper_vextuhrx_ppc
 #define helper_vextuwrx helper_vextuwrx_ppc
+#define helper_VEXTDUBVLX helper_VEXTDUBVLX_ppc
+#define helper_VEXTDUHVLX helper_VEXTDUHVLX_ppc
+#define helper_VEXTDUWVLX helper_VEXTDUWVLX_ppc
+#define helper_VEXTDDVLX helper_VEXTDDVLX_ppc
 #define helper_vslv helper_vslv_ppc
 #define helper_vsrv helper_vsrv_ppc
 #define helper_vsldoi helper_vsldoi_ppc
@@ -1487,6 +2274,10 @@
 #define helper_vinserth helper_vinserth_ppc
 #define helper_vinsertw helper_vinsertw_ppc
 #define helper_vinsertd helper_vinsertd_ppc
+#define helper_VINSBLX helper_VINSBLX_ppc
+#define helper_VINSHLX helper_VINSHLX_ppc
+#define helper_VINSWLX helper_VINSWLX_ppc
+#define helper_VINSDLX helper_VINSDLX_ppc
 #define helper_vextractub helper_vextractub_ppc
 #define helper_vextractuh helper_vextractuh_ppc
 #define helper_vextractuw helper_vextractuw_ppc
@@ -1882,6 +2673,14 @@
 #define helper_xsmsubdp helper_xsmsubdp_ppc
 #define helper_xsnmadddp helper_xsnmadddp_ppc
 #define helper_xsnmsubdp helper_xsnmsubdp_ppc
+#define helper_XSMADDQP helper_XSMADDQP_ppc
+#define helper_XSMADDQPO helper_XSMADDQPO_ppc
+#define helper_XSMSUBQP helper_XSMSUBQP_ppc
+#define helper_XSMSUBQPO helper_XSMSUBQPO_ppc
+#define helper_XSNMADDQP helper_XSNMADDQP_ppc
+#define helper_XSNMADDQPO helper_XSNMADDQPO_ppc
+#define helper_XSNMSUBQP helper_XSNMSUBQP_ppc
+#define helper_XSNMSUBQPO helper_XSNMSUBQPO_ppc
 #define helper_xsmaddsp helper_xsmaddsp_ppc
 #define helper_xsmsubsp helper_xsmsubsp_ppc
 #define helper_xsnmaddsp helper_xsnmaddsp_ppc
@@ -1898,6 +2697,9 @@
 #define helper_xscmpgedp helper_xscmpgedp_ppc
 #define helper_xscmpgtdp helper_xscmpgtdp_ppc
 #define helper_xscmpnedp helper_xscmpnedp_ppc
+#define helper_XSCMPEQQP helper_XSCMPEQQP_ppc
+#define helper_XSCMPGEQP helper_XSCMPGEQP_ppc
+#define helper_XSCMPGTQP helper_XSCMPGTQP_ppc
 #define helper_xscmpexpdp helper_xscmpexpdp_ppc
 #define helper_xscmpexpqp helper_xscmpexpqp_ppc
 #define helper_xscmpodp helper_xscmpodp_ppc
@@ -1914,6 +2716,8 @@
 #define helper_xsmincdp helper_xsmincdp_ppc
 #define helper_xsmaxjdp helper_xsmaxjdp_ppc
 #define helper_xsminjdp helper_xsminjdp_ppc
+#define helper_XSMAXCQP helper_XSMAXCQP_ppc
+#define helper_XSMINCQP helper_XSMINCQP_ppc
 #define helper_xvcmpeqdp helper_xvcmpeqdp_ppc
 #define helper_xvcmpgedp helper_xvcmpgedp_ppc
 #define helper_xvcmpgtdp helper_xvcmpgtdp_ppc
@@ -1931,7 +2735,98 @@
 #define helper_xscvhpdp helper_xscvhpdp_ppc
 #define helper_xvcvsphp helper_xvcvsphp_ppc
 #define helper_xvcvhpsp helper_xvcvhpsp_ppc
+#define helper_XVCVSPBF16 helper_XVCVSPBF16_ppc
+#define helper_XVCVBF16SPN helper_XVCVBF16SPN_ppc
+#define helper_dadd helper_dadd_ppc
+#define helper_daddq helper_daddq_ppc
+#define helper_dsub helper_dsub_ppc
+#define helper_dsubq helper_dsubq_ppc
+#define helper_dmul helper_dmul_ppc
+#define helper_dmulq helper_dmulq_ppc
+#define helper_ddiv helper_ddiv_ppc
+#define helper_ddivq helper_ddivq_ppc
+#define helper_dcmpo helper_dcmpo_ppc
+#define helper_dcmpoq helper_dcmpoq_ppc
+#define helper_dcmpu helper_dcmpu_ppc
+#define helper_dcmpuq helper_dcmpuq_ppc
+#define helper_dtstdc helper_dtstdc_ppc
+#define helper_dtstdcq helper_dtstdcq_ppc
+#define helper_dtstdg helper_dtstdg_ppc
+#define helper_dtstdgq helper_dtstdgq_ppc
+#define helper_dtstex helper_dtstex_ppc
+#define helper_dtstexq helper_dtstexq_ppc
+#define helper_dtstsf helper_dtstsf_ppc
+#define helper_dtstsfq helper_dtstsfq_ppc
+#define helper_dtstsfi helper_dtstsfi_ppc
+#define helper_dtstsfiq helper_dtstsfiq_ppc
+#define helper_dquai helper_dquai_ppc
+#define helper_dquaiq helper_dquaiq_ppc
+#define helper_dqua helper_dqua_ppc
+#define helper_dquaq helper_dquaq_ppc
+#define helper_drrnd helper_drrnd_ppc
+#define helper_drrndq helper_drrndq_ppc
+#define helper_drintx helper_drintx_ppc
+#define helper_drintxq helper_drintxq_ppc
+#define helper_drintn helper_drintn_ppc
+#define helper_drintnq helper_drintnq_ppc
+#define helper_dctdp helper_dctdp_ppc
+#define helper_dctqpq helper_dctqpq_ppc
+#define helper_drsp helper_drsp_ppc
+#define helper_drdpq helper_drdpq_ppc
+#define helper_dcffix helper_dcffix_ppc
+#define helper_dcffixq helper_dcffixq_ppc
+#define helper_DCFFIXQQ helper_DCFFIXQQ_ppc
+#define helper_dctfix helper_dctfix_ppc
+#define helper_dctfixq helper_dctfixq_ppc
+#define helper_DCTFIXQQ helper_DCTFIXQQ_ppc
+#define helper_ddedpd helper_ddedpd_ppc
+#define helper_ddedpdq helper_ddedpdq_ppc
+#define helper_denbcd helper_denbcd_ppc
+#define helper_denbcdq helper_denbcdq_ppc
+#define helper_dxex helper_dxex_ppc
+#define helper_dxexq helper_dxexq_ppc
+#define helper_diex helper_diex_ppc
+#define helper_diexq helper_diexq_ppc
+#define helper_dscri helper_dscri_ppc
+#define helper_dscriq helper_dscriq_ppc
+#define helper_dscli helper_dscli_ppc
+#define helper_dscliq helper_dscliq_ppc
+#define helper_CDTBCD helper_CDTBCD_ppc
+#define helper_CBCDTD helper_CBCDTD_ppc
+#define helper_XVI4GER8 helper_XVI4GER8_ppc
+#define helper_XVI4GER8PP helper_XVI4GER8PP_ppc
+#define helper_XVI8GER4 helper_XVI8GER4_ppc
+#define helper_XVI8GER4PP helper_XVI8GER4PP_ppc
+#define helper_XVI8GER4SPP helper_XVI8GER4SPP_ppc
+#define helper_XVI16GER2 helper_XVI16GER2_ppc
+#define helper_XVI16GER2S helper_XVI16GER2S_ppc
+#define helper_XVI16GER2PP helper_XVI16GER2PP_ppc
+#define helper_XVI16GER2SPP helper_XVI16GER2SPP_ppc
+#define helper_XVBF16GER2 helper_XVBF16GER2_ppc
+#define helper_XVBF16GER2PP helper_XVBF16GER2PP_ppc
+#define helper_XVBF16GER2PN helper_XVBF16GER2PN_ppc
+#define helper_XVBF16GER2NP helper_XVBF16GER2NP_ppc
+#define helper_XVBF16GER2NN helper_XVBF16GER2NN_ppc
+#define helper_XVF16GER2 helper_XVF16GER2_ppc
+#define helper_XVF16GER2PP helper_XVF16GER2PP_ppc
+#define helper_XVF16GER2PN helper_XVF16GER2PN_ppc
+#define helper_XVF16GER2NP helper_XVF16GER2NP_ppc
+#define helper_XVF16GER2NN helper_XVF16GER2NN_ppc
+#define helper_XVF32GER helper_XVF32GER_ppc
+#define helper_XVF32GERPP helper_XVF32GERPP_ppc
+#define helper_XVF32GERPN helper_XVF32GERPN_ppc
+#define helper_XVF32GERNP helper_XVF32GERNP_ppc
+#define helper_XVF32GERNN helper_XVF32GERNN_ppc
+#define helper_XVF64GER helper_XVF64GER_ppc
+#define helper_XVF64GERPP helper_XVF64GERPP_ppc
+#define helper_XVF64GERPN helper_XVF64GERPN_ppc
+#define helper_XVF64GERNP helper_XVF64GERNP_ppc
+#define helper_XVF64GERNN helper_XVF64GERNN_ppc
 #define helper_xscvqpdp helper_xscvqpdp_ppc
+#define helper_XSCVQPUQZ helper_XSCVQPUQZ_ppc
+#define helper_XSCVQPSQZ helper_XSCVQPSQZ_ppc
+#define helper_XSCVUQQP helper_XSCVUQQP_ppc
+#define helper_XSCVSQQP helper_XSCVSQQP_ppc
 #define helper_xscvdpspn helper_xscvdpspn_ppc
 #define helper_xscvspdpn helper_xscvspdpn_ppc
 #define helper_xscvdpsxds helper_xscvdpsxds_ppc
@@ -1982,6 +2877,23 @@
 #define helper_xsrsp helper_xsrsp_ppc
 #define helper_xxperm helper_xxperm_ppc
 #define helper_xxpermr helper_xxpermr_ppc
+#define helper_XXPERMX helper_XXPERMX_ppc
+#define helper_XXGENPCVBM_be_exp helper_XXGENPCVBM_be_exp_ppc
+#define helper_XXGENPCVBM_be_comp helper_XXGENPCVBM_be_comp_ppc
+#define helper_XXGENPCVBM_le_exp helper_XXGENPCVBM_le_exp_ppc
+#define helper_XXGENPCVBM_le_comp helper_XXGENPCVBM_le_comp_ppc
+#define helper_XXGENPCVHM_be_exp helper_XXGENPCVHM_be_exp_ppc
+#define helper_XXGENPCVHM_be_comp helper_XXGENPCVHM_be_comp_ppc
+#define helper_XXGENPCVHM_le_exp helper_XXGENPCVHM_le_exp_ppc
+#define helper_XXGENPCVHM_le_comp helper_XXGENPCVHM_le_comp_ppc
+#define helper_XXGENPCVWM_be_exp helper_XXGENPCVWM_be_exp_ppc
+#define helper_XXGENPCVWM_be_comp helper_XXGENPCVWM_be_comp_ppc
+#define helper_XXGENPCVWM_le_exp helper_XXGENPCVWM_le_exp_ppc
+#define helper_XXGENPCVWM_le_comp helper_XXGENPCVWM_le_comp_ppc
+#define helper_XXGENPCVDM_be_exp helper_XXGENPCVDM_be_exp_ppc
+#define helper_XXGENPCVDM_be_comp helper_XXGENPCVDM_be_comp_ppc
+#define helper_XXGENPCVDM_le_exp helper_XXGENPCVDM_le_exp_ppc
+#define helper_XXGENPCVDM_le_comp helper_XXGENPCVDM_le_comp_ppc
 #define helper_xvxsigsp helper_xvxsigsp_ppc
 #define helper_xvtstdcdp helper_xvtstdcdp_ppc
 #define helper_xvtstdcsp helper_xvtstdcsp_ppc
diff --git a/qemu/ppc64.h b/qemu/ppc64.h
index 2d995d5ece..a42068e85d 100644
--- a/qemu/ppc64.h
+++ b/qemu/ppc64.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_ppc64
 #define float32_squash_input_denormal float32_squash_input_denormal_ppc64
 #define float64_squash_input_denormal float64_squash_input_denormal_ppc64
+#define bfloat16_add bfloat16_add_ppc64
+#define bfloat16_compare bfloat16_compare_ppc64
+#define bfloat16_compare_quiet bfloat16_compare_quiet_ppc64
+#define bfloat16_default_nan bfloat16_default_nan_ppc64
+#define bfloat16_div bfloat16_div_ppc64
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_ppc64
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_ppc64
+#define bfloat16_max bfloat16_max_ppc64
+#define bfloat16_maximum_number bfloat16_maximum_number_ppc64
+#define bfloat16_maxnum bfloat16_maxnum_ppc64
+#define bfloat16_maxnummag bfloat16_maxnummag_ppc64
+#define bfloat16_min bfloat16_min_ppc64
+#define bfloat16_minimum_number bfloat16_minimum_number_ppc64
+#define bfloat16_minnum bfloat16_minnum_ppc64
+#define bfloat16_minnummag bfloat16_minnummag_ppc64
+#define bfloat16_mul bfloat16_mul_ppc64
+#define bfloat16_muladd bfloat16_muladd_ppc64
+#define bfloat16_round_to_int bfloat16_round_to_int_ppc64
+#define bfloat16_scalbn bfloat16_scalbn_ppc64
+#define bfloat16_silence_nan bfloat16_silence_nan_ppc64
+#define bfloat16_sqrt bfloat16_sqrt_ppc64
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_ppc64
+#define bfloat16_sub bfloat16_sub_ppc64
+#define bfloat16_to_float32 bfloat16_to_float32_ppc64
+#define bfloat16_to_float64 bfloat16_to_float64_ppc64
+#define bfloat16_to_int16 bfloat16_to_int16_ppc64
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_ppc64
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_ppc64
+#define bfloat16_to_int32 bfloat16_to_int32_ppc64
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_ppc64
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_ppc64
+#define bfloat16_to_int64 bfloat16_to_int64_ppc64
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_ppc64
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_ppc64
+#define bfloat16_to_uint16 bfloat16_to_uint16_ppc64
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_ppc64
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_ppc64
+#define bfloat16_to_uint32 bfloat16_to_uint32_ppc64
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_ppc64
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_ppc64
+#define bfloat16_to_uint64 bfloat16_to_uint64_ppc64
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_ppc64
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_ppc64
+#define float128_maximum_number float128_maximum_number_ppc64
+#define float128_max float128_max_ppc64
+#define float128_maxnum float128_maxnum_ppc64
+#define float128_maxnummag float128_maxnummag_ppc64
+#define float128_min float128_min_ppc64
+#define float128_minimum_number float128_minimum_number_ppc64
+#define float128_minnum float128_minnum_ppc64
+#define float128_minnummag float128_minnummag_ppc64
+#define float128_muladd float128_muladd_ppc64
+#define float128_to_int128 float128_to_int128_ppc64
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_ppc64
+#define float128_to_uint128 float128_to_uint128_ppc64
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_ppc64
+#define float16_maximum_number float16_maximum_number_ppc64
+#define float16_minimum_number float16_minimum_number_ppc64
+#define float16_to_int8 float16_to_int8_ppc64
+#define float16_to_int8_scalbn float16_to_int8_scalbn_ppc64
+#define float16_to_uint8 float16_to_uint8_ppc64
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_ppc64
+#define float32_maximum_number float32_maximum_number_ppc64
+#define float32_minimum_number float32_minimum_number_ppc64
+#define float32_to_bfloat16 float32_to_bfloat16_ppc64
+#define float64_maximum_number float64_maximum_number_ppc64
+#define float64_minimum_number float64_minimum_number_ppc64
+#define float64_to_bfloat16 float64_to_bfloat16_ppc64
+#define float64r32_add float64r32_add_ppc64
+#define float64r32_div float64r32_div_ppc64
+#define float64r32_mul float64r32_mul_ppc64
+#define float64r32_muladd float64r32_muladd_ppc64
+#define float64r32_sqrt float64r32_sqrt_ppc64
+#define float64r32_sub float64r32_sub_ppc64
+#define floatx80_mod floatx80_mod_ppc64
+#define floatx80_modrem floatx80_modrem_ppc64
+#define int128_to_float128 int128_to_float128_ppc64
+#define int16_to_bfloat16 int16_to_bfloat16_ppc64
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_ppc64
+#define int32_to_bfloat16 int32_to_bfloat16_ppc64
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_ppc64
+#define int64_to_bfloat16 int64_to_bfloat16_ppc64
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_ppc64
+#define int8_to_float16 int8_to_float16_ppc64
+#define uint128_to_float128 uint128_to_float128_ppc64
+#define uint16_to_bfloat16 uint16_to_bfloat16_ppc64
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_ppc64
+#define uint32_to_bfloat16 uint32_to_bfloat16_ppc64
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_ppc64
+#define uint64_to_bfloat16 uint64_to_bfloat16_ppc64
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_ppc64
+#define uint8_to_float16 uint8_to_float16_ppc64
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_ppc64
 #define roundAndPackFloatx80 roundAndPackFloatx80_ppc64
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_ppc64
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_ppc64
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_ppc64
 #define helper_exit_atomic helper_exit_atomic_ppc64
+#define helper_memset helper_memset_ppc64
+#define helper_emu_stop helper_emu_stop_ppc64
+#define tcg_remove_ops_after tcg_remove_ops_after_ppc64
+#define tcg_constant_vec_matching tcg_constant_vec_matching_ppc64
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_ppc64
 #define helper_gvec_add8 helper_gvec_add8_ppc64
 #define helper_gvec_add16 helper_gvec_add16_ppc64
 #define helper_gvec_add32 helper_gvec_add32_ppc64
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_ppc64
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_ppc64
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_ppc64
+#define gen_helper_mve_vctp gen_helper_mve_vctp_ppc64
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_ppc64
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_ppc64
+#define gen_helper_mve_vdup gen_helper_mve_vdup_ppc64
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_ppc64
+#define gen_helper_mve_vandi gen_helper_mve_vandi_ppc64
+#define gen_helper_mve_vorri gen_helper_mve_vorri_ppc64
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_ppc64
+#define gen_helper_mve_viduph gen_helper_mve_viduph_ppc64
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_ppc64
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_ppc64
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_ppc64
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_ppc64
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_ppc64
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_ppc64
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_ppc64
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_ppc64
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_ppc64
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_ppc64
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_ppc64
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_ppc64
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_ppc64
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_ppc64
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_ppc64
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_ppc64
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_ppc64
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_ppc64
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_ppc64
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_ppc64
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_ppc64
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_ppc64
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_ppc64
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_ppc64
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_ppc64
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_ppc64
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_ppc64
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_ppc64
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_ppc64
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_ppc64
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_ppc64
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_ppc64
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_ppc64
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_ppc64
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_ppc64
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_ppc64
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_ppc64
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_ppc64
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_ppc64
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_ppc64
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_ppc64
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_ppc64
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_ppc64
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_ppc64
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_ppc64
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_ppc64
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_ppc64
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_ppc64
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_ppc64
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_ppc64
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_ppc64
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_ppc64
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_ppc64
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_ppc64
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_ppc64
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_ppc64
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_ppc64
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_ppc64
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_ppc64
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_ppc64
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_ppc64
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_ppc64
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_ppc64
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_ppc64
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_ppc64
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_ppc64
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_ppc64
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_ppc64
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_ppc64
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_ppc64
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_ppc64
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_ppc64
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_ppc64
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_ppc64
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_ppc64
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_ppc64
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_ppc64
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_ppc64
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_ppc64
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_ppc64
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_ppc64
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_ppc64
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_ppc64
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_ppc64
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_ppc64
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_ppc64
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_ppc64
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_ppc64
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_ppc64
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_ppc64
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_ppc64
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_ppc64
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_ppc64
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_ppc64
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_ppc64
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_ppc64
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_ppc64
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_ppc64
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_ppc64
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_ppc64
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_ppc64
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_ppc64
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_ppc64
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_ppc64
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_ppc64
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_ppc64
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_ppc64
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_ppc64
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_ppc64
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_ppc64
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_ppc64
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_ppc64
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_ppc64
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_ppc64
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_ppc64
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_ppc64
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_ppc64
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_ppc64
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_ppc64
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_ppc64
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_ppc64
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_ppc64
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_ppc64
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_ppc64
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_ppc64
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_ppc64
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_ppc64
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_ppc64
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_ppc64
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_ppc64
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_ppc64
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_ppc64
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_ppc64
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_ppc64
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_ppc64
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_ppc64
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_ppc64
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_ppc64
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_ppc64
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_ppc64
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_ppc64
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_ppc64
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_ppc64
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_ppc64
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_ppc64
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_ppc64
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_ppc64
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_ppc64
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_ppc64
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_ppc64
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_ppc64
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_ppc64
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_ppc64
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_ppc64
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_ppc64
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_ppc64
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_ppc64
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_ppc64
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_ppc64
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_ppc64
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_ppc64
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_ppc64
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_ppc64
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_ppc64
+#define gen_helper_mve_vand gen_helper_mve_vand_ppc64
+#define gen_helper_mve_vbic gen_helper_mve_vbic_ppc64
+#define gen_helper_mve_vorr gen_helper_mve_vorr_ppc64
+#define gen_helper_mve_vorn gen_helper_mve_vorn_ppc64
+#define gen_helper_mve_veor gen_helper_mve_veor_ppc64
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_ppc64
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_ppc64
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_ppc64
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_ppc64
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_ppc64
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_ppc64
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_ppc64
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_ppc64
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_ppc64
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_ppc64
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_ppc64
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_ppc64
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_ppc64
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_ppc64
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_ppc64
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_ppc64
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_ppc64
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_ppc64
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_ppc64
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_ppc64
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_ppc64
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_ppc64
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_ppc64
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_ppc64
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_ppc64
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_ppc64
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_ppc64
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_ppc64
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_ppc64
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_ppc64
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_ppc64
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_ppc64
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_ppc64
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_ppc64
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_ppc64
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_ppc64
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_ppc64
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_ppc64
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_ppc64
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_ppc64
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_ppc64
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_ppc64
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_ppc64
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_ppc64
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_ppc64
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_ppc64
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_ppc64
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_ppc64
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_ppc64
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_ppc64
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_ppc64
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_ppc64
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_ppc64
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_ppc64
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_ppc64
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_ppc64
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_ppc64
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_ppc64
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_ppc64
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_ppc64
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_ppc64
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_ppc64
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_ppc64
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_ppc64
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_ppc64
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_ppc64
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_ppc64
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_ppc64
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_ppc64
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_ppc64
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_ppc64
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_ppc64
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_ppc64
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_ppc64
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_ppc64
+#define gen_helper_mve_vminub gen_helper_mve_vminub_ppc64
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_ppc64
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_ppc64
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_ppc64
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_ppc64
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_ppc64
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_ppc64
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_ppc64
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_ppc64
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_ppc64
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_ppc64
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_ppc64
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_ppc64
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_ppc64
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_ppc64
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_ppc64
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_ppc64
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_ppc64
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_ppc64
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_ppc64
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_ppc64
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_ppc64
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_ppc64
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_ppc64
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_ppc64
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_ppc64
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_ppc64
+#define gen_helper_mve_vadc gen_helper_mve_vadc_ppc64
+#define gen_helper_mve_vadci gen_helper_mve_vadci_ppc64
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_ppc64
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_ppc64
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_ppc64
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_ppc64
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_ppc64
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_ppc64
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_ppc64
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_ppc64
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_ppc64
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_ppc64
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_ppc64
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_ppc64
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_ppc64
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_ppc64
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_ppc64
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_ppc64
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_ppc64
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_ppc64
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_ppc64
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_ppc64
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_ppc64
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_ppc64
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_ppc64
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_ppc64
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_ppc64
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_ppc64
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_ppc64
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_ppc64
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_ppc64
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_ppc64
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_ppc64
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_ppc64
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_ppc64
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_ppc64
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_ppc64
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_ppc64
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_ppc64
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_ppc64
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_ppc64
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_ppc64
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_ppc64
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_ppc64
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_ppc64
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_ppc64
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_ppc64
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_ppc64
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_ppc64
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_ppc64
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_ppc64
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_ppc64
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_ppc64
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_ppc64
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_ppc64
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_ppc64
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_ppc64
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_ppc64
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_ppc64
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_ppc64
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_ppc64
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_ppc64
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_ppc64
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_ppc64
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_ppc64
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_ppc64
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_ppc64
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_ppc64
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_ppc64
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_ppc64
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_ppc64
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_ppc64
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_ppc64
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_ppc64
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_ppc64
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_ppc64
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_ppc64
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_ppc64
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_ppc64
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_ppc64
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_ppc64
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_ppc64
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_ppc64
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_ppc64
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_ppc64
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_ppc64
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_ppc64
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_ppc64
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_ppc64
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_ppc64
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_ppc64
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_ppc64
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_ppc64
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_ppc64
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_ppc64
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_ppc64
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_ppc64
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_ppc64
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_ppc64
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_ppc64
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_ppc64
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_ppc64
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_ppc64
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_ppc64
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_ppc64
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_ppc64
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_ppc64
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_ppc64
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_ppc64
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_ppc64
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_ppc64
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_ppc64
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_ppc64
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_ppc64
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_ppc64
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_ppc64
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_ppc64
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_ppc64
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_ppc64
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_ppc64
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_ppc64
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_ppc64
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_ppc64
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_ppc64
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_ppc64
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_ppc64
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_ppc64
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_ppc64
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_ppc64
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_ppc64
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_ppc64
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_ppc64
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_ppc64
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_ppc64
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_ppc64
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_ppc64
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_ppc64
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_ppc64
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_ppc64
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_ppc64
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_ppc64
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_ppc64
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_ppc64
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_ppc64
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_ppc64
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_ppc64
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_ppc64
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_ppc64
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_ppc64
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_ppc64
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_ppc64
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_ppc64
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_ppc64
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_ppc64
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_ppc64
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_ppc64
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_ppc64
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_ppc64
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_ppc64
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_ppc64
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_ppc64
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_ppc64
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_ppc64
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_ppc64
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_ppc64
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_ppc64
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_ppc64
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_ppc64
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_ppc64
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_ppc64
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_ppc64
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_ppc64
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_ppc64
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_ppc64
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_ppc64
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_ppc64
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_ppc64
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_ppc64
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_ppc64
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_ppc64
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_ppc64
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_ppc64
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_ppc64
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_ppc64
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_ppc64
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_ppc64
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_ppc64
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_ppc64
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_ppc64
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_ppc64
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_ppc64
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_ppc64
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_ppc64
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_ppc64
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_ppc64
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_ppc64
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_ppc64
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_ppc64
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_ppc64
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_ppc64
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_ppc64
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_ppc64
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_ppc64
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_ppc64
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_ppc64
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_ppc64
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_ppc64
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_ppc64
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_ppc64
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_ppc64
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_ppc64
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_ppc64
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_ppc64
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_ppc64
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_ppc64
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_ppc64
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_ppc64
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_ppc64
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_ppc64
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_ppc64
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_ppc64
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_ppc64
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_ppc64
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_ppc64
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_ppc64
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_ppc64
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_ppc64
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_ppc64
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_ppc64
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_ppc64
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_ppc64
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_ppc64
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_ppc64
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_ppc64
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_ppc64
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_ppc64
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_ppc64
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_ppc64
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_ppc64
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_ppc64
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_ppc64
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_ppc64
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_ppc64
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_ppc64
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_ppc64
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_ppc64
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_ppc64
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_ppc64
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_ppc64
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_ppc64
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_ppc64
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_ppc64
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_ppc64
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_ppc64
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_ppc64
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_ppc64
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_ppc64
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_ppc64
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_ppc64
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_ppc64
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_ppc64
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_ppc64
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_ppc64
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_ppc64
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_ppc64
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_ppc64
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_ppc64
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_ppc64
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_ppc64
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_ppc64
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_ppc64
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_ppc64
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_ppc64
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_ppc64
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_ppc64
+#define gen_helper_mve_ushll gen_helper_mve_ushll_ppc64
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_ppc64
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_ppc64
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_ppc64
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_ppc64
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_ppc64
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_ppc64
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_ppc64
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_ppc64
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_ppc64
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_ppc64
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_ppc64
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_ppc64
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_ppc64
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_ppc64
+#define gen_helper_mve_vslib gen_helper_mve_vslib_ppc64
+#define gen_helper_mve_vslih gen_helper_mve_vslih_ppc64
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_ppc64
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_ppc64
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_ppc64
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_ppc64
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_ppc64
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_ppc64
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_ppc64
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_ppc64
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_ppc64
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_ppc64
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_ppc64
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_ppc64
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_ppc64
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_ppc64
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_ppc64
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_ppc64
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_ppc64
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_ppc64
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_ppc64
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_ppc64
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_ppc64
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_ppc64
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_ppc64
+#define gen_helper_mve_vminab gen_helper_mve_vminab_ppc64
+#define gen_helper_mve_vminah gen_helper_mve_vminah_ppc64
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_ppc64
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_ppc64
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_ppc64
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_ppc64
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_ppc64
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_ppc64
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_ppc64
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_ppc64
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_ppc64
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_ppc64
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_ppc64
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_ppc64
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_ppc64
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_ppc64
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_ppc64
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_ppc64
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_ppc64
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_ppc64
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_ppc64
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_ppc64
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_ppc64
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_ppc64
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_ppc64
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_ppc64
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_ppc64
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_ppc64
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_ppc64
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_ppc64
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_ppc64
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_ppc64
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_ppc64
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_ppc64
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_ppc64
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_ppc64
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_ppc64
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_ppc64
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_ppc64
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_ppc64
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_ppc64
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_ppc64
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_ppc64
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_ppc64
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_ppc64
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_ppc64
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_ppc64
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_ppc64
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_ppc64
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_ppc64
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_ppc64
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_ppc64
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_ppc64
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_ppc64
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_ppc64
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_ppc64
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_ppc64
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_ppc64
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_ppc64
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_ppc64
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_ppc64
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_ppc64
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_ppc64
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_ppc64
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_ppc64
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_ppc64
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_ppc64
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_ppc64
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_ppc64
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_ppc64
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_ppc64
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_ppc64
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_ppc64
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_ppc64
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_ppc64
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_ppc64
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_ppc64
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_ppc64
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_ppc64
 #define gen_helper_cpsr_read gen_helper_cpsr_read_ppc64
 #define gen_helper_cpsr_write gen_helper_cpsr_write_ppc64
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_ppc64
@@ -1332,6 +2103,18 @@
 #define helper_vprtybd helper_vprtybd_ppc64
 #define helper_vprtybq helper_vprtybq_ppc64
 #define helper_vmuluwm helper_vmuluwm_ppc64
+#define helper_VDIVSQ helper_VDIVSQ_ppc64
+#define helper_VDIVUQ helper_VDIVUQ_ppc64
+#define helper_VDIVESD helper_VDIVESD_ppc64
+#define helper_VDIVEUD helper_VDIVEUD_ppc64
+#define helper_VDIVESQ helper_VDIVESQ_ppc64
+#define helper_VDIVEUQ helper_VDIVEUQ_ppc64
+#define helper_VMODSQ helper_VMODSQ_ppc64
+#define helper_VMODUQ helper_VMODUQ_ppc64
+#define helper_VSTRIBL helper_VSTRIBL_ppc64
+#define helper_VSTRIBR helper_VSTRIBR_ppc64
+#define helper_VSTRIHL helper_VSTRIHL_ppc64
+#define helper_VSTRIHR helper_VSTRIHR_ppc64
 #define helper_vaddfp helper_vaddfp_ppc64
 #define helper_vsubfp helper_vsubfp_ppc64
 #define helper_vminfp helper_vminfp_ppc64
@@ -1479,6 +2262,10 @@
 #define helper_vextubrx helper_vextubrx_ppc64
 #define helper_vextuhrx helper_vextuhrx_ppc64
 #define helper_vextuwrx helper_vextuwrx_ppc64
+#define helper_VEXTDUBVLX helper_VEXTDUBVLX_ppc64
+#define helper_VEXTDUHVLX helper_VEXTDUHVLX_ppc64
+#define helper_VEXTDUWVLX helper_VEXTDUWVLX_ppc64
+#define helper_VEXTDDVLX helper_VEXTDDVLX_ppc64
 #define helper_vslv helper_vslv_ppc64
 #define helper_vsrv helper_vsrv_ppc64
 #define helper_vsldoi helper_vsldoi_ppc64
@@ -1487,6 +2274,10 @@
 #define helper_vinserth helper_vinserth_ppc64
 #define helper_vinsertw helper_vinsertw_ppc64
 #define helper_vinsertd helper_vinsertd_ppc64
+#define helper_VINSBLX helper_VINSBLX_ppc64
+#define helper_VINSHLX helper_VINSHLX_ppc64
+#define helper_VINSWLX helper_VINSWLX_ppc64
+#define helper_VINSDLX helper_VINSDLX_ppc64
 #define helper_vextractub helper_vextractub_ppc64
 #define helper_vextractuh helper_vextractuh_ppc64
 #define helper_vextractuw helper_vextractuw_ppc64
@@ -1882,6 +2673,14 @@
 #define helper_xsmsubdp helper_xsmsubdp_ppc64
 #define helper_xsnmadddp helper_xsnmadddp_ppc64
 #define helper_xsnmsubdp helper_xsnmsubdp_ppc64
+#define helper_XSMADDQP helper_XSMADDQP_ppc64
+#define helper_XSMADDQPO helper_XSMADDQPO_ppc64
+#define helper_XSMSUBQP helper_XSMSUBQP_ppc64
+#define helper_XSMSUBQPO helper_XSMSUBQPO_ppc64
+#define helper_XSNMADDQP helper_XSNMADDQP_ppc64
+#define helper_XSNMADDQPO helper_XSNMADDQPO_ppc64
+#define helper_XSNMSUBQP helper_XSNMSUBQP_ppc64
+#define helper_XSNMSUBQPO helper_XSNMSUBQPO_ppc64
 #define helper_xsmaddsp helper_xsmaddsp_ppc64
 #define helper_xsmsubsp helper_xsmsubsp_ppc64
 #define helper_xsnmaddsp helper_xsnmaddsp_ppc64
@@ -1898,6 +2697,9 @@
 #define helper_xscmpgedp helper_xscmpgedp_ppc64
 #define helper_xscmpgtdp helper_xscmpgtdp_ppc64
 #define helper_xscmpnedp helper_xscmpnedp_ppc64
+#define helper_XSCMPEQQP helper_XSCMPEQQP_ppc64
+#define helper_XSCMPGEQP helper_XSCMPGEQP_ppc64
+#define helper_XSCMPGTQP helper_XSCMPGTQP_ppc64
 #define helper_xscmpexpdp helper_xscmpexpdp_ppc64
 #define helper_xscmpexpqp helper_xscmpexpqp_ppc64
 #define helper_xscmpodp helper_xscmpodp_ppc64
@@ -1914,6 +2716,8 @@
 #define helper_xsmincdp helper_xsmincdp_ppc64
 #define helper_xsmaxjdp helper_xsmaxjdp_ppc64
 #define helper_xsminjdp helper_xsminjdp_ppc64
+#define helper_XSMAXCQP helper_XSMAXCQP_ppc64
+#define helper_XSMINCQP helper_XSMINCQP_ppc64
 #define helper_xvcmpeqdp helper_xvcmpeqdp_ppc64
 #define helper_xvcmpgedp helper_xvcmpgedp_ppc64
 #define helper_xvcmpgtdp helper_xvcmpgtdp_ppc64
@@ -1931,7 +2735,98 @@
 #define helper_xscvhpdp helper_xscvhpdp_ppc64
 #define helper_xvcvsphp helper_xvcvsphp_ppc64
 #define helper_xvcvhpsp helper_xvcvhpsp_ppc64
+#define helper_XVCVSPBF16 helper_XVCVSPBF16_ppc64
+#define helper_XVCVBF16SPN helper_XVCVBF16SPN_ppc64
+#define helper_dadd helper_dadd_ppc64
+#define helper_daddq helper_daddq_ppc64
+#define helper_dsub helper_dsub_ppc64
+#define helper_dsubq helper_dsubq_ppc64
+#define helper_dmul helper_dmul_ppc64
+#define helper_dmulq helper_dmulq_ppc64
+#define helper_ddiv helper_ddiv_ppc64
+#define helper_ddivq helper_ddivq_ppc64
+#define helper_dcmpo helper_dcmpo_ppc64
+#define helper_dcmpoq helper_dcmpoq_ppc64
+#define helper_dcmpu helper_dcmpu_ppc64
+#define helper_dcmpuq helper_dcmpuq_ppc64
+#define helper_dtstdc helper_dtstdc_ppc64
+#define helper_dtstdcq helper_dtstdcq_ppc64
+#define helper_dtstdg helper_dtstdg_ppc64
+#define helper_dtstdgq helper_dtstdgq_ppc64
+#define helper_dtstex helper_dtstex_ppc64
+#define helper_dtstexq helper_dtstexq_ppc64
+#define helper_dtstsf helper_dtstsf_ppc64
+#define helper_dtstsfq helper_dtstsfq_ppc64
+#define helper_dtstsfi helper_dtstsfi_ppc64
+#define helper_dtstsfiq helper_dtstsfiq_ppc64
+#define helper_dquai helper_dquai_ppc64
+#define helper_dquaiq helper_dquaiq_ppc64
+#define helper_dqua helper_dqua_ppc64
+#define helper_dquaq helper_dquaq_ppc64
+#define helper_drrnd helper_drrnd_ppc64
+#define helper_drrndq helper_drrndq_ppc64
+#define helper_drintx helper_drintx_ppc64
+#define helper_drintxq helper_drintxq_ppc64
+#define helper_drintn helper_drintn_ppc64
+#define helper_drintnq helper_drintnq_ppc64
+#define helper_dctdp helper_dctdp_ppc64
+#define helper_dctqpq helper_dctqpq_ppc64
+#define helper_drsp helper_drsp_ppc64
+#define helper_drdpq helper_drdpq_ppc64
+#define helper_dcffix helper_dcffix_ppc64
+#define helper_dcffixq helper_dcffixq_ppc64
+#define helper_DCFFIXQQ helper_DCFFIXQQ_ppc64
+#define helper_dctfix helper_dctfix_ppc64
+#define helper_dctfixq helper_dctfixq_ppc64
+#define helper_DCTFIXQQ helper_DCTFIXQQ_ppc64
+#define helper_ddedpd helper_ddedpd_ppc64
+#define helper_ddedpdq helper_ddedpdq_ppc64
+#define helper_denbcd helper_denbcd_ppc64
+#define helper_denbcdq helper_denbcdq_ppc64
+#define helper_dxex helper_dxex_ppc64
+#define helper_dxexq helper_dxexq_ppc64
+#define helper_diex helper_diex_ppc64
+#define helper_diexq helper_diexq_ppc64
+#define helper_dscri helper_dscri_ppc64
+#define helper_dscriq helper_dscriq_ppc64
+#define helper_dscli helper_dscli_ppc64
+#define helper_dscliq helper_dscliq_ppc64
+#define helper_CDTBCD helper_CDTBCD_ppc64
+#define helper_CBCDTD helper_CBCDTD_ppc64
+#define helper_XVI4GER8 helper_XVI4GER8_ppc64
+#define helper_XVI4GER8PP helper_XVI4GER8PP_ppc64
+#define helper_XVI8GER4 helper_XVI8GER4_ppc64
+#define helper_XVI8GER4PP helper_XVI8GER4PP_ppc64
+#define helper_XVI8GER4SPP helper_XVI8GER4SPP_ppc64
+#define helper_XVI16GER2 helper_XVI16GER2_ppc64
+#define helper_XVI16GER2S helper_XVI16GER2S_ppc64
+#define helper_XVI16GER2PP helper_XVI16GER2PP_ppc64
+#define helper_XVI16GER2SPP helper_XVI16GER2SPP_ppc64
+#define helper_XVBF16GER2 helper_XVBF16GER2_ppc64
+#define helper_XVBF16GER2PP helper_XVBF16GER2PP_ppc64
+#define helper_XVBF16GER2PN helper_XVBF16GER2PN_ppc64
+#define helper_XVBF16GER2NP helper_XVBF16GER2NP_ppc64
+#define helper_XVBF16GER2NN helper_XVBF16GER2NN_ppc64
+#define helper_XVF16GER2 helper_XVF16GER2_ppc64
+#define helper_XVF16GER2PP helper_XVF16GER2PP_ppc64
+#define helper_XVF16GER2PN helper_XVF16GER2PN_ppc64
+#define helper_XVF16GER2NP helper_XVF16GER2NP_ppc64
+#define helper_XVF16GER2NN helper_XVF16GER2NN_ppc64
+#define helper_XVF32GER helper_XVF32GER_ppc64
+#define helper_XVF32GERPP helper_XVF32GERPP_ppc64
+#define helper_XVF32GERPN helper_XVF32GERPN_ppc64
+#define helper_XVF32GERNP helper_XVF32GERNP_ppc64
+#define helper_XVF32GERNN helper_XVF32GERNN_ppc64
+#define helper_XVF64GER helper_XVF64GER_ppc64
+#define helper_XVF64GERPP helper_XVF64GERPP_ppc64
+#define helper_XVF64GERPN helper_XVF64GERPN_ppc64
+#define helper_XVF64GERNP helper_XVF64GERNP_ppc64
+#define helper_XVF64GERNN helper_XVF64GERNN_ppc64
 #define helper_xscvqpdp helper_xscvqpdp_ppc64
+#define helper_XSCVQPUQZ helper_XSCVQPUQZ_ppc64
+#define helper_XSCVQPSQZ helper_XSCVQPSQZ_ppc64
+#define helper_XSCVUQQP helper_XSCVUQQP_ppc64
+#define helper_XSCVSQQP helper_XSCVSQQP_ppc64
 #define helper_xscvdpspn helper_xscvdpspn_ppc64
 #define helper_xscvspdpn helper_xscvspdpn_ppc64
 #define helper_xscvdpsxds helper_xscvdpsxds_ppc64
@@ -1982,6 +2877,23 @@
 #define helper_xsrsp helper_xsrsp_ppc64
 #define helper_xxperm helper_xxperm_ppc64
 #define helper_xxpermr helper_xxpermr_ppc64
+#define helper_XXPERMX helper_XXPERMX_ppc64
+#define helper_XXGENPCVBM_be_exp helper_XXGENPCVBM_be_exp_ppc64
+#define helper_XXGENPCVBM_be_comp helper_XXGENPCVBM_be_comp_ppc64
+#define helper_XXGENPCVBM_le_exp helper_XXGENPCVBM_le_exp_ppc64
+#define helper_XXGENPCVBM_le_comp helper_XXGENPCVBM_le_comp_ppc64
+#define helper_XXGENPCVHM_be_exp helper_XXGENPCVHM_be_exp_ppc64
+#define helper_XXGENPCVHM_be_comp helper_XXGENPCVHM_be_comp_ppc64
+#define helper_XXGENPCVHM_le_exp helper_XXGENPCVHM_le_exp_ppc64
+#define helper_XXGENPCVHM_le_comp helper_XXGENPCVHM_le_comp_ppc64
+#define helper_XXGENPCVWM_be_exp helper_XXGENPCVWM_be_exp_ppc64
+#define helper_XXGENPCVWM_be_comp helper_XXGENPCVWM_be_comp_ppc64
+#define helper_XXGENPCVWM_le_exp helper_XXGENPCVWM_le_exp_ppc64
+#define helper_XXGENPCVWM_le_comp helper_XXGENPCVWM_le_comp_ppc64
+#define helper_XXGENPCVDM_be_exp helper_XXGENPCVDM_be_exp_ppc64
+#define helper_XXGENPCVDM_be_comp helper_XXGENPCVDM_be_comp_ppc64
+#define helper_XXGENPCVDM_le_exp helper_XXGENPCVDM_le_exp_ppc64
+#define helper_XXGENPCVDM_le_comp helper_XXGENPCVDM_le_comp_ppc64
 #define helper_xvxsigsp helper_xvxsigsp_ppc64
 #define helper_xvtstdcdp helper_xvtstdcdp_ppc64
 #define helper_xvtstdcsp helper_xvtstdcsp_ppc64
diff --git a/qemu/riscv32.h b/qemu/riscv32.h
index 26c798b413..975e0893ae 100644
--- a/qemu/riscv32.h
+++ b/qemu/riscv32.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_riscv32
 #define float32_squash_input_denormal float32_squash_input_denormal_riscv32
 #define float64_squash_input_denormal float64_squash_input_denormal_riscv32
+#define bfloat16_add bfloat16_add_riscv32
+#define bfloat16_compare bfloat16_compare_riscv32
+#define bfloat16_compare_quiet bfloat16_compare_quiet_riscv32
+#define bfloat16_default_nan bfloat16_default_nan_riscv32
+#define bfloat16_div bfloat16_div_riscv32
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_riscv32
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_riscv32
+#define bfloat16_max bfloat16_max_riscv32
+#define bfloat16_maximum_number bfloat16_maximum_number_riscv32
+#define bfloat16_maxnum bfloat16_maxnum_riscv32
+#define bfloat16_maxnummag bfloat16_maxnummag_riscv32
+#define bfloat16_min bfloat16_min_riscv32
+#define bfloat16_minimum_number bfloat16_minimum_number_riscv32
+#define bfloat16_minnum bfloat16_minnum_riscv32
+#define bfloat16_minnummag bfloat16_minnummag_riscv32
+#define bfloat16_mul bfloat16_mul_riscv32
+#define bfloat16_muladd bfloat16_muladd_riscv32
+#define bfloat16_round_to_int bfloat16_round_to_int_riscv32
+#define bfloat16_scalbn bfloat16_scalbn_riscv32
+#define bfloat16_silence_nan bfloat16_silence_nan_riscv32
+#define bfloat16_sqrt bfloat16_sqrt_riscv32
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_riscv32
+#define bfloat16_sub bfloat16_sub_riscv32
+#define bfloat16_to_float32 bfloat16_to_float32_riscv32
+#define bfloat16_to_float64 bfloat16_to_float64_riscv32
+#define bfloat16_to_int16 bfloat16_to_int16_riscv32
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_riscv32
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_riscv32
+#define bfloat16_to_int32 bfloat16_to_int32_riscv32
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_riscv32
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_riscv32
+#define bfloat16_to_int64 bfloat16_to_int64_riscv32
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_riscv32
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_riscv32
+#define bfloat16_to_uint16 bfloat16_to_uint16_riscv32
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_riscv32
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_riscv32
+#define bfloat16_to_uint32 bfloat16_to_uint32_riscv32
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_riscv32
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_riscv32
+#define bfloat16_to_uint64 bfloat16_to_uint64_riscv32
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_riscv32
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_riscv32
+#define float128_maximum_number float128_maximum_number_riscv32
+#define float128_max float128_max_riscv32
+#define float128_maxnum float128_maxnum_riscv32
+#define float128_maxnummag float128_maxnummag_riscv32
+#define float128_min float128_min_riscv32
+#define float128_minimum_number float128_minimum_number_riscv32
+#define float128_minnum float128_minnum_riscv32
+#define float128_minnummag float128_minnummag_riscv32
+#define float128_muladd float128_muladd_riscv32
+#define float128_to_int128 float128_to_int128_riscv32
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_riscv32
+#define float128_to_uint128 float128_to_uint128_riscv32
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_riscv32
+#define float16_maximum_number float16_maximum_number_riscv32
+#define float16_minimum_number float16_minimum_number_riscv32
+#define float16_to_int8 float16_to_int8_riscv32
+#define float16_to_int8_scalbn float16_to_int8_scalbn_riscv32
+#define float16_to_uint8 float16_to_uint8_riscv32
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_riscv32
+#define float32_maximum_number float32_maximum_number_riscv32
+#define float32_minimum_number float32_minimum_number_riscv32
+#define float32_to_bfloat16 float32_to_bfloat16_riscv32
+#define float64_maximum_number float64_maximum_number_riscv32
+#define float64_minimum_number float64_minimum_number_riscv32
+#define float64_to_bfloat16 float64_to_bfloat16_riscv32
+#define float64r32_add float64r32_add_riscv32
+#define float64r32_div float64r32_div_riscv32
+#define float64r32_mul float64r32_mul_riscv32
+#define float64r32_muladd float64r32_muladd_riscv32
+#define float64r32_sqrt float64r32_sqrt_riscv32
+#define float64r32_sub float64r32_sub_riscv32
+#define floatx80_mod floatx80_mod_riscv32
+#define floatx80_modrem floatx80_modrem_riscv32
+#define int128_to_float128 int128_to_float128_riscv32
+#define int16_to_bfloat16 int16_to_bfloat16_riscv32
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_riscv32
+#define int32_to_bfloat16 int32_to_bfloat16_riscv32
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_riscv32
+#define int64_to_bfloat16 int64_to_bfloat16_riscv32
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_riscv32
+#define int8_to_float16 int8_to_float16_riscv32
+#define uint128_to_float128 uint128_to_float128_riscv32
+#define uint16_to_bfloat16 uint16_to_bfloat16_riscv32
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_riscv32
+#define uint32_to_bfloat16 uint32_to_bfloat16_riscv32
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_riscv32
+#define uint64_to_bfloat16 uint64_to_bfloat16_riscv32
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_riscv32
+#define uint8_to_float16 uint8_to_float16_riscv32
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_riscv32
 #define roundAndPackFloatx80 roundAndPackFloatx80_riscv32
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_riscv32
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_riscv32
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_riscv32
 #define helper_exit_atomic helper_exit_atomic_riscv32
+#define helper_memset helper_memset_riscv32
+#define helper_emu_stop helper_emu_stop_riscv32
+#define tcg_remove_ops_after tcg_remove_ops_after_riscv32
+#define tcg_constant_vec_matching tcg_constant_vec_matching_riscv32
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_riscv32
 #define helper_gvec_add8 helper_gvec_add8_riscv32
 #define helper_gvec_add16 helper_gvec_add16_riscv32
 #define helper_gvec_add32 helper_gvec_add32_riscv32
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_riscv32
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_riscv32
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_riscv32
+#define gen_helper_mve_vctp gen_helper_mve_vctp_riscv32
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_riscv32
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_riscv32
+#define gen_helper_mve_vdup gen_helper_mve_vdup_riscv32
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_riscv32
+#define gen_helper_mve_vandi gen_helper_mve_vandi_riscv32
+#define gen_helper_mve_vorri gen_helper_mve_vorri_riscv32
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_riscv32
+#define gen_helper_mve_viduph gen_helper_mve_viduph_riscv32
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_riscv32
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_riscv32
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_riscv32
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_riscv32
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_riscv32
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_riscv32
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_riscv32
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_riscv32
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_riscv32
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_riscv32
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_riscv32
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_riscv32
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_riscv32
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_riscv32
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_riscv32
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_riscv32
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_riscv32
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_riscv32
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_riscv32
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_riscv32
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_riscv32
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_riscv32
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_riscv32
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_riscv32
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_riscv32
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_riscv32
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_riscv32
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_riscv32
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_riscv32
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_riscv32
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_riscv32
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_riscv32
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_riscv32
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_riscv32
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_riscv32
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_riscv32
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_riscv32
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_riscv32
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_riscv32
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_riscv32
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_riscv32
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_riscv32
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_riscv32
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_riscv32
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_riscv32
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_riscv32
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_riscv32
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_riscv32
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_riscv32
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_riscv32
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_riscv32
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_riscv32
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_riscv32
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_riscv32
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_riscv32
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_riscv32
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_riscv32
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_riscv32
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_riscv32
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_riscv32
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_riscv32
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_riscv32
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_riscv32
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_riscv32
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_riscv32
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_riscv32
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_riscv32
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_riscv32
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_riscv32
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_riscv32
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_riscv32
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_riscv32
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_riscv32
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_riscv32
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_riscv32
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_riscv32
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_riscv32
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_riscv32
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_riscv32
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_riscv32
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_riscv32
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_riscv32
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_riscv32
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_riscv32
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_riscv32
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_riscv32
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_riscv32
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_riscv32
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_riscv32
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_riscv32
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_riscv32
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_riscv32
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_riscv32
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_riscv32
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_riscv32
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_riscv32
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_riscv32
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_riscv32
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_riscv32
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_riscv32
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_riscv32
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_riscv32
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_riscv32
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_riscv32
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_riscv32
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_riscv32
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_riscv32
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_riscv32
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_riscv32
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_riscv32
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_riscv32
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_riscv32
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_riscv32
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_riscv32
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_riscv32
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_riscv32
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_riscv32
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_riscv32
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_riscv32
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_riscv32
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_riscv32
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_riscv32
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_riscv32
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_riscv32
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_riscv32
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_riscv32
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_riscv32
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_riscv32
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_riscv32
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_riscv32
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_riscv32
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_riscv32
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_riscv32
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_riscv32
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_riscv32
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_riscv32
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_riscv32
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_riscv32
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_riscv32
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_riscv32
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_riscv32
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_riscv32
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_riscv32
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_riscv32
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_riscv32
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_riscv32
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_riscv32
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_riscv32
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_riscv32
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_riscv32
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_riscv32
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_riscv32
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_riscv32
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_riscv32
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_riscv32
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_riscv32
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_riscv32
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_riscv32
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_riscv32
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_riscv32
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_riscv32
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_riscv32
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_riscv32
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_riscv32
+#define gen_helper_mve_vand gen_helper_mve_vand_riscv32
+#define gen_helper_mve_vbic gen_helper_mve_vbic_riscv32
+#define gen_helper_mve_vorr gen_helper_mve_vorr_riscv32
+#define gen_helper_mve_vorn gen_helper_mve_vorn_riscv32
+#define gen_helper_mve_veor gen_helper_mve_veor_riscv32
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_riscv32
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_riscv32
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_riscv32
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_riscv32
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_riscv32
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_riscv32
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_riscv32
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_riscv32
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_riscv32
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_riscv32
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_riscv32
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_riscv32
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_riscv32
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_riscv32
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_riscv32
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_riscv32
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_riscv32
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_riscv32
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_riscv32
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_riscv32
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_riscv32
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_riscv32
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_riscv32
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_riscv32
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_riscv32
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_riscv32
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_riscv32
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_riscv32
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_riscv32
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_riscv32
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_riscv32
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_riscv32
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_riscv32
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_riscv32
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_riscv32
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_riscv32
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_riscv32
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_riscv32
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_riscv32
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_riscv32
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_riscv32
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_riscv32
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_riscv32
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_riscv32
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_riscv32
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_riscv32
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_riscv32
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_riscv32
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_riscv32
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_riscv32
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_riscv32
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_riscv32
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_riscv32
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_riscv32
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_riscv32
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_riscv32
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_riscv32
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_riscv32
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_riscv32
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_riscv32
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_riscv32
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_riscv32
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_riscv32
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_riscv32
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_riscv32
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_riscv32
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_riscv32
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_riscv32
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_riscv32
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_riscv32
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_riscv32
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_riscv32
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_riscv32
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_riscv32
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_riscv32
+#define gen_helper_mve_vminub gen_helper_mve_vminub_riscv32
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_riscv32
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_riscv32
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_riscv32
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_riscv32
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_riscv32
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_riscv32
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_riscv32
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_riscv32
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_riscv32
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_riscv32
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_riscv32
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_riscv32
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_riscv32
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_riscv32
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_riscv32
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_riscv32
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_riscv32
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_riscv32
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_riscv32
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_riscv32
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_riscv32
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_riscv32
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_riscv32
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_riscv32
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_riscv32
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_riscv32
+#define gen_helper_mve_vadc gen_helper_mve_vadc_riscv32
+#define gen_helper_mve_vadci gen_helper_mve_vadci_riscv32
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_riscv32
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_riscv32
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_riscv32
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_riscv32
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_riscv32
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_riscv32
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_riscv32
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_riscv32
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_riscv32
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_riscv32
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_riscv32
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_riscv32
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_riscv32
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_riscv32
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_riscv32
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_riscv32
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_riscv32
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_riscv32
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_riscv32
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_riscv32
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_riscv32
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_riscv32
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_riscv32
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_riscv32
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_riscv32
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_riscv32
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_riscv32
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_riscv32
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_riscv32
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_riscv32
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_riscv32
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_riscv32
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_riscv32
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_riscv32
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_riscv32
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_riscv32
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_riscv32
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_riscv32
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_riscv32
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_riscv32
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_riscv32
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_riscv32
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_riscv32
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_riscv32
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_riscv32
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_riscv32
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_riscv32
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_riscv32
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_riscv32
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_riscv32
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_riscv32
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_riscv32
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_riscv32
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_riscv32
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_riscv32
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_riscv32
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_riscv32
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_riscv32
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_riscv32
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_riscv32
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_riscv32
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_riscv32
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_riscv32
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_riscv32
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_riscv32
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_riscv32
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_riscv32
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_riscv32
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_riscv32
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_riscv32
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_riscv32
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_riscv32
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_riscv32
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_riscv32
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_riscv32
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_riscv32
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_riscv32
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_riscv32
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_riscv32
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_riscv32
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_riscv32
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_riscv32
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_riscv32
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_riscv32
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_riscv32
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_riscv32
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_riscv32
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_riscv32
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_riscv32
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_riscv32
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_riscv32
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_riscv32
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_riscv32
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_riscv32
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_riscv32
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_riscv32
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_riscv32
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_riscv32
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_riscv32
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_riscv32
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_riscv32
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_riscv32
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_riscv32
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_riscv32
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_riscv32
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_riscv32
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_riscv32
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_riscv32
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_riscv32
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_riscv32
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_riscv32
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_riscv32
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_riscv32
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_riscv32
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_riscv32
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_riscv32
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_riscv32
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_riscv32
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_riscv32
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_riscv32
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_riscv32
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_riscv32
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_riscv32
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_riscv32
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_riscv32
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_riscv32
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_riscv32
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_riscv32
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_riscv32
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_riscv32
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_riscv32
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_riscv32
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_riscv32
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_riscv32
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_riscv32
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_riscv32
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_riscv32
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_riscv32
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_riscv32
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_riscv32
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_riscv32
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_riscv32
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_riscv32
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_riscv32
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_riscv32
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_riscv32
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_riscv32
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_riscv32
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_riscv32
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_riscv32
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_riscv32
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_riscv32
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_riscv32
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_riscv32
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_riscv32
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_riscv32
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_riscv32
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_riscv32
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_riscv32
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_riscv32
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_riscv32
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_riscv32
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_riscv32
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_riscv32
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_riscv32
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_riscv32
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_riscv32
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_riscv32
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_riscv32
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_riscv32
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_riscv32
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_riscv32
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_riscv32
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_riscv32
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_riscv32
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_riscv32
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_riscv32
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_riscv32
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_riscv32
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_riscv32
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_riscv32
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_riscv32
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_riscv32
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_riscv32
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_riscv32
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_riscv32
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_riscv32
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_riscv32
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_riscv32
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_riscv32
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_riscv32
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_riscv32
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_riscv32
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_riscv32
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_riscv32
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_riscv32
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_riscv32
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_riscv32
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_riscv32
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_riscv32
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_riscv32
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_riscv32
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_riscv32
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_riscv32
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_riscv32
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_riscv32
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_riscv32
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_riscv32
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_riscv32
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_riscv32
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_riscv32
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_riscv32
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_riscv32
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_riscv32
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_riscv32
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_riscv32
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_riscv32
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_riscv32
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_riscv32
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_riscv32
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_riscv32
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_riscv32
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_riscv32
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_riscv32
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_riscv32
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_riscv32
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_riscv32
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_riscv32
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_riscv32
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_riscv32
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_riscv32
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_riscv32
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_riscv32
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_riscv32
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_riscv32
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_riscv32
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_riscv32
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_riscv32
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_riscv32
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_riscv32
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_riscv32
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_riscv32
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_riscv32
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_riscv32
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_riscv32
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_riscv32
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_riscv32
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_riscv32
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_riscv32
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_riscv32
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_riscv32
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_riscv32
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_riscv32
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_riscv32
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_riscv32
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_riscv32
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_riscv32
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_riscv32
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_riscv32
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_riscv32
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_riscv32
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_riscv32
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_riscv32
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_riscv32
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_riscv32
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_riscv32
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_riscv32
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_riscv32
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_riscv32
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_riscv32
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_riscv32
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_riscv32
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_riscv32
+#define gen_helper_mve_ushll gen_helper_mve_ushll_riscv32
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_riscv32
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_riscv32
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_riscv32
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_riscv32
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_riscv32
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_riscv32
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_riscv32
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_riscv32
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_riscv32
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_riscv32
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_riscv32
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_riscv32
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_riscv32
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_riscv32
+#define gen_helper_mve_vslib gen_helper_mve_vslib_riscv32
+#define gen_helper_mve_vslih gen_helper_mve_vslih_riscv32
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_riscv32
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_riscv32
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_riscv32
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_riscv32
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_riscv32
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_riscv32
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_riscv32
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_riscv32
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_riscv32
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_riscv32
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_riscv32
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_riscv32
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_riscv32
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_riscv32
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_riscv32
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_riscv32
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_riscv32
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_riscv32
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_riscv32
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_riscv32
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_riscv32
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_riscv32
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_riscv32
+#define gen_helper_mve_vminab gen_helper_mve_vminab_riscv32
+#define gen_helper_mve_vminah gen_helper_mve_vminah_riscv32
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_riscv32
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_riscv32
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_riscv32
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_riscv32
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_riscv32
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_riscv32
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_riscv32
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_riscv32
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_riscv32
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_riscv32
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_riscv32
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_riscv32
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_riscv32
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_riscv32
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_riscv32
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_riscv32
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_riscv32
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_riscv32
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_riscv32
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_riscv32
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_riscv32
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_riscv32
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_riscv32
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_riscv32
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_riscv32
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_riscv32
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_riscv32
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_riscv32
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_riscv32
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_riscv32
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_riscv32
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_riscv32
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_riscv32
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_riscv32
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_riscv32
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_riscv32
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_riscv32
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_riscv32
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_riscv32
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_riscv32
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_riscv32
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_riscv32
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_riscv32
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_riscv32
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_riscv32
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_riscv32
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_riscv32
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_riscv32
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_riscv32
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_riscv32
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_riscv32
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_riscv32
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_riscv32
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_riscv32
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_riscv32
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_riscv32
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_riscv32
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_riscv32
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_riscv32
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_riscv32
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_riscv32
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_riscv32
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_riscv32
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_riscv32
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_riscv32
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_riscv32
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_riscv32
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_riscv32
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_riscv32
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_riscv32
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_riscv32
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_riscv32
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_riscv32
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_riscv32
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_riscv32
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_riscv32
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_riscv32
 #define gen_helper_cpsr_read gen_helper_cpsr_read_riscv32
 #define gen_helper_cpsr_write gen_helper_cpsr_write_riscv32
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_riscv32
@@ -1318,14 +2089,19 @@
 #define riscv_cpu_get_fflags riscv_cpu_get_fflags_riscv32
 #define riscv_cpu_set_fflags riscv_cpu_set_fflags_riscv32
 #define helper_set_rounding_mode helper_set_rounding_mode_riscv32
+#define helper_set_rod_rounding_mode helper_set_rod_rounding_mode_riscv32
 #define helper_fmadd_s helper_fmadd_s_riscv32
 #define helper_fmadd_d helper_fmadd_d_riscv32
+#define helper_fmadd_h helper_fmadd_h_riscv32
 #define helper_fmsub_s helper_fmsub_s_riscv32
 #define helper_fmsub_d helper_fmsub_d_riscv32
+#define helper_fmsub_h helper_fmsub_h_riscv32
 #define helper_fnmsub_s helper_fnmsub_s_riscv32
 #define helper_fnmsub_d helper_fnmsub_d_riscv32
+#define helper_fnmsub_h helper_fnmsub_h_riscv32
 #define helper_fnmadd_s helper_fnmadd_s_riscv32
 #define helper_fnmadd_d helper_fnmadd_d_riscv32
+#define helper_fnmadd_h helper_fnmadd_h_riscv32
 #define helper_fadd_s helper_fadd_s_riscv32
 #define helper_fsub_s helper_fsub_s_riscv32
 #define helper_fmul_s helper_fmul_s_riscv32
@@ -1341,6 +2117,25 @@
 #define helper_fcvt_s_w helper_fcvt_s_w_riscv32
 #define helper_fcvt_s_wu helper_fcvt_s_wu_riscv32
 #define helper_fclass_s helper_fclass_s_riscv32
+#define helper_fadd_h helper_fadd_h_riscv32
+#define helper_fsub_h helper_fsub_h_riscv32
+#define helper_fmul_h helper_fmul_h_riscv32
+#define helper_fdiv_h helper_fdiv_h_riscv32
+#define helper_fmin_h helper_fmin_h_riscv32
+#define helper_fmax_h helper_fmax_h_riscv32
+#define helper_fsqrt_h helper_fsqrt_h_riscv32
+#define helper_fle_h helper_fle_h_riscv32
+#define helper_flt_h helper_flt_h_riscv32
+#define helper_feq_h helper_feq_h_riscv32
+#define helper_fcvt_s_h helper_fcvt_s_h_riscv32
+#define helper_fcvt_h_s helper_fcvt_h_s_riscv32
+#define helper_fcvt_d_h helper_fcvt_d_h_riscv32
+#define helper_fcvt_h_d helper_fcvt_h_d_riscv32
+#define helper_fcvt_w_h helper_fcvt_w_h_riscv32
+#define helper_fcvt_wu_h helper_fcvt_wu_h_riscv32
+#define helper_fcvt_h_w helper_fcvt_h_w_riscv32
+#define helper_fcvt_h_wu helper_fcvt_h_wu_riscv32
+#define helper_fclass_h helper_fclass_h_riscv32
 #define helper_fadd_d helper_fadd_d_riscv32
 #define helper_fsub_d helper_fsub_d_riscv32
 #define helper_fmul_d helper_fmul_d_riscv32
@@ -1364,15 +2159,965 @@
 #define helper_csrrw helper_csrrw_riscv32
 #define helper_csrrs helper_csrrs_riscv32
 #define helper_csrrc helper_csrrc_riscv32
+#define helper_vsetvl helper_vsetvl_riscv32
+#define helper_vle8_v helper_vle8_v_riscv32
+#define helper_vle16_v helper_vle16_v_riscv32
+#define helper_vle32_v helper_vle32_v_riscv32
+#define helper_vle64_v helper_vle64_v_riscv32
+#define helper_vse8_v helper_vse8_v_riscv32
+#define helper_vse16_v helper_vse16_v_riscv32
+#define helper_vse32_v helper_vse32_v_riscv32
+#define helper_vse64_v helper_vse64_v_riscv32
+#define helper_vlse8_v helper_vlse8_v_riscv32
+#define helper_vlse16_v helper_vlse16_v_riscv32
+#define helper_vlse32_v helper_vlse32_v_riscv32
+#define helper_vlse64_v helper_vlse64_v_riscv32
+#define helper_vsse8_v helper_vsse8_v_riscv32
+#define helper_vsse16_v helper_vsse16_v_riscv32
+#define helper_vsse32_v helper_vsse32_v_riscv32
+#define helper_vsse64_v helper_vsse64_v_riscv32
+#define helper_vlxei8_8_v helper_vlxei8_8_v_riscv32
+#define helper_vlxei8_16_v helper_vlxei8_16_v_riscv32
+#define helper_vlxei8_32_v helper_vlxei8_32_v_riscv32
+#define helper_vlxei8_64_v helper_vlxei8_64_v_riscv32
+#define helper_vlxei16_8_v helper_vlxei16_8_v_riscv32
+#define helper_vlxei16_16_v helper_vlxei16_16_v_riscv32
+#define helper_vlxei16_32_v helper_vlxei16_32_v_riscv32
+#define helper_vlxei16_64_v helper_vlxei16_64_v_riscv32
+#define helper_vlxei32_8_v helper_vlxei32_8_v_riscv32
+#define helper_vlxei32_16_v helper_vlxei32_16_v_riscv32
+#define helper_vlxei32_32_v helper_vlxei32_32_v_riscv32
+#define helper_vlxei32_64_v helper_vlxei32_64_v_riscv32
+#define helper_vlxei64_8_v helper_vlxei64_8_v_riscv32
+#define helper_vlxei64_16_v helper_vlxei64_16_v_riscv32
+#define helper_vlxei64_32_v helper_vlxei64_32_v_riscv32
+#define helper_vlxei64_64_v helper_vlxei64_64_v_riscv32
+#define helper_vsxei8_8_v helper_vsxei8_8_v_riscv32
+#define helper_vsxei8_16_v helper_vsxei8_16_v_riscv32
+#define helper_vsxei8_32_v helper_vsxei8_32_v_riscv32
+#define helper_vsxei8_64_v helper_vsxei8_64_v_riscv32
+#define helper_vsxei16_8_v helper_vsxei16_8_v_riscv32
+#define helper_vsxei16_16_v helper_vsxei16_16_v_riscv32
+#define helper_vsxei16_32_v helper_vsxei16_32_v_riscv32
+#define helper_vsxei16_64_v helper_vsxei16_64_v_riscv32
+#define helper_vsxei32_8_v helper_vsxei32_8_v_riscv32
+#define helper_vsxei32_16_v helper_vsxei32_16_v_riscv32
+#define helper_vsxei32_32_v helper_vsxei32_32_v_riscv32
+#define helper_vsxei32_64_v helper_vsxei32_64_v_riscv32
+#define helper_vsxei64_8_v helper_vsxei64_8_v_riscv32
+#define helper_vsxei64_16_v helper_vsxei64_16_v_riscv32
+#define helper_vsxei64_32_v helper_vsxei64_32_v_riscv32
+#define helper_vsxei64_64_v helper_vsxei64_64_v_riscv32
+#define helper_vle8ff_v helper_vle8ff_v_riscv32
+#define helper_vle16ff_v helper_vle16ff_v_riscv32
+#define helper_vle32ff_v helper_vle32ff_v_riscv32
+#define helper_vle64ff_v helper_vle64ff_v_riscv32
+#define helper_vl1re8_v helper_vl1re8_v_riscv32
+#define helper_vl1re16_v helper_vl1re16_v_riscv32
+#define helper_vl1re32_v helper_vl1re32_v_riscv32
+#define helper_vl1re64_v helper_vl1re64_v_riscv32
+#define helper_vl2re8_v helper_vl2re8_v_riscv32
+#define helper_vl2re16_v helper_vl2re16_v_riscv32
+#define helper_vl2re32_v helper_vl2re32_v_riscv32
+#define helper_vl2re64_v helper_vl2re64_v_riscv32
+#define helper_vl4re8_v helper_vl4re8_v_riscv32
+#define helper_vl4re16_v helper_vl4re16_v_riscv32
+#define helper_vl4re32_v helper_vl4re32_v_riscv32
+#define helper_vl4re64_v helper_vl4re64_v_riscv32
+#define helper_vl8re8_v helper_vl8re8_v_riscv32
+#define helper_vl8re16_v helper_vl8re16_v_riscv32
+#define helper_vl8re32_v helper_vl8re32_v_riscv32
+#define helper_vl8re64_v helper_vl8re64_v_riscv32
+#define helper_vs1r_v helper_vs1r_v_riscv32
+#define helper_vs2r_v helper_vs2r_v_riscv32
+#define helper_vs4r_v helper_vs4r_v_riscv32
+#define helper_vs8r_v helper_vs8r_v_riscv32
+#define helper_vlm_v helper_vlm_v_riscv32
+#define helper_vsm_v helper_vsm_v_riscv32
+#define helper_vadd_vv_b helper_vadd_vv_b_riscv32
+#define helper_vadd_vv_h helper_vadd_vv_h_riscv32
+#define helper_vadd_vv_w helper_vadd_vv_w_riscv32
+#define helper_vadd_vv_d helper_vadd_vv_d_riscv32
+#define helper_vsub_vv_b helper_vsub_vv_b_riscv32
+#define helper_vsub_vv_h helper_vsub_vv_h_riscv32
+#define helper_vsub_vv_w helper_vsub_vv_w_riscv32
+#define helper_vsub_vv_d helper_vsub_vv_d_riscv32
+#define helper_vfadd_vv_h helper_vfadd_vv_h_riscv32
+#define helper_vfadd_vv_w helper_vfadd_vv_w_riscv32
+#define helper_vfadd_vv_d helper_vfadd_vv_d_riscv32
+#define helper_vfsub_vv_h helper_vfsub_vv_h_riscv32
+#define helper_vfsub_vv_w helper_vfsub_vv_w_riscv32
+#define helper_vfsub_vv_d helper_vfsub_vv_d_riscv32
+#define helper_vfmul_vv_h helper_vfmul_vv_h_riscv32
+#define helper_vfmul_vv_w helper_vfmul_vv_w_riscv32
+#define helper_vfmul_vv_d helper_vfmul_vv_d_riscv32
+#define helper_vfdiv_vv_h helper_vfdiv_vv_h_riscv32
+#define helper_vfdiv_vv_w helper_vfdiv_vv_w_riscv32
+#define helper_vfdiv_vv_d helper_vfdiv_vv_d_riscv32
+#define helper_vfwadd_vv_h helper_vfwadd_vv_h_riscv32
+#define helper_vfwadd_vv_w helper_vfwadd_vv_w_riscv32
+#define helper_vfwsub_vv_h helper_vfwsub_vv_h_riscv32
+#define helper_vfwsub_vv_w helper_vfwsub_vv_w_riscv32
+#define helper_vfwadd_wv_h helper_vfwadd_wv_h_riscv32
+#define helper_vfwadd_wv_w helper_vfwadd_wv_w_riscv32
+#define helper_vfwsub_wv_h helper_vfwsub_wv_h_riscv32
+#define helper_vfwsub_wv_w helper_vfwsub_wv_w_riscv32
+#define helper_vfwmul_vv_h helper_vfwmul_vv_h_riscv32
+#define helper_vfwmul_vv_w helper_vfwmul_vv_w_riscv32
+#define helper_vfmacc_vv_h helper_vfmacc_vv_h_riscv32
+#define helper_vfmacc_vv_w helper_vfmacc_vv_w_riscv32
+#define helper_vfmacc_vv_d helper_vfmacc_vv_d_riscv32
+#define helper_vfnmacc_vv_h helper_vfnmacc_vv_h_riscv32
+#define helper_vfnmacc_vv_w helper_vfnmacc_vv_w_riscv32
+#define helper_vfnmacc_vv_d helper_vfnmacc_vv_d_riscv32
+#define helper_vfmsac_vv_h helper_vfmsac_vv_h_riscv32
+#define helper_vfmsac_vv_w helper_vfmsac_vv_w_riscv32
+#define helper_vfmsac_vv_d helper_vfmsac_vv_d_riscv32
+#define helper_vfnmsac_vv_h helper_vfnmsac_vv_h_riscv32
+#define helper_vfnmsac_vv_w helper_vfnmsac_vv_w_riscv32
+#define helper_vfnmsac_vv_d helper_vfnmsac_vv_d_riscv32
+#define helper_vfmadd_vv_h helper_vfmadd_vv_h_riscv32
+#define helper_vfmadd_vv_w helper_vfmadd_vv_w_riscv32
+#define helper_vfmadd_vv_d helper_vfmadd_vv_d_riscv32
+#define helper_vfnmadd_vv_h helper_vfnmadd_vv_h_riscv32
+#define helper_vfnmadd_vv_w helper_vfnmadd_vv_w_riscv32
+#define helper_vfnmadd_vv_d helper_vfnmadd_vv_d_riscv32
+#define helper_vfmsub_vv_h helper_vfmsub_vv_h_riscv32
+#define helper_vfmsub_vv_w helper_vfmsub_vv_w_riscv32
+#define helper_vfmsub_vv_d helper_vfmsub_vv_d_riscv32
+#define helper_vfnmsub_vv_h helper_vfnmsub_vv_h_riscv32
+#define helper_vfnmsub_vv_w helper_vfnmsub_vv_w_riscv32
+#define helper_vfnmsub_vv_d helper_vfnmsub_vv_d_riscv32
+#define helper_vfwmacc_vv_h helper_vfwmacc_vv_h_riscv32
+#define helper_vfwmacc_vv_w helper_vfwmacc_vv_w_riscv32
+#define helper_vfwnmacc_vv_h helper_vfwnmacc_vv_h_riscv32
+#define helper_vfwnmacc_vv_w helper_vfwnmacc_vv_w_riscv32
+#define helper_vfwmsac_vv_h helper_vfwmsac_vv_h_riscv32
+#define helper_vfwmsac_vv_w helper_vfwmsac_vv_w_riscv32
+#define helper_vfwnmsac_vv_h helper_vfwnmsac_vv_h_riscv32
+#define helper_vfwnmsac_vv_w helper_vfwnmsac_vv_w_riscv32
+#define helper_vfmin_vv_h helper_vfmin_vv_h_riscv32
+#define helper_vfmin_vv_w helper_vfmin_vv_w_riscv32
+#define helper_vfmin_vv_d helper_vfmin_vv_d_riscv32
+#define helper_vfmax_vv_h helper_vfmax_vv_h_riscv32
+#define helper_vfmax_vv_w helper_vfmax_vv_w_riscv32
+#define helper_vfmax_vv_d helper_vfmax_vv_d_riscv32
+#define helper_vfsgnj_vv_h helper_vfsgnj_vv_h_riscv32
+#define helper_vfsgnj_vv_w helper_vfsgnj_vv_w_riscv32
+#define helper_vfsgnj_vv_d helper_vfsgnj_vv_d_riscv32
+#define helper_vfsgnjn_vv_h helper_vfsgnjn_vv_h_riscv32
+#define helper_vfsgnjn_vv_w helper_vfsgnjn_vv_w_riscv32
+#define helper_vfsgnjn_vv_d helper_vfsgnjn_vv_d_riscv32
+#define helper_vfsgnjx_vv_h helper_vfsgnjx_vv_h_riscv32
+#define helper_vfsgnjx_vv_w helper_vfsgnjx_vv_w_riscv32
+#define helper_vfsgnjx_vv_d helper_vfsgnjx_vv_d_riscv32
+#define helper_vadc_vvm_b helper_vadc_vvm_b_riscv32
+#define helper_vadc_vvm_h helper_vadc_vvm_h_riscv32
+#define helper_vadc_vvm_w helper_vadc_vvm_w_riscv32
+#define helper_vadc_vvm_d helper_vadc_vvm_d_riscv32
+#define helper_vsbc_vvm_b helper_vsbc_vvm_b_riscv32
+#define helper_vsbc_vvm_h helper_vsbc_vvm_h_riscv32
+#define helper_vsbc_vvm_w helper_vsbc_vvm_w_riscv32
+#define helper_vsbc_vvm_d helper_vsbc_vvm_d_riscv32
+#define helper_vmadc_vvm_b helper_vmadc_vvm_b_riscv32
+#define helper_vmadc_vvm_h helper_vmadc_vvm_h_riscv32
+#define helper_vmadc_vvm_w helper_vmadc_vvm_w_riscv32
+#define helper_vmadc_vvm_d helper_vmadc_vvm_d_riscv32
+#define helper_vmsbc_vvm_b helper_vmsbc_vvm_b_riscv32
+#define helper_vmsbc_vvm_h helper_vmsbc_vvm_h_riscv32
+#define helper_vmsbc_vvm_w helper_vmsbc_vvm_w_riscv32
+#define helper_vmsbc_vvm_d helper_vmsbc_vvm_d_riscv32
+#define helper_vand_vv_b helper_vand_vv_b_riscv32
+#define helper_vand_vv_h helper_vand_vv_h_riscv32
+#define helper_vand_vv_w helper_vand_vv_w_riscv32
+#define helper_vand_vv_d helper_vand_vv_d_riscv32
+#define helper_vor_vv_b helper_vor_vv_b_riscv32
+#define helper_vor_vv_h helper_vor_vv_h_riscv32
+#define helper_vor_vv_w helper_vor_vv_w_riscv32
+#define helper_vor_vv_d helper_vor_vv_d_riscv32
+#define helper_vxor_vv_b helper_vxor_vv_b_riscv32
+#define helper_vxor_vv_h helper_vxor_vv_h_riscv32
+#define helper_vxor_vv_w helper_vxor_vv_w_riscv32
+#define helper_vxor_vv_d helper_vxor_vv_d_riscv32
+#define helper_vminu_vv_b helper_vminu_vv_b_riscv32
+#define helper_vminu_vv_h helper_vminu_vv_h_riscv32
+#define helper_vminu_vv_w helper_vminu_vv_w_riscv32
+#define helper_vminu_vv_d helper_vminu_vv_d_riscv32
+#define helper_vmin_vv_b helper_vmin_vv_b_riscv32
+#define helper_vmin_vv_h helper_vmin_vv_h_riscv32
+#define helper_vmin_vv_w helper_vmin_vv_w_riscv32
+#define helper_vmin_vv_d helper_vmin_vv_d_riscv32
+#define helper_vmaxu_vv_b helper_vmaxu_vv_b_riscv32
+#define helper_vmaxu_vv_h helper_vmaxu_vv_h_riscv32
+#define helper_vmaxu_vv_w helper_vmaxu_vv_w_riscv32
+#define helper_vmaxu_vv_d helper_vmaxu_vv_d_riscv32
+#define helper_vmax_vv_b helper_vmax_vv_b_riscv32
+#define helper_vmax_vv_h helper_vmax_vv_h_riscv32
+#define helper_vmax_vv_w helper_vmax_vv_w_riscv32
+#define helper_vmax_vv_d helper_vmax_vv_d_riscv32
+#define helper_vmseq_vv_b helper_vmseq_vv_b_riscv32
+#define helper_vmseq_vv_h helper_vmseq_vv_h_riscv32
+#define helper_vmseq_vv_w helper_vmseq_vv_w_riscv32
+#define helper_vmseq_vv_d helper_vmseq_vv_d_riscv32
+#define helper_vmsne_vv_b helper_vmsne_vv_b_riscv32
+#define helper_vmsne_vv_h helper_vmsne_vv_h_riscv32
+#define helper_vmsne_vv_w helper_vmsne_vv_w_riscv32
+#define helper_vmsne_vv_d helper_vmsne_vv_d_riscv32
+#define helper_vmsltu_vv_b helper_vmsltu_vv_b_riscv32
+#define helper_vmsltu_vv_h helper_vmsltu_vv_h_riscv32
+#define helper_vmsltu_vv_w helper_vmsltu_vv_w_riscv32
+#define helper_vmsltu_vv_d helper_vmsltu_vv_d_riscv32
+#define helper_vmslt_vv_b helper_vmslt_vv_b_riscv32
+#define helper_vmslt_vv_h helper_vmslt_vv_h_riscv32
+#define helper_vmslt_vv_w helper_vmslt_vv_w_riscv32
+#define helper_vmslt_vv_d helper_vmslt_vv_d_riscv32
+#define helper_vmsleu_vv_b helper_vmsleu_vv_b_riscv32
+#define helper_vmsleu_vv_h helper_vmsleu_vv_h_riscv32
+#define helper_vmsleu_vv_w helper_vmsleu_vv_w_riscv32
+#define helper_vmsleu_vv_d helper_vmsleu_vv_d_riscv32
+#define helper_vmsle_vv_b helper_vmsle_vv_b_riscv32
+#define helper_vmsle_vv_h helper_vmsle_vv_h_riscv32
+#define helper_vmsle_vv_w helper_vmsle_vv_w_riscv32
+#define helper_vmsle_vv_d helper_vmsle_vv_d_riscv32
+#define helper_vmfeq_vv_h helper_vmfeq_vv_h_riscv32
+#define helper_vmfeq_vv_w helper_vmfeq_vv_w_riscv32
+#define helper_vmfeq_vv_d helper_vmfeq_vv_d_riscv32
+#define helper_vmfne_vv_h helper_vmfne_vv_h_riscv32
+#define helper_vmfne_vv_w helper_vmfne_vv_w_riscv32
+#define helper_vmfne_vv_d helper_vmfne_vv_d_riscv32
+#define helper_vmflt_vv_h helper_vmflt_vv_h_riscv32
+#define helper_vmflt_vv_w helper_vmflt_vv_w_riscv32
+#define helper_vmflt_vv_d helper_vmflt_vv_d_riscv32
+#define helper_vmfle_vv_h helper_vmfle_vv_h_riscv32
+#define helper_vmfle_vv_w helper_vmfle_vv_w_riscv32
+#define helper_vmfle_vv_d helper_vmfle_vv_d_riscv32
+#define helper_vfsqrt_v_h helper_vfsqrt_v_h_riscv32
+#define helper_vfsqrt_v_w helper_vfsqrt_v_w_riscv32
+#define helper_vfsqrt_v_d helper_vfsqrt_v_d_riscv32
+#define helper_vfrsqrt7_v_h helper_vfrsqrt7_v_h_riscv32
+#define helper_vfrsqrt7_v_w helper_vfrsqrt7_v_w_riscv32
+#define helper_vfrsqrt7_v_d helper_vfrsqrt7_v_d_riscv32
+#define helper_vfrec7_v_h helper_vfrec7_v_h_riscv32
+#define helper_vfrec7_v_w helper_vfrec7_v_w_riscv32
+#define helper_vfrec7_v_d helper_vfrec7_v_d_riscv32
+#define helper_vfcvt_xu_f_v_h helper_vfcvt_xu_f_v_h_riscv32
+#define helper_vfcvt_xu_f_v_w helper_vfcvt_xu_f_v_w_riscv32
+#define helper_vfcvt_xu_f_v_d helper_vfcvt_xu_f_v_d_riscv32
+#define helper_vfcvt_x_f_v_h helper_vfcvt_x_f_v_h_riscv32
+#define helper_vfcvt_x_f_v_w helper_vfcvt_x_f_v_w_riscv32
+#define helper_vfcvt_x_f_v_d helper_vfcvt_x_f_v_d_riscv32
+#define helper_vfcvt_f_xu_v_h helper_vfcvt_f_xu_v_h_riscv32
+#define helper_vfcvt_f_xu_v_w helper_vfcvt_f_xu_v_w_riscv32
+#define helper_vfcvt_f_xu_v_d helper_vfcvt_f_xu_v_d_riscv32
+#define helper_vfcvt_f_x_v_h helper_vfcvt_f_x_v_h_riscv32
+#define helper_vfcvt_f_x_v_w helper_vfcvt_f_x_v_w_riscv32
+#define helper_vfcvt_f_x_v_d helper_vfcvt_f_x_v_d_riscv32
+#define helper_vfwcvt_xu_f_v_h helper_vfwcvt_xu_f_v_h_riscv32
+#define helper_vfwcvt_xu_f_v_w helper_vfwcvt_xu_f_v_w_riscv32
+#define helper_vfwcvt_x_f_v_h helper_vfwcvt_x_f_v_h_riscv32
+#define helper_vfwcvt_x_f_v_w helper_vfwcvt_x_f_v_w_riscv32
+#define helper_vfwcvt_f_xu_v_b helper_vfwcvt_f_xu_v_b_riscv32
+#define helper_vfwcvt_f_xu_v_h helper_vfwcvt_f_xu_v_h_riscv32
+#define helper_vfwcvt_f_xu_v_w helper_vfwcvt_f_xu_v_w_riscv32
+#define helper_vfwcvt_f_x_v_b helper_vfwcvt_f_x_v_b_riscv32
+#define helper_vfwcvt_f_x_v_h helper_vfwcvt_f_x_v_h_riscv32
+#define helper_vfwcvt_f_x_v_w helper_vfwcvt_f_x_v_w_riscv32
+#define helper_vfwcvt_f_f_v_h helper_vfwcvt_f_f_v_h_riscv32
+#define helper_vfwcvt_f_f_v_w helper_vfwcvt_f_f_v_w_riscv32
+#define helper_vfncvt_xu_f_w_b helper_vfncvt_xu_f_w_b_riscv32
+#define helper_vfncvt_xu_f_w_h helper_vfncvt_xu_f_w_h_riscv32
+#define helper_vfncvt_xu_f_w_w helper_vfncvt_xu_f_w_w_riscv32
+#define helper_vfncvt_x_f_w_b helper_vfncvt_x_f_w_b_riscv32
+#define helper_vfncvt_x_f_w_h helper_vfncvt_x_f_w_h_riscv32
+#define helper_vfncvt_x_f_w_w helper_vfncvt_x_f_w_w_riscv32
+#define helper_vfncvt_f_xu_w_h helper_vfncvt_f_xu_w_h_riscv32
+#define helper_vfncvt_f_xu_w_w helper_vfncvt_f_xu_w_w_riscv32
+#define helper_vfncvt_f_x_w_h helper_vfncvt_f_x_w_h_riscv32
+#define helper_vfncvt_f_x_w_w helper_vfncvt_f_x_w_w_riscv32
+#define helper_vfncvt_f_f_w_h helper_vfncvt_f_f_w_h_riscv32
+#define helper_vfncvt_f_f_w_w helper_vfncvt_f_f_w_w_riscv32
+#define helper_vfclass_v_h helper_vfclass_v_h_riscv32
+#define helper_vfclass_v_w helper_vfclass_v_w_riscv32
+#define helper_vfclass_v_d helper_vfclass_v_d_riscv32
+#define helper_vmand_mm helper_vmand_mm_riscv32
+#define helper_vmnand_mm helper_vmnand_mm_riscv32
+#define helper_vmandn_mm helper_vmandn_mm_riscv32
+#define helper_vmxor_mm helper_vmxor_mm_riscv32
+#define helper_vmor_mm helper_vmor_mm_riscv32
+#define helper_vmnor_mm helper_vmnor_mm_riscv32
+#define helper_vmorn_mm helper_vmorn_mm_riscv32
+#define helper_vmxnor_mm helper_vmxnor_mm_riscv32
+#define helper_vcpop_m helper_vcpop_m_riscv32
+#define helper_vfirst_m helper_vfirst_m_riscv32
+#define helper_vmsbf_m helper_vmsbf_m_riscv32
+#define helper_vmsif_m helper_vmsif_m_riscv32
+#define helper_vmsof_m helper_vmsof_m_riscv32
+#define helper_viota_m_b helper_viota_m_b_riscv32
+#define helper_viota_m_h helper_viota_m_h_riscv32
+#define helper_viota_m_w helper_viota_m_w_riscv32
+#define helper_viota_m_d helper_viota_m_d_riscv32
+#define helper_vid_v_b helper_vid_v_b_riscv32
+#define helper_vid_v_h helper_vid_v_h_riscv32
+#define helper_vid_v_w helper_vid_v_w_riscv32
+#define helper_vid_v_d helper_vid_v_d_riscv32
+#define helper_vredsum_vs_b helper_vredsum_vs_b_riscv32
+#define helper_vredsum_vs_h helper_vredsum_vs_h_riscv32
+#define helper_vredsum_vs_w helper_vredsum_vs_w_riscv32
+#define helper_vredsum_vs_d helper_vredsum_vs_d_riscv32
+#define helper_vredand_vs_b helper_vredand_vs_b_riscv32
+#define helper_vredand_vs_h helper_vredand_vs_h_riscv32
+#define helper_vredand_vs_w helper_vredand_vs_w_riscv32
+#define helper_vredand_vs_d helper_vredand_vs_d_riscv32
+#define helper_vredor_vs_b helper_vredor_vs_b_riscv32
+#define helper_vredor_vs_h helper_vredor_vs_h_riscv32
+#define helper_vredor_vs_w helper_vredor_vs_w_riscv32
+#define helper_vredor_vs_d helper_vredor_vs_d_riscv32
+#define helper_vredxor_vs_b helper_vredxor_vs_b_riscv32
+#define helper_vredxor_vs_h helper_vredxor_vs_h_riscv32
+#define helper_vredxor_vs_w helper_vredxor_vs_w_riscv32
+#define helper_vredxor_vs_d helper_vredxor_vs_d_riscv32
+#define helper_vredminu_vs_b helper_vredminu_vs_b_riscv32
+#define helper_vredminu_vs_h helper_vredminu_vs_h_riscv32
+#define helper_vredminu_vs_w helper_vredminu_vs_w_riscv32
+#define helper_vredminu_vs_d helper_vredminu_vs_d_riscv32
+#define helper_vredmin_vs_b helper_vredmin_vs_b_riscv32
+#define helper_vredmin_vs_h helper_vredmin_vs_h_riscv32
+#define helper_vredmin_vs_w helper_vredmin_vs_w_riscv32
+#define helper_vredmin_vs_d helper_vredmin_vs_d_riscv32
+#define helper_vredmaxu_vs_b helper_vredmaxu_vs_b_riscv32
+#define helper_vredmaxu_vs_h helper_vredmaxu_vs_h_riscv32
+#define helper_vredmaxu_vs_w helper_vredmaxu_vs_w_riscv32
+#define helper_vredmaxu_vs_d helper_vredmaxu_vs_d_riscv32
+#define helper_vredmax_vs_b helper_vredmax_vs_b_riscv32
+#define helper_vredmax_vs_h helper_vredmax_vs_h_riscv32
+#define helper_vredmax_vs_w helper_vredmax_vs_w_riscv32
+#define helper_vredmax_vs_d helper_vredmax_vs_d_riscv32
+#define helper_vwredsumu_vs_b helper_vwredsumu_vs_b_riscv32
+#define helper_vwredsumu_vs_h helper_vwredsumu_vs_h_riscv32
+#define helper_vwredsumu_vs_w helper_vwredsumu_vs_w_riscv32
+#define helper_vwredsum_vs_b helper_vwredsum_vs_b_riscv32
+#define helper_vwredsum_vs_h helper_vwredsum_vs_h_riscv32
+#define helper_vwredsum_vs_w helper_vwredsum_vs_w_riscv32
+#define helper_vfredusum_vs_h helper_vfredusum_vs_h_riscv32
+#define helper_vfredusum_vs_w helper_vfredusum_vs_w_riscv32
+#define helper_vfredusum_vs_d helper_vfredusum_vs_d_riscv32
+#define helper_vfredosum_vs_h helper_vfredosum_vs_h_riscv32
+#define helper_vfredosum_vs_w helper_vfredosum_vs_w_riscv32
+#define helper_vfredosum_vs_d helper_vfredosum_vs_d_riscv32
+#define helper_vfredmin_vs_h helper_vfredmin_vs_h_riscv32
+#define helper_vfredmin_vs_w helper_vfredmin_vs_w_riscv32
+#define helper_vfredmin_vs_d helper_vfredmin_vs_d_riscv32
+#define helper_vfredmax_vs_h helper_vfredmax_vs_h_riscv32
+#define helper_vfredmax_vs_w helper_vfredmax_vs_w_riscv32
+#define helper_vfredmax_vs_d helper_vfredmax_vs_d_riscv32
+#define helper_vfwredusum_vs_h helper_vfwredusum_vs_h_riscv32
+#define helper_vfwredusum_vs_w helper_vfwredusum_vs_w_riscv32
+#define helper_vfwredosum_vs_h helper_vfwredosum_vs_h_riscv32
+#define helper_vfwredosum_vs_w helper_vfwredosum_vs_w_riscv32
+#define helper_vadd_vx_b helper_vadd_vx_b_riscv32
+#define helper_vadd_vx_h helper_vadd_vx_h_riscv32
+#define helper_vadd_vx_w helper_vadd_vx_w_riscv32
+#define helper_vadd_vx_d helper_vadd_vx_d_riscv32
+#define helper_vsub_vx_b helper_vsub_vx_b_riscv32
+#define helper_vsub_vx_h helper_vsub_vx_h_riscv32
+#define helper_vsub_vx_w helper_vsub_vx_w_riscv32
+#define helper_vsub_vx_d helper_vsub_vx_d_riscv32
+#define helper_vrsub_vx_b helper_vrsub_vx_b_riscv32
+#define helper_vrsub_vx_h helper_vrsub_vx_h_riscv32
+#define helper_vrsub_vx_w helper_vrsub_vx_w_riscv32
+#define helper_vrsub_vx_d helper_vrsub_vx_d_riscv32
+#define helper_vfadd_vf_h helper_vfadd_vf_h_riscv32
+#define helper_vfadd_vf_w helper_vfadd_vf_w_riscv32
+#define helper_vfadd_vf_d helper_vfadd_vf_d_riscv32
+#define helper_vfsub_vf_h helper_vfsub_vf_h_riscv32
+#define helper_vfsub_vf_w helper_vfsub_vf_w_riscv32
+#define helper_vfsub_vf_d helper_vfsub_vf_d_riscv32
+#define helper_vfrsub_vf_h helper_vfrsub_vf_h_riscv32
+#define helper_vfrsub_vf_w helper_vfrsub_vf_w_riscv32
+#define helper_vfrsub_vf_d helper_vfrsub_vf_d_riscv32
+#define helper_vfmul_vf_h helper_vfmul_vf_h_riscv32
+#define helper_vfmul_vf_w helper_vfmul_vf_w_riscv32
+#define helper_vfmul_vf_d helper_vfmul_vf_d_riscv32
+#define helper_vfdiv_vf_h helper_vfdiv_vf_h_riscv32
+#define helper_vfdiv_vf_w helper_vfdiv_vf_w_riscv32
+#define helper_vfdiv_vf_d helper_vfdiv_vf_d_riscv32
+#define helper_vfrdiv_vf_h helper_vfrdiv_vf_h_riscv32
+#define helper_vfrdiv_vf_w helper_vfrdiv_vf_w_riscv32
+#define helper_vfrdiv_vf_d helper_vfrdiv_vf_d_riscv32
+#define helper_vfwadd_vf_h helper_vfwadd_vf_h_riscv32
+#define helper_vfwadd_vf_w helper_vfwadd_vf_w_riscv32
+#define helper_vfwsub_vf_h helper_vfwsub_vf_h_riscv32
+#define helper_vfwsub_vf_w helper_vfwsub_vf_w_riscv32
+#define helper_vfwadd_wf_h helper_vfwadd_wf_h_riscv32
+#define helper_vfwadd_wf_w helper_vfwadd_wf_w_riscv32
+#define helper_vfwsub_wf_h helper_vfwsub_wf_h_riscv32
+#define helper_vfwsub_wf_w helper_vfwsub_wf_w_riscv32
+#define helper_vfwmul_vf_h helper_vfwmul_vf_h_riscv32
+#define helper_vfwmul_vf_w helper_vfwmul_vf_w_riscv32
+#define helper_vfmacc_vf_h helper_vfmacc_vf_h_riscv32
+#define helper_vfmacc_vf_w helper_vfmacc_vf_w_riscv32
+#define helper_vfmacc_vf_d helper_vfmacc_vf_d_riscv32
+#define helper_vfnmacc_vf_h helper_vfnmacc_vf_h_riscv32
+#define helper_vfnmacc_vf_w helper_vfnmacc_vf_w_riscv32
+#define helper_vfnmacc_vf_d helper_vfnmacc_vf_d_riscv32
+#define helper_vfmsac_vf_h helper_vfmsac_vf_h_riscv32
+#define helper_vfmsac_vf_w helper_vfmsac_vf_w_riscv32
+#define helper_vfmsac_vf_d helper_vfmsac_vf_d_riscv32
+#define helper_vfnmsac_vf_h helper_vfnmsac_vf_h_riscv32
+#define helper_vfnmsac_vf_w helper_vfnmsac_vf_w_riscv32
+#define helper_vfnmsac_vf_d helper_vfnmsac_vf_d_riscv32
+#define helper_vfmadd_vf_h helper_vfmadd_vf_h_riscv32
+#define helper_vfmadd_vf_w helper_vfmadd_vf_w_riscv32
+#define helper_vfmadd_vf_d helper_vfmadd_vf_d_riscv32
+#define helper_vfnmadd_vf_h helper_vfnmadd_vf_h_riscv32
+#define helper_vfnmadd_vf_w helper_vfnmadd_vf_w_riscv32
+#define helper_vfnmadd_vf_d helper_vfnmadd_vf_d_riscv32
+#define helper_vfmsub_vf_h helper_vfmsub_vf_h_riscv32
+#define helper_vfmsub_vf_w helper_vfmsub_vf_w_riscv32
+#define helper_vfmsub_vf_d helper_vfmsub_vf_d_riscv32
+#define helper_vfnmsub_vf_h helper_vfnmsub_vf_h_riscv32
+#define helper_vfnmsub_vf_w helper_vfnmsub_vf_w_riscv32
+#define helper_vfnmsub_vf_d helper_vfnmsub_vf_d_riscv32
+#define helper_vfwmacc_vf_h helper_vfwmacc_vf_h_riscv32
+#define helper_vfwmacc_vf_w helper_vfwmacc_vf_w_riscv32
+#define helper_vfwnmacc_vf_h helper_vfwnmacc_vf_h_riscv32
+#define helper_vfwnmacc_vf_w helper_vfwnmacc_vf_w_riscv32
+#define helper_vfwmsac_vf_h helper_vfwmsac_vf_h_riscv32
+#define helper_vfwmsac_vf_w helper_vfwmsac_vf_w_riscv32
+#define helper_vfwnmsac_vf_h helper_vfwnmsac_vf_h_riscv32
+#define helper_vfwnmsac_vf_w helper_vfwnmsac_vf_w_riscv32
+#define helper_vfmin_vf_h helper_vfmin_vf_h_riscv32
+#define helper_vfmin_vf_w helper_vfmin_vf_w_riscv32
+#define helper_vfmin_vf_d helper_vfmin_vf_d_riscv32
+#define helper_vfmax_vf_h helper_vfmax_vf_h_riscv32
+#define helper_vfmax_vf_w helper_vfmax_vf_w_riscv32
+#define helper_vfmax_vf_d helper_vfmax_vf_d_riscv32
+#define helper_vfsgnj_vf_h helper_vfsgnj_vf_h_riscv32
+#define helper_vfsgnj_vf_w helper_vfsgnj_vf_w_riscv32
+#define helper_vfsgnj_vf_d helper_vfsgnj_vf_d_riscv32
+#define helper_vfsgnjn_vf_h helper_vfsgnjn_vf_h_riscv32
+#define helper_vfsgnjn_vf_w helper_vfsgnjn_vf_w_riscv32
+#define helper_vfsgnjn_vf_d helper_vfsgnjn_vf_d_riscv32
+#define helper_vfsgnjx_vf_h helper_vfsgnjx_vf_h_riscv32
+#define helper_vfsgnjx_vf_w helper_vfsgnjx_vf_w_riscv32
+#define helper_vfsgnjx_vf_d helper_vfsgnjx_vf_d_riscv32
+#define helper_vmfeq_vf_h helper_vmfeq_vf_h_riscv32
+#define helper_vmfeq_vf_w helper_vmfeq_vf_w_riscv32
+#define helper_vmfeq_vf_d helper_vmfeq_vf_d_riscv32
+#define helper_vmfne_vf_h helper_vmfne_vf_h_riscv32
+#define helper_vmfne_vf_w helper_vmfne_vf_w_riscv32
+#define helper_vmfne_vf_d helper_vmfne_vf_d_riscv32
+#define helper_vmflt_vf_h helper_vmflt_vf_h_riscv32
+#define helper_vmflt_vf_w helper_vmflt_vf_w_riscv32
+#define helper_vmflt_vf_d helper_vmflt_vf_d_riscv32
+#define helper_vmfle_vf_h helper_vmfle_vf_h_riscv32
+#define helper_vmfle_vf_w helper_vmfle_vf_w_riscv32
+#define helper_vmfle_vf_d helper_vmfle_vf_d_riscv32
+#define helper_vmfgt_vf_h helper_vmfgt_vf_h_riscv32
+#define helper_vmfgt_vf_w helper_vmfgt_vf_w_riscv32
+#define helper_vmfgt_vf_d helper_vmfgt_vf_d_riscv32
+#define helper_vmfge_vf_h helper_vmfge_vf_h_riscv32
+#define helper_vmfge_vf_w helper_vmfge_vf_w_riscv32
+#define helper_vmfge_vf_d helper_vmfge_vf_d_riscv32
+#define helper_vfmerge_vfm_h helper_vfmerge_vfm_h_riscv32
+#define helper_vfmerge_vfm_w helper_vfmerge_vfm_w_riscv32
+#define helper_vfmerge_vfm_d helper_vfmerge_vfm_d_riscv32
+#define helper_vslideup_vx_b helper_vslideup_vx_b_riscv32
+#define helper_vslideup_vx_h helper_vslideup_vx_h_riscv32
+#define helper_vslideup_vx_w helper_vslideup_vx_w_riscv32
+#define helper_vslideup_vx_d helper_vslideup_vx_d_riscv32
+#define helper_vslidedown_vx_b helper_vslidedown_vx_b_riscv32
+#define helper_vslidedown_vx_h helper_vslidedown_vx_h_riscv32
+#define helper_vslidedown_vx_w helper_vslidedown_vx_w_riscv32
+#define helper_vslidedown_vx_d helper_vslidedown_vx_d_riscv32
+#define helper_vslide1up_vx_b helper_vslide1up_vx_b_riscv32
+#define helper_vslide1up_vx_h helper_vslide1up_vx_h_riscv32
+#define helper_vslide1up_vx_w helper_vslide1up_vx_w_riscv32
+#define helper_vslide1up_vx_d helper_vslide1up_vx_d_riscv32
+#define helper_vslide1down_vx_b helper_vslide1down_vx_b_riscv32
+#define helper_vslide1down_vx_h helper_vslide1down_vx_h_riscv32
+#define helper_vslide1down_vx_w helper_vslide1down_vx_w_riscv32
+#define helper_vslide1down_vx_d helper_vslide1down_vx_d_riscv32
+#define helper_vrgather_vv_b helper_vrgather_vv_b_riscv32
+#define helper_vrgather_vv_h helper_vrgather_vv_h_riscv32
+#define helper_vrgather_vv_w helper_vrgather_vv_w_riscv32
+#define helper_vrgather_vv_d helper_vrgather_vv_d_riscv32
+#define helper_vrgatherei16_vv_b helper_vrgatherei16_vv_b_riscv32
+#define helper_vrgatherei16_vv_h helper_vrgatherei16_vv_h_riscv32
+#define helper_vrgatherei16_vv_w helper_vrgatherei16_vv_w_riscv32
+#define helper_vrgatherei16_vv_d helper_vrgatherei16_vv_d_riscv32
+#define helper_vrgather_vx_b helper_vrgather_vx_b_riscv32
+#define helper_vrgather_vx_h helper_vrgather_vx_h_riscv32
+#define helper_vrgather_vx_w helper_vrgather_vx_w_riscv32
+#define helper_vrgather_vx_d helper_vrgather_vx_d_riscv32
+#define helper_vcompress_vm_b helper_vcompress_vm_b_riscv32
+#define helper_vcompress_vm_h helper_vcompress_vm_h_riscv32
+#define helper_vcompress_vm_w helper_vcompress_vm_w_riscv32
+#define helper_vcompress_vm_d helper_vcompress_vm_d_riscv32
+#define helper_vmvr_v helper_vmvr_v_riscv32
+#define helper_vfslide1up_vf_h helper_vfslide1up_vf_h_riscv32
+#define helper_vfslide1up_vf_w helper_vfslide1up_vf_w_riscv32
+#define helper_vfslide1up_vf_d helper_vfslide1up_vf_d_riscv32
+#define helper_vfslide1down_vf_h helper_vfslide1down_vf_h_riscv32
+#define helper_vfslide1down_vf_w helper_vfslide1down_vf_w_riscv32
+#define helper_vfslide1down_vf_d helper_vfslide1down_vf_d_riscv32
+#define helper_vadc_vxm_b helper_vadc_vxm_b_riscv32
+#define helper_vadc_vxm_h helper_vadc_vxm_h_riscv32
+#define helper_vadc_vxm_w helper_vadc_vxm_w_riscv32
+#define helper_vadc_vxm_d helper_vadc_vxm_d_riscv32
+#define helper_vsbc_vxm_b helper_vsbc_vxm_b_riscv32
+#define helper_vsbc_vxm_h helper_vsbc_vxm_h_riscv32
+#define helper_vsbc_vxm_w helper_vsbc_vxm_w_riscv32
+#define helper_vsbc_vxm_d helper_vsbc_vxm_d_riscv32
+#define helper_vmadc_vxm_b helper_vmadc_vxm_b_riscv32
+#define helper_vmadc_vxm_h helper_vmadc_vxm_h_riscv32
+#define helper_vmadc_vxm_w helper_vmadc_vxm_w_riscv32
+#define helper_vmadc_vxm_d helper_vmadc_vxm_d_riscv32
+#define helper_vmsbc_vxm_b helper_vmsbc_vxm_b_riscv32
+#define helper_vmsbc_vxm_h helper_vmsbc_vxm_h_riscv32
+#define helper_vmsbc_vxm_w helper_vmsbc_vxm_w_riscv32
+#define helper_vmsbc_vxm_d helper_vmsbc_vxm_d_riscv32
+#define helper_vand_vx_b helper_vand_vx_b_riscv32
+#define helper_vand_vx_h helper_vand_vx_h_riscv32
+#define helper_vand_vx_w helper_vand_vx_w_riscv32
+#define helper_vand_vx_d helper_vand_vx_d_riscv32
+#define helper_vor_vx_b helper_vor_vx_b_riscv32
+#define helper_vor_vx_h helper_vor_vx_h_riscv32
+#define helper_vor_vx_w helper_vor_vx_w_riscv32
+#define helper_vor_vx_d helper_vor_vx_d_riscv32
+#define helper_vxor_vx_b helper_vxor_vx_b_riscv32
+#define helper_vxor_vx_h helper_vxor_vx_h_riscv32
+#define helper_vxor_vx_w helper_vxor_vx_w_riscv32
+#define helper_vxor_vx_d helper_vxor_vx_d_riscv32
+#define helper_vsaddu_vv_b helper_vsaddu_vv_b_riscv32
+#define helper_vsaddu_vv_h helper_vsaddu_vv_h_riscv32
+#define helper_vsaddu_vv_w helper_vsaddu_vv_w_riscv32
+#define helper_vsaddu_vv_d helper_vsaddu_vv_d_riscv32
+#define helper_vsaddu_vx_b helper_vsaddu_vx_b_riscv32
+#define helper_vsaddu_vx_h helper_vsaddu_vx_h_riscv32
+#define helper_vsaddu_vx_w helper_vsaddu_vx_w_riscv32
+#define helper_vsaddu_vx_d helper_vsaddu_vx_d_riscv32
+#define helper_vsadd_vv_b helper_vsadd_vv_b_riscv32
+#define helper_vsadd_vv_h helper_vsadd_vv_h_riscv32
+#define helper_vsadd_vv_w helper_vsadd_vv_w_riscv32
+#define helper_vsadd_vv_d helper_vsadd_vv_d_riscv32
+#define helper_vsadd_vx_b helper_vsadd_vx_b_riscv32
+#define helper_vsadd_vx_h helper_vsadd_vx_h_riscv32
+#define helper_vsadd_vx_w helper_vsadd_vx_w_riscv32
+#define helper_vsadd_vx_d helper_vsadd_vx_d_riscv32
+#define helper_vssubu_vv_b helper_vssubu_vv_b_riscv32
+#define helper_vssubu_vv_h helper_vssubu_vv_h_riscv32
+#define helper_vssubu_vv_w helper_vssubu_vv_w_riscv32
+#define helper_vssubu_vv_d helper_vssubu_vv_d_riscv32
+#define helper_vssubu_vx_b helper_vssubu_vx_b_riscv32
+#define helper_vssubu_vx_h helper_vssubu_vx_h_riscv32
+#define helper_vssubu_vx_w helper_vssubu_vx_w_riscv32
+#define helper_vssubu_vx_d helper_vssubu_vx_d_riscv32
+#define helper_vssub_vv_b helper_vssub_vv_b_riscv32
+#define helper_vssub_vv_h helper_vssub_vv_h_riscv32
+#define helper_vssub_vv_w helper_vssub_vv_w_riscv32
+#define helper_vssub_vv_d helper_vssub_vv_d_riscv32
+#define helper_vssub_vx_b helper_vssub_vx_b_riscv32
+#define helper_vssub_vx_h helper_vssub_vx_h_riscv32
+#define helper_vssub_vx_w helper_vssub_vx_w_riscv32
+#define helper_vssub_vx_d helper_vssub_vx_d_riscv32
+#define helper_vaadd_vv_b helper_vaadd_vv_b_riscv32
+#define helper_vaadd_vv_h helper_vaadd_vv_h_riscv32
+#define helper_vaadd_vv_w helper_vaadd_vv_w_riscv32
+#define helper_vaadd_vv_d helper_vaadd_vv_d_riscv32
+#define helper_vaadd_vx_b helper_vaadd_vx_b_riscv32
+#define helper_vaadd_vx_h helper_vaadd_vx_h_riscv32
+#define helper_vaadd_vx_w helper_vaadd_vx_w_riscv32
+#define helper_vaadd_vx_d helper_vaadd_vx_d_riscv32
+#define helper_vaaddu_vv_b helper_vaaddu_vv_b_riscv32
+#define helper_vaaddu_vv_h helper_vaaddu_vv_h_riscv32
+#define helper_vaaddu_vv_w helper_vaaddu_vv_w_riscv32
+#define helper_vaaddu_vv_d helper_vaaddu_vv_d_riscv32
+#define helper_vaaddu_vx_b helper_vaaddu_vx_b_riscv32
+#define helper_vaaddu_vx_h helper_vaaddu_vx_h_riscv32
+#define helper_vaaddu_vx_w helper_vaaddu_vx_w_riscv32
+#define helper_vaaddu_vx_d helper_vaaddu_vx_d_riscv32
+#define helper_vasub_vv_b helper_vasub_vv_b_riscv32
+#define helper_vasub_vv_h helper_vasub_vv_h_riscv32
+#define helper_vasub_vv_w helper_vasub_vv_w_riscv32
+#define helper_vasub_vv_d helper_vasub_vv_d_riscv32
+#define helper_vasub_vx_b helper_vasub_vx_b_riscv32
+#define helper_vasub_vx_h helper_vasub_vx_h_riscv32
+#define helper_vasub_vx_w helper_vasub_vx_w_riscv32
+#define helper_vasub_vx_d helper_vasub_vx_d_riscv32
+#define helper_vasubu_vv_b helper_vasubu_vv_b_riscv32
+#define helper_vasubu_vv_h helper_vasubu_vv_h_riscv32
+#define helper_vasubu_vv_w helper_vasubu_vv_w_riscv32
+#define helper_vasubu_vv_d helper_vasubu_vv_d_riscv32
+#define helper_vasubu_vx_b helper_vasubu_vx_b_riscv32
+#define helper_vasubu_vx_h helper_vasubu_vx_h_riscv32
+#define helper_vasubu_vx_w helper_vasubu_vx_w_riscv32
+#define helper_vasubu_vx_d helper_vasubu_vx_d_riscv32
+#define helper_vsmul_vv_b helper_vsmul_vv_b_riscv32
+#define helper_vsmul_vv_h helper_vsmul_vv_h_riscv32
+#define helper_vsmul_vv_w helper_vsmul_vv_w_riscv32
+#define helper_vsmul_vv_d helper_vsmul_vv_d_riscv32
+#define helper_vsmul_vx_b helper_vsmul_vx_b_riscv32
+#define helper_vsmul_vx_h helper_vsmul_vx_h_riscv32
+#define helper_vsmul_vx_w helper_vsmul_vx_w_riscv32
+#define helper_vsmul_vx_d helper_vsmul_vx_d_riscv32
+#define helper_vssrl_vv_b helper_vssrl_vv_b_riscv32
+#define helper_vssrl_vv_h helper_vssrl_vv_h_riscv32
+#define helper_vssrl_vv_w helper_vssrl_vv_w_riscv32
+#define helper_vssrl_vv_d helper_vssrl_vv_d_riscv32
+#define helper_vssrl_vx_b helper_vssrl_vx_b_riscv32
+#define helper_vssrl_vx_h helper_vssrl_vx_h_riscv32
+#define helper_vssrl_vx_w helper_vssrl_vx_w_riscv32
+#define helper_vssrl_vx_d helper_vssrl_vx_d_riscv32
+#define helper_vssra_vv_b helper_vssra_vv_b_riscv32
+#define helper_vssra_vv_h helper_vssra_vv_h_riscv32
+#define helper_vssra_vv_w helper_vssra_vv_w_riscv32
+#define helper_vssra_vv_d helper_vssra_vv_d_riscv32
+#define helper_vssra_vx_b helper_vssra_vx_b_riscv32
+#define helper_vssra_vx_h helper_vssra_vx_h_riscv32
+#define helper_vssra_vx_w helper_vssra_vx_w_riscv32
+#define helper_vssra_vx_d helper_vssra_vx_d_riscv32
+#define helper_vnclip_wv_b helper_vnclip_wv_b_riscv32
+#define helper_vnclip_wv_h helper_vnclip_wv_h_riscv32
+#define helper_vnclip_wv_w helper_vnclip_wv_w_riscv32
+#define helper_vnclip_wx_b helper_vnclip_wx_b_riscv32
+#define helper_vnclip_wx_h helper_vnclip_wx_h_riscv32
+#define helper_vnclip_wx_w helper_vnclip_wx_w_riscv32
+#define helper_vnclipu_wv_b helper_vnclipu_wv_b_riscv32
+#define helper_vnclipu_wv_h helper_vnclipu_wv_h_riscv32
+#define helper_vnclipu_wv_w helper_vnclipu_wv_w_riscv32
+#define helper_vnclipu_wx_b helper_vnclipu_wx_b_riscv32
+#define helper_vnclipu_wx_h helper_vnclipu_wx_h_riscv32
+#define helper_vnclipu_wx_w helper_vnclipu_wx_w_riscv32
+#define helper_vminu_vx_b helper_vminu_vx_b_riscv32
+#define helper_vminu_vx_h helper_vminu_vx_h_riscv32
+#define helper_vminu_vx_w helper_vminu_vx_w_riscv32
+#define helper_vminu_vx_d helper_vminu_vx_d_riscv32
+#define helper_vmin_vx_b helper_vmin_vx_b_riscv32
+#define helper_vmin_vx_h helper_vmin_vx_h_riscv32
+#define helper_vmin_vx_w helper_vmin_vx_w_riscv32
+#define helper_vmin_vx_d helper_vmin_vx_d_riscv32
+#define helper_vmaxu_vx_b helper_vmaxu_vx_b_riscv32
+#define helper_vmaxu_vx_h helper_vmaxu_vx_h_riscv32
+#define helper_vmaxu_vx_w helper_vmaxu_vx_w_riscv32
+#define helper_vmaxu_vx_d helper_vmaxu_vx_d_riscv32
+#define helper_vmax_vx_b helper_vmax_vx_b_riscv32
+#define helper_vmax_vx_h helper_vmax_vx_h_riscv32
+#define helper_vmax_vx_w helper_vmax_vx_w_riscv32
+#define helper_vmax_vx_d helper_vmax_vx_d_riscv32
+#define helper_vwaddu_vv_b helper_vwaddu_vv_b_riscv32
+#define helper_vwaddu_vv_h helper_vwaddu_vv_h_riscv32
+#define helper_vwaddu_vv_w helper_vwaddu_vv_w_riscv32
+#define helper_vwsubu_vv_b helper_vwsubu_vv_b_riscv32
+#define helper_vwsubu_vv_h helper_vwsubu_vv_h_riscv32
+#define helper_vwsubu_vv_w helper_vwsubu_vv_w_riscv32
+#define helper_vwadd_vv_b helper_vwadd_vv_b_riscv32
+#define helper_vwadd_vv_h helper_vwadd_vv_h_riscv32
+#define helper_vwadd_vv_w helper_vwadd_vv_w_riscv32
+#define helper_vwsub_vv_b helper_vwsub_vv_b_riscv32
+#define helper_vwsub_vv_h helper_vwsub_vv_h_riscv32
+#define helper_vwsub_vv_w helper_vwsub_vv_w_riscv32
+#define helper_vwaddu_vx_b helper_vwaddu_vx_b_riscv32
+#define helper_vwaddu_vx_h helper_vwaddu_vx_h_riscv32
+#define helper_vwaddu_vx_w helper_vwaddu_vx_w_riscv32
+#define helper_vwsubu_vx_b helper_vwsubu_vx_b_riscv32
+#define helper_vwsubu_vx_h helper_vwsubu_vx_h_riscv32
+#define helper_vwsubu_vx_w helper_vwsubu_vx_w_riscv32
+#define helper_vwadd_vx_b helper_vwadd_vx_b_riscv32
+#define helper_vwadd_vx_h helper_vwadd_vx_h_riscv32
+#define helper_vwadd_vx_w helper_vwadd_vx_w_riscv32
+#define helper_vwsub_vx_b helper_vwsub_vx_b_riscv32
+#define helper_vwsub_vx_h helper_vwsub_vx_h_riscv32
+#define helper_vwsub_vx_w helper_vwsub_vx_w_riscv32
+#define helper_vwaddu_wv_b helper_vwaddu_wv_b_riscv32
+#define helper_vwaddu_wv_h helper_vwaddu_wv_h_riscv32
+#define helper_vwaddu_wv_w helper_vwaddu_wv_w_riscv32
+#define helper_vwsubu_wv_b helper_vwsubu_wv_b_riscv32
+#define helper_vwsubu_wv_h helper_vwsubu_wv_h_riscv32
+#define helper_vwsubu_wv_w helper_vwsubu_wv_w_riscv32
+#define helper_vwadd_wv_b helper_vwadd_wv_b_riscv32
+#define helper_vwadd_wv_h helper_vwadd_wv_h_riscv32
+#define helper_vwadd_wv_w helper_vwadd_wv_w_riscv32
+#define helper_vwsub_wv_b helper_vwsub_wv_b_riscv32
+#define helper_vwsub_wv_h helper_vwsub_wv_h_riscv32
+#define helper_vwsub_wv_w helper_vwsub_wv_w_riscv32
+#define helper_vwaddu_wx_b helper_vwaddu_wx_b_riscv32
+#define helper_vwaddu_wx_h helper_vwaddu_wx_h_riscv32
+#define helper_vwaddu_wx_w helper_vwaddu_wx_w_riscv32
+#define helper_vwsubu_wx_b helper_vwsubu_wx_b_riscv32
+#define helper_vwsubu_wx_h helper_vwsubu_wx_h_riscv32
+#define helper_vwsubu_wx_w helper_vwsubu_wx_w_riscv32
+#define helper_vwadd_wx_b helper_vwadd_wx_b_riscv32
+#define helper_vwadd_wx_h helper_vwadd_wx_h_riscv32
+#define helper_vwadd_wx_w helper_vwadd_wx_w_riscv32
+#define helper_vwsub_wx_b helper_vwsub_wx_b_riscv32
+#define helper_vwsub_wx_h helper_vwsub_wx_h_riscv32
+#define helper_vwsub_wx_w helper_vwsub_wx_w_riscv32
+#define helper_vwmul_vv_b helper_vwmul_vv_b_riscv32
+#define helper_vwmul_vv_h helper_vwmul_vv_h_riscv32
+#define helper_vwmul_vv_w helper_vwmul_vv_w_riscv32
+#define helper_vwmulu_vv_b helper_vwmulu_vv_b_riscv32
+#define helper_vwmulu_vv_h helper_vwmulu_vv_h_riscv32
+#define helper_vwmulu_vv_w helper_vwmulu_vv_w_riscv32
+#define helper_vwmulsu_vv_b helper_vwmulsu_vv_b_riscv32
+#define helper_vwmulsu_vv_h helper_vwmulsu_vv_h_riscv32
+#define helper_vwmulsu_vv_w helper_vwmulsu_vv_w_riscv32
+#define helper_vwmul_vx_b helper_vwmul_vx_b_riscv32
+#define helper_vwmul_vx_h helper_vwmul_vx_h_riscv32
+#define helper_vwmul_vx_w helper_vwmul_vx_w_riscv32
+#define helper_vwmulu_vx_b helper_vwmulu_vx_b_riscv32
+#define helper_vwmulu_vx_h helper_vwmulu_vx_h_riscv32
+#define helper_vwmulu_vx_w helper_vwmulu_vx_w_riscv32
+#define helper_vwmulsu_vx_b helper_vwmulsu_vx_b_riscv32
+#define helper_vwmulsu_vx_h helper_vwmulsu_vx_h_riscv32
+#define helper_vwmulsu_vx_w helper_vwmulsu_vx_w_riscv32
+#define helper_vmacc_vv_b helper_vmacc_vv_b_riscv32
+#define helper_vmacc_vv_h helper_vmacc_vv_h_riscv32
+#define helper_vmacc_vv_w helper_vmacc_vv_w_riscv32
+#define helper_vmacc_vv_d helper_vmacc_vv_d_riscv32
+#define helper_vnmsac_vv_b helper_vnmsac_vv_b_riscv32
+#define helper_vnmsac_vv_h helper_vnmsac_vv_h_riscv32
+#define helper_vnmsac_vv_w helper_vnmsac_vv_w_riscv32
+#define helper_vnmsac_vv_d helper_vnmsac_vv_d_riscv32
+#define helper_vmadd_vv_b helper_vmadd_vv_b_riscv32
+#define helper_vmadd_vv_h helper_vmadd_vv_h_riscv32
+#define helper_vmadd_vv_w helper_vmadd_vv_w_riscv32
+#define helper_vmadd_vv_d helper_vmadd_vv_d_riscv32
+#define helper_vnmsub_vv_b helper_vnmsub_vv_b_riscv32
+#define helper_vnmsub_vv_h helper_vnmsub_vv_h_riscv32
+#define helper_vnmsub_vv_w helper_vnmsub_vv_w_riscv32
+#define helper_vnmsub_vv_d helper_vnmsub_vv_d_riscv32
+#define helper_vmacc_vx_b helper_vmacc_vx_b_riscv32
+#define helper_vmacc_vx_h helper_vmacc_vx_h_riscv32
+#define helper_vmacc_vx_w helper_vmacc_vx_w_riscv32
+#define helper_vmacc_vx_d helper_vmacc_vx_d_riscv32
+#define helper_vnmsac_vx_b helper_vnmsac_vx_b_riscv32
+#define helper_vnmsac_vx_h helper_vnmsac_vx_h_riscv32
+#define helper_vnmsac_vx_w helper_vnmsac_vx_w_riscv32
+#define helper_vnmsac_vx_d helper_vnmsac_vx_d_riscv32
+#define helper_vmadd_vx_b helper_vmadd_vx_b_riscv32
+#define helper_vmadd_vx_h helper_vmadd_vx_h_riscv32
+#define helper_vmadd_vx_w helper_vmadd_vx_w_riscv32
+#define helper_vmadd_vx_d helper_vmadd_vx_d_riscv32
+#define helper_vnmsub_vx_b helper_vnmsub_vx_b_riscv32
+#define helper_vnmsub_vx_h helper_vnmsub_vx_h_riscv32
+#define helper_vnmsub_vx_w helper_vnmsub_vx_w_riscv32
+#define helper_vnmsub_vx_d helper_vnmsub_vx_d_riscv32
+#define helper_vwmaccu_vv_b helper_vwmaccu_vv_b_riscv32
+#define helper_vwmaccu_vv_h helper_vwmaccu_vv_h_riscv32
+#define helper_vwmaccu_vv_w helper_vwmaccu_vv_w_riscv32
+#define helper_vwmacc_vv_b helper_vwmacc_vv_b_riscv32
+#define helper_vwmacc_vv_h helper_vwmacc_vv_h_riscv32
+#define helper_vwmacc_vv_w helper_vwmacc_vv_w_riscv32
+#define helper_vwmaccsu_vv_b helper_vwmaccsu_vv_b_riscv32
+#define helper_vwmaccsu_vv_h helper_vwmaccsu_vv_h_riscv32
+#define helper_vwmaccsu_vv_w helper_vwmaccsu_vv_w_riscv32
+#define helper_vwmaccu_vx_b helper_vwmaccu_vx_b_riscv32
+#define helper_vwmaccu_vx_h helper_vwmaccu_vx_h_riscv32
+#define helper_vwmaccu_vx_w helper_vwmaccu_vx_w_riscv32
+#define helper_vwmacc_vx_b helper_vwmacc_vx_b_riscv32
+#define helper_vwmacc_vx_h helper_vwmacc_vx_h_riscv32
+#define helper_vwmacc_vx_w helper_vwmacc_vx_w_riscv32
+#define helper_vwmaccsu_vx_b helper_vwmaccsu_vx_b_riscv32
+#define helper_vwmaccsu_vx_h helper_vwmaccsu_vx_h_riscv32
+#define helper_vwmaccsu_vx_w helper_vwmaccsu_vx_w_riscv32
+#define helper_vwmaccus_vx_b helper_vwmaccus_vx_b_riscv32
+#define helper_vwmaccus_vx_h helper_vwmaccus_vx_h_riscv32
+#define helper_vwmaccus_vx_w helper_vwmaccus_vx_w_riscv32
+#define helper_vmul_vv_b helper_vmul_vv_b_riscv32
+#define helper_vmul_vv_h helper_vmul_vv_h_riscv32
+#define helper_vmul_vv_w helper_vmul_vv_w_riscv32
+#define helper_vmul_vv_d helper_vmul_vv_d_riscv32
+#define helper_vmulh_vv_b helper_vmulh_vv_b_riscv32
+#define helper_vmulh_vv_h helper_vmulh_vv_h_riscv32
+#define helper_vmulh_vv_w helper_vmulh_vv_w_riscv32
+#define helper_vmulh_vv_d helper_vmulh_vv_d_riscv32
+#define helper_vmulhu_vv_b helper_vmulhu_vv_b_riscv32
+#define helper_vmulhu_vv_h helper_vmulhu_vv_h_riscv32
+#define helper_vmulhu_vv_w helper_vmulhu_vv_w_riscv32
+#define helper_vmulhu_vv_d helper_vmulhu_vv_d_riscv32
+#define helper_vmulhsu_vv_b helper_vmulhsu_vv_b_riscv32
+#define helper_vmulhsu_vv_h helper_vmulhsu_vv_h_riscv32
+#define helper_vmulhsu_vv_w helper_vmulhsu_vv_w_riscv32
+#define helper_vmulhsu_vv_d helper_vmulhsu_vv_d_riscv32
+#define helper_vmul_vx_b helper_vmul_vx_b_riscv32
+#define helper_vmul_vx_h helper_vmul_vx_h_riscv32
+#define helper_vmul_vx_w helper_vmul_vx_w_riscv32
+#define helper_vmul_vx_d helper_vmul_vx_d_riscv32
+#define helper_vmulh_vx_b helper_vmulh_vx_b_riscv32
+#define helper_vmulh_vx_h helper_vmulh_vx_h_riscv32
+#define helper_vmulh_vx_w helper_vmulh_vx_w_riscv32
+#define helper_vmulh_vx_d helper_vmulh_vx_d_riscv32
+#define helper_vmulhu_vx_b helper_vmulhu_vx_b_riscv32
+#define helper_vmulhu_vx_h helper_vmulhu_vx_h_riscv32
+#define helper_vmulhu_vx_w helper_vmulhu_vx_w_riscv32
+#define helper_vmulhu_vx_d helper_vmulhu_vx_d_riscv32
+#define helper_vmulhsu_vx_b helper_vmulhsu_vx_b_riscv32
+#define helper_vmulhsu_vx_h helper_vmulhsu_vx_h_riscv32
+#define helper_vmulhsu_vx_w helper_vmulhsu_vx_w_riscv32
+#define helper_vmulhsu_vx_d helper_vmulhsu_vx_d_riscv32
+#define helper_vdivu_vv_b helper_vdivu_vv_b_riscv32
+#define helper_vdivu_vv_h helper_vdivu_vv_h_riscv32
+#define helper_vdivu_vv_w helper_vdivu_vv_w_riscv32
+#define helper_vdivu_vv_d helper_vdivu_vv_d_riscv32
+#define helper_vdiv_vv_b helper_vdiv_vv_b_riscv32
+#define helper_vdiv_vv_h helper_vdiv_vv_h_riscv32
+#define helper_vdiv_vv_w helper_vdiv_vv_w_riscv32
+#define helper_vdiv_vv_d helper_vdiv_vv_d_riscv32
+#define helper_vremu_vv_b helper_vremu_vv_b_riscv32
+#define helper_vremu_vv_h helper_vremu_vv_h_riscv32
+#define helper_vremu_vv_w helper_vremu_vv_w_riscv32
+#define helper_vremu_vv_d helper_vremu_vv_d_riscv32
+#define helper_vrem_vv_b helper_vrem_vv_b_riscv32
+#define helper_vrem_vv_h helper_vrem_vv_h_riscv32
+#define helper_vrem_vv_w helper_vrem_vv_w_riscv32
+#define helper_vrem_vv_d helper_vrem_vv_d_riscv32
+#define helper_vdivu_vx_b helper_vdivu_vx_b_riscv32
+#define helper_vdivu_vx_h helper_vdivu_vx_h_riscv32
+#define helper_vdivu_vx_w helper_vdivu_vx_w_riscv32
+#define helper_vdivu_vx_d helper_vdivu_vx_d_riscv32
+#define helper_vdiv_vx_b helper_vdiv_vx_b_riscv32
+#define helper_vdiv_vx_h helper_vdiv_vx_h_riscv32
+#define helper_vdiv_vx_w helper_vdiv_vx_w_riscv32
+#define helper_vdiv_vx_d helper_vdiv_vx_d_riscv32
+#define helper_vremu_vx_b helper_vremu_vx_b_riscv32
+#define helper_vremu_vx_h helper_vremu_vx_h_riscv32
+#define helper_vremu_vx_w helper_vremu_vx_w_riscv32
+#define helper_vremu_vx_d helper_vremu_vx_d_riscv32
+#define helper_vrem_vx_b helper_vrem_vx_b_riscv32
+#define helper_vrem_vx_h helper_vrem_vx_h_riscv32
+#define helper_vrem_vx_w helper_vrem_vx_w_riscv32
+#define helper_vrem_vx_d helper_vrem_vx_d_riscv32
+#define helper_vsll_vv_b helper_vsll_vv_b_riscv32
+#define helper_vsll_vv_h helper_vsll_vv_h_riscv32
+#define helper_vsll_vv_w helper_vsll_vv_w_riscv32
+#define helper_vsll_vv_d helper_vsll_vv_d_riscv32
+#define helper_vsrl_vv_b helper_vsrl_vv_b_riscv32
+#define helper_vsrl_vv_h helper_vsrl_vv_h_riscv32
+#define helper_vsrl_vv_w helper_vsrl_vv_w_riscv32
+#define helper_vsrl_vv_d helper_vsrl_vv_d_riscv32
+#define helper_vsra_vv_b helper_vsra_vv_b_riscv32
+#define helper_vsra_vv_h helper_vsra_vv_h_riscv32
+#define helper_vsra_vv_w helper_vsra_vv_w_riscv32
+#define helper_vsra_vv_d helper_vsra_vv_d_riscv32
+#define helper_vsll_vx_b helper_vsll_vx_b_riscv32
+#define helper_vsll_vx_h helper_vsll_vx_h_riscv32
+#define helper_vsll_vx_w helper_vsll_vx_w_riscv32
+#define helper_vsll_vx_d helper_vsll_vx_d_riscv32
+#define helper_vsrl_vx_b helper_vsrl_vx_b_riscv32
+#define helper_vsrl_vx_h helper_vsrl_vx_h_riscv32
+#define helper_vsrl_vx_w helper_vsrl_vx_w_riscv32
+#define helper_vsrl_vx_d helper_vsrl_vx_d_riscv32
+#define helper_vsra_vx_b helper_vsra_vx_b_riscv32
+#define helper_vsra_vx_h helper_vsra_vx_h_riscv32
+#define helper_vsra_vx_w helper_vsra_vx_w_riscv32
+#define helper_vsra_vx_d helper_vsra_vx_d_riscv32
+#define helper_vnsrl_wv_b helper_vnsrl_wv_b_riscv32
+#define helper_vnsrl_wv_h helper_vnsrl_wv_h_riscv32
+#define helper_vnsrl_wv_w helper_vnsrl_wv_w_riscv32
+#define helper_vnsra_wv_b helper_vnsra_wv_b_riscv32
+#define helper_vnsra_wv_h helper_vnsra_wv_h_riscv32
+#define helper_vnsra_wv_w helper_vnsra_wv_w_riscv32
+#define helper_vnsrl_wx_b helper_vnsrl_wx_b_riscv32
+#define helper_vnsrl_wx_h helper_vnsrl_wx_h_riscv32
+#define helper_vnsrl_wx_w helper_vnsrl_wx_w_riscv32
+#define helper_vnsra_wx_b helper_vnsra_wx_b_riscv32
+#define helper_vnsra_wx_h helper_vnsra_wx_h_riscv32
+#define helper_vnsra_wx_w helper_vnsra_wx_w_riscv32
+#define helper_vzext_vf2_h helper_vzext_vf2_h_riscv32
+#define helper_vzext_vf2_w helper_vzext_vf2_w_riscv32
+#define helper_vzext_vf2_d helper_vzext_vf2_d_riscv32
+#define helper_vzext_vf4_w helper_vzext_vf4_w_riscv32
+#define helper_vzext_vf4_d helper_vzext_vf4_d_riscv32
+#define helper_vzext_vf8_d helper_vzext_vf8_d_riscv32
+#define helper_vsext_vf2_h helper_vsext_vf2_h_riscv32
+#define helper_vsext_vf2_w helper_vsext_vf2_w_riscv32
+#define helper_vsext_vf2_d helper_vsext_vf2_d_riscv32
+#define helper_vsext_vf4_w helper_vsext_vf4_w_riscv32
+#define helper_vsext_vf4_d helper_vsext_vf4_d_riscv32
+#define helper_vsext_vf8_d helper_vsext_vf8_d_riscv32
+#define helper_vmseq_vx_b helper_vmseq_vx_b_riscv32
+#define helper_vmseq_vx_h helper_vmseq_vx_h_riscv32
+#define helper_vmseq_vx_w helper_vmseq_vx_w_riscv32
+#define helper_vmseq_vx_d helper_vmseq_vx_d_riscv32
+#define helper_vmsne_vx_b helper_vmsne_vx_b_riscv32
+#define helper_vmsne_vx_h helper_vmsne_vx_h_riscv32
+#define helper_vmsne_vx_w helper_vmsne_vx_w_riscv32
+#define helper_vmsne_vx_d helper_vmsne_vx_d_riscv32
+#define helper_vmsltu_vx_b helper_vmsltu_vx_b_riscv32
+#define helper_vmsltu_vx_h helper_vmsltu_vx_h_riscv32
+#define helper_vmsltu_vx_w helper_vmsltu_vx_w_riscv32
+#define helper_vmsltu_vx_d helper_vmsltu_vx_d_riscv32
+#define helper_vmslt_vx_b helper_vmslt_vx_b_riscv32
+#define helper_vmslt_vx_h helper_vmslt_vx_h_riscv32
+#define helper_vmslt_vx_w helper_vmslt_vx_w_riscv32
+#define helper_vmslt_vx_d helper_vmslt_vx_d_riscv32
+#define helper_vmsleu_vx_b helper_vmsleu_vx_b_riscv32
+#define helper_vmsleu_vx_h helper_vmsleu_vx_h_riscv32
+#define helper_vmsleu_vx_w helper_vmsleu_vx_w_riscv32
+#define helper_vmsleu_vx_d helper_vmsleu_vx_d_riscv32
+#define helper_vmsle_vx_b helper_vmsle_vx_b_riscv32
+#define helper_vmsle_vx_h helper_vmsle_vx_h_riscv32
+#define helper_vmsle_vx_w helper_vmsle_vx_w_riscv32
+#define helper_vmsle_vx_d helper_vmsle_vx_d_riscv32
+#define helper_vmsgtu_vx_b helper_vmsgtu_vx_b_riscv32
+#define helper_vmsgtu_vx_h helper_vmsgtu_vx_h_riscv32
+#define helper_vmsgtu_vx_w helper_vmsgtu_vx_w_riscv32
+#define helper_vmsgtu_vx_d helper_vmsgtu_vx_d_riscv32
+#define helper_vmsgt_vx_b helper_vmsgt_vx_b_riscv32
+#define helper_vmsgt_vx_h helper_vmsgt_vx_h_riscv32
+#define helper_vmsgt_vx_w helper_vmsgt_vx_w_riscv32
+#define helper_vmsgt_vx_d helper_vmsgt_vx_d_riscv32
+#define helper_vmv_v_v_b helper_vmv_v_v_b_riscv32
+#define helper_vmv_v_v_h helper_vmv_v_v_h_riscv32
+#define helper_vmv_v_v_w helper_vmv_v_v_w_riscv32
+#define helper_vmv_v_v_d helper_vmv_v_v_d_riscv32
+#define helper_vmv_v_x_b helper_vmv_v_x_b_riscv32
+#define helper_vmv_v_x_h helper_vmv_v_x_h_riscv32
+#define helper_vmv_v_x_w helper_vmv_v_x_w_riscv32
+#define helper_vmv_v_x_d helper_vmv_v_x_d_riscv32
+#define helper_vmerge_vvm_b helper_vmerge_vvm_b_riscv32
+#define helper_vmerge_vvm_h helper_vmerge_vvm_h_riscv32
+#define helper_vmerge_vvm_w helper_vmerge_vvm_w_riscv32
+#define helper_vmerge_vvm_d helper_vmerge_vvm_d_riscv32
+#define helper_vmerge_vxm_b helper_vmerge_vxm_b_riscv32
+#define helper_vmerge_vxm_h helper_vmerge_vxm_h_riscv32
+#define helper_vmerge_vxm_w helper_vmerge_vxm_w_riscv32
+#define helper_vmerge_vxm_d helper_vmerge_vxm_d_riscv32
+#define helper_clmul helper_clmul_riscv32
+#define helper_clmulr helper_clmulr_riscv32
+#define helper_brev8 helper_brev8_riscv32
+#define helper_unzip helper_unzip_riscv32
+#define helper_zip helper_zip_riscv32
+#define helper_xperm4 helper_xperm4_riscv32
+#define helper_xperm8 helper_xperm8_riscv32
+#define helper_aes32esmi helper_aes32esmi_riscv32
+#define helper_aes32esi helper_aes32esi_riscv32
+#define helper_aes32dsmi helper_aes32dsmi_riscv32
+#define helper_aes32dsi helper_aes32dsi_riscv32
+#define helper_aes64esm helper_aes64esm_riscv32
+#define helper_aes64es helper_aes64es_riscv32
+#define helper_aes64ds helper_aes64ds_riscv32
+#define helper_aes64dsm helper_aes64dsm_riscv32
+#define helper_aes64ks2 helper_aes64ks2_riscv32
+#define helper_aes64ks1i helper_aes64ks1i_riscv32
+#define helper_aes64im helper_aes64im_riscv32
+#define helper_sm4ed helper_sm4ed_riscv32
+#define helper_sm4ks helper_sm4ks_riscv32
 #define helper_sret helper_sret_riscv32
 #define helper_mret helper_mret_riscv32
 #define helper_wfi helper_wfi_riscv32
 #define helper_tlb_flush helper_tlb_flush_riscv32
+#define helper_hyp_tlb_flush helper_hyp_tlb_flush_riscv32
+#define helper_hyp_gvma_tlb_flush helper_hyp_gvma_tlb_flush_riscv32
+#define helper_hyp_hlvx_hu helper_hyp_hlvx_hu_riscv32
+#define helper_hyp_hlvx_wu helper_hyp_hlvx_wu_riscv32
 #define pmp_hart_has_privs pmp_hart_has_privs_riscv32
 #define pmpcfg_csr_write pmpcfg_csr_write_riscv32
 #define pmpcfg_csr_read pmpcfg_csr_read_riscv32
 #define pmpaddr_csr_write pmpaddr_csr_write_riscv32
 #define pmpaddr_csr_read pmpaddr_csr_read_riscv32
+#define riscv_cpu_vector_enabled riscv_cpu_vector_enabled_riscv32
 #define gen_intermediate_code gen_intermediate_code_riscv32
 #define riscv_translate_init riscv_translate_init_riscv32
 #define restore_state_to_opc restore_state_to_opc_riscv32
@@ -1385,7 +3130,15 @@
 #define helper_fcvt_lu_d helper_fcvt_lu_d_riscv32
 #define helper_fcvt_d_l helper_fcvt_d_l_riscv32
 #define helper_fcvt_d_lu helper_fcvt_d_lu_riscv32
+#define helper_fcvt_l_h helper_fcvt_l_h_riscv32
+#define helper_fcvt_lu_h helper_fcvt_lu_h_riscv32
+#define helper_fcvt_h_l helper_fcvt_h_l_riscv32
+#define helper_fcvt_h_lu helper_fcvt_h_lu_riscv32
 #define gen_helper_tlb_flush gen_helper_tlb_flush_riscv32
+#define gen_helper_hyp_tlb_flush gen_helper_hyp_tlb_flush_riscv32
+#define gen_helper_hyp_gvma_tlb_flush gen_helper_hyp_gvma_tlb_flush_riscv32
+#define gen_helper_hyp_hlvx_hu gen_helper_hyp_hlvx_hu_riscv32
+#define gen_helper_hyp_hlvx_wu gen_helper_hyp_hlvx_wu_riscv32
 #define riscv_fpr_regnames riscv_fpr_regnames_riscv32
 #define riscv_int_regnames riscv_int_regnames_riscv32
 #endif
diff --git a/qemu/riscv64.h b/qemu/riscv64.h
index 8332ce6dab..90cc133731 100644
--- a/qemu/riscv64.h
+++ b/qemu/riscv64.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_riscv64
 #define float32_squash_input_denormal float32_squash_input_denormal_riscv64
 #define float64_squash_input_denormal float64_squash_input_denormal_riscv64
+#define bfloat16_add bfloat16_add_riscv64
+#define bfloat16_compare bfloat16_compare_riscv64
+#define bfloat16_compare_quiet bfloat16_compare_quiet_riscv64
+#define bfloat16_default_nan bfloat16_default_nan_riscv64
+#define bfloat16_div bfloat16_div_riscv64
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_riscv64
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_riscv64
+#define bfloat16_max bfloat16_max_riscv64
+#define bfloat16_maximum_number bfloat16_maximum_number_riscv64
+#define bfloat16_maxnum bfloat16_maxnum_riscv64
+#define bfloat16_maxnummag bfloat16_maxnummag_riscv64
+#define bfloat16_min bfloat16_min_riscv64
+#define bfloat16_minimum_number bfloat16_minimum_number_riscv64
+#define bfloat16_minnum bfloat16_minnum_riscv64
+#define bfloat16_minnummag bfloat16_minnummag_riscv64
+#define bfloat16_mul bfloat16_mul_riscv64
+#define bfloat16_muladd bfloat16_muladd_riscv64
+#define bfloat16_round_to_int bfloat16_round_to_int_riscv64
+#define bfloat16_scalbn bfloat16_scalbn_riscv64
+#define bfloat16_silence_nan bfloat16_silence_nan_riscv64
+#define bfloat16_sqrt bfloat16_sqrt_riscv64
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_riscv64
+#define bfloat16_sub bfloat16_sub_riscv64
+#define bfloat16_to_float32 bfloat16_to_float32_riscv64
+#define bfloat16_to_float64 bfloat16_to_float64_riscv64
+#define bfloat16_to_int16 bfloat16_to_int16_riscv64
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_riscv64
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_riscv64
+#define bfloat16_to_int32 bfloat16_to_int32_riscv64
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_riscv64
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_riscv64
+#define bfloat16_to_int64 bfloat16_to_int64_riscv64
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_riscv64
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_riscv64
+#define bfloat16_to_uint16 bfloat16_to_uint16_riscv64
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_riscv64
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_riscv64
+#define bfloat16_to_uint32 bfloat16_to_uint32_riscv64
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_riscv64
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_riscv64
+#define bfloat16_to_uint64 bfloat16_to_uint64_riscv64
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_riscv64
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_riscv64
+#define float128_maximum_number float128_maximum_number_riscv64
+#define float128_max float128_max_riscv64
+#define float128_maxnum float128_maxnum_riscv64
+#define float128_maxnummag float128_maxnummag_riscv64
+#define float128_min float128_min_riscv64
+#define float128_minimum_number float128_minimum_number_riscv64
+#define float128_minnum float128_minnum_riscv64
+#define float128_minnummag float128_minnummag_riscv64
+#define float128_muladd float128_muladd_riscv64
+#define float128_to_int128 float128_to_int128_riscv64
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_riscv64
+#define float128_to_uint128 float128_to_uint128_riscv64
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_riscv64
+#define float16_maximum_number float16_maximum_number_riscv64
+#define float16_minimum_number float16_minimum_number_riscv64
+#define float16_to_int8 float16_to_int8_riscv64
+#define float16_to_int8_scalbn float16_to_int8_scalbn_riscv64
+#define float16_to_uint8 float16_to_uint8_riscv64
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_riscv64
+#define float32_maximum_number float32_maximum_number_riscv64
+#define float32_minimum_number float32_minimum_number_riscv64
+#define float32_to_bfloat16 float32_to_bfloat16_riscv64
+#define float64_maximum_number float64_maximum_number_riscv64
+#define float64_minimum_number float64_minimum_number_riscv64
+#define float64_to_bfloat16 float64_to_bfloat16_riscv64
+#define float64r32_add float64r32_add_riscv64
+#define float64r32_div float64r32_div_riscv64
+#define float64r32_mul float64r32_mul_riscv64
+#define float64r32_muladd float64r32_muladd_riscv64
+#define float64r32_sqrt float64r32_sqrt_riscv64
+#define float64r32_sub float64r32_sub_riscv64
+#define floatx80_mod floatx80_mod_riscv64
+#define floatx80_modrem floatx80_modrem_riscv64
+#define int128_to_float128 int128_to_float128_riscv64
+#define int16_to_bfloat16 int16_to_bfloat16_riscv64
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_riscv64
+#define int32_to_bfloat16 int32_to_bfloat16_riscv64
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_riscv64
+#define int64_to_bfloat16 int64_to_bfloat16_riscv64
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_riscv64
+#define int8_to_float16 int8_to_float16_riscv64
+#define uint128_to_float128 uint128_to_float128_riscv64
+#define uint16_to_bfloat16 uint16_to_bfloat16_riscv64
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_riscv64
+#define uint32_to_bfloat16 uint32_to_bfloat16_riscv64
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_riscv64
+#define uint64_to_bfloat16 uint64_to_bfloat16_riscv64
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_riscv64
+#define uint8_to_float16 uint8_to_float16_riscv64
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_riscv64
 #define roundAndPackFloatx80 roundAndPackFloatx80_riscv64
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_riscv64
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_riscv64
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_riscv64
 #define helper_exit_atomic helper_exit_atomic_riscv64
+#define helper_memset helper_memset_riscv64
+#define helper_emu_stop helper_emu_stop_riscv64
+#define tcg_remove_ops_after tcg_remove_ops_after_riscv64
+#define tcg_constant_vec_matching tcg_constant_vec_matching_riscv64
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_riscv64
 #define helper_gvec_add8 helper_gvec_add8_riscv64
 #define helper_gvec_add16 helper_gvec_add16_riscv64
 #define helper_gvec_add32 helper_gvec_add32_riscv64
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_riscv64
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_riscv64
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_riscv64
+#define gen_helper_mve_vctp gen_helper_mve_vctp_riscv64
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_riscv64
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_riscv64
+#define gen_helper_mve_vdup gen_helper_mve_vdup_riscv64
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_riscv64
+#define gen_helper_mve_vandi gen_helper_mve_vandi_riscv64
+#define gen_helper_mve_vorri gen_helper_mve_vorri_riscv64
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_riscv64
+#define gen_helper_mve_viduph gen_helper_mve_viduph_riscv64
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_riscv64
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_riscv64
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_riscv64
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_riscv64
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_riscv64
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_riscv64
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_riscv64
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_riscv64
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_riscv64
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_riscv64
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_riscv64
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_riscv64
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_riscv64
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_riscv64
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_riscv64
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_riscv64
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_riscv64
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_riscv64
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_riscv64
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_riscv64
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_riscv64
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_riscv64
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_riscv64
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_riscv64
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_riscv64
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_riscv64
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_riscv64
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_riscv64
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_riscv64
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_riscv64
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_riscv64
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_riscv64
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_riscv64
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_riscv64
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_riscv64
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_riscv64
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_riscv64
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_riscv64
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_riscv64
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_riscv64
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_riscv64
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_riscv64
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_riscv64
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_riscv64
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_riscv64
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_riscv64
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_riscv64
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_riscv64
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_riscv64
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_riscv64
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_riscv64
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_riscv64
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_riscv64
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_riscv64
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_riscv64
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_riscv64
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_riscv64
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_riscv64
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_riscv64
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_riscv64
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_riscv64
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_riscv64
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_riscv64
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_riscv64
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_riscv64
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_riscv64
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_riscv64
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_riscv64
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_riscv64
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_riscv64
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_riscv64
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_riscv64
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_riscv64
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_riscv64
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_riscv64
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_riscv64
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_riscv64
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_riscv64
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_riscv64
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_riscv64
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_riscv64
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_riscv64
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_riscv64
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_riscv64
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_riscv64
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_riscv64
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_riscv64
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_riscv64
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_riscv64
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_riscv64
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_riscv64
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_riscv64
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_riscv64
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_riscv64
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_riscv64
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_riscv64
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_riscv64
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_riscv64
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_riscv64
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_riscv64
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_riscv64
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_riscv64
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_riscv64
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_riscv64
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_riscv64
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_riscv64
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_riscv64
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_riscv64
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_riscv64
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_riscv64
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_riscv64
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_riscv64
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_riscv64
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_riscv64
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_riscv64
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_riscv64
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_riscv64
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_riscv64
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_riscv64
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_riscv64
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_riscv64
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_riscv64
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_riscv64
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_riscv64
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_riscv64
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_riscv64
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_riscv64
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_riscv64
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_riscv64
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_riscv64
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_riscv64
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_riscv64
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_riscv64
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_riscv64
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_riscv64
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_riscv64
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_riscv64
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_riscv64
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_riscv64
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_riscv64
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_riscv64
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_riscv64
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_riscv64
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_riscv64
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_riscv64
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_riscv64
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_riscv64
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_riscv64
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_riscv64
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_riscv64
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_riscv64
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_riscv64
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_riscv64
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_riscv64
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_riscv64
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_riscv64
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_riscv64
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_riscv64
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_riscv64
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_riscv64
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_riscv64
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_riscv64
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_riscv64
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_riscv64
+#define gen_helper_mve_vand gen_helper_mve_vand_riscv64
+#define gen_helper_mve_vbic gen_helper_mve_vbic_riscv64
+#define gen_helper_mve_vorr gen_helper_mve_vorr_riscv64
+#define gen_helper_mve_vorn gen_helper_mve_vorn_riscv64
+#define gen_helper_mve_veor gen_helper_mve_veor_riscv64
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_riscv64
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_riscv64
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_riscv64
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_riscv64
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_riscv64
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_riscv64
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_riscv64
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_riscv64
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_riscv64
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_riscv64
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_riscv64
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_riscv64
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_riscv64
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_riscv64
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_riscv64
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_riscv64
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_riscv64
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_riscv64
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_riscv64
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_riscv64
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_riscv64
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_riscv64
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_riscv64
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_riscv64
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_riscv64
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_riscv64
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_riscv64
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_riscv64
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_riscv64
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_riscv64
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_riscv64
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_riscv64
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_riscv64
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_riscv64
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_riscv64
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_riscv64
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_riscv64
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_riscv64
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_riscv64
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_riscv64
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_riscv64
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_riscv64
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_riscv64
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_riscv64
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_riscv64
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_riscv64
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_riscv64
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_riscv64
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_riscv64
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_riscv64
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_riscv64
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_riscv64
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_riscv64
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_riscv64
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_riscv64
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_riscv64
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_riscv64
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_riscv64
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_riscv64
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_riscv64
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_riscv64
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_riscv64
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_riscv64
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_riscv64
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_riscv64
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_riscv64
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_riscv64
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_riscv64
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_riscv64
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_riscv64
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_riscv64
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_riscv64
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_riscv64
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_riscv64
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_riscv64
+#define gen_helper_mve_vminub gen_helper_mve_vminub_riscv64
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_riscv64
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_riscv64
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_riscv64
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_riscv64
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_riscv64
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_riscv64
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_riscv64
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_riscv64
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_riscv64
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_riscv64
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_riscv64
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_riscv64
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_riscv64
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_riscv64
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_riscv64
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_riscv64
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_riscv64
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_riscv64
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_riscv64
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_riscv64
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_riscv64
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_riscv64
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_riscv64
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_riscv64
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_riscv64
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_riscv64
+#define gen_helper_mve_vadc gen_helper_mve_vadc_riscv64
+#define gen_helper_mve_vadci gen_helper_mve_vadci_riscv64
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_riscv64
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_riscv64
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_riscv64
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_riscv64
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_riscv64
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_riscv64
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_riscv64
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_riscv64
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_riscv64
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_riscv64
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_riscv64
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_riscv64
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_riscv64
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_riscv64
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_riscv64
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_riscv64
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_riscv64
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_riscv64
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_riscv64
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_riscv64
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_riscv64
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_riscv64
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_riscv64
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_riscv64
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_riscv64
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_riscv64
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_riscv64
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_riscv64
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_riscv64
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_riscv64
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_riscv64
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_riscv64
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_riscv64
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_riscv64
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_riscv64
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_riscv64
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_riscv64
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_riscv64
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_riscv64
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_riscv64
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_riscv64
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_riscv64
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_riscv64
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_riscv64
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_riscv64
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_riscv64
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_riscv64
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_riscv64
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_riscv64
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_riscv64
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_riscv64
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_riscv64
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_riscv64
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_riscv64
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_riscv64
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_riscv64
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_riscv64
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_riscv64
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_riscv64
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_riscv64
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_riscv64
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_riscv64
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_riscv64
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_riscv64
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_riscv64
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_riscv64
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_riscv64
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_riscv64
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_riscv64
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_riscv64
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_riscv64
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_riscv64
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_riscv64
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_riscv64
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_riscv64
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_riscv64
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_riscv64
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_riscv64
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_riscv64
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_riscv64
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_riscv64
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_riscv64
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_riscv64
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_riscv64
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_riscv64
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_riscv64
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_riscv64
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_riscv64
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_riscv64
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_riscv64
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_riscv64
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_riscv64
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_riscv64
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_riscv64
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_riscv64
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_riscv64
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_riscv64
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_riscv64
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_riscv64
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_riscv64
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_riscv64
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_riscv64
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_riscv64
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_riscv64
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_riscv64
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_riscv64
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_riscv64
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_riscv64
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_riscv64
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_riscv64
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_riscv64
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_riscv64
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_riscv64
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_riscv64
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_riscv64
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_riscv64
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_riscv64
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_riscv64
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_riscv64
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_riscv64
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_riscv64
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_riscv64
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_riscv64
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_riscv64
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_riscv64
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_riscv64
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_riscv64
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_riscv64
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_riscv64
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_riscv64
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_riscv64
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_riscv64
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_riscv64
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_riscv64
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_riscv64
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_riscv64
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_riscv64
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_riscv64
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_riscv64
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_riscv64
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_riscv64
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_riscv64
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_riscv64
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_riscv64
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_riscv64
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_riscv64
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_riscv64
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_riscv64
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_riscv64
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_riscv64
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_riscv64
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_riscv64
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_riscv64
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_riscv64
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_riscv64
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_riscv64
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_riscv64
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_riscv64
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_riscv64
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_riscv64
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_riscv64
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_riscv64
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_riscv64
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_riscv64
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_riscv64
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_riscv64
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_riscv64
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_riscv64
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_riscv64
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_riscv64
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_riscv64
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_riscv64
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_riscv64
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_riscv64
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_riscv64
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_riscv64
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_riscv64
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_riscv64
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_riscv64
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_riscv64
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_riscv64
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_riscv64
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_riscv64
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_riscv64
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_riscv64
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_riscv64
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_riscv64
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_riscv64
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_riscv64
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_riscv64
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_riscv64
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_riscv64
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_riscv64
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_riscv64
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_riscv64
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_riscv64
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_riscv64
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_riscv64
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_riscv64
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_riscv64
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_riscv64
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_riscv64
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_riscv64
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_riscv64
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_riscv64
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_riscv64
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_riscv64
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_riscv64
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_riscv64
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_riscv64
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_riscv64
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_riscv64
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_riscv64
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_riscv64
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_riscv64
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_riscv64
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_riscv64
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_riscv64
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_riscv64
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_riscv64
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_riscv64
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_riscv64
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_riscv64
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_riscv64
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_riscv64
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_riscv64
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_riscv64
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_riscv64
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_riscv64
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_riscv64
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_riscv64
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_riscv64
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_riscv64
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_riscv64
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_riscv64
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_riscv64
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_riscv64
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_riscv64
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_riscv64
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_riscv64
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_riscv64
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_riscv64
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_riscv64
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_riscv64
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_riscv64
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_riscv64
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_riscv64
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_riscv64
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_riscv64
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_riscv64
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_riscv64
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_riscv64
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_riscv64
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_riscv64
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_riscv64
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_riscv64
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_riscv64
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_riscv64
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_riscv64
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_riscv64
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_riscv64
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_riscv64
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_riscv64
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_riscv64
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_riscv64
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_riscv64
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_riscv64
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_riscv64
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_riscv64
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_riscv64
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_riscv64
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_riscv64
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_riscv64
+#define gen_helper_mve_ushll gen_helper_mve_ushll_riscv64
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_riscv64
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_riscv64
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_riscv64
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_riscv64
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_riscv64
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_riscv64
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_riscv64
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_riscv64
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_riscv64
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_riscv64
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_riscv64
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_riscv64
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_riscv64
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_riscv64
+#define gen_helper_mve_vslib gen_helper_mve_vslib_riscv64
+#define gen_helper_mve_vslih gen_helper_mve_vslih_riscv64
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_riscv64
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_riscv64
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_riscv64
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_riscv64
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_riscv64
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_riscv64
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_riscv64
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_riscv64
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_riscv64
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_riscv64
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_riscv64
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_riscv64
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_riscv64
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_riscv64
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_riscv64
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_riscv64
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_riscv64
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_riscv64
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_riscv64
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_riscv64
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_riscv64
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_riscv64
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_riscv64
+#define gen_helper_mve_vminab gen_helper_mve_vminab_riscv64
+#define gen_helper_mve_vminah gen_helper_mve_vminah_riscv64
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_riscv64
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_riscv64
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_riscv64
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_riscv64
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_riscv64
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_riscv64
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_riscv64
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_riscv64
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_riscv64
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_riscv64
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_riscv64
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_riscv64
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_riscv64
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_riscv64
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_riscv64
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_riscv64
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_riscv64
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_riscv64
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_riscv64
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_riscv64
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_riscv64
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_riscv64
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_riscv64
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_riscv64
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_riscv64
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_riscv64
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_riscv64
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_riscv64
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_riscv64
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_riscv64
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_riscv64
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_riscv64
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_riscv64
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_riscv64
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_riscv64
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_riscv64
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_riscv64
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_riscv64
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_riscv64
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_riscv64
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_riscv64
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_riscv64
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_riscv64
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_riscv64
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_riscv64
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_riscv64
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_riscv64
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_riscv64
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_riscv64
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_riscv64
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_riscv64
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_riscv64
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_riscv64
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_riscv64
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_riscv64
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_riscv64
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_riscv64
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_riscv64
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_riscv64
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_riscv64
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_riscv64
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_riscv64
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_riscv64
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_riscv64
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_riscv64
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_riscv64
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_riscv64
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_riscv64
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_riscv64
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_riscv64
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_riscv64
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_riscv64
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_riscv64
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_riscv64
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_riscv64
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_riscv64
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_riscv64
 #define gen_helper_cpsr_read gen_helper_cpsr_read_riscv64
 #define gen_helper_cpsr_write gen_helper_cpsr_write_riscv64
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_riscv64
@@ -1318,14 +2089,19 @@
 #define riscv_cpu_get_fflags riscv_cpu_get_fflags_riscv64
 #define riscv_cpu_set_fflags riscv_cpu_set_fflags_riscv64
 #define helper_set_rounding_mode helper_set_rounding_mode_riscv64
+#define helper_set_rod_rounding_mode helper_set_rod_rounding_mode_riscv64
 #define helper_fmadd_s helper_fmadd_s_riscv64
 #define helper_fmadd_d helper_fmadd_d_riscv64
+#define helper_fmadd_h helper_fmadd_h_riscv64
 #define helper_fmsub_s helper_fmsub_s_riscv64
 #define helper_fmsub_d helper_fmsub_d_riscv64
+#define helper_fmsub_h helper_fmsub_h_riscv64
 #define helper_fnmsub_s helper_fnmsub_s_riscv64
 #define helper_fnmsub_d helper_fnmsub_d_riscv64
+#define helper_fnmsub_h helper_fnmsub_h_riscv64
 #define helper_fnmadd_s helper_fnmadd_s_riscv64
 #define helper_fnmadd_d helper_fnmadd_d_riscv64
+#define helper_fnmadd_h helper_fnmadd_h_riscv64
 #define helper_fadd_s helper_fadd_s_riscv64
 #define helper_fsub_s helper_fsub_s_riscv64
 #define helper_fmul_s helper_fmul_s_riscv64
@@ -1341,6 +2117,25 @@
 #define helper_fcvt_s_w helper_fcvt_s_w_riscv64
 #define helper_fcvt_s_wu helper_fcvt_s_wu_riscv64
 #define helper_fclass_s helper_fclass_s_riscv64
+#define helper_fadd_h helper_fadd_h_riscv64
+#define helper_fsub_h helper_fsub_h_riscv64
+#define helper_fmul_h helper_fmul_h_riscv64
+#define helper_fdiv_h helper_fdiv_h_riscv64
+#define helper_fmin_h helper_fmin_h_riscv64
+#define helper_fmax_h helper_fmax_h_riscv64
+#define helper_fsqrt_h helper_fsqrt_h_riscv64
+#define helper_fle_h helper_fle_h_riscv64
+#define helper_flt_h helper_flt_h_riscv64
+#define helper_feq_h helper_feq_h_riscv64
+#define helper_fcvt_s_h helper_fcvt_s_h_riscv64
+#define helper_fcvt_h_s helper_fcvt_h_s_riscv64
+#define helper_fcvt_d_h helper_fcvt_d_h_riscv64
+#define helper_fcvt_h_d helper_fcvt_h_d_riscv64
+#define helper_fcvt_w_h helper_fcvt_w_h_riscv64
+#define helper_fcvt_wu_h helper_fcvt_wu_h_riscv64
+#define helper_fcvt_h_w helper_fcvt_h_w_riscv64
+#define helper_fcvt_h_wu helper_fcvt_h_wu_riscv64
+#define helper_fclass_h helper_fclass_h_riscv64
 #define helper_fadd_d helper_fadd_d_riscv64
 #define helper_fsub_d helper_fsub_d_riscv64
 #define helper_fmul_d helper_fmul_d_riscv64
@@ -1364,15 +2159,965 @@
 #define helper_csrrw helper_csrrw_riscv64
 #define helper_csrrs helper_csrrs_riscv64
 #define helper_csrrc helper_csrrc_riscv64
+#define helper_vsetvl helper_vsetvl_riscv64
+#define helper_vle8_v helper_vle8_v_riscv64
+#define helper_vle16_v helper_vle16_v_riscv64
+#define helper_vle32_v helper_vle32_v_riscv64
+#define helper_vle64_v helper_vle64_v_riscv64
+#define helper_vse8_v helper_vse8_v_riscv64
+#define helper_vse16_v helper_vse16_v_riscv64
+#define helper_vse32_v helper_vse32_v_riscv64
+#define helper_vse64_v helper_vse64_v_riscv64
+#define helper_vlse8_v helper_vlse8_v_riscv64
+#define helper_vlse16_v helper_vlse16_v_riscv64
+#define helper_vlse32_v helper_vlse32_v_riscv64
+#define helper_vlse64_v helper_vlse64_v_riscv64
+#define helper_vsse8_v helper_vsse8_v_riscv64
+#define helper_vsse16_v helper_vsse16_v_riscv64
+#define helper_vsse32_v helper_vsse32_v_riscv64
+#define helper_vsse64_v helper_vsse64_v_riscv64
+#define helper_vlxei8_8_v helper_vlxei8_8_v_riscv64
+#define helper_vlxei8_16_v helper_vlxei8_16_v_riscv64
+#define helper_vlxei8_32_v helper_vlxei8_32_v_riscv64
+#define helper_vlxei8_64_v helper_vlxei8_64_v_riscv64
+#define helper_vlxei16_8_v helper_vlxei16_8_v_riscv64
+#define helper_vlxei16_16_v helper_vlxei16_16_v_riscv64
+#define helper_vlxei16_32_v helper_vlxei16_32_v_riscv64
+#define helper_vlxei16_64_v helper_vlxei16_64_v_riscv64
+#define helper_vlxei32_8_v helper_vlxei32_8_v_riscv64
+#define helper_vlxei32_16_v helper_vlxei32_16_v_riscv64
+#define helper_vlxei32_32_v helper_vlxei32_32_v_riscv64
+#define helper_vlxei32_64_v helper_vlxei32_64_v_riscv64
+#define helper_vlxei64_8_v helper_vlxei64_8_v_riscv64
+#define helper_vlxei64_16_v helper_vlxei64_16_v_riscv64
+#define helper_vlxei64_32_v helper_vlxei64_32_v_riscv64
+#define helper_vlxei64_64_v helper_vlxei64_64_v_riscv64
+#define helper_vsxei8_8_v helper_vsxei8_8_v_riscv64
+#define helper_vsxei8_16_v helper_vsxei8_16_v_riscv64
+#define helper_vsxei8_32_v helper_vsxei8_32_v_riscv64
+#define helper_vsxei8_64_v helper_vsxei8_64_v_riscv64
+#define helper_vsxei16_8_v helper_vsxei16_8_v_riscv64
+#define helper_vsxei16_16_v helper_vsxei16_16_v_riscv64
+#define helper_vsxei16_32_v helper_vsxei16_32_v_riscv64
+#define helper_vsxei16_64_v helper_vsxei16_64_v_riscv64
+#define helper_vsxei32_8_v helper_vsxei32_8_v_riscv64
+#define helper_vsxei32_16_v helper_vsxei32_16_v_riscv64
+#define helper_vsxei32_32_v helper_vsxei32_32_v_riscv64
+#define helper_vsxei32_64_v helper_vsxei32_64_v_riscv64
+#define helper_vsxei64_8_v helper_vsxei64_8_v_riscv64
+#define helper_vsxei64_16_v helper_vsxei64_16_v_riscv64
+#define helper_vsxei64_32_v helper_vsxei64_32_v_riscv64
+#define helper_vsxei64_64_v helper_vsxei64_64_v_riscv64
+#define helper_vle8ff_v helper_vle8ff_v_riscv64
+#define helper_vle16ff_v helper_vle16ff_v_riscv64
+#define helper_vle32ff_v helper_vle32ff_v_riscv64
+#define helper_vle64ff_v helper_vle64ff_v_riscv64
+#define helper_vl1re8_v helper_vl1re8_v_riscv64
+#define helper_vl1re16_v helper_vl1re16_v_riscv64
+#define helper_vl1re32_v helper_vl1re32_v_riscv64
+#define helper_vl1re64_v helper_vl1re64_v_riscv64
+#define helper_vl2re8_v helper_vl2re8_v_riscv64
+#define helper_vl2re16_v helper_vl2re16_v_riscv64
+#define helper_vl2re32_v helper_vl2re32_v_riscv64
+#define helper_vl2re64_v helper_vl2re64_v_riscv64
+#define helper_vl4re8_v helper_vl4re8_v_riscv64
+#define helper_vl4re16_v helper_vl4re16_v_riscv64
+#define helper_vl4re32_v helper_vl4re32_v_riscv64
+#define helper_vl4re64_v helper_vl4re64_v_riscv64
+#define helper_vl8re8_v helper_vl8re8_v_riscv64
+#define helper_vl8re16_v helper_vl8re16_v_riscv64
+#define helper_vl8re32_v helper_vl8re32_v_riscv64
+#define helper_vl8re64_v helper_vl8re64_v_riscv64
+#define helper_vs1r_v helper_vs1r_v_riscv64
+#define helper_vs2r_v helper_vs2r_v_riscv64
+#define helper_vs4r_v helper_vs4r_v_riscv64
+#define helper_vs8r_v helper_vs8r_v_riscv64
+#define helper_vlm_v helper_vlm_v_riscv64
+#define helper_vsm_v helper_vsm_v_riscv64
+#define helper_vadd_vv_b helper_vadd_vv_b_riscv64
+#define helper_vadd_vv_h helper_vadd_vv_h_riscv64
+#define helper_vadd_vv_w helper_vadd_vv_w_riscv64
+#define helper_vadd_vv_d helper_vadd_vv_d_riscv64
+#define helper_vsub_vv_b helper_vsub_vv_b_riscv64
+#define helper_vsub_vv_h helper_vsub_vv_h_riscv64
+#define helper_vsub_vv_w helper_vsub_vv_w_riscv64
+#define helper_vsub_vv_d helper_vsub_vv_d_riscv64
+#define helper_vfadd_vv_h helper_vfadd_vv_h_riscv64
+#define helper_vfadd_vv_w helper_vfadd_vv_w_riscv64
+#define helper_vfadd_vv_d helper_vfadd_vv_d_riscv64
+#define helper_vfsub_vv_h helper_vfsub_vv_h_riscv64
+#define helper_vfsub_vv_w helper_vfsub_vv_w_riscv64
+#define helper_vfsub_vv_d helper_vfsub_vv_d_riscv64
+#define helper_vfmul_vv_h helper_vfmul_vv_h_riscv64
+#define helper_vfmul_vv_w helper_vfmul_vv_w_riscv64
+#define helper_vfmul_vv_d helper_vfmul_vv_d_riscv64
+#define helper_vfdiv_vv_h helper_vfdiv_vv_h_riscv64
+#define helper_vfdiv_vv_w helper_vfdiv_vv_w_riscv64
+#define helper_vfdiv_vv_d helper_vfdiv_vv_d_riscv64
+#define helper_vfwadd_vv_h helper_vfwadd_vv_h_riscv64
+#define helper_vfwadd_vv_w helper_vfwadd_vv_w_riscv64
+#define helper_vfwsub_vv_h helper_vfwsub_vv_h_riscv64
+#define helper_vfwsub_vv_w helper_vfwsub_vv_w_riscv64
+#define helper_vfwadd_wv_h helper_vfwadd_wv_h_riscv64
+#define helper_vfwadd_wv_w helper_vfwadd_wv_w_riscv64
+#define helper_vfwsub_wv_h helper_vfwsub_wv_h_riscv64
+#define helper_vfwsub_wv_w helper_vfwsub_wv_w_riscv64
+#define helper_vfwmul_vv_h helper_vfwmul_vv_h_riscv64
+#define helper_vfwmul_vv_w helper_vfwmul_vv_w_riscv64
+#define helper_vfmacc_vv_h helper_vfmacc_vv_h_riscv64
+#define helper_vfmacc_vv_w helper_vfmacc_vv_w_riscv64
+#define helper_vfmacc_vv_d helper_vfmacc_vv_d_riscv64
+#define helper_vfnmacc_vv_h helper_vfnmacc_vv_h_riscv64
+#define helper_vfnmacc_vv_w helper_vfnmacc_vv_w_riscv64
+#define helper_vfnmacc_vv_d helper_vfnmacc_vv_d_riscv64
+#define helper_vfmsac_vv_h helper_vfmsac_vv_h_riscv64
+#define helper_vfmsac_vv_w helper_vfmsac_vv_w_riscv64
+#define helper_vfmsac_vv_d helper_vfmsac_vv_d_riscv64
+#define helper_vfnmsac_vv_h helper_vfnmsac_vv_h_riscv64
+#define helper_vfnmsac_vv_w helper_vfnmsac_vv_w_riscv64
+#define helper_vfnmsac_vv_d helper_vfnmsac_vv_d_riscv64
+#define helper_vfmadd_vv_h helper_vfmadd_vv_h_riscv64
+#define helper_vfmadd_vv_w helper_vfmadd_vv_w_riscv64
+#define helper_vfmadd_vv_d helper_vfmadd_vv_d_riscv64
+#define helper_vfnmadd_vv_h helper_vfnmadd_vv_h_riscv64
+#define helper_vfnmadd_vv_w helper_vfnmadd_vv_w_riscv64
+#define helper_vfnmadd_vv_d helper_vfnmadd_vv_d_riscv64
+#define helper_vfmsub_vv_h helper_vfmsub_vv_h_riscv64
+#define helper_vfmsub_vv_w helper_vfmsub_vv_w_riscv64
+#define helper_vfmsub_vv_d helper_vfmsub_vv_d_riscv64
+#define helper_vfnmsub_vv_h helper_vfnmsub_vv_h_riscv64
+#define helper_vfnmsub_vv_w helper_vfnmsub_vv_w_riscv64
+#define helper_vfnmsub_vv_d helper_vfnmsub_vv_d_riscv64
+#define helper_vfwmacc_vv_h helper_vfwmacc_vv_h_riscv64
+#define helper_vfwmacc_vv_w helper_vfwmacc_vv_w_riscv64
+#define helper_vfwnmacc_vv_h helper_vfwnmacc_vv_h_riscv64
+#define helper_vfwnmacc_vv_w helper_vfwnmacc_vv_w_riscv64
+#define helper_vfwmsac_vv_h helper_vfwmsac_vv_h_riscv64
+#define helper_vfwmsac_vv_w helper_vfwmsac_vv_w_riscv64
+#define helper_vfwnmsac_vv_h helper_vfwnmsac_vv_h_riscv64
+#define helper_vfwnmsac_vv_w helper_vfwnmsac_vv_w_riscv64
+#define helper_vfmin_vv_h helper_vfmin_vv_h_riscv64
+#define helper_vfmin_vv_w helper_vfmin_vv_w_riscv64
+#define helper_vfmin_vv_d helper_vfmin_vv_d_riscv64
+#define helper_vfmax_vv_h helper_vfmax_vv_h_riscv64
+#define helper_vfmax_vv_w helper_vfmax_vv_w_riscv64
+#define helper_vfmax_vv_d helper_vfmax_vv_d_riscv64
+#define helper_vfsgnj_vv_h helper_vfsgnj_vv_h_riscv64
+#define helper_vfsgnj_vv_w helper_vfsgnj_vv_w_riscv64
+#define helper_vfsgnj_vv_d helper_vfsgnj_vv_d_riscv64
+#define helper_vfsgnjn_vv_h helper_vfsgnjn_vv_h_riscv64
+#define helper_vfsgnjn_vv_w helper_vfsgnjn_vv_w_riscv64
+#define helper_vfsgnjn_vv_d helper_vfsgnjn_vv_d_riscv64
+#define helper_vfsgnjx_vv_h helper_vfsgnjx_vv_h_riscv64
+#define helper_vfsgnjx_vv_w helper_vfsgnjx_vv_w_riscv64
+#define helper_vfsgnjx_vv_d helper_vfsgnjx_vv_d_riscv64
+#define helper_vadc_vvm_b helper_vadc_vvm_b_riscv64
+#define helper_vadc_vvm_h helper_vadc_vvm_h_riscv64
+#define helper_vadc_vvm_w helper_vadc_vvm_w_riscv64
+#define helper_vadc_vvm_d helper_vadc_vvm_d_riscv64
+#define helper_vsbc_vvm_b helper_vsbc_vvm_b_riscv64
+#define helper_vsbc_vvm_h helper_vsbc_vvm_h_riscv64
+#define helper_vsbc_vvm_w helper_vsbc_vvm_w_riscv64
+#define helper_vsbc_vvm_d helper_vsbc_vvm_d_riscv64
+#define helper_vmadc_vvm_b helper_vmadc_vvm_b_riscv64
+#define helper_vmadc_vvm_h helper_vmadc_vvm_h_riscv64
+#define helper_vmadc_vvm_w helper_vmadc_vvm_w_riscv64
+#define helper_vmadc_vvm_d helper_vmadc_vvm_d_riscv64
+#define helper_vmsbc_vvm_b helper_vmsbc_vvm_b_riscv64
+#define helper_vmsbc_vvm_h helper_vmsbc_vvm_h_riscv64
+#define helper_vmsbc_vvm_w helper_vmsbc_vvm_w_riscv64
+#define helper_vmsbc_vvm_d helper_vmsbc_vvm_d_riscv64
+#define helper_vand_vv_b helper_vand_vv_b_riscv64
+#define helper_vand_vv_h helper_vand_vv_h_riscv64
+#define helper_vand_vv_w helper_vand_vv_w_riscv64
+#define helper_vand_vv_d helper_vand_vv_d_riscv64
+#define helper_vor_vv_b helper_vor_vv_b_riscv64
+#define helper_vor_vv_h helper_vor_vv_h_riscv64
+#define helper_vor_vv_w helper_vor_vv_w_riscv64
+#define helper_vor_vv_d helper_vor_vv_d_riscv64
+#define helper_vxor_vv_b helper_vxor_vv_b_riscv64
+#define helper_vxor_vv_h helper_vxor_vv_h_riscv64
+#define helper_vxor_vv_w helper_vxor_vv_w_riscv64
+#define helper_vxor_vv_d helper_vxor_vv_d_riscv64
+#define helper_vminu_vv_b helper_vminu_vv_b_riscv64
+#define helper_vminu_vv_h helper_vminu_vv_h_riscv64
+#define helper_vminu_vv_w helper_vminu_vv_w_riscv64
+#define helper_vminu_vv_d helper_vminu_vv_d_riscv64
+#define helper_vmin_vv_b helper_vmin_vv_b_riscv64
+#define helper_vmin_vv_h helper_vmin_vv_h_riscv64
+#define helper_vmin_vv_w helper_vmin_vv_w_riscv64
+#define helper_vmin_vv_d helper_vmin_vv_d_riscv64
+#define helper_vmaxu_vv_b helper_vmaxu_vv_b_riscv64
+#define helper_vmaxu_vv_h helper_vmaxu_vv_h_riscv64
+#define helper_vmaxu_vv_w helper_vmaxu_vv_w_riscv64
+#define helper_vmaxu_vv_d helper_vmaxu_vv_d_riscv64
+#define helper_vmax_vv_b helper_vmax_vv_b_riscv64
+#define helper_vmax_vv_h helper_vmax_vv_h_riscv64
+#define helper_vmax_vv_w helper_vmax_vv_w_riscv64
+#define helper_vmax_vv_d helper_vmax_vv_d_riscv64
+#define helper_vmseq_vv_b helper_vmseq_vv_b_riscv64
+#define helper_vmseq_vv_h helper_vmseq_vv_h_riscv64
+#define helper_vmseq_vv_w helper_vmseq_vv_w_riscv64
+#define helper_vmseq_vv_d helper_vmseq_vv_d_riscv64
+#define helper_vmsne_vv_b helper_vmsne_vv_b_riscv64
+#define helper_vmsne_vv_h helper_vmsne_vv_h_riscv64
+#define helper_vmsne_vv_w helper_vmsne_vv_w_riscv64
+#define helper_vmsne_vv_d helper_vmsne_vv_d_riscv64
+#define helper_vmsltu_vv_b helper_vmsltu_vv_b_riscv64
+#define helper_vmsltu_vv_h helper_vmsltu_vv_h_riscv64
+#define helper_vmsltu_vv_w helper_vmsltu_vv_w_riscv64
+#define helper_vmsltu_vv_d helper_vmsltu_vv_d_riscv64
+#define helper_vmslt_vv_b helper_vmslt_vv_b_riscv64
+#define helper_vmslt_vv_h helper_vmslt_vv_h_riscv64
+#define helper_vmslt_vv_w helper_vmslt_vv_w_riscv64
+#define helper_vmslt_vv_d helper_vmslt_vv_d_riscv64
+#define helper_vmsleu_vv_b helper_vmsleu_vv_b_riscv64
+#define helper_vmsleu_vv_h helper_vmsleu_vv_h_riscv64
+#define helper_vmsleu_vv_w helper_vmsleu_vv_w_riscv64
+#define helper_vmsleu_vv_d helper_vmsleu_vv_d_riscv64
+#define helper_vmsle_vv_b helper_vmsle_vv_b_riscv64
+#define helper_vmsle_vv_h helper_vmsle_vv_h_riscv64
+#define helper_vmsle_vv_w helper_vmsle_vv_w_riscv64
+#define helper_vmsle_vv_d helper_vmsle_vv_d_riscv64
+#define helper_vmfeq_vv_h helper_vmfeq_vv_h_riscv64
+#define helper_vmfeq_vv_w helper_vmfeq_vv_w_riscv64
+#define helper_vmfeq_vv_d helper_vmfeq_vv_d_riscv64
+#define helper_vmfne_vv_h helper_vmfne_vv_h_riscv64
+#define helper_vmfne_vv_w helper_vmfne_vv_w_riscv64
+#define helper_vmfne_vv_d helper_vmfne_vv_d_riscv64
+#define helper_vmflt_vv_h helper_vmflt_vv_h_riscv64
+#define helper_vmflt_vv_w helper_vmflt_vv_w_riscv64
+#define helper_vmflt_vv_d helper_vmflt_vv_d_riscv64
+#define helper_vmfle_vv_h helper_vmfle_vv_h_riscv64
+#define helper_vmfle_vv_w helper_vmfle_vv_w_riscv64
+#define helper_vmfle_vv_d helper_vmfle_vv_d_riscv64
+#define helper_vfsqrt_v_h helper_vfsqrt_v_h_riscv64
+#define helper_vfsqrt_v_w helper_vfsqrt_v_w_riscv64
+#define helper_vfsqrt_v_d helper_vfsqrt_v_d_riscv64
+#define helper_vfrsqrt7_v_h helper_vfrsqrt7_v_h_riscv64
+#define helper_vfrsqrt7_v_w helper_vfrsqrt7_v_w_riscv64
+#define helper_vfrsqrt7_v_d helper_vfrsqrt7_v_d_riscv64
+#define helper_vfrec7_v_h helper_vfrec7_v_h_riscv64
+#define helper_vfrec7_v_w helper_vfrec7_v_w_riscv64
+#define helper_vfrec7_v_d helper_vfrec7_v_d_riscv64
+#define helper_vfcvt_xu_f_v_h helper_vfcvt_xu_f_v_h_riscv64
+#define helper_vfcvt_xu_f_v_w helper_vfcvt_xu_f_v_w_riscv64
+#define helper_vfcvt_xu_f_v_d helper_vfcvt_xu_f_v_d_riscv64
+#define helper_vfcvt_x_f_v_h helper_vfcvt_x_f_v_h_riscv64
+#define helper_vfcvt_x_f_v_w helper_vfcvt_x_f_v_w_riscv64
+#define helper_vfcvt_x_f_v_d helper_vfcvt_x_f_v_d_riscv64
+#define helper_vfcvt_f_xu_v_h helper_vfcvt_f_xu_v_h_riscv64
+#define helper_vfcvt_f_xu_v_w helper_vfcvt_f_xu_v_w_riscv64
+#define helper_vfcvt_f_xu_v_d helper_vfcvt_f_xu_v_d_riscv64
+#define helper_vfcvt_f_x_v_h helper_vfcvt_f_x_v_h_riscv64
+#define helper_vfcvt_f_x_v_w helper_vfcvt_f_x_v_w_riscv64
+#define helper_vfcvt_f_x_v_d helper_vfcvt_f_x_v_d_riscv64
+#define helper_vfwcvt_xu_f_v_h helper_vfwcvt_xu_f_v_h_riscv64
+#define helper_vfwcvt_xu_f_v_w helper_vfwcvt_xu_f_v_w_riscv64
+#define helper_vfwcvt_x_f_v_h helper_vfwcvt_x_f_v_h_riscv64
+#define helper_vfwcvt_x_f_v_w helper_vfwcvt_x_f_v_w_riscv64
+#define helper_vfwcvt_f_xu_v_b helper_vfwcvt_f_xu_v_b_riscv64
+#define helper_vfwcvt_f_xu_v_h helper_vfwcvt_f_xu_v_h_riscv64
+#define helper_vfwcvt_f_xu_v_w helper_vfwcvt_f_xu_v_w_riscv64
+#define helper_vfwcvt_f_x_v_b helper_vfwcvt_f_x_v_b_riscv64
+#define helper_vfwcvt_f_x_v_h helper_vfwcvt_f_x_v_h_riscv64
+#define helper_vfwcvt_f_x_v_w helper_vfwcvt_f_x_v_w_riscv64
+#define helper_vfwcvt_f_f_v_h helper_vfwcvt_f_f_v_h_riscv64
+#define helper_vfwcvt_f_f_v_w helper_vfwcvt_f_f_v_w_riscv64
+#define helper_vfncvt_xu_f_w_b helper_vfncvt_xu_f_w_b_riscv64
+#define helper_vfncvt_xu_f_w_h helper_vfncvt_xu_f_w_h_riscv64
+#define helper_vfncvt_xu_f_w_w helper_vfncvt_xu_f_w_w_riscv64
+#define helper_vfncvt_x_f_w_b helper_vfncvt_x_f_w_b_riscv64
+#define helper_vfncvt_x_f_w_h helper_vfncvt_x_f_w_h_riscv64
+#define helper_vfncvt_x_f_w_w helper_vfncvt_x_f_w_w_riscv64
+#define helper_vfncvt_f_xu_w_h helper_vfncvt_f_xu_w_h_riscv64
+#define helper_vfncvt_f_xu_w_w helper_vfncvt_f_xu_w_w_riscv64
+#define helper_vfncvt_f_x_w_h helper_vfncvt_f_x_w_h_riscv64
+#define helper_vfncvt_f_x_w_w helper_vfncvt_f_x_w_w_riscv64
+#define helper_vfncvt_f_f_w_h helper_vfncvt_f_f_w_h_riscv64
+#define helper_vfncvt_f_f_w_w helper_vfncvt_f_f_w_w_riscv64
+#define helper_vfclass_v_h helper_vfclass_v_h_riscv64
+#define helper_vfclass_v_w helper_vfclass_v_w_riscv64
+#define helper_vfclass_v_d helper_vfclass_v_d_riscv64
+#define helper_vmand_mm helper_vmand_mm_riscv64
+#define helper_vmnand_mm helper_vmnand_mm_riscv64
+#define helper_vmandn_mm helper_vmandn_mm_riscv64
+#define helper_vmxor_mm helper_vmxor_mm_riscv64
+#define helper_vmor_mm helper_vmor_mm_riscv64
+#define helper_vmnor_mm helper_vmnor_mm_riscv64
+#define helper_vmorn_mm helper_vmorn_mm_riscv64
+#define helper_vmxnor_mm helper_vmxnor_mm_riscv64
+#define helper_vcpop_m helper_vcpop_m_riscv64
+#define helper_vfirst_m helper_vfirst_m_riscv64
+#define helper_vmsbf_m helper_vmsbf_m_riscv64
+#define helper_vmsif_m helper_vmsif_m_riscv64
+#define helper_vmsof_m helper_vmsof_m_riscv64
+#define helper_viota_m_b helper_viota_m_b_riscv64
+#define helper_viota_m_h helper_viota_m_h_riscv64
+#define helper_viota_m_w helper_viota_m_w_riscv64
+#define helper_viota_m_d helper_viota_m_d_riscv64
+#define helper_vid_v_b helper_vid_v_b_riscv64
+#define helper_vid_v_h helper_vid_v_h_riscv64
+#define helper_vid_v_w helper_vid_v_w_riscv64
+#define helper_vid_v_d helper_vid_v_d_riscv64
+#define helper_vredsum_vs_b helper_vredsum_vs_b_riscv64
+#define helper_vredsum_vs_h helper_vredsum_vs_h_riscv64
+#define helper_vredsum_vs_w helper_vredsum_vs_w_riscv64
+#define helper_vredsum_vs_d helper_vredsum_vs_d_riscv64
+#define helper_vredand_vs_b helper_vredand_vs_b_riscv64
+#define helper_vredand_vs_h helper_vredand_vs_h_riscv64
+#define helper_vredand_vs_w helper_vredand_vs_w_riscv64
+#define helper_vredand_vs_d helper_vredand_vs_d_riscv64
+#define helper_vredor_vs_b helper_vredor_vs_b_riscv64
+#define helper_vredor_vs_h helper_vredor_vs_h_riscv64
+#define helper_vredor_vs_w helper_vredor_vs_w_riscv64
+#define helper_vredor_vs_d helper_vredor_vs_d_riscv64
+#define helper_vredxor_vs_b helper_vredxor_vs_b_riscv64
+#define helper_vredxor_vs_h helper_vredxor_vs_h_riscv64
+#define helper_vredxor_vs_w helper_vredxor_vs_w_riscv64
+#define helper_vredxor_vs_d helper_vredxor_vs_d_riscv64
+#define helper_vredminu_vs_b helper_vredminu_vs_b_riscv64
+#define helper_vredminu_vs_h helper_vredminu_vs_h_riscv64
+#define helper_vredminu_vs_w helper_vredminu_vs_w_riscv64
+#define helper_vredminu_vs_d helper_vredminu_vs_d_riscv64
+#define helper_vredmin_vs_b helper_vredmin_vs_b_riscv64
+#define helper_vredmin_vs_h helper_vredmin_vs_h_riscv64
+#define helper_vredmin_vs_w helper_vredmin_vs_w_riscv64
+#define helper_vredmin_vs_d helper_vredmin_vs_d_riscv64
+#define helper_vredmaxu_vs_b helper_vredmaxu_vs_b_riscv64
+#define helper_vredmaxu_vs_h helper_vredmaxu_vs_h_riscv64
+#define helper_vredmaxu_vs_w helper_vredmaxu_vs_w_riscv64
+#define helper_vredmaxu_vs_d helper_vredmaxu_vs_d_riscv64
+#define helper_vredmax_vs_b helper_vredmax_vs_b_riscv64
+#define helper_vredmax_vs_h helper_vredmax_vs_h_riscv64
+#define helper_vredmax_vs_w helper_vredmax_vs_w_riscv64
+#define helper_vredmax_vs_d helper_vredmax_vs_d_riscv64
+#define helper_vwredsumu_vs_b helper_vwredsumu_vs_b_riscv64
+#define helper_vwredsumu_vs_h helper_vwredsumu_vs_h_riscv64
+#define helper_vwredsumu_vs_w helper_vwredsumu_vs_w_riscv64
+#define helper_vwredsum_vs_b helper_vwredsum_vs_b_riscv64
+#define helper_vwredsum_vs_h helper_vwredsum_vs_h_riscv64
+#define helper_vwredsum_vs_w helper_vwredsum_vs_w_riscv64
+#define helper_vfredusum_vs_h helper_vfredusum_vs_h_riscv64
+#define helper_vfredusum_vs_w helper_vfredusum_vs_w_riscv64
+#define helper_vfredusum_vs_d helper_vfredusum_vs_d_riscv64
+#define helper_vfredosum_vs_h helper_vfredosum_vs_h_riscv64
+#define helper_vfredosum_vs_w helper_vfredosum_vs_w_riscv64
+#define helper_vfredosum_vs_d helper_vfredosum_vs_d_riscv64
+#define helper_vfredmin_vs_h helper_vfredmin_vs_h_riscv64
+#define helper_vfredmin_vs_w helper_vfredmin_vs_w_riscv64
+#define helper_vfredmin_vs_d helper_vfredmin_vs_d_riscv64
+#define helper_vfredmax_vs_h helper_vfredmax_vs_h_riscv64
+#define helper_vfredmax_vs_w helper_vfredmax_vs_w_riscv64
+#define helper_vfredmax_vs_d helper_vfredmax_vs_d_riscv64
+#define helper_vfwredusum_vs_h helper_vfwredusum_vs_h_riscv64
+#define helper_vfwredusum_vs_w helper_vfwredusum_vs_w_riscv64
+#define helper_vfwredosum_vs_h helper_vfwredosum_vs_h_riscv64
+#define helper_vfwredosum_vs_w helper_vfwredosum_vs_w_riscv64
+#define helper_vadd_vx_b helper_vadd_vx_b_riscv64
+#define helper_vadd_vx_h helper_vadd_vx_h_riscv64
+#define helper_vadd_vx_w helper_vadd_vx_w_riscv64
+#define helper_vadd_vx_d helper_vadd_vx_d_riscv64
+#define helper_vsub_vx_b helper_vsub_vx_b_riscv64
+#define helper_vsub_vx_h helper_vsub_vx_h_riscv64
+#define helper_vsub_vx_w helper_vsub_vx_w_riscv64
+#define helper_vsub_vx_d helper_vsub_vx_d_riscv64
+#define helper_vrsub_vx_b helper_vrsub_vx_b_riscv64
+#define helper_vrsub_vx_h helper_vrsub_vx_h_riscv64
+#define helper_vrsub_vx_w helper_vrsub_vx_w_riscv64
+#define helper_vrsub_vx_d helper_vrsub_vx_d_riscv64
+#define helper_vfadd_vf_h helper_vfadd_vf_h_riscv64
+#define helper_vfadd_vf_w helper_vfadd_vf_w_riscv64
+#define helper_vfadd_vf_d helper_vfadd_vf_d_riscv64
+#define helper_vfsub_vf_h helper_vfsub_vf_h_riscv64
+#define helper_vfsub_vf_w helper_vfsub_vf_w_riscv64
+#define helper_vfsub_vf_d helper_vfsub_vf_d_riscv64
+#define helper_vfrsub_vf_h helper_vfrsub_vf_h_riscv64
+#define helper_vfrsub_vf_w helper_vfrsub_vf_w_riscv64
+#define helper_vfrsub_vf_d helper_vfrsub_vf_d_riscv64
+#define helper_vfmul_vf_h helper_vfmul_vf_h_riscv64
+#define helper_vfmul_vf_w helper_vfmul_vf_w_riscv64
+#define helper_vfmul_vf_d helper_vfmul_vf_d_riscv64
+#define helper_vfdiv_vf_h helper_vfdiv_vf_h_riscv64
+#define helper_vfdiv_vf_w helper_vfdiv_vf_w_riscv64
+#define helper_vfdiv_vf_d helper_vfdiv_vf_d_riscv64
+#define helper_vfrdiv_vf_h helper_vfrdiv_vf_h_riscv64
+#define helper_vfrdiv_vf_w helper_vfrdiv_vf_w_riscv64
+#define helper_vfrdiv_vf_d helper_vfrdiv_vf_d_riscv64
+#define helper_vfwadd_vf_h helper_vfwadd_vf_h_riscv64
+#define helper_vfwadd_vf_w helper_vfwadd_vf_w_riscv64
+#define helper_vfwsub_vf_h helper_vfwsub_vf_h_riscv64
+#define helper_vfwsub_vf_w helper_vfwsub_vf_w_riscv64
+#define helper_vfwadd_wf_h helper_vfwadd_wf_h_riscv64
+#define helper_vfwadd_wf_w helper_vfwadd_wf_w_riscv64
+#define helper_vfwsub_wf_h helper_vfwsub_wf_h_riscv64
+#define helper_vfwsub_wf_w helper_vfwsub_wf_w_riscv64
+#define helper_vfwmul_vf_h helper_vfwmul_vf_h_riscv64
+#define helper_vfwmul_vf_w helper_vfwmul_vf_w_riscv64
+#define helper_vfmacc_vf_h helper_vfmacc_vf_h_riscv64
+#define helper_vfmacc_vf_w helper_vfmacc_vf_w_riscv64
+#define helper_vfmacc_vf_d helper_vfmacc_vf_d_riscv64
+#define helper_vfnmacc_vf_h helper_vfnmacc_vf_h_riscv64
+#define helper_vfnmacc_vf_w helper_vfnmacc_vf_w_riscv64
+#define helper_vfnmacc_vf_d helper_vfnmacc_vf_d_riscv64
+#define helper_vfmsac_vf_h helper_vfmsac_vf_h_riscv64
+#define helper_vfmsac_vf_w helper_vfmsac_vf_w_riscv64
+#define helper_vfmsac_vf_d helper_vfmsac_vf_d_riscv64
+#define helper_vfnmsac_vf_h helper_vfnmsac_vf_h_riscv64
+#define helper_vfnmsac_vf_w helper_vfnmsac_vf_w_riscv64
+#define helper_vfnmsac_vf_d helper_vfnmsac_vf_d_riscv64
+#define helper_vfmadd_vf_h helper_vfmadd_vf_h_riscv64
+#define helper_vfmadd_vf_w helper_vfmadd_vf_w_riscv64
+#define helper_vfmadd_vf_d helper_vfmadd_vf_d_riscv64
+#define helper_vfnmadd_vf_h helper_vfnmadd_vf_h_riscv64
+#define helper_vfnmadd_vf_w helper_vfnmadd_vf_w_riscv64
+#define helper_vfnmadd_vf_d helper_vfnmadd_vf_d_riscv64
+#define helper_vfmsub_vf_h helper_vfmsub_vf_h_riscv64
+#define helper_vfmsub_vf_w helper_vfmsub_vf_w_riscv64
+#define helper_vfmsub_vf_d helper_vfmsub_vf_d_riscv64
+#define helper_vfnmsub_vf_h helper_vfnmsub_vf_h_riscv64
+#define helper_vfnmsub_vf_w helper_vfnmsub_vf_w_riscv64
+#define helper_vfnmsub_vf_d helper_vfnmsub_vf_d_riscv64
+#define helper_vfwmacc_vf_h helper_vfwmacc_vf_h_riscv64
+#define helper_vfwmacc_vf_w helper_vfwmacc_vf_w_riscv64
+#define helper_vfwnmacc_vf_h helper_vfwnmacc_vf_h_riscv64
+#define helper_vfwnmacc_vf_w helper_vfwnmacc_vf_w_riscv64
+#define helper_vfwmsac_vf_h helper_vfwmsac_vf_h_riscv64
+#define helper_vfwmsac_vf_w helper_vfwmsac_vf_w_riscv64
+#define helper_vfwnmsac_vf_h helper_vfwnmsac_vf_h_riscv64
+#define helper_vfwnmsac_vf_w helper_vfwnmsac_vf_w_riscv64
+#define helper_vfmin_vf_h helper_vfmin_vf_h_riscv64
+#define helper_vfmin_vf_w helper_vfmin_vf_w_riscv64
+#define helper_vfmin_vf_d helper_vfmin_vf_d_riscv64
+#define helper_vfmax_vf_h helper_vfmax_vf_h_riscv64
+#define helper_vfmax_vf_w helper_vfmax_vf_w_riscv64
+#define helper_vfmax_vf_d helper_vfmax_vf_d_riscv64
+#define helper_vfsgnj_vf_h helper_vfsgnj_vf_h_riscv64
+#define helper_vfsgnj_vf_w helper_vfsgnj_vf_w_riscv64
+#define helper_vfsgnj_vf_d helper_vfsgnj_vf_d_riscv64
+#define helper_vfsgnjn_vf_h helper_vfsgnjn_vf_h_riscv64
+#define helper_vfsgnjn_vf_w helper_vfsgnjn_vf_w_riscv64
+#define helper_vfsgnjn_vf_d helper_vfsgnjn_vf_d_riscv64
+#define helper_vfsgnjx_vf_h helper_vfsgnjx_vf_h_riscv64
+#define helper_vfsgnjx_vf_w helper_vfsgnjx_vf_w_riscv64
+#define helper_vfsgnjx_vf_d helper_vfsgnjx_vf_d_riscv64
+#define helper_vmfeq_vf_h helper_vmfeq_vf_h_riscv64
+#define helper_vmfeq_vf_w helper_vmfeq_vf_w_riscv64
+#define helper_vmfeq_vf_d helper_vmfeq_vf_d_riscv64
+#define helper_vmfne_vf_h helper_vmfne_vf_h_riscv64
+#define helper_vmfne_vf_w helper_vmfne_vf_w_riscv64
+#define helper_vmfne_vf_d helper_vmfne_vf_d_riscv64
+#define helper_vmflt_vf_h helper_vmflt_vf_h_riscv64
+#define helper_vmflt_vf_w helper_vmflt_vf_w_riscv64
+#define helper_vmflt_vf_d helper_vmflt_vf_d_riscv64
+#define helper_vmfle_vf_h helper_vmfle_vf_h_riscv64
+#define helper_vmfle_vf_w helper_vmfle_vf_w_riscv64
+#define helper_vmfle_vf_d helper_vmfle_vf_d_riscv64
+#define helper_vmfgt_vf_h helper_vmfgt_vf_h_riscv64
+#define helper_vmfgt_vf_w helper_vmfgt_vf_w_riscv64
+#define helper_vmfgt_vf_d helper_vmfgt_vf_d_riscv64
+#define helper_vmfge_vf_h helper_vmfge_vf_h_riscv64
+#define helper_vmfge_vf_w helper_vmfge_vf_w_riscv64
+#define helper_vmfge_vf_d helper_vmfge_vf_d_riscv64
+#define helper_vfmerge_vfm_h helper_vfmerge_vfm_h_riscv64
+#define helper_vfmerge_vfm_w helper_vfmerge_vfm_w_riscv64
+#define helper_vfmerge_vfm_d helper_vfmerge_vfm_d_riscv64
+#define helper_vslideup_vx_b helper_vslideup_vx_b_riscv64
+#define helper_vslideup_vx_h helper_vslideup_vx_h_riscv64
+#define helper_vslideup_vx_w helper_vslideup_vx_w_riscv64
+#define helper_vslideup_vx_d helper_vslideup_vx_d_riscv64
+#define helper_vslidedown_vx_b helper_vslidedown_vx_b_riscv64
+#define helper_vslidedown_vx_h helper_vslidedown_vx_h_riscv64
+#define helper_vslidedown_vx_w helper_vslidedown_vx_w_riscv64
+#define helper_vslidedown_vx_d helper_vslidedown_vx_d_riscv64
+#define helper_vslide1up_vx_b helper_vslide1up_vx_b_riscv64
+#define helper_vslide1up_vx_h helper_vslide1up_vx_h_riscv64
+#define helper_vslide1up_vx_w helper_vslide1up_vx_w_riscv64
+#define helper_vslide1up_vx_d helper_vslide1up_vx_d_riscv64
+#define helper_vslide1down_vx_b helper_vslide1down_vx_b_riscv64
+#define helper_vslide1down_vx_h helper_vslide1down_vx_h_riscv64
+#define helper_vslide1down_vx_w helper_vslide1down_vx_w_riscv64
+#define helper_vslide1down_vx_d helper_vslide1down_vx_d_riscv64
+#define helper_vrgather_vv_b helper_vrgather_vv_b_riscv64
+#define helper_vrgather_vv_h helper_vrgather_vv_h_riscv64
+#define helper_vrgather_vv_w helper_vrgather_vv_w_riscv64
+#define helper_vrgather_vv_d helper_vrgather_vv_d_riscv64
+#define helper_vrgatherei16_vv_b helper_vrgatherei16_vv_b_riscv64
+#define helper_vrgatherei16_vv_h helper_vrgatherei16_vv_h_riscv64
+#define helper_vrgatherei16_vv_w helper_vrgatherei16_vv_w_riscv64
+#define helper_vrgatherei16_vv_d helper_vrgatherei16_vv_d_riscv64
+#define helper_vrgather_vx_b helper_vrgather_vx_b_riscv64
+#define helper_vrgather_vx_h helper_vrgather_vx_h_riscv64
+#define helper_vrgather_vx_w helper_vrgather_vx_w_riscv64
+#define helper_vrgather_vx_d helper_vrgather_vx_d_riscv64
+#define helper_vcompress_vm_b helper_vcompress_vm_b_riscv64
+#define helper_vcompress_vm_h helper_vcompress_vm_h_riscv64
+#define helper_vcompress_vm_w helper_vcompress_vm_w_riscv64
+#define helper_vcompress_vm_d helper_vcompress_vm_d_riscv64
+#define helper_vmvr_v helper_vmvr_v_riscv64
+#define helper_vfslide1up_vf_h helper_vfslide1up_vf_h_riscv64
+#define helper_vfslide1up_vf_w helper_vfslide1up_vf_w_riscv64
+#define helper_vfslide1up_vf_d helper_vfslide1up_vf_d_riscv64
+#define helper_vfslide1down_vf_h helper_vfslide1down_vf_h_riscv64
+#define helper_vfslide1down_vf_w helper_vfslide1down_vf_w_riscv64
+#define helper_vfslide1down_vf_d helper_vfslide1down_vf_d_riscv64
+#define helper_vadc_vxm_b helper_vadc_vxm_b_riscv64
+#define helper_vadc_vxm_h helper_vadc_vxm_h_riscv64
+#define helper_vadc_vxm_w helper_vadc_vxm_w_riscv64
+#define helper_vadc_vxm_d helper_vadc_vxm_d_riscv64
+#define helper_vsbc_vxm_b helper_vsbc_vxm_b_riscv64
+#define helper_vsbc_vxm_h helper_vsbc_vxm_h_riscv64
+#define helper_vsbc_vxm_w helper_vsbc_vxm_w_riscv64
+#define helper_vsbc_vxm_d helper_vsbc_vxm_d_riscv64
+#define helper_vmadc_vxm_b helper_vmadc_vxm_b_riscv64
+#define helper_vmadc_vxm_h helper_vmadc_vxm_h_riscv64
+#define helper_vmadc_vxm_w helper_vmadc_vxm_w_riscv64
+#define helper_vmadc_vxm_d helper_vmadc_vxm_d_riscv64
+#define helper_vmsbc_vxm_b helper_vmsbc_vxm_b_riscv64
+#define helper_vmsbc_vxm_h helper_vmsbc_vxm_h_riscv64
+#define helper_vmsbc_vxm_w helper_vmsbc_vxm_w_riscv64
+#define helper_vmsbc_vxm_d helper_vmsbc_vxm_d_riscv64
+#define helper_vand_vx_b helper_vand_vx_b_riscv64
+#define helper_vand_vx_h helper_vand_vx_h_riscv64
+#define helper_vand_vx_w helper_vand_vx_w_riscv64
+#define helper_vand_vx_d helper_vand_vx_d_riscv64
+#define helper_vor_vx_b helper_vor_vx_b_riscv64
+#define helper_vor_vx_h helper_vor_vx_h_riscv64
+#define helper_vor_vx_w helper_vor_vx_w_riscv64
+#define helper_vor_vx_d helper_vor_vx_d_riscv64
+#define helper_vxor_vx_b helper_vxor_vx_b_riscv64
+#define helper_vxor_vx_h helper_vxor_vx_h_riscv64
+#define helper_vxor_vx_w helper_vxor_vx_w_riscv64
+#define helper_vxor_vx_d helper_vxor_vx_d_riscv64
+#define helper_vsaddu_vv_b helper_vsaddu_vv_b_riscv64
+#define helper_vsaddu_vv_h helper_vsaddu_vv_h_riscv64
+#define helper_vsaddu_vv_w helper_vsaddu_vv_w_riscv64
+#define helper_vsaddu_vv_d helper_vsaddu_vv_d_riscv64
+#define helper_vsaddu_vx_b helper_vsaddu_vx_b_riscv64
+#define helper_vsaddu_vx_h helper_vsaddu_vx_h_riscv64
+#define helper_vsaddu_vx_w helper_vsaddu_vx_w_riscv64
+#define helper_vsaddu_vx_d helper_vsaddu_vx_d_riscv64
+#define helper_vsadd_vv_b helper_vsadd_vv_b_riscv64
+#define helper_vsadd_vv_h helper_vsadd_vv_h_riscv64
+#define helper_vsadd_vv_w helper_vsadd_vv_w_riscv64
+#define helper_vsadd_vv_d helper_vsadd_vv_d_riscv64
+#define helper_vsadd_vx_b helper_vsadd_vx_b_riscv64
+#define helper_vsadd_vx_h helper_vsadd_vx_h_riscv64
+#define helper_vsadd_vx_w helper_vsadd_vx_w_riscv64
+#define helper_vsadd_vx_d helper_vsadd_vx_d_riscv64
+#define helper_vssubu_vv_b helper_vssubu_vv_b_riscv64
+#define helper_vssubu_vv_h helper_vssubu_vv_h_riscv64
+#define helper_vssubu_vv_w helper_vssubu_vv_w_riscv64
+#define helper_vssubu_vv_d helper_vssubu_vv_d_riscv64
+#define helper_vssubu_vx_b helper_vssubu_vx_b_riscv64
+#define helper_vssubu_vx_h helper_vssubu_vx_h_riscv64
+#define helper_vssubu_vx_w helper_vssubu_vx_w_riscv64
+#define helper_vssubu_vx_d helper_vssubu_vx_d_riscv64
+#define helper_vssub_vv_b helper_vssub_vv_b_riscv64
+#define helper_vssub_vv_h helper_vssub_vv_h_riscv64
+#define helper_vssub_vv_w helper_vssub_vv_w_riscv64
+#define helper_vssub_vv_d helper_vssub_vv_d_riscv64
+#define helper_vssub_vx_b helper_vssub_vx_b_riscv64
+#define helper_vssub_vx_h helper_vssub_vx_h_riscv64
+#define helper_vssub_vx_w helper_vssub_vx_w_riscv64
+#define helper_vssub_vx_d helper_vssub_vx_d_riscv64
+#define helper_vaadd_vv_b helper_vaadd_vv_b_riscv64
+#define helper_vaadd_vv_h helper_vaadd_vv_h_riscv64
+#define helper_vaadd_vv_w helper_vaadd_vv_w_riscv64
+#define helper_vaadd_vv_d helper_vaadd_vv_d_riscv64
+#define helper_vaadd_vx_b helper_vaadd_vx_b_riscv64
+#define helper_vaadd_vx_h helper_vaadd_vx_h_riscv64
+#define helper_vaadd_vx_w helper_vaadd_vx_w_riscv64
+#define helper_vaadd_vx_d helper_vaadd_vx_d_riscv64
+#define helper_vaaddu_vv_b helper_vaaddu_vv_b_riscv64
+#define helper_vaaddu_vv_h helper_vaaddu_vv_h_riscv64
+#define helper_vaaddu_vv_w helper_vaaddu_vv_w_riscv64
+#define helper_vaaddu_vv_d helper_vaaddu_vv_d_riscv64
+#define helper_vaaddu_vx_b helper_vaaddu_vx_b_riscv64
+#define helper_vaaddu_vx_h helper_vaaddu_vx_h_riscv64
+#define helper_vaaddu_vx_w helper_vaaddu_vx_w_riscv64
+#define helper_vaaddu_vx_d helper_vaaddu_vx_d_riscv64
+#define helper_vasub_vv_b helper_vasub_vv_b_riscv64
+#define helper_vasub_vv_h helper_vasub_vv_h_riscv64
+#define helper_vasub_vv_w helper_vasub_vv_w_riscv64
+#define helper_vasub_vv_d helper_vasub_vv_d_riscv64
+#define helper_vasub_vx_b helper_vasub_vx_b_riscv64
+#define helper_vasub_vx_h helper_vasub_vx_h_riscv64
+#define helper_vasub_vx_w helper_vasub_vx_w_riscv64
+#define helper_vasub_vx_d helper_vasub_vx_d_riscv64
+#define helper_vasubu_vv_b helper_vasubu_vv_b_riscv64
+#define helper_vasubu_vv_h helper_vasubu_vv_h_riscv64
+#define helper_vasubu_vv_w helper_vasubu_vv_w_riscv64
+#define helper_vasubu_vv_d helper_vasubu_vv_d_riscv64
+#define helper_vasubu_vx_b helper_vasubu_vx_b_riscv64
+#define helper_vasubu_vx_h helper_vasubu_vx_h_riscv64
+#define helper_vasubu_vx_w helper_vasubu_vx_w_riscv64
+#define helper_vasubu_vx_d helper_vasubu_vx_d_riscv64
+#define helper_vsmul_vv_b helper_vsmul_vv_b_riscv64
+#define helper_vsmul_vv_h helper_vsmul_vv_h_riscv64
+#define helper_vsmul_vv_w helper_vsmul_vv_w_riscv64
+#define helper_vsmul_vv_d helper_vsmul_vv_d_riscv64
+#define helper_vsmul_vx_b helper_vsmul_vx_b_riscv64
+#define helper_vsmul_vx_h helper_vsmul_vx_h_riscv64
+#define helper_vsmul_vx_w helper_vsmul_vx_w_riscv64
+#define helper_vsmul_vx_d helper_vsmul_vx_d_riscv64
+#define helper_vssrl_vv_b helper_vssrl_vv_b_riscv64
+#define helper_vssrl_vv_h helper_vssrl_vv_h_riscv64
+#define helper_vssrl_vv_w helper_vssrl_vv_w_riscv64
+#define helper_vssrl_vv_d helper_vssrl_vv_d_riscv64
+#define helper_vssrl_vx_b helper_vssrl_vx_b_riscv64
+#define helper_vssrl_vx_h helper_vssrl_vx_h_riscv64
+#define helper_vssrl_vx_w helper_vssrl_vx_w_riscv64
+#define helper_vssrl_vx_d helper_vssrl_vx_d_riscv64
+#define helper_vssra_vv_b helper_vssra_vv_b_riscv64
+#define helper_vssra_vv_h helper_vssra_vv_h_riscv64
+#define helper_vssra_vv_w helper_vssra_vv_w_riscv64
+#define helper_vssra_vv_d helper_vssra_vv_d_riscv64
+#define helper_vssra_vx_b helper_vssra_vx_b_riscv64
+#define helper_vssra_vx_h helper_vssra_vx_h_riscv64
+#define helper_vssra_vx_w helper_vssra_vx_w_riscv64
+#define helper_vssra_vx_d helper_vssra_vx_d_riscv64
+#define helper_vnclip_wv_b helper_vnclip_wv_b_riscv64
+#define helper_vnclip_wv_h helper_vnclip_wv_h_riscv64
+#define helper_vnclip_wv_w helper_vnclip_wv_w_riscv64
+#define helper_vnclip_wx_b helper_vnclip_wx_b_riscv64
+#define helper_vnclip_wx_h helper_vnclip_wx_h_riscv64
+#define helper_vnclip_wx_w helper_vnclip_wx_w_riscv64
+#define helper_vnclipu_wv_b helper_vnclipu_wv_b_riscv64
+#define helper_vnclipu_wv_h helper_vnclipu_wv_h_riscv64
+#define helper_vnclipu_wv_w helper_vnclipu_wv_w_riscv64
+#define helper_vnclipu_wx_b helper_vnclipu_wx_b_riscv64
+#define helper_vnclipu_wx_h helper_vnclipu_wx_h_riscv64
+#define helper_vnclipu_wx_w helper_vnclipu_wx_w_riscv64
+#define helper_vminu_vx_b helper_vminu_vx_b_riscv64
+#define helper_vminu_vx_h helper_vminu_vx_h_riscv64
+#define helper_vminu_vx_w helper_vminu_vx_w_riscv64
+#define helper_vminu_vx_d helper_vminu_vx_d_riscv64
+#define helper_vmin_vx_b helper_vmin_vx_b_riscv64
+#define helper_vmin_vx_h helper_vmin_vx_h_riscv64
+#define helper_vmin_vx_w helper_vmin_vx_w_riscv64
+#define helper_vmin_vx_d helper_vmin_vx_d_riscv64
+#define helper_vmaxu_vx_b helper_vmaxu_vx_b_riscv64
+#define helper_vmaxu_vx_h helper_vmaxu_vx_h_riscv64
+#define helper_vmaxu_vx_w helper_vmaxu_vx_w_riscv64
+#define helper_vmaxu_vx_d helper_vmaxu_vx_d_riscv64
+#define helper_vmax_vx_b helper_vmax_vx_b_riscv64
+#define helper_vmax_vx_h helper_vmax_vx_h_riscv64
+#define helper_vmax_vx_w helper_vmax_vx_w_riscv64
+#define helper_vmax_vx_d helper_vmax_vx_d_riscv64
+#define helper_vwaddu_vv_b helper_vwaddu_vv_b_riscv64
+#define helper_vwaddu_vv_h helper_vwaddu_vv_h_riscv64
+#define helper_vwaddu_vv_w helper_vwaddu_vv_w_riscv64
+#define helper_vwsubu_vv_b helper_vwsubu_vv_b_riscv64
+#define helper_vwsubu_vv_h helper_vwsubu_vv_h_riscv64
+#define helper_vwsubu_vv_w helper_vwsubu_vv_w_riscv64
+#define helper_vwadd_vv_b helper_vwadd_vv_b_riscv64
+#define helper_vwadd_vv_h helper_vwadd_vv_h_riscv64
+#define helper_vwadd_vv_w helper_vwadd_vv_w_riscv64
+#define helper_vwsub_vv_b helper_vwsub_vv_b_riscv64
+#define helper_vwsub_vv_h helper_vwsub_vv_h_riscv64
+#define helper_vwsub_vv_w helper_vwsub_vv_w_riscv64
+#define helper_vwaddu_vx_b helper_vwaddu_vx_b_riscv64
+#define helper_vwaddu_vx_h helper_vwaddu_vx_h_riscv64
+#define helper_vwaddu_vx_w helper_vwaddu_vx_w_riscv64
+#define helper_vwsubu_vx_b helper_vwsubu_vx_b_riscv64
+#define helper_vwsubu_vx_h helper_vwsubu_vx_h_riscv64
+#define helper_vwsubu_vx_w helper_vwsubu_vx_w_riscv64
+#define helper_vwadd_vx_b helper_vwadd_vx_b_riscv64
+#define helper_vwadd_vx_h helper_vwadd_vx_h_riscv64
+#define helper_vwadd_vx_w helper_vwadd_vx_w_riscv64
+#define helper_vwsub_vx_b helper_vwsub_vx_b_riscv64
+#define helper_vwsub_vx_h helper_vwsub_vx_h_riscv64
+#define helper_vwsub_vx_w helper_vwsub_vx_w_riscv64
+#define helper_vwaddu_wv_b helper_vwaddu_wv_b_riscv64
+#define helper_vwaddu_wv_h helper_vwaddu_wv_h_riscv64
+#define helper_vwaddu_wv_w helper_vwaddu_wv_w_riscv64
+#define helper_vwsubu_wv_b helper_vwsubu_wv_b_riscv64
+#define helper_vwsubu_wv_h helper_vwsubu_wv_h_riscv64
+#define helper_vwsubu_wv_w helper_vwsubu_wv_w_riscv64
+#define helper_vwadd_wv_b helper_vwadd_wv_b_riscv64
+#define helper_vwadd_wv_h helper_vwadd_wv_h_riscv64
+#define helper_vwadd_wv_w helper_vwadd_wv_w_riscv64
+#define helper_vwsub_wv_b helper_vwsub_wv_b_riscv64
+#define helper_vwsub_wv_h helper_vwsub_wv_h_riscv64
+#define helper_vwsub_wv_w helper_vwsub_wv_w_riscv64
+#define helper_vwaddu_wx_b helper_vwaddu_wx_b_riscv64
+#define helper_vwaddu_wx_h helper_vwaddu_wx_h_riscv64
+#define helper_vwaddu_wx_w helper_vwaddu_wx_w_riscv64
+#define helper_vwsubu_wx_b helper_vwsubu_wx_b_riscv64
+#define helper_vwsubu_wx_h helper_vwsubu_wx_h_riscv64
+#define helper_vwsubu_wx_w helper_vwsubu_wx_w_riscv64
+#define helper_vwadd_wx_b helper_vwadd_wx_b_riscv64
+#define helper_vwadd_wx_h helper_vwadd_wx_h_riscv64
+#define helper_vwadd_wx_w helper_vwadd_wx_w_riscv64
+#define helper_vwsub_wx_b helper_vwsub_wx_b_riscv64
+#define helper_vwsub_wx_h helper_vwsub_wx_h_riscv64
+#define helper_vwsub_wx_w helper_vwsub_wx_w_riscv64
+#define helper_vwmul_vv_b helper_vwmul_vv_b_riscv64
+#define helper_vwmul_vv_h helper_vwmul_vv_h_riscv64
+#define helper_vwmul_vv_w helper_vwmul_vv_w_riscv64
+#define helper_vwmulu_vv_b helper_vwmulu_vv_b_riscv64
+#define helper_vwmulu_vv_h helper_vwmulu_vv_h_riscv64
+#define helper_vwmulu_vv_w helper_vwmulu_vv_w_riscv64
+#define helper_vwmulsu_vv_b helper_vwmulsu_vv_b_riscv64
+#define helper_vwmulsu_vv_h helper_vwmulsu_vv_h_riscv64
+#define helper_vwmulsu_vv_w helper_vwmulsu_vv_w_riscv64
+#define helper_vwmul_vx_b helper_vwmul_vx_b_riscv64
+#define helper_vwmul_vx_h helper_vwmul_vx_h_riscv64
+#define helper_vwmul_vx_w helper_vwmul_vx_w_riscv64
+#define helper_vwmulu_vx_b helper_vwmulu_vx_b_riscv64
+#define helper_vwmulu_vx_h helper_vwmulu_vx_h_riscv64
+#define helper_vwmulu_vx_w helper_vwmulu_vx_w_riscv64
+#define helper_vwmulsu_vx_b helper_vwmulsu_vx_b_riscv64
+#define helper_vwmulsu_vx_h helper_vwmulsu_vx_h_riscv64
+#define helper_vwmulsu_vx_w helper_vwmulsu_vx_w_riscv64
+#define helper_vmacc_vv_b helper_vmacc_vv_b_riscv64
+#define helper_vmacc_vv_h helper_vmacc_vv_h_riscv64
+#define helper_vmacc_vv_w helper_vmacc_vv_w_riscv64
+#define helper_vmacc_vv_d helper_vmacc_vv_d_riscv64
+#define helper_vnmsac_vv_b helper_vnmsac_vv_b_riscv64
+#define helper_vnmsac_vv_h helper_vnmsac_vv_h_riscv64
+#define helper_vnmsac_vv_w helper_vnmsac_vv_w_riscv64
+#define helper_vnmsac_vv_d helper_vnmsac_vv_d_riscv64
+#define helper_vmadd_vv_b helper_vmadd_vv_b_riscv64
+#define helper_vmadd_vv_h helper_vmadd_vv_h_riscv64
+#define helper_vmadd_vv_w helper_vmadd_vv_w_riscv64
+#define helper_vmadd_vv_d helper_vmadd_vv_d_riscv64
+#define helper_vnmsub_vv_b helper_vnmsub_vv_b_riscv64
+#define helper_vnmsub_vv_h helper_vnmsub_vv_h_riscv64
+#define helper_vnmsub_vv_w helper_vnmsub_vv_w_riscv64
+#define helper_vnmsub_vv_d helper_vnmsub_vv_d_riscv64
+#define helper_vmacc_vx_b helper_vmacc_vx_b_riscv64
+#define helper_vmacc_vx_h helper_vmacc_vx_h_riscv64
+#define helper_vmacc_vx_w helper_vmacc_vx_w_riscv64
+#define helper_vmacc_vx_d helper_vmacc_vx_d_riscv64
+#define helper_vnmsac_vx_b helper_vnmsac_vx_b_riscv64
+#define helper_vnmsac_vx_h helper_vnmsac_vx_h_riscv64
+#define helper_vnmsac_vx_w helper_vnmsac_vx_w_riscv64
+#define helper_vnmsac_vx_d helper_vnmsac_vx_d_riscv64
+#define helper_vmadd_vx_b helper_vmadd_vx_b_riscv64
+#define helper_vmadd_vx_h helper_vmadd_vx_h_riscv64
+#define helper_vmadd_vx_w helper_vmadd_vx_w_riscv64
+#define helper_vmadd_vx_d helper_vmadd_vx_d_riscv64
+#define helper_vnmsub_vx_b helper_vnmsub_vx_b_riscv64
+#define helper_vnmsub_vx_h helper_vnmsub_vx_h_riscv64
+#define helper_vnmsub_vx_w helper_vnmsub_vx_w_riscv64
+#define helper_vnmsub_vx_d helper_vnmsub_vx_d_riscv64
+#define helper_vwmaccu_vv_b helper_vwmaccu_vv_b_riscv64
+#define helper_vwmaccu_vv_h helper_vwmaccu_vv_h_riscv64
+#define helper_vwmaccu_vv_w helper_vwmaccu_vv_w_riscv64
+#define helper_vwmacc_vv_b helper_vwmacc_vv_b_riscv64
+#define helper_vwmacc_vv_h helper_vwmacc_vv_h_riscv64
+#define helper_vwmacc_vv_w helper_vwmacc_vv_w_riscv64
+#define helper_vwmaccsu_vv_b helper_vwmaccsu_vv_b_riscv64
+#define helper_vwmaccsu_vv_h helper_vwmaccsu_vv_h_riscv64
+#define helper_vwmaccsu_vv_w helper_vwmaccsu_vv_w_riscv64
+#define helper_vwmaccu_vx_b helper_vwmaccu_vx_b_riscv64
+#define helper_vwmaccu_vx_h helper_vwmaccu_vx_h_riscv64
+#define helper_vwmaccu_vx_w helper_vwmaccu_vx_w_riscv64
+#define helper_vwmacc_vx_b helper_vwmacc_vx_b_riscv64
+#define helper_vwmacc_vx_h helper_vwmacc_vx_h_riscv64
+#define helper_vwmacc_vx_w helper_vwmacc_vx_w_riscv64
+#define helper_vwmaccsu_vx_b helper_vwmaccsu_vx_b_riscv64
+#define helper_vwmaccsu_vx_h helper_vwmaccsu_vx_h_riscv64
+#define helper_vwmaccsu_vx_w helper_vwmaccsu_vx_w_riscv64
+#define helper_vwmaccus_vx_b helper_vwmaccus_vx_b_riscv64
+#define helper_vwmaccus_vx_h helper_vwmaccus_vx_h_riscv64
+#define helper_vwmaccus_vx_w helper_vwmaccus_vx_w_riscv64
+#define helper_vmul_vv_b helper_vmul_vv_b_riscv64
+#define helper_vmul_vv_h helper_vmul_vv_h_riscv64
+#define helper_vmul_vv_w helper_vmul_vv_w_riscv64
+#define helper_vmul_vv_d helper_vmul_vv_d_riscv64
+#define helper_vmulh_vv_b helper_vmulh_vv_b_riscv64
+#define helper_vmulh_vv_h helper_vmulh_vv_h_riscv64
+#define helper_vmulh_vv_w helper_vmulh_vv_w_riscv64
+#define helper_vmulh_vv_d helper_vmulh_vv_d_riscv64
+#define helper_vmulhu_vv_b helper_vmulhu_vv_b_riscv64
+#define helper_vmulhu_vv_h helper_vmulhu_vv_h_riscv64
+#define helper_vmulhu_vv_w helper_vmulhu_vv_w_riscv64
+#define helper_vmulhu_vv_d helper_vmulhu_vv_d_riscv64
+#define helper_vmulhsu_vv_b helper_vmulhsu_vv_b_riscv64
+#define helper_vmulhsu_vv_h helper_vmulhsu_vv_h_riscv64
+#define helper_vmulhsu_vv_w helper_vmulhsu_vv_w_riscv64
+#define helper_vmulhsu_vv_d helper_vmulhsu_vv_d_riscv64
+#define helper_vmul_vx_b helper_vmul_vx_b_riscv64
+#define helper_vmul_vx_h helper_vmul_vx_h_riscv64
+#define helper_vmul_vx_w helper_vmul_vx_w_riscv64
+#define helper_vmul_vx_d helper_vmul_vx_d_riscv64
+#define helper_vmulh_vx_b helper_vmulh_vx_b_riscv64
+#define helper_vmulh_vx_h helper_vmulh_vx_h_riscv64
+#define helper_vmulh_vx_w helper_vmulh_vx_w_riscv64
+#define helper_vmulh_vx_d helper_vmulh_vx_d_riscv64
+#define helper_vmulhu_vx_b helper_vmulhu_vx_b_riscv64
+#define helper_vmulhu_vx_h helper_vmulhu_vx_h_riscv64
+#define helper_vmulhu_vx_w helper_vmulhu_vx_w_riscv64
+#define helper_vmulhu_vx_d helper_vmulhu_vx_d_riscv64
+#define helper_vmulhsu_vx_b helper_vmulhsu_vx_b_riscv64
+#define helper_vmulhsu_vx_h helper_vmulhsu_vx_h_riscv64
+#define helper_vmulhsu_vx_w helper_vmulhsu_vx_w_riscv64
+#define helper_vmulhsu_vx_d helper_vmulhsu_vx_d_riscv64
+#define helper_vdivu_vv_b helper_vdivu_vv_b_riscv64
+#define helper_vdivu_vv_h helper_vdivu_vv_h_riscv64
+#define helper_vdivu_vv_w helper_vdivu_vv_w_riscv64
+#define helper_vdivu_vv_d helper_vdivu_vv_d_riscv64
+#define helper_vdiv_vv_b helper_vdiv_vv_b_riscv64
+#define helper_vdiv_vv_h helper_vdiv_vv_h_riscv64
+#define helper_vdiv_vv_w helper_vdiv_vv_w_riscv64
+#define helper_vdiv_vv_d helper_vdiv_vv_d_riscv64
+#define helper_vremu_vv_b helper_vremu_vv_b_riscv64
+#define helper_vremu_vv_h helper_vremu_vv_h_riscv64
+#define helper_vremu_vv_w helper_vremu_vv_w_riscv64
+#define helper_vremu_vv_d helper_vremu_vv_d_riscv64
+#define helper_vrem_vv_b helper_vrem_vv_b_riscv64
+#define helper_vrem_vv_h helper_vrem_vv_h_riscv64
+#define helper_vrem_vv_w helper_vrem_vv_w_riscv64
+#define helper_vrem_vv_d helper_vrem_vv_d_riscv64
+#define helper_vdivu_vx_b helper_vdivu_vx_b_riscv64
+#define helper_vdivu_vx_h helper_vdivu_vx_h_riscv64
+#define helper_vdivu_vx_w helper_vdivu_vx_w_riscv64
+#define helper_vdivu_vx_d helper_vdivu_vx_d_riscv64
+#define helper_vdiv_vx_b helper_vdiv_vx_b_riscv64
+#define helper_vdiv_vx_h helper_vdiv_vx_h_riscv64
+#define helper_vdiv_vx_w helper_vdiv_vx_w_riscv64
+#define helper_vdiv_vx_d helper_vdiv_vx_d_riscv64
+#define helper_vremu_vx_b helper_vremu_vx_b_riscv64
+#define helper_vremu_vx_h helper_vremu_vx_h_riscv64
+#define helper_vremu_vx_w helper_vremu_vx_w_riscv64
+#define helper_vremu_vx_d helper_vremu_vx_d_riscv64
+#define helper_vrem_vx_b helper_vrem_vx_b_riscv64
+#define helper_vrem_vx_h helper_vrem_vx_h_riscv64
+#define helper_vrem_vx_w helper_vrem_vx_w_riscv64
+#define helper_vrem_vx_d helper_vrem_vx_d_riscv64
+#define helper_vsll_vv_b helper_vsll_vv_b_riscv64
+#define helper_vsll_vv_h helper_vsll_vv_h_riscv64
+#define helper_vsll_vv_w helper_vsll_vv_w_riscv64
+#define helper_vsll_vv_d helper_vsll_vv_d_riscv64
+#define helper_vsrl_vv_b helper_vsrl_vv_b_riscv64
+#define helper_vsrl_vv_h helper_vsrl_vv_h_riscv64
+#define helper_vsrl_vv_w helper_vsrl_vv_w_riscv64
+#define helper_vsrl_vv_d helper_vsrl_vv_d_riscv64
+#define helper_vsra_vv_b helper_vsra_vv_b_riscv64
+#define helper_vsra_vv_h helper_vsra_vv_h_riscv64
+#define helper_vsra_vv_w helper_vsra_vv_w_riscv64
+#define helper_vsra_vv_d helper_vsra_vv_d_riscv64
+#define helper_vsll_vx_b helper_vsll_vx_b_riscv64
+#define helper_vsll_vx_h helper_vsll_vx_h_riscv64
+#define helper_vsll_vx_w helper_vsll_vx_w_riscv64
+#define helper_vsll_vx_d helper_vsll_vx_d_riscv64
+#define helper_vsrl_vx_b helper_vsrl_vx_b_riscv64
+#define helper_vsrl_vx_h helper_vsrl_vx_h_riscv64
+#define helper_vsrl_vx_w helper_vsrl_vx_w_riscv64
+#define helper_vsrl_vx_d helper_vsrl_vx_d_riscv64
+#define helper_vsra_vx_b helper_vsra_vx_b_riscv64
+#define helper_vsra_vx_h helper_vsra_vx_h_riscv64
+#define helper_vsra_vx_w helper_vsra_vx_w_riscv64
+#define helper_vsra_vx_d helper_vsra_vx_d_riscv64
+#define helper_vnsrl_wv_b helper_vnsrl_wv_b_riscv64
+#define helper_vnsrl_wv_h helper_vnsrl_wv_h_riscv64
+#define helper_vnsrl_wv_w helper_vnsrl_wv_w_riscv64
+#define helper_vnsra_wv_b helper_vnsra_wv_b_riscv64
+#define helper_vnsra_wv_h helper_vnsra_wv_h_riscv64
+#define helper_vnsra_wv_w helper_vnsra_wv_w_riscv64
+#define helper_vnsrl_wx_b helper_vnsrl_wx_b_riscv64
+#define helper_vnsrl_wx_h helper_vnsrl_wx_h_riscv64
+#define helper_vnsrl_wx_w helper_vnsrl_wx_w_riscv64
+#define helper_vnsra_wx_b helper_vnsra_wx_b_riscv64
+#define helper_vnsra_wx_h helper_vnsra_wx_h_riscv64
+#define helper_vnsra_wx_w helper_vnsra_wx_w_riscv64
+#define helper_vzext_vf2_h helper_vzext_vf2_h_riscv64
+#define helper_vzext_vf2_w helper_vzext_vf2_w_riscv64
+#define helper_vzext_vf2_d helper_vzext_vf2_d_riscv64
+#define helper_vzext_vf4_w helper_vzext_vf4_w_riscv64
+#define helper_vzext_vf4_d helper_vzext_vf4_d_riscv64
+#define helper_vzext_vf8_d helper_vzext_vf8_d_riscv64
+#define helper_vsext_vf2_h helper_vsext_vf2_h_riscv64
+#define helper_vsext_vf2_w helper_vsext_vf2_w_riscv64
+#define helper_vsext_vf2_d helper_vsext_vf2_d_riscv64
+#define helper_vsext_vf4_w helper_vsext_vf4_w_riscv64
+#define helper_vsext_vf4_d helper_vsext_vf4_d_riscv64
+#define helper_vsext_vf8_d helper_vsext_vf8_d_riscv64
+#define helper_vmseq_vx_b helper_vmseq_vx_b_riscv64
+#define helper_vmseq_vx_h helper_vmseq_vx_h_riscv64
+#define helper_vmseq_vx_w helper_vmseq_vx_w_riscv64
+#define helper_vmseq_vx_d helper_vmseq_vx_d_riscv64
+#define helper_vmsne_vx_b helper_vmsne_vx_b_riscv64
+#define helper_vmsne_vx_h helper_vmsne_vx_h_riscv64
+#define helper_vmsne_vx_w helper_vmsne_vx_w_riscv64
+#define helper_vmsne_vx_d helper_vmsne_vx_d_riscv64
+#define helper_vmsltu_vx_b helper_vmsltu_vx_b_riscv64
+#define helper_vmsltu_vx_h helper_vmsltu_vx_h_riscv64
+#define helper_vmsltu_vx_w helper_vmsltu_vx_w_riscv64
+#define helper_vmsltu_vx_d helper_vmsltu_vx_d_riscv64
+#define helper_vmslt_vx_b helper_vmslt_vx_b_riscv64
+#define helper_vmslt_vx_h helper_vmslt_vx_h_riscv64
+#define helper_vmslt_vx_w helper_vmslt_vx_w_riscv64
+#define helper_vmslt_vx_d helper_vmslt_vx_d_riscv64
+#define helper_vmsleu_vx_b helper_vmsleu_vx_b_riscv64
+#define helper_vmsleu_vx_h helper_vmsleu_vx_h_riscv64
+#define helper_vmsleu_vx_w helper_vmsleu_vx_w_riscv64
+#define helper_vmsleu_vx_d helper_vmsleu_vx_d_riscv64
+#define helper_vmsle_vx_b helper_vmsle_vx_b_riscv64
+#define helper_vmsle_vx_h helper_vmsle_vx_h_riscv64
+#define helper_vmsle_vx_w helper_vmsle_vx_w_riscv64
+#define helper_vmsle_vx_d helper_vmsle_vx_d_riscv64
+#define helper_vmsgtu_vx_b helper_vmsgtu_vx_b_riscv64
+#define helper_vmsgtu_vx_h helper_vmsgtu_vx_h_riscv64
+#define helper_vmsgtu_vx_w helper_vmsgtu_vx_w_riscv64
+#define helper_vmsgtu_vx_d helper_vmsgtu_vx_d_riscv64
+#define helper_vmsgt_vx_b helper_vmsgt_vx_b_riscv64
+#define helper_vmsgt_vx_h helper_vmsgt_vx_h_riscv64
+#define helper_vmsgt_vx_w helper_vmsgt_vx_w_riscv64
+#define helper_vmsgt_vx_d helper_vmsgt_vx_d_riscv64
+#define helper_vmv_v_v_b helper_vmv_v_v_b_riscv64
+#define helper_vmv_v_v_h helper_vmv_v_v_h_riscv64
+#define helper_vmv_v_v_w helper_vmv_v_v_w_riscv64
+#define helper_vmv_v_v_d helper_vmv_v_v_d_riscv64
+#define helper_vmv_v_x_b helper_vmv_v_x_b_riscv64
+#define helper_vmv_v_x_h helper_vmv_v_x_h_riscv64
+#define helper_vmv_v_x_w helper_vmv_v_x_w_riscv64
+#define helper_vmv_v_x_d helper_vmv_v_x_d_riscv64
+#define helper_vmerge_vvm_b helper_vmerge_vvm_b_riscv64
+#define helper_vmerge_vvm_h helper_vmerge_vvm_h_riscv64
+#define helper_vmerge_vvm_w helper_vmerge_vvm_w_riscv64
+#define helper_vmerge_vvm_d helper_vmerge_vvm_d_riscv64
+#define helper_vmerge_vxm_b helper_vmerge_vxm_b_riscv64
+#define helper_vmerge_vxm_h helper_vmerge_vxm_h_riscv64
+#define helper_vmerge_vxm_w helper_vmerge_vxm_w_riscv64
+#define helper_vmerge_vxm_d helper_vmerge_vxm_d_riscv64
+#define helper_clmul helper_clmul_riscv64
+#define helper_clmulr helper_clmulr_riscv64
+#define helper_brev8 helper_brev8_riscv64
+#define helper_unzip helper_unzip_riscv64
+#define helper_zip helper_zip_riscv64
+#define helper_xperm4 helper_xperm4_riscv64
+#define helper_xperm8 helper_xperm8_riscv64
+#define helper_aes32esmi helper_aes32esmi_riscv64
+#define helper_aes32esi helper_aes32esi_riscv64
+#define helper_aes32dsmi helper_aes32dsmi_riscv64
+#define helper_aes32dsi helper_aes32dsi_riscv64
+#define helper_aes64esm helper_aes64esm_riscv64
+#define helper_aes64es helper_aes64es_riscv64
+#define helper_aes64ds helper_aes64ds_riscv64
+#define helper_aes64dsm helper_aes64dsm_riscv64
+#define helper_aes64ks2 helper_aes64ks2_riscv64
+#define helper_aes64ks1i helper_aes64ks1i_riscv64
+#define helper_aes64im helper_aes64im_riscv64
+#define helper_sm4ed helper_sm4ed_riscv64
+#define helper_sm4ks helper_sm4ks_riscv64
 #define helper_sret helper_sret_riscv64
 #define helper_mret helper_mret_riscv64
 #define helper_wfi helper_wfi_riscv64
 #define helper_tlb_flush helper_tlb_flush_riscv64
+#define helper_hyp_tlb_flush helper_hyp_tlb_flush_riscv64
+#define helper_hyp_gvma_tlb_flush helper_hyp_gvma_tlb_flush_riscv64
+#define helper_hyp_hlvx_hu helper_hyp_hlvx_hu_riscv64
+#define helper_hyp_hlvx_wu helper_hyp_hlvx_wu_riscv64
 #define pmp_hart_has_privs pmp_hart_has_privs_riscv64
 #define pmpcfg_csr_write pmpcfg_csr_write_riscv64
 #define pmpcfg_csr_read pmpcfg_csr_read_riscv64
 #define pmpaddr_csr_write pmpaddr_csr_write_riscv64
 #define pmpaddr_csr_read pmpaddr_csr_read_riscv64
+#define riscv_cpu_vector_enabled riscv_cpu_vector_enabled_riscv64
 #define gen_intermediate_code gen_intermediate_code_riscv64
 #define riscv_translate_init riscv_translate_init_riscv64
 #define restore_state_to_opc restore_state_to_opc_riscv64
@@ -1385,7 +3130,15 @@
 #define helper_fcvt_lu_d helper_fcvt_lu_d_riscv64
 #define helper_fcvt_d_l helper_fcvt_d_l_riscv64
 #define helper_fcvt_d_lu helper_fcvt_d_lu_riscv64
+#define helper_fcvt_l_h helper_fcvt_l_h_riscv64
+#define helper_fcvt_lu_h helper_fcvt_lu_h_riscv64
+#define helper_fcvt_h_l helper_fcvt_h_l_riscv64
+#define helper_fcvt_h_lu helper_fcvt_h_lu_riscv64
 #define gen_helper_tlb_flush gen_helper_tlb_flush_riscv64
+#define gen_helper_hyp_tlb_flush gen_helper_hyp_tlb_flush_riscv64
+#define gen_helper_hyp_gvma_tlb_flush gen_helper_hyp_gvma_tlb_flush_riscv64
+#define gen_helper_hyp_hlvx_hu gen_helper_hyp_hlvx_hu_riscv64
+#define gen_helper_hyp_hlvx_wu gen_helper_hyp_hlvx_wu_riscv64
 #define riscv_fpr_regnames riscv_fpr_regnames_riscv64
 #define riscv_int_regnames riscv_int_regnames_riscv64
 #endif
diff --git a/qemu/s390x.h b/qemu/s390x.h
index 5daa819afa..e5e85370af 100644
--- a/qemu/s390x.h
+++ b/qemu/s390x.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_s390x
 #define float32_squash_input_denormal float32_squash_input_denormal_s390x
 #define float64_squash_input_denormal float64_squash_input_denormal_s390x
+#define bfloat16_add bfloat16_add_s390x
+#define bfloat16_compare bfloat16_compare_s390x
+#define bfloat16_compare_quiet bfloat16_compare_quiet_s390x
+#define bfloat16_default_nan bfloat16_default_nan_s390x
+#define bfloat16_div bfloat16_div_s390x
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_s390x
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_s390x
+#define bfloat16_max bfloat16_max_s390x
+#define bfloat16_maximum_number bfloat16_maximum_number_s390x
+#define bfloat16_maxnum bfloat16_maxnum_s390x
+#define bfloat16_maxnummag bfloat16_maxnummag_s390x
+#define bfloat16_min bfloat16_min_s390x
+#define bfloat16_minimum_number bfloat16_minimum_number_s390x
+#define bfloat16_minnum bfloat16_minnum_s390x
+#define bfloat16_minnummag bfloat16_minnummag_s390x
+#define bfloat16_mul bfloat16_mul_s390x
+#define bfloat16_muladd bfloat16_muladd_s390x
+#define bfloat16_round_to_int bfloat16_round_to_int_s390x
+#define bfloat16_scalbn bfloat16_scalbn_s390x
+#define bfloat16_silence_nan bfloat16_silence_nan_s390x
+#define bfloat16_sqrt bfloat16_sqrt_s390x
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_s390x
+#define bfloat16_sub bfloat16_sub_s390x
+#define bfloat16_to_float32 bfloat16_to_float32_s390x
+#define bfloat16_to_float64 bfloat16_to_float64_s390x
+#define bfloat16_to_int16 bfloat16_to_int16_s390x
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_s390x
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_s390x
+#define bfloat16_to_int32 bfloat16_to_int32_s390x
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_s390x
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_s390x
+#define bfloat16_to_int64 bfloat16_to_int64_s390x
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_s390x
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_s390x
+#define bfloat16_to_uint16 bfloat16_to_uint16_s390x
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_s390x
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_s390x
+#define bfloat16_to_uint32 bfloat16_to_uint32_s390x
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_s390x
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_s390x
+#define bfloat16_to_uint64 bfloat16_to_uint64_s390x
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_s390x
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_s390x
+#define float128_maximum_number float128_maximum_number_s390x
+#define float128_max float128_max_s390x
+#define float128_maxnum float128_maxnum_s390x
+#define float128_maxnummag float128_maxnummag_s390x
+#define float128_min float128_min_s390x
+#define float128_minimum_number float128_minimum_number_s390x
+#define float128_minnum float128_minnum_s390x
+#define float128_minnummag float128_minnummag_s390x
+#define float128_muladd float128_muladd_s390x
+#define float128_to_int128 float128_to_int128_s390x
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_s390x
+#define float128_to_uint128 float128_to_uint128_s390x
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_s390x
+#define float16_maximum_number float16_maximum_number_s390x
+#define float16_minimum_number float16_minimum_number_s390x
+#define float16_to_int8 float16_to_int8_s390x
+#define float16_to_int8_scalbn float16_to_int8_scalbn_s390x
+#define float16_to_uint8 float16_to_uint8_s390x
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_s390x
+#define float32_maximum_number float32_maximum_number_s390x
+#define float32_minimum_number float32_minimum_number_s390x
+#define float32_to_bfloat16 float32_to_bfloat16_s390x
+#define float64_maximum_number float64_maximum_number_s390x
+#define float64_minimum_number float64_minimum_number_s390x
+#define float64_to_bfloat16 float64_to_bfloat16_s390x
+#define float64r32_add float64r32_add_s390x
+#define float64r32_div float64r32_div_s390x
+#define float64r32_mul float64r32_mul_s390x
+#define float64r32_muladd float64r32_muladd_s390x
+#define float64r32_sqrt float64r32_sqrt_s390x
+#define float64r32_sub float64r32_sub_s390x
+#define floatx80_mod floatx80_mod_s390x
+#define floatx80_modrem floatx80_modrem_s390x
+#define int128_to_float128 int128_to_float128_s390x
+#define int16_to_bfloat16 int16_to_bfloat16_s390x
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_s390x
+#define int32_to_bfloat16 int32_to_bfloat16_s390x
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_s390x
+#define int64_to_bfloat16 int64_to_bfloat16_s390x
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_s390x
+#define int8_to_float16 int8_to_float16_s390x
+#define uint128_to_float128 uint128_to_float128_s390x
+#define uint16_to_bfloat16 uint16_to_bfloat16_s390x
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_s390x
+#define uint32_to_bfloat16 uint32_to_bfloat16_s390x
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_s390x
+#define uint64_to_bfloat16 uint64_to_bfloat16_s390x
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_s390x
+#define uint8_to_float16 uint8_to_float16_s390x
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_s390x
 #define roundAndPackFloatx80 roundAndPackFloatx80_s390x
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_s390x
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_s390x
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_s390x
 #define helper_exit_atomic helper_exit_atomic_s390x
+#define helper_memset helper_memset_s390x
+#define helper_emu_stop helper_emu_stop_s390x
+#define tcg_remove_ops_after tcg_remove_ops_after_s390x
+#define tcg_constant_vec_matching tcg_constant_vec_matching_s390x
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_s390x
 #define helper_gvec_add8 helper_gvec_add8_s390x
 #define helper_gvec_add16 helper_gvec_add16_s390x
 #define helper_gvec_add32 helper_gvec_add32_s390x
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_s390x
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_s390x
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_s390x
+#define gen_helper_mve_vctp gen_helper_mve_vctp_s390x
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_s390x
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_s390x
+#define gen_helper_mve_vdup gen_helper_mve_vdup_s390x
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_s390x
+#define gen_helper_mve_vandi gen_helper_mve_vandi_s390x
+#define gen_helper_mve_vorri gen_helper_mve_vorri_s390x
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_s390x
+#define gen_helper_mve_viduph gen_helper_mve_viduph_s390x
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_s390x
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_s390x
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_s390x
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_s390x
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_s390x
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_s390x
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_s390x
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_s390x
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_s390x
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_s390x
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_s390x
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_s390x
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_s390x
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_s390x
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_s390x
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_s390x
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_s390x
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_s390x
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_s390x
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_s390x
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_s390x
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_s390x
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_s390x
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_s390x
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_s390x
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_s390x
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_s390x
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_s390x
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_s390x
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_s390x
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_s390x
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_s390x
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_s390x
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_s390x
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_s390x
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_s390x
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_s390x
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_s390x
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_s390x
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_s390x
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_s390x
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_s390x
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_s390x
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_s390x
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_s390x
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_s390x
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_s390x
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_s390x
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_s390x
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_s390x
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_s390x
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_s390x
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_s390x
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_s390x
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_s390x
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_s390x
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_s390x
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_s390x
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_s390x
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_s390x
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_s390x
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_s390x
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_s390x
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_s390x
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_s390x
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_s390x
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_s390x
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_s390x
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_s390x
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_s390x
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_s390x
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_s390x
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_s390x
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_s390x
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_s390x
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_s390x
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_s390x
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_s390x
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_s390x
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_s390x
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_s390x
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_s390x
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_s390x
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_s390x
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_s390x
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_s390x
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_s390x
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_s390x
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_s390x
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_s390x
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_s390x
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_s390x
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_s390x
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_s390x
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_s390x
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_s390x
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_s390x
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_s390x
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_s390x
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_s390x
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_s390x
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_s390x
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_s390x
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_s390x
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_s390x
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_s390x
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_s390x
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_s390x
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_s390x
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_s390x
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_s390x
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_s390x
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_s390x
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_s390x
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_s390x
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_s390x
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_s390x
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_s390x
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_s390x
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_s390x
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_s390x
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_s390x
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_s390x
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_s390x
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_s390x
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_s390x
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_s390x
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_s390x
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_s390x
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_s390x
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_s390x
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_s390x
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_s390x
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_s390x
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_s390x
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_s390x
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_s390x
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_s390x
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_s390x
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_s390x
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_s390x
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_s390x
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_s390x
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_s390x
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_s390x
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_s390x
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_s390x
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_s390x
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_s390x
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_s390x
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_s390x
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_s390x
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_s390x
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_s390x
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_s390x
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_s390x
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_s390x
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_s390x
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_s390x
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_s390x
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_s390x
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_s390x
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_s390x
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_s390x
+#define gen_helper_mve_vand gen_helper_mve_vand_s390x
+#define gen_helper_mve_vbic gen_helper_mve_vbic_s390x
+#define gen_helper_mve_vorr gen_helper_mve_vorr_s390x
+#define gen_helper_mve_vorn gen_helper_mve_vorn_s390x
+#define gen_helper_mve_veor gen_helper_mve_veor_s390x
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_s390x
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_s390x
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_s390x
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_s390x
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_s390x
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_s390x
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_s390x
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_s390x
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_s390x
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_s390x
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_s390x
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_s390x
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_s390x
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_s390x
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_s390x
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_s390x
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_s390x
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_s390x
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_s390x
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_s390x
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_s390x
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_s390x
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_s390x
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_s390x
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_s390x
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_s390x
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_s390x
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_s390x
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_s390x
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_s390x
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_s390x
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_s390x
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_s390x
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_s390x
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_s390x
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_s390x
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_s390x
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_s390x
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_s390x
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_s390x
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_s390x
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_s390x
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_s390x
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_s390x
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_s390x
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_s390x
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_s390x
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_s390x
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_s390x
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_s390x
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_s390x
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_s390x
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_s390x
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_s390x
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_s390x
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_s390x
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_s390x
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_s390x
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_s390x
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_s390x
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_s390x
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_s390x
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_s390x
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_s390x
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_s390x
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_s390x
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_s390x
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_s390x
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_s390x
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_s390x
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_s390x
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_s390x
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_s390x
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_s390x
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_s390x
+#define gen_helper_mve_vminub gen_helper_mve_vminub_s390x
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_s390x
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_s390x
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_s390x
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_s390x
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_s390x
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_s390x
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_s390x
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_s390x
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_s390x
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_s390x
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_s390x
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_s390x
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_s390x
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_s390x
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_s390x
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_s390x
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_s390x
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_s390x
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_s390x
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_s390x
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_s390x
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_s390x
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_s390x
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_s390x
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_s390x
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_s390x
+#define gen_helper_mve_vadc gen_helper_mve_vadc_s390x
+#define gen_helper_mve_vadci gen_helper_mve_vadci_s390x
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_s390x
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_s390x
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_s390x
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_s390x
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_s390x
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_s390x
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_s390x
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_s390x
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_s390x
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_s390x
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_s390x
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_s390x
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_s390x
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_s390x
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_s390x
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_s390x
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_s390x
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_s390x
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_s390x
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_s390x
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_s390x
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_s390x
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_s390x
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_s390x
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_s390x
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_s390x
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_s390x
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_s390x
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_s390x
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_s390x
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_s390x
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_s390x
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_s390x
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_s390x
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_s390x
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_s390x
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_s390x
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_s390x
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_s390x
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_s390x
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_s390x
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_s390x
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_s390x
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_s390x
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_s390x
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_s390x
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_s390x
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_s390x
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_s390x
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_s390x
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_s390x
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_s390x
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_s390x
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_s390x
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_s390x
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_s390x
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_s390x
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_s390x
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_s390x
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_s390x
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_s390x
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_s390x
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_s390x
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_s390x
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_s390x
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_s390x
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_s390x
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_s390x
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_s390x
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_s390x
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_s390x
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_s390x
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_s390x
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_s390x
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_s390x
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_s390x
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_s390x
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_s390x
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_s390x
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_s390x
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_s390x
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_s390x
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_s390x
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_s390x
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_s390x
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_s390x
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_s390x
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_s390x
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_s390x
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_s390x
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_s390x
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_s390x
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_s390x
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_s390x
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_s390x
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_s390x
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_s390x
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_s390x
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_s390x
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_s390x
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_s390x
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_s390x
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_s390x
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_s390x
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_s390x
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_s390x
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_s390x
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_s390x
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_s390x
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_s390x
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_s390x
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_s390x
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_s390x
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_s390x
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_s390x
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_s390x
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_s390x
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_s390x
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_s390x
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_s390x
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_s390x
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_s390x
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_s390x
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_s390x
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_s390x
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_s390x
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_s390x
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_s390x
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_s390x
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_s390x
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_s390x
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_s390x
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_s390x
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_s390x
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_s390x
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_s390x
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_s390x
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_s390x
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_s390x
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_s390x
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_s390x
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_s390x
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_s390x
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_s390x
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_s390x
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_s390x
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_s390x
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_s390x
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_s390x
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_s390x
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_s390x
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_s390x
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_s390x
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_s390x
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_s390x
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_s390x
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_s390x
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_s390x
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_s390x
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_s390x
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_s390x
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_s390x
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_s390x
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_s390x
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_s390x
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_s390x
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_s390x
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_s390x
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_s390x
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_s390x
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_s390x
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_s390x
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_s390x
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_s390x
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_s390x
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_s390x
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_s390x
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_s390x
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_s390x
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_s390x
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_s390x
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_s390x
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_s390x
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_s390x
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_s390x
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_s390x
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_s390x
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_s390x
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_s390x
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_s390x
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_s390x
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_s390x
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_s390x
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_s390x
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_s390x
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_s390x
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_s390x
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_s390x
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_s390x
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_s390x
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_s390x
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_s390x
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_s390x
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_s390x
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_s390x
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_s390x
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_s390x
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_s390x
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_s390x
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_s390x
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_s390x
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_s390x
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_s390x
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_s390x
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_s390x
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_s390x
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_s390x
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_s390x
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_s390x
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_s390x
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_s390x
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_s390x
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_s390x
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_s390x
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_s390x
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_s390x
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_s390x
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_s390x
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_s390x
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_s390x
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_s390x
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_s390x
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_s390x
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_s390x
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_s390x
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_s390x
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_s390x
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_s390x
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_s390x
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_s390x
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_s390x
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_s390x
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_s390x
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_s390x
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_s390x
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_s390x
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_s390x
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_s390x
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_s390x
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_s390x
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_s390x
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_s390x
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_s390x
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_s390x
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_s390x
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_s390x
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_s390x
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_s390x
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_s390x
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_s390x
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_s390x
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_s390x
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_s390x
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_s390x
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_s390x
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_s390x
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_s390x
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_s390x
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_s390x
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_s390x
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_s390x
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_s390x
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_s390x
+#define gen_helper_mve_ushll gen_helper_mve_ushll_s390x
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_s390x
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_s390x
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_s390x
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_s390x
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_s390x
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_s390x
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_s390x
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_s390x
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_s390x
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_s390x
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_s390x
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_s390x
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_s390x
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_s390x
+#define gen_helper_mve_vslib gen_helper_mve_vslib_s390x
+#define gen_helper_mve_vslih gen_helper_mve_vslih_s390x
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_s390x
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_s390x
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_s390x
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_s390x
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_s390x
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_s390x
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_s390x
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_s390x
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_s390x
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_s390x
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_s390x
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_s390x
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_s390x
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_s390x
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_s390x
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_s390x
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_s390x
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_s390x
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_s390x
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_s390x
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_s390x
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_s390x
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_s390x
+#define gen_helper_mve_vminab gen_helper_mve_vminab_s390x
+#define gen_helper_mve_vminah gen_helper_mve_vminah_s390x
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_s390x
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_s390x
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_s390x
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_s390x
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_s390x
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_s390x
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_s390x
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_s390x
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_s390x
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_s390x
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_s390x
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_s390x
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_s390x
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_s390x
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_s390x
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_s390x
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_s390x
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_s390x
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_s390x
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_s390x
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_s390x
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_s390x
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_s390x
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_s390x
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_s390x
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_s390x
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_s390x
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_s390x
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_s390x
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_s390x
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_s390x
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_s390x
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_s390x
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_s390x
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_s390x
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_s390x
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_s390x
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_s390x
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_s390x
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_s390x
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_s390x
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_s390x
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_s390x
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_s390x
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_s390x
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_s390x
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_s390x
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_s390x
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_s390x
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_s390x
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_s390x
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_s390x
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_s390x
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_s390x
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_s390x
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_s390x
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_s390x
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_s390x
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_s390x
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_s390x
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_s390x
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_s390x
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_s390x
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_s390x
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_s390x
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_s390x
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_s390x
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_s390x
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_s390x
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_s390x
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_s390x
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_s390x
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_s390x
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_s390x
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_s390x
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_s390x
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_s390x
 #define gen_helper_cpsr_read gen_helper_cpsr_read_s390x
 #define gen_helper_cpsr_write gen_helper_cpsr_write_s390x
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_s390x
diff --git a/qemu/softmmu/cpus.c b/qemu/softmmu/cpus.c
index afb8bbedff..6207dd5da5 100644
--- a/qemu/softmmu/cpus.c
+++ b/qemu/softmmu/cpus.c
@@ -189,15 +189,20 @@ static inline gboolean uc_exit_invalidate_iter(gpointer key, gpointer val, gpoin
     uc_engine *uc = (uc_engine*)data;
     
     if (exit != 0) {
-        // Unicorn: Why addr - 1?
-        // 
-        // 0: INC ecx
-        // 1: DEC edx <--- We put exit here, then the range of TB is [0, 1)
-        //
-        // While tb_invalidate_phys_range invalides [start, end)
-        //
-        // This function is designed to used with g_tree_foreach
-        uc->uc_invalidate_tb(uc, exit - 1, 1);
+        /*
+         * Unicorn: Why addr - 1 through addr?
+         *
+         * 0: INC ecx
+         * 1: DEC edx <--- We put exit here, then the range of TB is [0, 1).
+         *
+         * Nested emulation may also cache a normal TB starting at the active
+         * exit address, so the byte at addr has to be invalidated too.
+         *
+         * While tb_invalidate_phys_range invalidates [start, end).
+         *
+         * This function is designed to be used with g_tree_foreach.
+         */
+        uc->uc_invalidate_tb(uc, exit - 1, 2);
     }
 
     return false;
@@ -224,7 +229,12 @@ void resume_all_vcpus(struct uc_struct* uc)
     if (uc->use_exits) {
         g_tree_foreach(uc->ctl_exits, uc_exit_invalidate_iter, (void*)uc);
     } else {
-        uc_exit_invalidate_iter((gpointer)&uc->exits[uc->nested_level - 1], NULL, (gpointer)uc);
+        int i;
+
+        for (i = 0; i < uc->nested_level; i++) {
+            uc_exit_invalidate_iter((gpointer)&uc->exits[i], NULL,
+                                    (gpointer)uc);
+        }
     }
 
     cpu->created = false;
diff --git a/qemu/softmmu/memory.c b/qemu/softmmu/memory.c
index 08e747e8e1..c2d4244504 100644
--- a/qemu/softmmu/memory.c
+++ b/qemu/softmmu/memory.c
@@ -26,6 +26,8 @@
 
 //#define DEBUG_UNASSIGNED
 
+#define MTE_TAG_STORAGE_GRANULE 32
+
 void memory_region_transaction_begin(void);
 static void memory_region_transaction_commit(MemoryRegion *mr);
 static void memory_region_destructor_container(MemoryRegion *mr);
@@ -94,6 +96,29 @@ static void make_contained(struct uc_struct *uc, MemoryRegion *current)
     memory_region_add_subregion(uc->system_memory, addr, container);
 }
 
+static void memory_cow_copy_mte_tags(RAMBlock *dst, RAMBlock *src,
+                                     hwaddr src_offset, size_t size)
+{
+    ram_addr_t src_tag_offset, tag_size;
+
+    if (!src->mte_tags) {
+        return;
+    }
+
+    src_tag_offset = src_offset / MTE_TAG_STORAGE_GRANULE;
+    if (src_tag_offset >= src->mte_tags_size) {
+        return;
+    }
+
+    tag_size = DIV_ROUND_UP(size, MTE_TAG_STORAGE_GRANULE);
+    tag_size = MIN(tag_size, src->mte_tags_size - src_tag_offset);
+    dst->mte_tags_size = DIV_ROUND_UP(dst->max_length,
+                                      MTE_TAG_STORAGE_GRANULE);
+    dst->mte_tags = g_malloc0(dst->mte_tags_size);
+    memcpy(dst->mte_tags, src->mte_tags + src_tag_offset,
+           MIN(tag_size, dst->mte_tags_size));
+}
+
 MemoryRegion *memory_cow(struct uc_struct *uc, MemoryRegion *current, hwaddr begin, size_t size)
 {
     hwaddr addr;
@@ -117,6 +142,8 @@ MemoryRegion *memory_cow(struct uc_struct *uc, MemoryRegion *current, hwaddr beg
     }
 
     memcpy(ramblock_ptr(ram->ram_block, 0), ramblock_ptr(current->ram_block, current_offset), size);
+    memory_cow_copy_mte_tags(ram->ram_block, current->ram_block,
+                             current_offset, size);
     memory_region_add_subregion_overlap(current->container, offset, ram, uc->snapshot_level);
 
     if (uc->cpu) {
diff --git a/qemu/sparc.h b/qemu/sparc.h
index 5f7c4689b7..76b9eea506 100644
--- a/qemu/sparc.h
+++ b/qemu/sparc.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_sparc
 #define float32_squash_input_denormal float32_squash_input_denormal_sparc
 #define float64_squash_input_denormal float64_squash_input_denormal_sparc
+#define bfloat16_add bfloat16_add_sparc
+#define bfloat16_compare bfloat16_compare_sparc
+#define bfloat16_compare_quiet bfloat16_compare_quiet_sparc
+#define bfloat16_default_nan bfloat16_default_nan_sparc
+#define bfloat16_div bfloat16_div_sparc
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_sparc
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_sparc
+#define bfloat16_max bfloat16_max_sparc
+#define bfloat16_maximum_number bfloat16_maximum_number_sparc
+#define bfloat16_maxnum bfloat16_maxnum_sparc
+#define bfloat16_maxnummag bfloat16_maxnummag_sparc
+#define bfloat16_min bfloat16_min_sparc
+#define bfloat16_minimum_number bfloat16_minimum_number_sparc
+#define bfloat16_minnum bfloat16_minnum_sparc
+#define bfloat16_minnummag bfloat16_minnummag_sparc
+#define bfloat16_mul bfloat16_mul_sparc
+#define bfloat16_muladd bfloat16_muladd_sparc
+#define bfloat16_round_to_int bfloat16_round_to_int_sparc
+#define bfloat16_scalbn bfloat16_scalbn_sparc
+#define bfloat16_silence_nan bfloat16_silence_nan_sparc
+#define bfloat16_sqrt bfloat16_sqrt_sparc
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_sparc
+#define bfloat16_sub bfloat16_sub_sparc
+#define bfloat16_to_float32 bfloat16_to_float32_sparc
+#define bfloat16_to_float64 bfloat16_to_float64_sparc
+#define bfloat16_to_int16 bfloat16_to_int16_sparc
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_sparc
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_sparc
+#define bfloat16_to_int32 bfloat16_to_int32_sparc
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_sparc
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_sparc
+#define bfloat16_to_int64 bfloat16_to_int64_sparc
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_sparc
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_sparc
+#define bfloat16_to_uint16 bfloat16_to_uint16_sparc
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_sparc
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_sparc
+#define bfloat16_to_uint32 bfloat16_to_uint32_sparc
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_sparc
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_sparc
+#define bfloat16_to_uint64 bfloat16_to_uint64_sparc
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_sparc
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_sparc
+#define float128_maximum_number float128_maximum_number_sparc
+#define float128_max float128_max_sparc
+#define float128_maxnum float128_maxnum_sparc
+#define float128_maxnummag float128_maxnummag_sparc
+#define float128_min float128_min_sparc
+#define float128_minimum_number float128_minimum_number_sparc
+#define float128_minnum float128_minnum_sparc
+#define float128_minnummag float128_minnummag_sparc
+#define float128_muladd float128_muladd_sparc
+#define float128_to_int128 float128_to_int128_sparc
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_sparc
+#define float128_to_uint128 float128_to_uint128_sparc
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_sparc
+#define float16_maximum_number float16_maximum_number_sparc
+#define float16_minimum_number float16_minimum_number_sparc
+#define float16_to_int8 float16_to_int8_sparc
+#define float16_to_int8_scalbn float16_to_int8_scalbn_sparc
+#define float16_to_uint8 float16_to_uint8_sparc
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_sparc
+#define float32_maximum_number float32_maximum_number_sparc
+#define float32_minimum_number float32_minimum_number_sparc
+#define float32_to_bfloat16 float32_to_bfloat16_sparc
+#define float64_maximum_number float64_maximum_number_sparc
+#define float64_minimum_number float64_minimum_number_sparc
+#define float64_to_bfloat16 float64_to_bfloat16_sparc
+#define float64r32_add float64r32_add_sparc
+#define float64r32_div float64r32_div_sparc
+#define float64r32_mul float64r32_mul_sparc
+#define float64r32_muladd float64r32_muladd_sparc
+#define float64r32_sqrt float64r32_sqrt_sparc
+#define float64r32_sub float64r32_sub_sparc
+#define floatx80_mod floatx80_mod_sparc
+#define floatx80_modrem floatx80_modrem_sparc
+#define int128_to_float128 int128_to_float128_sparc
+#define int16_to_bfloat16 int16_to_bfloat16_sparc
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_sparc
+#define int32_to_bfloat16 int32_to_bfloat16_sparc
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_sparc
+#define int64_to_bfloat16 int64_to_bfloat16_sparc
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_sparc
+#define int8_to_float16 int8_to_float16_sparc
+#define uint128_to_float128 uint128_to_float128_sparc
+#define uint16_to_bfloat16 uint16_to_bfloat16_sparc
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_sparc
+#define uint32_to_bfloat16 uint32_to_bfloat16_sparc
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_sparc
+#define uint64_to_bfloat16 uint64_to_bfloat16_sparc
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_sparc
+#define uint8_to_float16 uint8_to_float16_sparc
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_sparc
 #define roundAndPackFloatx80 roundAndPackFloatx80_sparc
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_sparc
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_sparc
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_sparc
 #define helper_exit_atomic helper_exit_atomic_sparc
+#define helper_memset helper_memset_sparc
+#define helper_emu_stop helper_emu_stop_sparc
+#define tcg_remove_ops_after tcg_remove_ops_after_sparc
+#define tcg_constant_vec_matching tcg_constant_vec_matching_sparc
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_sparc
 #define helper_gvec_add8 helper_gvec_add8_sparc
 #define helper_gvec_add16 helper_gvec_add16_sparc
 #define helper_gvec_add32 helper_gvec_add32_sparc
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_sparc
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_sparc
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_sparc
+#define gen_helper_mve_vctp gen_helper_mve_vctp_sparc
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_sparc
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_sparc
+#define gen_helper_mve_vdup gen_helper_mve_vdup_sparc
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_sparc
+#define gen_helper_mve_vandi gen_helper_mve_vandi_sparc
+#define gen_helper_mve_vorri gen_helper_mve_vorri_sparc
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_sparc
+#define gen_helper_mve_viduph gen_helper_mve_viduph_sparc
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_sparc
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_sparc
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_sparc
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_sparc
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_sparc
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_sparc
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_sparc
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_sparc
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_sparc
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_sparc
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_sparc
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_sparc
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_sparc
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_sparc
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_sparc
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_sparc
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_sparc
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_sparc
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_sparc
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_sparc
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_sparc
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_sparc
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_sparc
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_sparc
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_sparc
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_sparc
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_sparc
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_sparc
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_sparc
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_sparc
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_sparc
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_sparc
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_sparc
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_sparc
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_sparc
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_sparc
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_sparc
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_sparc
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_sparc
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_sparc
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_sparc
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_sparc
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_sparc
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_sparc
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_sparc
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_sparc
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_sparc
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_sparc
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_sparc
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_sparc
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_sparc
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_sparc
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_sparc
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_sparc
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_sparc
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_sparc
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_sparc
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_sparc
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_sparc
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_sparc
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_sparc
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_sparc
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_sparc
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_sparc
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_sparc
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_sparc
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_sparc
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_sparc
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_sparc
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_sparc
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_sparc
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_sparc
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_sparc
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_sparc
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_sparc
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_sparc
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_sparc
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_sparc
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_sparc
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_sparc
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_sparc
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_sparc
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_sparc
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_sparc
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_sparc
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_sparc
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_sparc
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_sparc
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_sparc
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_sparc
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_sparc
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_sparc
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_sparc
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_sparc
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_sparc
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_sparc
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_sparc
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_sparc
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_sparc
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_sparc
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_sparc
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_sparc
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_sparc
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_sparc
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_sparc
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_sparc
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_sparc
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_sparc
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_sparc
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_sparc
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_sparc
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_sparc
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_sparc
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_sparc
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_sparc
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_sparc
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_sparc
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_sparc
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_sparc
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_sparc
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_sparc
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_sparc
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_sparc
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_sparc
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_sparc
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_sparc
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_sparc
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_sparc
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_sparc
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_sparc
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_sparc
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_sparc
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_sparc
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_sparc
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_sparc
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_sparc
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_sparc
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_sparc
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_sparc
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_sparc
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_sparc
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_sparc
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_sparc
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_sparc
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_sparc
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_sparc
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_sparc
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_sparc
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_sparc
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_sparc
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_sparc
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_sparc
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_sparc
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_sparc
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_sparc
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_sparc
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_sparc
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_sparc
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_sparc
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_sparc
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_sparc
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_sparc
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_sparc
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_sparc
+#define gen_helper_mve_vand gen_helper_mve_vand_sparc
+#define gen_helper_mve_vbic gen_helper_mve_vbic_sparc
+#define gen_helper_mve_vorr gen_helper_mve_vorr_sparc
+#define gen_helper_mve_vorn gen_helper_mve_vorn_sparc
+#define gen_helper_mve_veor gen_helper_mve_veor_sparc
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_sparc
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_sparc
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_sparc
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_sparc
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_sparc
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_sparc
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_sparc
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_sparc
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_sparc
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_sparc
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_sparc
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_sparc
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_sparc
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_sparc
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_sparc
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_sparc
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_sparc
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_sparc
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_sparc
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_sparc
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_sparc
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_sparc
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_sparc
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_sparc
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_sparc
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_sparc
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_sparc
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_sparc
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_sparc
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_sparc
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_sparc
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_sparc
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_sparc
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_sparc
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_sparc
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_sparc
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_sparc
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_sparc
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_sparc
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_sparc
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_sparc
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_sparc
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_sparc
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_sparc
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_sparc
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_sparc
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_sparc
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_sparc
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_sparc
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_sparc
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_sparc
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_sparc
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_sparc
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_sparc
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_sparc
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_sparc
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_sparc
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_sparc
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_sparc
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_sparc
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_sparc
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_sparc
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_sparc
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_sparc
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_sparc
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_sparc
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_sparc
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_sparc
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_sparc
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_sparc
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_sparc
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_sparc
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_sparc
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_sparc
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_sparc
+#define gen_helper_mve_vminub gen_helper_mve_vminub_sparc
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_sparc
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_sparc
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_sparc
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_sparc
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_sparc
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_sparc
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_sparc
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_sparc
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_sparc
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_sparc
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_sparc
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_sparc
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_sparc
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_sparc
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_sparc
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_sparc
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_sparc
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_sparc
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_sparc
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_sparc
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_sparc
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_sparc
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_sparc
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_sparc
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_sparc
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_sparc
+#define gen_helper_mve_vadc gen_helper_mve_vadc_sparc
+#define gen_helper_mve_vadci gen_helper_mve_vadci_sparc
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_sparc
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_sparc
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_sparc
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_sparc
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_sparc
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_sparc
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_sparc
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_sparc
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_sparc
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_sparc
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_sparc
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_sparc
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_sparc
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_sparc
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_sparc
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_sparc
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_sparc
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_sparc
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_sparc
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_sparc
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_sparc
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_sparc
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_sparc
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_sparc
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_sparc
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_sparc
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_sparc
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_sparc
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_sparc
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_sparc
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_sparc
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_sparc
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_sparc
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_sparc
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_sparc
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_sparc
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_sparc
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_sparc
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_sparc
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_sparc
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_sparc
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_sparc
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_sparc
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_sparc
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_sparc
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_sparc
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_sparc
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_sparc
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_sparc
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_sparc
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_sparc
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_sparc
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_sparc
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_sparc
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_sparc
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_sparc
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_sparc
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_sparc
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_sparc
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_sparc
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_sparc
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_sparc
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_sparc
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_sparc
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_sparc
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_sparc
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_sparc
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_sparc
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_sparc
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_sparc
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_sparc
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_sparc
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_sparc
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_sparc
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_sparc
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_sparc
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_sparc
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_sparc
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_sparc
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_sparc
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_sparc
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_sparc
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_sparc
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_sparc
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_sparc
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_sparc
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_sparc
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_sparc
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_sparc
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_sparc
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_sparc
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_sparc
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_sparc
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_sparc
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_sparc
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_sparc
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_sparc
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_sparc
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_sparc
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_sparc
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_sparc
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_sparc
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_sparc
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_sparc
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_sparc
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_sparc
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_sparc
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_sparc
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_sparc
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_sparc
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_sparc
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_sparc
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_sparc
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_sparc
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_sparc
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_sparc
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_sparc
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_sparc
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_sparc
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_sparc
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_sparc
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_sparc
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_sparc
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_sparc
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_sparc
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_sparc
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_sparc
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_sparc
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_sparc
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_sparc
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_sparc
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_sparc
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_sparc
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_sparc
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_sparc
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_sparc
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_sparc
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_sparc
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_sparc
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_sparc
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_sparc
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_sparc
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_sparc
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_sparc
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_sparc
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_sparc
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_sparc
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_sparc
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_sparc
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_sparc
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_sparc
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_sparc
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_sparc
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_sparc
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_sparc
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_sparc
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_sparc
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_sparc
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_sparc
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_sparc
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_sparc
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_sparc
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_sparc
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_sparc
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_sparc
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_sparc
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_sparc
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_sparc
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_sparc
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_sparc
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_sparc
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_sparc
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_sparc
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_sparc
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_sparc
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_sparc
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_sparc
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_sparc
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_sparc
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_sparc
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_sparc
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_sparc
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_sparc
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_sparc
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_sparc
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_sparc
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_sparc
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_sparc
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_sparc
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_sparc
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_sparc
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_sparc
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_sparc
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_sparc
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_sparc
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_sparc
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_sparc
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_sparc
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_sparc
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_sparc
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_sparc
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_sparc
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_sparc
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_sparc
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_sparc
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_sparc
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_sparc
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_sparc
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_sparc
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_sparc
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_sparc
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_sparc
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_sparc
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_sparc
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_sparc
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_sparc
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_sparc
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_sparc
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_sparc
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_sparc
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_sparc
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_sparc
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_sparc
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_sparc
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_sparc
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_sparc
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_sparc
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_sparc
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_sparc
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_sparc
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_sparc
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_sparc
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_sparc
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_sparc
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_sparc
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_sparc
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_sparc
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_sparc
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_sparc
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_sparc
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_sparc
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_sparc
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_sparc
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_sparc
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_sparc
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_sparc
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_sparc
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_sparc
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_sparc
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_sparc
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_sparc
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_sparc
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_sparc
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_sparc
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_sparc
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_sparc
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_sparc
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_sparc
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_sparc
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_sparc
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_sparc
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_sparc
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_sparc
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_sparc
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_sparc
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_sparc
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_sparc
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_sparc
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_sparc
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_sparc
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_sparc
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_sparc
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_sparc
+#define gen_helper_mve_ushll gen_helper_mve_ushll_sparc
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_sparc
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_sparc
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_sparc
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_sparc
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_sparc
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_sparc
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_sparc
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_sparc
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_sparc
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_sparc
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_sparc
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_sparc
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_sparc
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_sparc
+#define gen_helper_mve_vslib gen_helper_mve_vslib_sparc
+#define gen_helper_mve_vslih gen_helper_mve_vslih_sparc
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_sparc
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_sparc
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_sparc
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_sparc
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_sparc
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_sparc
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_sparc
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_sparc
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_sparc
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_sparc
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_sparc
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_sparc
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_sparc
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_sparc
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_sparc
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_sparc
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_sparc
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_sparc
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_sparc
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_sparc
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_sparc
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_sparc
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_sparc
+#define gen_helper_mve_vminab gen_helper_mve_vminab_sparc
+#define gen_helper_mve_vminah gen_helper_mve_vminah_sparc
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_sparc
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_sparc
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_sparc
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_sparc
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_sparc
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_sparc
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_sparc
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_sparc
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_sparc
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_sparc
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_sparc
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_sparc
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_sparc
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_sparc
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_sparc
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_sparc
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_sparc
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_sparc
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_sparc
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_sparc
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_sparc
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_sparc
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_sparc
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_sparc
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_sparc
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_sparc
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_sparc
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_sparc
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_sparc
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_sparc
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_sparc
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_sparc
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_sparc
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_sparc
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_sparc
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_sparc
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_sparc
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_sparc
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_sparc
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_sparc
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_sparc
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_sparc
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_sparc
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_sparc
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_sparc
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_sparc
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_sparc
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_sparc
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_sparc
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_sparc
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_sparc
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_sparc
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_sparc
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_sparc
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_sparc
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_sparc
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_sparc
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_sparc
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_sparc
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_sparc
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_sparc
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_sparc
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_sparc
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_sparc
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_sparc
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_sparc
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_sparc
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_sparc
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_sparc
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_sparc
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_sparc
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_sparc
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_sparc
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_sparc
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_sparc
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_sparc
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_sparc
 #define gen_helper_cpsr_read gen_helper_cpsr_read_sparc
 #define gen_helper_cpsr_write gen_helper_cpsr_write_sparc
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_sparc
diff --git a/qemu/sparc64.h b/qemu/sparc64.h
index 33003493db..4d4b0317aa 100644
--- a/qemu/sparc64.h
+++ b/qemu/sparc64.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_sparc64
 #define float32_squash_input_denormal float32_squash_input_denormal_sparc64
 #define float64_squash_input_denormal float64_squash_input_denormal_sparc64
+#define bfloat16_add bfloat16_add_sparc64
+#define bfloat16_compare bfloat16_compare_sparc64
+#define bfloat16_compare_quiet bfloat16_compare_quiet_sparc64
+#define bfloat16_default_nan bfloat16_default_nan_sparc64
+#define bfloat16_div bfloat16_div_sparc64
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_sparc64
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_sparc64
+#define bfloat16_max bfloat16_max_sparc64
+#define bfloat16_maximum_number bfloat16_maximum_number_sparc64
+#define bfloat16_maxnum bfloat16_maxnum_sparc64
+#define bfloat16_maxnummag bfloat16_maxnummag_sparc64
+#define bfloat16_min bfloat16_min_sparc64
+#define bfloat16_minimum_number bfloat16_minimum_number_sparc64
+#define bfloat16_minnum bfloat16_minnum_sparc64
+#define bfloat16_minnummag bfloat16_minnummag_sparc64
+#define bfloat16_mul bfloat16_mul_sparc64
+#define bfloat16_muladd bfloat16_muladd_sparc64
+#define bfloat16_round_to_int bfloat16_round_to_int_sparc64
+#define bfloat16_scalbn bfloat16_scalbn_sparc64
+#define bfloat16_silence_nan bfloat16_silence_nan_sparc64
+#define bfloat16_sqrt bfloat16_sqrt_sparc64
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_sparc64
+#define bfloat16_sub bfloat16_sub_sparc64
+#define bfloat16_to_float32 bfloat16_to_float32_sparc64
+#define bfloat16_to_float64 bfloat16_to_float64_sparc64
+#define bfloat16_to_int16 bfloat16_to_int16_sparc64
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_sparc64
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_sparc64
+#define bfloat16_to_int32 bfloat16_to_int32_sparc64
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_sparc64
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_sparc64
+#define bfloat16_to_int64 bfloat16_to_int64_sparc64
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_sparc64
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_sparc64
+#define bfloat16_to_uint16 bfloat16_to_uint16_sparc64
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_sparc64
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_sparc64
+#define bfloat16_to_uint32 bfloat16_to_uint32_sparc64
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_sparc64
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_sparc64
+#define bfloat16_to_uint64 bfloat16_to_uint64_sparc64
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_sparc64
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_sparc64
+#define float128_maximum_number float128_maximum_number_sparc64
+#define float128_max float128_max_sparc64
+#define float128_maxnum float128_maxnum_sparc64
+#define float128_maxnummag float128_maxnummag_sparc64
+#define float128_min float128_min_sparc64
+#define float128_minimum_number float128_minimum_number_sparc64
+#define float128_minnum float128_minnum_sparc64
+#define float128_minnummag float128_minnummag_sparc64
+#define float128_muladd float128_muladd_sparc64
+#define float128_to_int128 float128_to_int128_sparc64
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_sparc64
+#define float128_to_uint128 float128_to_uint128_sparc64
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_sparc64
+#define float16_maximum_number float16_maximum_number_sparc64
+#define float16_minimum_number float16_minimum_number_sparc64
+#define float16_to_int8 float16_to_int8_sparc64
+#define float16_to_int8_scalbn float16_to_int8_scalbn_sparc64
+#define float16_to_uint8 float16_to_uint8_sparc64
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_sparc64
+#define float32_maximum_number float32_maximum_number_sparc64
+#define float32_minimum_number float32_minimum_number_sparc64
+#define float32_to_bfloat16 float32_to_bfloat16_sparc64
+#define float64_maximum_number float64_maximum_number_sparc64
+#define float64_minimum_number float64_minimum_number_sparc64
+#define float64_to_bfloat16 float64_to_bfloat16_sparc64
+#define float64r32_add float64r32_add_sparc64
+#define float64r32_div float64r32_div_sparc64
+#define float64r32_mul float64r32_mul_sparc64
+#define float64r32_muladd float64r32_muladd_sparc64
+#define float64r32_sqrt float64r32_sqrt_sparc64
+#define float64r32_sub float64r32_sub_sparc64
+#define floatx80_mod floatx80_mod_sparc64
+#define floatx80_modrem floatx80_modrem_sparc64
+#define int128_to_float128 int128_to_float128_sparc64
+#define int16_to_bfloat16 int16_to_bfloat16_sparc64
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_sparc64
+#define int32_to_bfloat16 int32_to_bfloat16_sparc64
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_sparc64
+#define int64_to_bfloat16 int64_to_bfloat16_sparc64
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_sparc64
+#define int8_to_float16 int8_to_float16_sparc64
+#define uint128_to_float128 uint128_to_float128_sparc64
+#define uint16_to_bfloat16 uint16_to_bfloat16_sparc64
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_sparc64
+#define uint32_to_bfloat16 uint32_to_bfloat16_sparc64
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_sparc64
+#define uint64_to_bfloat16 uint64_to_bfloat16_sparc64
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_sparc64
+#define uint8_to_float16 uint8_to_float16_sparc64
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_sparc64
 #define roundAndPackFloatx80 roundAndPackFloatx80_sparc64
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_sparc64
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_sparc64
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_sparc64
 #define helper_exit_atomic helper_exit_atomic_sparc64
+#define helper_memset helper_memset_sparc64
+#define helper_emu_stop helper_emu_stop_sparc64
+#define tcg_remove_ops_after tcg_remove_ops_after_sparc64
+#define tcg_constant_vec_matching tcg_constant_vec_matching_sparc64
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_sparc64
 #define helper_gvec_add8 helper_gvec_add8_sparc64
 #define helper_gvec_add16 helper_gvec_add16_sparc64
 #define helper_gvec_add32 helper_gvec_add32_sparc64
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_sparc64
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_sparc64
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_sparc64
+#define gen_helper_mve_vctp gen_helper_mve_vctp_sparc64
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_sparc64
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_sparc64
+#define gen_helper_mve_vdup gen_helper_mve_vdup_sparc64
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_sparc64
+#define gen_helper_mve_vandi gen_helper_mve_vandi_sparc64
+#define gen_helper_mve_vorri gen_helper_mve_vorri_sparc64
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_sparc64
+#define gen_helper_mve_viduph gen_helper_mve_viduph_sparc64
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_sparc64
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_sparc64
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_sparc64
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_sparc64
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_sparc64
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_sparc64
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_sparc64
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_sparc64
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_sparc64
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_sparc64
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_sparc64
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_sparc64
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_sparc64
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_sparc64
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_sparc64
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_sparc64
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_sparc64
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_sparc64
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_sparc64
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_sparc64
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_sparc64
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_sparc64
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_sparc64
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_sparc64
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_sparc64
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_sparc64
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_sparc64
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_sparc64
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_sparc64
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_sparc64
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_sparc64
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_sparc64
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_sparc64
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_sparc64
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_sparc64
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_sparc64
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_sparc64
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_sparc64
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_sparc64
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_sparc64
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_sparc64
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_sparc64
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_sparc64
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_sparc64
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_sparc64
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_sparc64
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_sparc64
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_sparc64
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_sparc64
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_sparc64
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_sparc64
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_sparc64
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_sparc64
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_sparc64
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_sparc64
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_sparc64
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_sparc64
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_sparc64
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_sparc64
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_sparc64
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_sparc64
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_sparc64
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_sparc64
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_sparc64
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_sparc64
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_sparc64
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_sparc64
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_sparc64
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_sparc64
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_sparc64
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_sparc64
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_sparc64
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_sparc64
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_sparc64
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_sparc64
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_sparc64
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_sparc64
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_sparc64
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_sparc64
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_sparc64
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_sparc64
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_sparc64
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_sparc64
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_sparc64
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_sparc64
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_sparc64
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_sparc64
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_sparc64
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_sparc64
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_sparc64
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_sparc64
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_sparc64
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_sparc64
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_sparc64
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_sparc64
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_sparc64
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_sparc64
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_sparc64
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_sparc64
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_sparc64
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_sparc64
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_sparc64
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_sparc64
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_sparc64
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_sparc64
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_sparc64
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_sparc64
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_sparc64
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_sparc64
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_sparc64
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_sparc64
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_sparc64
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_sparc64
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_sparc64
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_sparc64
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_sparc64
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_sparc64
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_sparc64
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_sparc64
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_sparc64
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_sparc64
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_sparc64
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_sparc64
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_sparc64
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_sparc64
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_sparc64
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_sparc64
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_sparc64
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_sparc64
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_sparc64
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_sparc64
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_sparc64
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_sparc64
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_sparc64
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_sparc64
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_sparc64
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_sparc64
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_sparc64
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_sparc64
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_sparc64
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_sparc64
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_sparc64
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_sparc64
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_sparc64
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_sparc64
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_sparc64
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_sparc64
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_sparc64
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_sparc64
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_sparc64
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_sparc64
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_sparc64
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_sparc64
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_sparc64
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_sparc64
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_sparc64
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_sparc64
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_sparc64
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_sparc64
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_sparc64
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_sparc64
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_sparc64
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_sparc64
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_sparc64
+#define gen_helper_mve_vand gen_helper_mve_vand_sparc64
+#define gen_helper_mve_vbic gen_helper_mve_vbic_sparc64
+#define gen_helper_mve_vorr gen_helper_mve_vorr_sparc64
+#define gen_helper_mve_vorn gen_helper_mve_vorn_sparc64
+#define gen_helper_mve_veor gen_helper_mve_veor_sparc64
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_sparc64
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_sparc64
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_sparc64
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_sparc64
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_sparc64
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_sparc64
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_sparc64
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_sparc64
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_sparc64
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_sparc64
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_sparc64
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_sparc64
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_sparc64
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_sparc64
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_sparc64
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_sparc64
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_sparc64
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_sparc64
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_sparc64
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_sparc64
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_sparc64
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_sparc64
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_sparc64
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_sparc64
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_sparc64
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_sparc64
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_sparc64
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_sparc64
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_sparc64
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_sparc64
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_sparc64
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_sparc64
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_sparc64
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_sparc64
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_sparc64
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_sparc64
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_sparc64
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_sparc64
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_sparc64
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_sparc64
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_sparc64
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_sparc64
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_sparc64
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_sparc64
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_sparc64
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_sparc64
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_sparc64
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_sparc64
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_sparc64
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_sparc64
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_sparc64
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_sparc64
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_sparc64
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_sparc64
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_sparc64
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_sparc64
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_sparc64
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_sparc64
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_sparc64
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_sparc64
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_sparc64
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_sparc64
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_sparc64
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_sparc64
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_sparc64
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_sparc64
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_sparc64
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_sparc64
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_sparc64
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_sparc64
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_sparc64
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_sparc64
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_sparc64
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_sparc64
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_sparc64
+#define gen_helper_mve_vminub gen_helper_mve_vminub_sparc64
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_sparc64
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_sparc64
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_sparc64
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_sparc64
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_sparc64
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_sparc64
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_sparc64
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_sparc64
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_sparc64
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_sparc64
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_sparc64
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_sparc64
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_sparc64
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_sparc64
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_sparc64
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_sparc64
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_sparc64
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_sparc64
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_sparc64
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_sparc64
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_sparc64
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_sparc64
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_sparc64
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_sparc64
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_sparc64
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_sparc64
+#define gen_helper_mve_vadc gen_helper_mve_vadc_sparc64
+#define gen_helper_mve_vadci gen_helper_mve_vadci_sparc64
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_sparc64
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_sparc64
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_sparc64
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_sparc64
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_sparc64
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_sparc64
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_sparc64
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_sparc64
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_sparc64
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_sparc64
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_sparc64
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_sparc64
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_sparc64
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_sparc64
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_sparc64
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_sparc64
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_sparc64
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_sparc64
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_sparc64
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_sparc64
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_sparc64
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_sparc64
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_sparc64
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_sparc64
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_sparc64
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_sparc64
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_sparc64
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_sparc64
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_sparc64
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_sparc64
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_sparc64
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_sparc64
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_sparc64
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_sparc64
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_sparc64
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_sparc64
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_sparc64
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_sparc64
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_sparc64
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_sparc64
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_sparc64
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_sparc64
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_sparc64
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_sparc64
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_sparc64
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_sparc64
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_sparc64
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_sparc64
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_sparc64
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_sparc64
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_sparc64
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_sparc64
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_sparc64
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_sparc64
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_sparc64
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_sparc64
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_sparc64
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_sparc64
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_sparc64
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_sparc64
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_sparc64
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_sparc64
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_sparc64
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_sparc64
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_sparc64
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_sparc64
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_sparc64
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_sparc64
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_sparc64
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_sparc64
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_sparc64
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_sparc64
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_sparc64
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_sparc64
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_sparc64
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_sparc64
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_sparc64
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_sparc64
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_sparc64
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_sparc64
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_sparc64
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_sparc64
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_sparc64
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_sparc64
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_sparc64
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_sparc64
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_sparc64
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_sparc64
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_sparc64
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_sparc64
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_sparc64
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_sparc64
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_sparc64
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_sparc64
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_sparc64
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_sparc64
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_sparc64
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_sparc64
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_sparc64
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_sparc64
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_sparc64
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_sparc64
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_sparc64
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_sparc64
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_sparc64
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_sparc64
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_sparc64
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_sparc64
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_sparc64
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_sparc64
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_sparc64
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_sparc64
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_sparc64
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_sparc64
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_sparc64
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_sparc64
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_sparc64
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_sparc64
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_sparc64
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_sparc64
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_sparc64
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_sparc64
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_sparc64
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_sparc64
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_sparc64
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_sparc64
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_sparc64
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_sparc64
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_sparc64
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_sparc64
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_sparc64
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_sparc64
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_sparc64
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_sparc64
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_sparc64
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_sparc64
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_sparc64
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_sparc64
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_sparc64
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_sparc64
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_sparc64
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_sparc64
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_sparc64
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_sparc64
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_sparc64
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_sparc64
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_sparc64
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_sparc64
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_sparc64
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_sparc64
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_sparc64
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_sparc64
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_sparc64
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_sparc64
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_sparc64
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_sparc64
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_sparc64
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_sparc64
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_sparc64
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_sparc64
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_sparc64
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_sparc64
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_sparc64
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_sparc64
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_sparc64
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_sparc64
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_sparc64
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_sparc64
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_sparc64
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_sparc64
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_sparc64
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_sparc64
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_sparc64
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_sparc64
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_sparc64
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_sparc64
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_sparc64
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_sparc64
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_sparc64
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_sparc64
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_sparc64
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_sparc64
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_sparc64
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_sparc64
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_sparc64
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_sparc64
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_sparc64
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_sparc64
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_sparc64
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_sparc64
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_sparc64
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_sparc64
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_sparc64
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_sparc64
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_sparc64
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_sparc64
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_sparc64
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_sparc64
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_sparc64
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_sparc64
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_sparc64
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_sparc64
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_sparc64
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_sparc64
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_sparc64
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_sparc64
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_sparc64
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_sparc64
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_sparc64
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_sparc64
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_sparc64
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_sparc64
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_sparc64
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_sparc64
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_sparc64
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_sparc64
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_sparc64
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_sparc64
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_sparc64
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_sparc64
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_sparc64
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_sparc64
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_sparc64
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_sparc64
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_sparc64
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_sparc64
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_sparc64
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_sparc64
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_sparc64
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_sparc64
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_sparc64
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_sparc64
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_sparc64
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_sparc64
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_sparc64
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_sparc64
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_sparc64
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_sparc64
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_sparc64
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_sparc64
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_sparc64
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_sparc64
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_sparc64
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_sparc64
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_sparc64
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_sparc64
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_sparc64
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_sparc64
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_sparc64
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_sparc64
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_sparc64
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_sparc64
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_sparc64
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_sparc64
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_sparc64
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_sparc64
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_sparc64
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_sparc64
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_sparc64
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_sparc64
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_sparc64
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_sparc64
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_sparc64
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_sparc64
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_sparc64
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_sparc64
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_sparc64
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_sparc64
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_sparc64
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_sparc64
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_sparc64
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_sparc64
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_sparc64
+#define gen_helper_mve_ushll gen_helper_mve_ushll_sparc64
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_sparc64
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_sparc64
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_sparc64
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_sparc64
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_sparc64
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_sparc64
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_sparc64
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_sparc64
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_sparc64
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_sparc64
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_sparc64
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_sparc64
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_sparc64
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_sparc64
+#define gen_helper_mve_vslib gen_helper_mve_vslib_sparc64
+#define gen_helper_mve_vslih gen_helper_mve_vslih_sparc64
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_sparc64
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_sparc64
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_sparc64
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_sparc64
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_sparc64
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_sparc64
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_sparc64
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_sparc64
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_sparc64
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_sparc64
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_sparc64
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_sparc64
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_sparc64
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_sparc64
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_sparc64
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_sparc64
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_sparc64
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_sparc64
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_sparc64
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_sparc64
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_sparc64
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_sparc64
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_sparc64
+#define gen_helper_mve_vminab gen_helper_mve_vminab_sparc64
+#define gen_helper_mve_vminah gen_helper_mve_vminah_sparc64
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_sparc64
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_sparc64
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_sparc64
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_sparc64
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_sparc64
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_sparc64
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_sparc64
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_sparc64
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_sparc64
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_sparc64
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_sparc64
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_sparc64
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_sparc64
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_sparc64
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_sparc64
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_sparc64
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_sparc64
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_sparc64
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_sparc64
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_sparc64
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_sparc64
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_sparc64
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_sparc64
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_sparc64
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_sparc64
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_sparc64
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_sparc64
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_sparc64
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_sparc64
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_sparc64
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_sparc64
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_sparc64
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_sparc64
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_sparc64
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_sparc64
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_sparc64
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_sparc64
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_sparc64
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_sparc64
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_sparc64
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_sparc64
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_sparc64
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_sparc64
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_sparc64
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_sparc64
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_sparc64
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_sparc64
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_sparc64
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_sparc64
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_sparc64
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_sparc64
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_sparc64
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_sparc64
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_sparc64
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_sparc64
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_sparc64
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_sparc64
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_sparc64
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_sparc64
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_sparc64
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_sparc64
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_sparc64
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_sparc64
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_sparc64
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_sparc64
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_sparc64
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_sparc64
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_sparc64
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_sparc64
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_sparc64
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_sparc64
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_sparc64
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_sparc64
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_sparc64
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_sparc64
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_sparc64
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_sparc64
 #define gen_helper_cpsr_read gen_helper_cpsr_read_sparc64
 #define gen_helper_cpsr_write gen_helper_cpsr_write_sparc64
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_sparc64
diff --git a/qemu/target/arm/cpu.c b/qemu/target/arm/cpu.c
index b1a813ba8c..2f4745efec 100644
--- a/qemu/target/arm/cpu.c
+++ b/qemu/target/arm/cpu.c
@@ -200,12 +200,42 @@ static void arm_cpu_reset(CPUState *dev)
     }
     env->daif = PSTATE_D | PSTATE_A | PSTATE_I | PSTATE_F;
 
+#ifdef TARGET_AARCH64
+    if (cpu_isar_feature(aa64_sme, cpu)) {
+        uint32_t vq = cpu->sme_default_vq ? cpu->sme_default_vq : 1;
+        uint64_t smcr = vq - 1;
+
+        if (cpu_isar_feature(aa64_sme_fa64, cpu)) {
+            FIELD_DP64(smcr, SMCR, FA64, 1, smcr);
+        }
+        env->vfp.smcr_el[1] = smcr;
+        if (arm_feature(env, ARM_FEATURE_EL2)) {
+            env->vfp.smcr_el[2] = smcr;
+        }
+        if (arm_feature(env, ARM_FEATURE_EL3)) {
+            env->vfp.smcr_el[3] = smcr;
+            FIELD_DP64(env->cp15.cptr_el[3], CPTR_EL3, ESM, 1,
+                       env->cp15.cptr_el[3]);
+        }
+        FIELD_DP64(env->cp15.cpacr_el1, CPACR_EL1, SMEN, 3,
+                   env->cp15.cpacr_el1);
+        FIELD_DP64(env->cp15.cptr_el[2], CPTR_EL2, SMEN, 3,
+                   env->cp15.cptr_el[2]);
+    }
+#endif
+
     if (arm_feature(env, ARM_FEATURE_M)) {
         uint32_t initial_msp; /* Loaded from 0x0 */
         uint32_t initial_pc; /* Loaded from 0x4 */
         // uint8_t *rom;
         uint32_t vecbase;
 
+        if (arm_feature(env, ARM_FEATURE_V8_1M)) {
+            env->v7m.ltpsize = 4;
+            env->v7m.fpdscr[M_REG_NS] = 4 << FPCR_LTPSIZE_SHIFT;
+            env->v7m.fpdscr[M_REG_S] = 4 << FPCR_LTPSIZE_SHIFT;
+        }
+
         if (arm_feature(env, ARM_FEATURE_M_SECURITY)) {
             env->v7m.secure = true;
         } else {
@@ -348,12 +378,15 @@ static void arm_cpu_reset(CPUState *dev)
     set_flush_to_zero(1, &env->vfp.standard_fp_status);
     set_flush_inputs_to_zero(1, &env->vfp.standard_fp_status);
     set_default_nan_mode(1, &env->vfp.standard_fp_status);
+    set_default_nan_mode(1, &env->vfp.standard_fp_status_f16);
     set_float_detect_tininess(float_tininess_before_rounding,
                               &env->vfp.fp_status);
     set_float_detect_tininess(float_tininess_before_rounding,
                               &env->vfp.standard_fp_status);
     set_float_detect_tininess(float_tininess_before_rounding,
                               &env->vfp.fp_status_f16);
+    set_float_detect_tininess(float_tininess_before_rounding,
+                              &env->vfp.standard_fp_status_f16);
 
     hw_breakpoint_update_all(cpu);
     hw_watchpoint_update_all(cpu);
@@ -812,6 +845,7 @@ void arm_cpu_realizefn(struct uc_struct *uc, CPUState *dev)
 
         u = cpu->isar.id_isar6;
         FIELD_DP32(u, ID_ISAR6, JSCVT, 0, u);
+        FIELD_DP32(u, ID_ISAR6, BF16, 0, u);
         cpu->isar.id_isar6 = u;
 
         u = cpu->isar.mvfr0;
@@ -861,18 +895,21 @@ void arm_cpu_realizefn(struct uc_struct *uc, CPUState *dev)
         u = cpu->isar.id_isar6;
         FIELD_DP32(u, ID_ISAR6, DP, 0, u);
         FIELD_DP32(u, ID_ISAR6, FHM, 0, u);
+        FIELD_DP32(u, ID_ISAR6, BF16, 0, u);
         cpu->isar.id_isar6 = u;
 
-        u = cpu->isar.mvfr1;
-        FIELD_DP32(u, MVFR1, SIMDLS, 0, u);
-        FIELD_DP32(u, MVFR1, SIMDINT, 0, u);
-        FIELD_DP32(u, MVFR1, SIMDSP, 0, u);
-        FIELD_DP32(u, MVFR1, SIMDHP, 0, u);
-        cpu->isar.mvfr1 = u;
-
-        u = cpu->isar.mvfr2;
-        FIELD_DP32(u, MVFR2, SIMDMISC, 0, u);
-        cpu->isar.mvfr2 = u;
+        if (!arm_feature(env, ARM_FEATURE_M)) {
+            u = cpu->isar.mvfr1;
+            FIELD_DP32(u, MVFR1, SIMDLS, 0, u);
+            FIELD_DP32(u, MVFR1, SIMDINT, 0, u);
+            FIELD_DP32(u, MVFR1, SIMDSP, 0, u);
+            FIELD_DP32(u, MVFR1, SIMDHP, 0, u);
+            cpu->isar.mvfr1 = u;
+
+            u = cpu->isar.mvfr2;
+            FIELD_DP32(u, MVFR2, SIMDMISC, 0, u);
+            cpu->isar.mvfr2 = u;
+        }
     }
 
     if (!cpu->has_neon && !cpu->has_vfp) {
@@ -1508,6 +1545,42 @@ static void cortex_m33_initfn(struct uc_struct *uc, CPUState *obj)
     cpu->ctr = 0x8000c000;
 }
 
+static void cortex_m55_initfn(struct uc_struct *uc, CPUState *obj)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+
+    set_feature(&cpu->env, ARM_FEATURE_V8);
+    set_feature(&cpu->env, ARM_FEATURE_V8_1M);
+    set_feature(&cpu->env, ARM_FEATURE_M);
+    set_feature(&cpu->env, ARM_FEATURE_M_MAIN);
+    set_feature(&cpu->env, ARM_FEATURE_M_SECURITY);
+    set_feature(&cpu->env, ARM_FEATURE_THUMB_DSP);
+    cpu->midr = 0x410fd221; /* r0p1 */
+    cpu->revidr = 0;
+    cpu->pmsav7_dregion = 16;
+    cpu->sau_sregion = 8;
+    cpu->isar.mvfr0 = 0x10110221;
+    cpu->isar.mvfr1 = 0x12100211;
+    cpu->isar.mvfr2 = 0x00000040;
+    cpu->id_pfr0 = 0x20000030;
+    cpu->id_pfr1 = 0x00000230;
+    cpu->isar.id_dfr0 = 0x10200000;
+    cpu->id_afr0 = 0x00000000;
+    cpu->isar.id_mmfr0 = 0x00111040;
+    cpu->isar.id_mmfr1 = 0x00000000;
+    cpu->isar.id_mmfr2 = 0x01000000;
+    cpu->isar.id_mmfr3 = 0x00000011;
+    cpu->isar.id_isar0 = 0x01103110;
+    cpu->isar.id_isar1 = 0x02212000;
+    cpu->isar.id_isar2 = 0x20232232;
+    cpu->isar.id_isar3 = 0x01111131;
+    cpu->isar.id_isar4 = 0x01310132;
+    cpu->isar.id_isar5 = 0x00000000;
+    cpu->isar.id_isar6 = 0x00000000;
+    cpu->clidr = 0x00000000;
+    cpu->ctr = 0x8303c003;
+}
+
 static void arm_v7m_class_init(struct uc_struct *uc, CPUClass *oc, void *data)
 {
     ARMCPUClass *acc = ARM_CPU_CLASS(oc);
@@ -1987,6 +2060,8 @@ static void arm_max_initfn(struct uc_struct *uc, CPUState *obj)
             FIELD_DP32(t, ID_ISAR6, FHM, 1, t);
             FIELD_DP32(t, ID_ISAR6, SB, 1, t);
             FIELD_DP32(t, ID_ISAR6, SPECRES, 1, t);
+            FIELD_DP32(t, ID_ISAR6, BF16, 1, t);
+            FIELD_DP32(t, ID_ISAR6, I8MM, 1, t);
             cpu->isar.id_isar6 = t;
 
             t = cpu->isar.mvfr1;
@@ -2061,8 +2136,24 @@ static struct ARMCPUInfo arm_cpus[] = {
     { "pxa270-b1",   pxa270b1_initfn },
     { "pxa270-c0",   pxa270c0_initfn },
     { "pxa270-c5",   pxa270c5_initfn },
+    { "cortex-m55",  cortex_m55_initfn, arm_v7m_class_init },
     { "max", arm_max_initfn },
 };
+
+static bool arm_cpu_model_is_mprofile(int cpu_model)
+{
+    switch (cpu_model) {
+    case UC_CPU_ARM_CORTEX_M0:
+    case UC_CPU_ARM_CORTEX_M3:
+    case UC_CPU_ARM_CORTEX_M4:
+    case UC_CPU_ARM_CORTEX_M7:
+    case UC_CPU_ARM_CORTEX_M33:
+    case UC_CPU_ARM_CORTEX_M55:
+        return true;
+    default:
+        return false;
+    }
+}
 #endif
 
 void arm_cpu_class_init(struct uc_struct *uc, CPUClass *oc)
@@ -2113,7 +2204,9 @@ ARMCPU *cpu_arm_init(struct uc_struct *uc)
     memset((void*)cpu, 0, sizeof(*cpu));
 
 #if !defined(TARGET_AARCH64)
-    if (uc->mode & UC_MODE_MCLASS) {
+    if ((uc->mode & UC_MODE_MCLASS) &&
+        (uc->cpu_model == INT_MAX ||
+         !arm_cpu_model_is_mprofile(uc->cpu_model))) {
         uc->cpu_model = UC_CPU_ARM_CORTEX_M33;
     } else if (uc->mode & UC_MODE_ARM926) {
         uc->cpu_model = UC_CPU_ARM_926;
diff --git a/qemu/target/arm/cpu.h b/qemu/target/arm/cpu.h
index f857850cfc..12efa8e856 100644
--- a/qemu/target/arm/cpu.h
+++ b/qemu/target/arm/cpu.h
@@ -180,13 +180,13 @@ typedef struct {
 #endif
 
 typedef struct ARMVectorReg {
-    uint64_t d[2 * ARM_MAX_VQ] QEMU_ALIGNED(16);
+    QEMU_ALIGN(16, uint64_t d[2 * ARM_MAX_VQ]);
 } ARMVectorReg;
 
 #ifdef TARGET_AARCH64
 /* In AArch32 mode, predicate registers do not exist at all.  */
 typedef struct ARMPredicateReg {
-    uint64_t p[DIV_ROUND_UP(2 * ARM_MAX_VQ, 8)] QEMU_ALIGNED(16);
+    QEMU_ALIGN(16, uint64_t p[DIV_ROUND_UP(2 * ARM_MAX_VQ, 8)]);
 } ARMPredicateReg;
 
 /* In AArch32 mode, PAC keys do not exist at all.  */
@@ -224,6 +224,7 @@ typedef struct CPUARMState {
 
     /* Cached TBFLAGS state.  See below for which bits are included.  */
     uint32_t hflags;
+    uint32_t hflags2;
 
     /* Frequently accessed CPSR bits are stored separately for efficiency.
        This contains all the other bits.  Use cpsr_{read,write} to access
@@ -251,6 +252,9 @@ typedef struct CPUARMState {
     uint32_t condexec_bits; /* IT bits.  cpsr[15:10,26:25].  */
     uint32_t btype;  /* BTI branch type.  spsr[11:10].  */
     uint64_t daif; /* exception masks, in the bits they are in PSTATE */
+#ifdef TARGET_AARCH64
+    uint64_t svcr; /* PSTATE.{SM,ZA} in the bits they are in SVCR */
+#endif
 
     uint64_t elr_el[4]; /* AArch64 exception link regs  */
     uint64_t sp_el[4]; /* AArch64 banked stack pointers */
@@ -429,6 +433,7 @@ typedef struct CPUARMState {
             };
             uint64_t tpidr_el[4];
         };
+        uint64_t tpidr2_el0;
         /* The secure banks of these registers don't map anywhere */
         uint64_t tpidrurw_s;
         uint64_t tpidrprw_s;
@@ -480,6 +485,9 @@ typedef struct CPUARMState {
         uint64_t pmccfiltr_el0; /* Performance Monitor Filter Register */
         uint64_t vpidr_el2; /* Virtualization Processor ID Register */
         uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */
+        uint64_t tfsr_el[4]; /* tfsre0_el1 is index 0. */
+        uint64_t rgsr_el1; /* Random Allocation Tag Seed Register */
+        uint64_t gcr_el1; /* Tag Control Register */
     } cp15;
 
     struct {
@@ -523,6 +531,8 @@ typedef struct CPUARMState {
         uint32_t fpdscr[M_REG_NUM_BANKS];
         uint32_t cpacr[M_REG_NUM_BANKS];
         uint32_t nsacr;
+        uint32_t ltpsize;
+        uint32_t vpr;
     } v7m;
 
     /* Information associated with an exception about to be taken:
@@ -568,7 +578,7 @@ typedef struct CPUARMState {
 #endif
 
         /* We store these fpcsr fields separately for convenience.  */
-        uint32_t qc[4] QEMU_ALIGNED(16);
+        QEMU_ALIGN(16, uint32_t qc[4]);
         int vec_len;
         int vec_stride;
 
@@ -582,6 +592,8 @@ typedef struct CPUARMState {
          *  fp_status: is the "normal" fp status.
          *  fp_status_fp16: used for half-precision calculations
          *  standard_fp_status : the ARM "Standard FPSCR Value"
+         *  standard_fp_status_fp16 : used for half-precision
+         *       calculations with the ARM "Standard FPSCR Value"
          *
          * Half-precision operations are governed by a separate
          * flush-to-zero control bit in FPSCR:FZ16. We pass a separate
@@ -592,18 +604,27 @@ typedef struct CPUARMState {
          * Neon) which the architecture defines as controlled by the
          * standard FPSCR value rather than the FPSCR.
          *
+         * The "standard FPSCR but for fp16 ops" is needed because
+         * the "standard FPSCR" tracks the FPSCR.FZ16 bit rather than
+         * using a fixed value for it.
+         *
          * To avoid having to transfer exception bits around, we simply
          * say that the FPSCR cumulative exception flags are the logical
-         * OR of the flags in the three fp statuses. This relies on the
+         * OR of the flags in the four fp statuses. This relies on the
          * only thing which needs to read the exception flags being
          * an explicit FPSCR read.
          */
         float_status fp_status;
         float_status fp_status_f16;
         float_status standard_fp_status;
+        float_status standard_fp_status_f16;
 
         /* ZCR_EL[1-3] */
         uint64_t zcr_el[4];
+#ifdef TARGET_AARCH64
+        /* SMCR_EL[1-3] */
+        uint64_t smcr_el[4];
+#endif
     } vfp;
     uint64_t exclusive_addr;
     uint64_t exclusive_val;
@@ -625,6 +646,14 @@ typedef struct CPUARMState {
         ARMPACKey apdb;
         ARMPACKey apga;
     } keys;
+
+    uint64_t scxtnum_el[4];
+
+    /*
+     * SME ZA storage, laid out like upstream QEMU: ZA[N] lives in the low
+     * bytes of zarray[N], with the visible square restricted by SVL.
+     */
+    ARMVectorReg zarray[ARM_MAX_VQ * 16];
 #endif
 
     /* Fields up to this point are cleared by a CPU reset */
@@ -715,7 +744,7 @@ struct ARMCPU {
     /*< public >*/
 
     CPUNegativeOffsetState neg;
-    CPUARMState env;
+    QEMU_ALIGN(16, CPUARMState env);
 
     /* Coprocessor information */
     GHashTable *cp_regs;
@@ -857,6 +886,8 @@ struct ARMCPU {
         uint64_t id_aa64mmfr2;
         uint64_t id_aa64dfr0;
         uint64_t id_aa64dfr1;
+        uint64_t id_aa64zfr0;
+        uint64_t id_aa64smfr0;
     } isar;
     uint32_t midr;
     uint32_t revidr;
@@ -905,6 +936,7 @@ struct ARMCPU {
 
     /* Used to set the maximum vector length the cpu will support.  */
     uint32_t sve_max_vq;
+    uint32_t sme_default_vq;
 
     /*
      * In sve_vq_map each set bit is a supported vector length of
@@ -917,6 +949,8 @@ struct ARMCPU {
      */
     DECLARE_BITMAP(sve_vq_map, ARM_MAX_VQ);
     DECLARE_BITMAP(sve_vq_init, ARM_MAX_VQ);
+    DECLARE_BITMAP(sme_vq_map, ARM_MAX_VQ);
+    DECLARE_BITMAP(sme_vq_init, ARM_MAX_VQ);
 
     /* Generic timer counter frequency, in Hz */
     uint64_t gt_cntfrq_hz;
@@ -944,6 +978,7 @@ int arm_gen_dynamic_svereg_xml(CPUState *cpu, int base_reg);
 void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq);
 void aarch64_sve_change_el(CPUARMState *env, int old_el,
                            int new_el, bool el0_a64);
+void aarch64_set_svcr(CPUARMState *env, uint64_t new, uint64_t mask);
 void aarch64_add_sve_properties(void *obj);
 
 /*
@@ -1075,6 +1110,7 @@ void pmu_init(ARMCPU *cpu);
 #define SCTLR_WXN     (1U << 19)
 #define SCTLR_ST      (1U << 20) /* up to ??, RAZ in v6 */
 #define SCTLR_UWXN    (1U << 20) /* v7 onward, AArch32 only */
+#define SCTLR_TSCXT   (1U << 20) /* FEAT_CSV2_1p2, AArch64 only */
 #define SCTLR_FI      (1U << 21) /* up to v7, v8 RES0 */
 #define SCTLR_IESB    (1U << 21) /* v8.2-IESB, AArch64 only */
 #define SCTLR_U       (1U << 22) /* up to v6, RAO in v7 */
@@ -1103,6 +1139,10 @@ void pmu_init(ARMCPU *cpu);
 #define SCTLR_ATA0    (1ULL << 42) /* v8.5-MemTag */
 #define SCTLR_ATA     (1ULL << 43) /* v8.5-MemTag */
 #define SCTLR_DSSBS   (1ULL << 44) /* v8.5 */
+#define SCTLR_EnTP2   (1ULL << 60) /* FEAT_SME */
+
+#define LOG2_TAG_GRANULE 4
+#define TAG_GRANULE (1 << LOG2_TAG_GRANULE)
 
 #define CPTR_TCPAC    (1U << 31)
 #define CPTR_TTA      (1U << 20)
@@ -1110,9 +1150,39 @@ void pmu_init(ARMCPU *cpu);
 #define CPTR_TZ       (1U << 8)   /* CPTR_EL2 */
 #define CPTR_EZ       (1U << 8)   /* CPTR_EL3 */
 
+FIELD(CPACR_EL1, ZEN, 16, 2)
+FIELD(CPACR_EL1, FPEN, 20, 2)
+FIELD(CPACR_EL1, SMEN, 24, 2)
+
+FIELD(CPTR_EL2, TZ, 8, 1)
+FIELD(CPTR_EL2, TFP, 10, 1)
+FIELD(CPTR_EL2, TSM, 12, 1)
+FIELD(CPTR_EL2, ZEN, 16, 2)
+FIELD(CPTR_EL2, FPEN, 20, 2)
+FIELD(CPTR_EL2, SMEN, 24, 2)
+FIELD(CPTR_EL2, TTA, 28, 1)
+FIELD(CPTR_EL2, TAM, 30, 1)
+FIELD(CPTR_EL2, TCPAC, 31, 1)
+
+FIELD(CPTR_EL3, EZ, 8, 1)
+FIELD(CPTR_EL3, TFP, 10, 1)
+FIELD(CPTR_EL3, ESM, 12, 1)
+FIELD(CPTR_EL3, TTA, 20, 1)
+FIELD(CPTR_EL3, TAM, 30, 1)
+FIELD(CPTR_EL3, TCPAC, 31, 1)
+
+FIELD(SVCR, SM, 0, 1)
+FIELD(SVCR, ZA, 1, 1)
+
+FIELD(SMCR, LEN, 0, 4)
+FIELD(SMCR, FA64, 31, 1)
+
 #define MDCR_EPMAD    (1U << 21)
 #define MDCR_EDAD     (1U << 20)
+#define MDCR_SCCD     (1U << 23)  /* MDCR_EL3 */
+#define MDCR_HCCD     (1U << 23)  /* MDCR_EL2 */
 #define MDCR_SPME     (1U << 17)  /* MDCR_EL3 */
+#define MDCR_HLP      (1U << 26)  /* MDCR_EL2 */
 #define MDCR_HPMD     (1U << 17)  /* MDCR_EL2 */
 #define MDCR_SDD      (1U << 16)
 #define MDCR_SPD      (3U << 14)
@@ -1126,7 +1196,8 @@ void pmu_init(ARMCPU *cpu);
 #define MDCR_HPMN     (0x1fU)
 
 /* Not all of the MDCR_EL3 bits are present in the 32-bit SDCR */
-#define SDCR_VALID_MASK (MDCR_EPMAD | MDCR_EDAD | MDCR_SPME | MDCR_SPD)
+#define SDCR_VALID_MASK \
+    (MDCR_EPMAD | MDCR_EDAD | MDCR_SPME | MDCR_SCCD | MDCR_SPD)
 
 #define CPSR_M (0x1fU)
 #define CPSR_T (1U << 5)
@@ -1204,6 +1275,7 @@ void pmu_init(ARMCPU *cpu);
 #define PSTATE_SS (1U << 21)
 #define PSTATE_PAN (1U << 22)
 #define PSTATE_UAO (1U << 23)
+#define PSTATE_TCO (1U << 25)
 #define PSTATE_V (1U << 28)
 #define PSTATE_C (1U << 29)
 #define PSTATE_Z (1U << 30)
@@ -1401,6 +1473,7 @@ static inline void xpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
 #define SCR_FIEN              (1U << 21)
 #define SCR_ENSCXT            (1U << 25)
 #define SCR_ATA               (1U << 26)
+#define SCR_ENTP2             (1ULL << 41)
 
 /* Return the current FPSCR value.  */
 uint32_t vfp_get_fpscr(CPUARMState *env);
@@ -1427,6 +1500,10 @@ void vfp_set_fpscr(CPUARMState *env, uint32_t val);
 #define FPCR_DN     (1 << 25)   /* Default NaN enable bit */
 #define FPCR_QC     (1 << 27)   /* Cumulative saturation bit */
 
+#define FPCR_LTPSIZE_SHIFT 16   /* LTPSIZE, M-profile only */
+#define FPCR_LTPSIZE_MASK (7 << FPCR_LTPSIZE_SHIFT)
+#define FPCR_LTPSIZE_LENGTH 3
+
 static inline uint32_t vfp_get_fpsr(CPUARMState *env)
 {
     return vfp_get_fpscr(env) & FPSR_MASK;
@@ -1470,6 +1547,19 @@ enum arm_cpu_mode {
 #define ARM_VFP_FPEXC   8
 #define ARM_VFP_FPINST  9
 #define ARM_VFP_FPINST2 10
+#define FPCR_V          (1U << 28)
+#define FPCR_C          (1U << 29)
+#define FPCR_Z          (1U << 30)
+#define FPCR_N          (1U << 31)
+#define FPCR_NZCV_MASK  (FPCR_N | FPCR_Z | FPCR_C | FPCR_V)
+#define FPCR_NZCVQC_MASK (FPCR_NZCV_MASK | FPCR_QC)
+#define FPCR_AHP        (1 << 26)
+/* These ones are M-profile only */
+#define ARM_VFP_FPSCR_NZCVQC 2
+#define ARM_VFP_VPR 12
+#define ARM_VFP_P0 13
+#define ARM_VFP_FPCXT_NS 14
+#define ARM_VFP_FPCXT_S 15
 
 /* iwMMXt coprocessor control registers.  */
 #define ARM_IWMMXT_wCID  0
@@ -1602,6 +1692,12 @@ FIELD(V7M_FPCCR, CLRONRET, 28, 1)
 FIELD(V7M_FPCCR, LSPENS, 29, 1)
 FIELD(V7M_FPCCR, LSPEN, 30, 1)
 FIELD(V7M_FPCCR, ASPEN, 31, 1)
+
+/* v7M VPR bits */
+FIELD(V7M_VPR, P0, 0, 16)
+FIELD(V7M_VPR, MASK01, 16, 4)
+FIELD(V7M_VPR, MASK23, 20, 4)
+
 /* These bits are banked. Others are non-banked and live in the M_REG_S bank */
 #define R_V7M_FPCCR_BANKED_MASK                 \
     (R_V7M_FPCCR_LSPACT_MASK |                  \
@@ -1678,6 +1774,8 @@ FIELD(ID_ISAR6, DP, 4, 4)
 FIELD(ID_ISAR6, FHM, 8, 4)
 FIELD(ID_ISAR6, SB, 12, 4)
 FIELD(ID_ISAR6, SPECRES, 16, 4)
+FIELD(ID_ISAR6, BF16, 20, 4)
+FIELD(ID_ISAR6, I8MM, 24, 4)
 
 FIELD(ID_MMFR3, CMAINTVA, 0, 4)
 FIELD(ID_MMFR3, CMAINTSW, 4, 4)
@@ -1759,10 +1857,22 @@ FIELD(ID_AA64ISAR1, GPI, 28, 4)
 #define R_ID_AA64ISAR1_SPECRES_SHIFT    40
 #define R_ID_AA64ISAR1_SPECRES_LENGTH   4
 #define R_ID_AA64ISAR1_SPECRES_MASK     MAKE_64BIT_MASK(R_ID_AA64ISAR1_SPECRES_SHIFT, R_ID_AA64ISAR1_SPECRES_LENGTH)
+#define R_ID_AA64ISAR1_BF16_SHIFT       44
+#define R_ID_AA64ISAR1_BF16_LENGTH      4
+#define R_ID_AA64ISAR1_BF16_MASK        MAKE_64BIT_MASK(R_ID_AA64ISAR1_BF16_SHIFT, R_ID_AA64ISAR1_BF16_LENGTH)
+#define R_ID_AA64ISAR1_DGH_SHIFT        48
+#define R_ID_AA64ISAR1_DGH_LENGTH       4
+#define R_ID_AA64ISAR1_DGH_MASK         MAKE_64BIT_MASK(R_ID_AA64ISAR1_DGH_SHIFT, R_ID_AA64ISAR1_DGH_LENGTH)
+#define R_ID_AA64ISAR1_I8MM_SHIFT       52
+#define R_ID_AA64ISAR1_I8MM_LENGTH      4
+#define R_ID_AA64ISAR1_I8MM_MASK        MAKE_64BIT_MASK(R_ID_AA64ISAR1_I8MM_SHIFT, R_ID_AA64ISAR1_I8MM_LENGTH)
 #else
 FIELD(ID_AA64ISAR1, FRINTTS, 32, 4)
 FIELD(ID_AA64ISAR1, SB, 36, 4)
 FIELD(ID_AA64ISAR1, SPECRES, 40, 4)
+FIELD(ID_AA64ISAR1, BF16, 44, 4)
+FIELD(ID_AA64ISAR1, DGH, 48, 4)
+FIELD(ID_AA64ISAR1, I8MM, 52, 4)
 #endif
 
 FIELD(ID_AA64PFR0, EL0, 0, 4)
@@ -1774,17 +1884,33 @@ FIELD(ID_AA64PFR0, ADVSIMD, 20, 4)
 FIELD(ID_AA64PFR0, GIC, 24, 4)
 FIELD(ID_AA64PFR0, RAS, 28, 4)
 #ifdef _MSC_VER
-#define R_ID_AA64PFR0_SVE_SHIFT     60
+#define R_ID_AA64PFR0_SVE_SHIFT     32
 #define R_ID_AA64PFR0_SVE_LENGTH    4
 #define R_ID_AA64PFR0_SVE_MASK      MAKE_64BIT_MASK(R_ID_AA64PFR0_SVE_SHIFT, R_ID_AA64PFR0_SVE_LENGTH)
+#define R_ID_AA64PFR0_CSV2_SHIFT    56
+#define R_ID_AA64PFR0_CSV2_LENGTH   4
+#define R_ID_AA64PFR0_CSV2_MASK     MAKE_64BIT_MASK(R_ID_AA64PFR0_CSV2_SHIFT, R_ID_AA64PFR0_CSV2_LENGTH)
+#define R_ID_AA64PFR0_CSV3_SHIFT    60
+#define R_ID_AA64PFR0_CSV3_LENGTH   4
+#define R_ID_AA64PFR0_CSV3_MASK     MAKE_64BIT_MASK(R_ID_AA64PFR0_CSV3_SHIFT, R_ID_AA64PFR0_CSV3_LENGTH)
 #else
 FIELD(ID_AA64PFR0, SVE, 32, 4)
+FIELD(ID_AA64PFR0, CSV2, 56, 4)
+FIELD(ID_AA64PFR0, CSV3, 60, 4)
 #endif
 
 FIELD(ID_AA64PFR1, BT, 0, 4)
 FIELD(ID_AA64PFR1, SBSS, 4, 4)
 FIELD(ID_AA64PFR1, MTE, 8, 4)
 FIELD(ID_AA64PFR1, RAS_FRAC, 12, 4)
+FIELD(ID_AA64PFR1, SME, 24, 4)
+#ifdef _MSC_VER
+#define R_ID_AA64PFR1_CSV2_FRAC_SHIFT   32
+#define R_ID_AA64PFR1_CSV2_FRAC_LENGTH  4
+#define R_ID_AA64PFR1_CSV2_FRAC_MASK    MAKE_64BIT_MASK(R_ID_AA64PFR1_CSV2_FRAC_SHIFT, R_ID_AA64PFR1_CSV2_FRAC_LENGTH)
+#else
+FIELD(ID_AA64PFR1, CSV2_FRAC, 32, 4)
+#endif
 
 FIELD(ID_AA64MMFR0, PARANGE, 0, 4)
 FIELD(ID_AA64MMFR0, ASIDBITS, 4, 4)
@@ -1885,6 +2011,82 @@ FIELD(ID_AA64DFR0, DOUBLELOCK, 36, 4)
 FIELD(ID_AA64DFR0, TRACEFILT, 40, 4)
 #endif
 
+#ifdef _MSC_VER
+#define R_ID_AA64ZFR0_SVEVER_SHIFT      0
+#define R_ID_AA64ZFR0_SVEVER_LENGTH     4
+#define R_ID_AA64ZFR0_SVEVER_MASK       MAKE_64BIT_MASK(R_ID_AA64ZFR0_SVEVER_SHIFT, R_ID_AA64ZFR0_SVEVER_LENGTH)
+#define R_ID_AA64ZFR0_AES_SHIFT         4
+#define R_ID_AA64ZFR0_AES_LENGTH        4
+#define R_ID_AA64ZFR0_AES_MASK          MAKE_64BIT_MASK(R_ID_AA64ZFR0_AES_SHIFT, R_ID_AA64ZFR0_AES_LENGTH)
+#define R_ID_AA64ZFR0_BITPERM_SHIFT     16
+#define R_ID_AA64ZFR0_BITPERM_LENGTH    4
+#define R_ID_AA64ZFR0_BITPERM_MASK      MAKE_64BIT_MASK(R_ID_AA64ZFR0_BITPERM_SHIFT, R_ID_AA64ZFR0_BITPERM_LENGTH)
+#define R_ID_AA64ZFR0_BFLOAT16_SHIFT    20
+#define R_ID_AA64ZFR0_BFLOAT16_LENGTH   4
+#define R_ID_AA64ZFR0_BFLOAT16_MASK     MAKE_64BIT_MASK(R_ID_AA64ZFR0_BFLOAT16_SHIFT, R_ID_AA64ZFR0_BFLOAT16_LENGTH)
+#define R_ID_AA64ZFR0_SHA3_SHIFT        32
+#define R_ID_AA64ZFR0_SHA3_LENGTH       4
+#define R_ID_AA64ZFR0_SHA3_MASK         MAKE_64BIT_MASK(R_ID_AA64ZFR0_SHA3_SHIFT, R_ID_AA64ZFR0_SHA3_LENGTH)
+#define R_ID_AA64ZFR0_SM4_SHIFT         40
+#define R_ID_AA64ZFR0_SM4_LENGTH        4
+#define R_ID_AA64ZFR0_SM4_MASK          MAKE_64BIT_MASK(R_ID_AA64ZFR0_SM4_SHIFT, R_ID_AA64ZFR0_SM4_LENGTH)
+#define R_ID_AA64ZFR0_I8MM_SHIFT        44
+#define R_ID_AA64ZFR0_I8MM_LENGTH       4
+#define R_ID_AA64ZFR0_I8MM_MASK         MAKE_64BIT_MASK(R_ID_AA64ZFR0_I8MM_SHIFT, R_ID_AA64ZFR0_I8MM_LENGTH)
+#define R_ID_AA64ZFR0_F32MM_SHIFT       52
+#define R_ID_AA64ZFR0_F32MM_LENGTH      4
+#define R_ID_AA64ZFR0_F32MM_MASK        MAKE_64BIT_MASK(R_ID_AA64ZFR0_F32MM_SHIFT, R_ID_AA64ZFR0_F32MM_LENGTH)
+#define R_ID_AA64ZFR0_F64MM_SHIFT       56
+#define R_ID_AA64ZFR0_F64MM_LENGTH      4
+#define R_ID_AA64ZFR0_F64MM_MASK        MAKE_64BIT_MASK(R_ID_AA64ZFR0_F64MM_SHIFT, R_ID_AA64ZFR0_F64MM_LENGTH)
+#else
+FIELD(ID_AA64ZFR0, SVEVER, 0, 4)
+FIELD(ID_AA64ZFR0, AES, 4, 4)
+FIELD(ID_AA64ZFR0, BITPERM, 16, 4)
+FIELD(ID_AA64ZFR0, BFLOAT16, 20, 4)
+FIELD(ID_AA64ZFR0, SHA3, 32, 4)
+FIELD(ID_AA64ZFR0, SM4, 40, 4)
+FIELD(ID_AA64ZFR0, I8MM, 44, 4)
+FIELD(ID_AA64ZFR0, F32MM, 52, 4)
+FIELD(ID_AA64ZFR0, F64MM, 56, 4)
+#endif
+
+#ifdef _MSC_VER
+#define R_ID_AA64SMFR0_F32F32_SHIFT     32
+#define R_ID_AA64SMFR0_F32F32_LENGTH    1
+#define R_ID_AA64SMFR0_F32F32_MASK      MAKE_64BIT_MASK(R_ID_AA64SMFR0_F32F32_SHIFT, R_ID_AA64SMFR0_F32F32_LENGTH)
+#define R_ID_AA64SMFR0_B16F32_SHIFT     34
+#define R_ID_AA64SMFR0_B16F32_LENGTH    1
+#define R_ID_AA64SMFR0_B16F32_MASK      MAKE_64BIT_MASK(R_ID_AA64SMFR0_B16F32_SHIFT, R_ID_AA64SMFR0_B16F32_LENGTH)
+#define R_ID_AA64SMFR0_F16F32_SHIFT     35
+#define R_ID_AA64SMFR0_F16F32_LENGTH    1
+#define R_ID_AA64SMFR0_F16F32_MASK      MAKE_64BIT_MASK(R_ID_AA64SMFR0_F16F32_SHIFT, R_ID_AA64SMFR0_F16F32_LENGTH)
+#define R_ID_AA64SMFR0_I8I32_SHIFT      36
+#define R_ID_AA64SMFR0_I8I32_LENGTH     4
+#define R_ID_AA64SMFR0_I8I32_MASK       MAKE_64BIT_MASK(R_ID_AA64SMFR0_I8I32_SHIFT, R_ID_AA64SMFR0_I8I32_LENGTH)
+#define R_ID_AA64SMFR0_F64F64_SHIFT     48
+#define R_ID_AA64SMFR0_F64F64_LENGTH    1
+#define R_ID_AA64SMFR0_F64F64_MASK      MAKE_64BIT_MASK(R_ID_AA64SMFR0_F64F64_SHIFT, R_ID_AA64SMFR0_F64F64_LENGTH)
+#define R_ID_AA64SMFR0_I16I64_SHIFT     52
+#define R_ID_AA64SMFR0_I16I64_LENGTH    4
+#define R_ID_AA64SMFR0_I16I64_MASK      MAKE_64BIT_MASK(R_ID_AA64SMFR0_I16I64_SHIFT, R_ID_AA64SMFR0_I16I64_LENGTH)
+#define R_ID_AA64SMFR0_SMEVER_SHIFT     56
+#define R_ID_AA64SMFR0_SMEVER_LENGTH    4
+#define R_ID_AA64SMFR0_SMEVER_MASK      MAKE_64BIT_MASK(R_ID_AA64SMFR0_SMEVER_SHIFT, R_ID_AA64SMFR0_SMEVER_LENGTH)
+#define R_ID_AA64SMFR0_FA64_SHIFT       63
+#define R_ID_AA64SMFR0_FA64_LENGTH      1
+#define R_ID_AA64SMFR0_FA64_MASK        MAKE_64BIT_MASK(R_ID_AA64SMFR0_FA64_SHIFT, R_ID_AA64SMFR0_FA64_LENGTH)
+#else
+FIELD(ID_AA64SMFR0, F32F32, 32, 1)
+FIELD(ID_AA64SMFR0, B16F32, 34, 1)
+FIELD(ID_AA64SMFR0, F16F32, 35, 1)
+FIELD(ID_AA64SMFR0, I8I32, 36, 4)
+FIELD(ID_AA64SMFR0, F64F64, 48, 1)
+FIELD(ID_AA64SMFR0, I16I64, 52, 4)
+FIELD(ID_AA64SMFR0, SMEVER, 56, 4)
+FIELD(ID_AA64SMFR0, FA64, 63, 1)
+#endif
+
 FIELD(ID_DFR0, COPDBG, 0, 4)
 FIELD(ID_DFR0, COPSDBG, 4, 4)
 FIELD(ID_DFR0, MMAPDBG, 8, 4)
@@ -1912,10 +2114,12 @@ FIELD(MVFR0, FPROUND, 28, 4)
 
 FIELD(MVFR1, FPFTZ, 0, 4)
 FIELD(MVFR1, FPDNAN, 4, 4)
-FIELD(MVFR1, SIMDLS, 8, 4)
-FIELD(MVFR1, SIMDINT, 12, 4)
-FIELD(MVFR1, SIMDSP, 16, 4)
-FIELD(MVFR1, SIMDHP, 20, 4)
+FIELD(MVFR1, SIMDLS, 8, 4) /* A-profile only */
+FIELD(MVFR1, SIMDINT, 12, 4) /* A-profile only */
+FIELD(MVFR1, SIMDSP, 16, 4) /* A-profile only */
+FIELD(MVFR1, SIMDHP, 20, 4) /* A-profile only */
+FIELD(MVFR1, MVE, 8, 4) /* M-profile only */
+FIELD(MVFR1, FP16, 20, 4) /* M-profile only */
 FIELD(MVFR1, FPHP, 24, 4)
 FIELD(MVFR1, SIMDFMAC, 28, 4)
 
@@ -1968,6 +2172,7 @@ enum arm_features {
     ARM_FEATURE_VBAR, /* has cp15 VBAR */
     ARM_FEATURE_M_SECURITY, /* M profile Security Extension */
     ARM_FEATURE_M_MAIN, /* M profile Main Extension */
+    ARM_FEATURE_V8_1M, /* M profile v8.1-M Extension */
 };
 
 static inline int arm_feature(CPUARMState *env, int feature)
@@ -2018,6 +2223,17 @@ static inline bool arm_is_secure(CPUARMState *env)
     return arm_is_secure_below_el3(env);
 }
 
+static inline bool arm_is_el2_enabled_secstate(CPUARMState *env, bool secure)
+{
+    return arm_feature(env, ARM_FEATURE_EL2) &&
+        (!secure || (env->cp15.scr_el3 & SCR_EEL2));
+}
+
+static inline bool arm_is_el2_enabled(CPUARMState *env)
+{
+    return arm_is_el2_enabled_secstate(env, arm_is_secure_below_el3(env));
+}
+
 /**
  * arm_hcr_el2_eff(): Return the effective value of HCR_EL2.
  * E.g. when in secure state, fields in HCR_EL2 are suppressed,
@@ -2349,16 +2565,19 @@ static inline uint64_t cpreg_to_kvm_id(uint32_t cpregid)
 #define ARM_CP_NZCV              (ARM_CP_SPECIAL | 0x0300)
 #define ARM_CP_CURRENTEL         (ARM_CP_SPECIAL | 0x0400)
 #define ARM_CP_DC_ZVA            (ARM_CP_SPECIAL | 0x0500)
-#define ARM_LAST_SPECIAL         ARM_CP_DC_ZVA
+#define ARM_CP_DC_GVA            (ARM_CP_SPECIAL | 0x0600)
+#define ARM_CP_DC_GZVA           (ARM_CP_SPECIAL | 0x0700)
+#define ARM_LAST_SPECIAL         ARM_CP_DC_GZVA
 #define ARM_CP_FPU               0x1000
 #define ARM_CP_SVE               0x2000
 #define ARM_CP_NO_GDB            0x4000
 #define ARM_CP_RAISES_EXC        0x8000
 #define ARM_CP_NEWEL             0x10000
+#define ARM_CP_SME               0x20000
 /* Used only as a terminator for ARMCPRegInfo lists */
 #define ARM_CP_SENTINEL          0xfffff
 /* Mask of only the flag bits in a type field */
-#define ARM_CP_FLAG_MASK         0x1f0ff
+#define ARM_CP_FLAG_MASK         0x3f0ff
 
 /* Valid values for ARMCPRegInfo state field, indicating which of
  * the AArch32 and AArch64 execution states this register is visible in.
@@ -3203,6 +3422,8 @@ FIELD(TBFLAG_M32, LSPACT, 11, 1)                 /* Not cached. */
 FIELD(TBFLAG_M32, NEW_FP_CTXT_NEEDED, 12, 1)     /* Not cached. */
 /* Set if FPCCR.S does not match current security state */
 FIELD(TBFLAG_M32, FPCCR_S_WRONG, 13, 1)          /* Not cached. */
+/* Set if MVE insns are definitely not predicated by VPR or LTPSIZE */
+FIELD(TBFLAG_M32, MVE_NO_PRED, 14, 1)
 
 /*
  * Bit usage when in AArch64 state
@@ -3215,6 +3436,17 @@ FIELD(TBFLAG_A64, BT, 9, 1)
 FIELD(TBFLAG_A64, BTYPE, 10, 2)         /* Not cached. */
 FIELD(TBFLAG_A64, TBID, 12, 2)
 FIELD(TBFLAG_A64, UNPRIV, 14, 1)
+FIELD(TBFLAG_A64, ATA, 15, 1)
+FIELD(TBFLAG_A64, TCMA, 16, 2)
+FIELD(TBFLAG_A64, MTE_ACTIVE, 18, 1)
+FIELD(TBFLAG_A64, MTE0_ACTIVE, 19, 1)
+
+/* Extra AArch64-only flags stored in tb->cs_base. */
+FIELD(TBFLAG_A64_2, SMEEXC_EL, 0, 2)
+FIELD(TBFLAG_A64_2, PSTATE_SM, 2, 1)
+FIELD(TBFLAG_A64_2, PSTATE_ZA, 3, 1)
+FIELD(TBFLAG_A64_2, SVL, 4, 4)
+FIELD(TBFLAG_A64_2, SME_TRAP_NONSTREAMING, 8, 1)
 
 /**
  * cpu_mmu_index:
@@ -3379,6 +3611,16 @@ static inline bool isar_feature_aa32_dp(const ARMISARegisters *id)
     return FIELD_EX32(id->id_isar6, ID_ISAR6, DP) != 0;
 }
 
+static inline bool isar_feature_aa32_bf16(const ARMISARegisters *id)
+{
+    return FIELD_EX32(id->id_isar6, ID_ISAR6, BF16) != 0;
+}
+
+static inline bool isar_feature_aa32_i8mm(const ARMISARegisters *id)
+{
+    return FIELD_EX32(id->id_isar6, ID_ISAR6, I8MM) != 0;
+}
+
 static inline bool isar_feature_aa32_fhm(const ARMISARegisters *id)
 {
     return FIELD_EX32(id->id_isar6, ID_ISAR6, FHM) != 0;
@@ -3404,6 +3646,16 @@ static inline bool isar_feature_aa32_fp16_arith(const ARMISARegisters *id)
     return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, FP) == 1;
 }
 
+static inline bool isar_feature_aa32_mve(const ARMISARegisters *id)
+{
+    return FIELD_EX32(id->mvfr1, MVFR1, MVE) > 0;
+}
+
+static inline bool isar_feature_aa32_mve_fp(const ARMISARegisters *id)
+{
+    return FIELD_EX32(id->mvfr1, MVFR1, MVE) >= 2;
+}
+
 static inline bool isar_feature_aa32_vfp_simd(const ARMISARegisters *id)
 {
     /*
@@ -3524,6 +3776,13 @@ static inline bool isar_feature_aa32_pmu_8_4(const ARMISARegisters *id)
         FIELD_EX32(id->id_dfr0, ID_DFR0, PERFMON) != 0xf;
 }
 
+static inline bool isar_feature_aa32_pmu_8_5(const ARMISARegisters *id)
+{
+    /* 0xf means "non-standard IMPDEF PMU" */
+    return FIELD_EX32(id->id_dfr0, ID_DFR0, PERFMON) >= 6 &&
+        FIELD_EX32(id->id_dfr0, ID_DFR0, PERFMON) != 0xf;
+}
+
 static inline bool isar_feature_aa32_hpd(const ARMISARegisters *id)
 {
     return FIELD_EX32(id->id_mmfr4, ID_MMFR4, HPDS) != 0;
@@ -3665,6 +3924,16 @@ static inline bool isar_feature_aa64_frint(const ARMISARegisters *id)
     return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, FRINTTS) != 0;
 }
 
+static inline bool isar_feature_aa64_i8mm(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, I8MM) != 0;
+}
+
+static inline bool isar_feature_aa64_bf16(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, BF16) != 0;
+}
+
 static inline bool isar_feature_aa64_dcpop(const ARMISARegisters *id)
 {
     return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, DPB) != 0;
@@ -3697,6 +3966,56 @@ static inline bool isar_feature_aa64_sve(const ARMISARegisters *id)
     return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, SVE) != 0;
 }
 
+static inline bool isar_feature_aa64_sve2(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SVEVER) != 0;
+}
+
+static inline bool isar_feature_aa64_sve2_aes(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) != 0;
+}
+
+static inline bool isar_feature_aa64_sve2_pmull128(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, AES) >= 2;
+}
+
+static inline bool isar_feature_aa64_sve2_bitperm(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BITPERM) != 0;
+}
+
+static inline bool isar_feature_aa64_sve_bf16(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, BFLOAT16) != 0;
+}
+
+static inline bool isar_feature_aa64_sve2_sha3(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SHA3) != 0;
+}
+
+static inline bool isar_feature_aa64_sve2_sm4(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, SM4) != 0;
+}
+
+static inline bool isar_feature_aa64_sve_i8mm(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, I8MM) != 0;
+}
+
+static inline bool isar_feature_aa64_sve_f32mm(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F32MM) != 0;
+}
+
+static inline bool isar_feature_aa64_sve_f64mm(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F64MM) != 0;
+}
+
 static inline bool isar_feature_aa64_vh(const ARMISARegisters *id)
 {
     return FIELD_EX64(id->id_aa64mmfr1, ID_AA64MMFR1, VH) != 0;
@@ -3727,6 +4046,36 @@ static inline bool isar_feature_aa64_bti(const ARMISARegisters *id)
     return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, BT) != 0;
 }
 
+static inline bool isar_feature_aa64_mte_insn_reg(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, MTE) != 0;
+}
+
+static inline bool isar_feature_aa64_mte(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, MTE) >= 2;
+}
+
+static inline bool isar_feature_aa64_sme(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, SME) != 0;
+}
+
+static inline bool isar_feature_aa64_sme_f64f64(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, F64F64) != 0;
+}
+
+static inline bool isar_feature_aa64_sme_i16i64(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, I16I64) == 0xf;
+}
+
+static inline bool isar_feature_aa64_sme_fa64(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, FA64) != 0;
+}
+
 static inline bool isar_feature_aa64_pmu_8_1(const ARMISARegisters *id)
 {
     return FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) >= 4 &&
@@ -3739,6 +4088,12 @@ static inline bool isar_feature_aa64_pmu_8_4(const ARMISARegisters *id)
         FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) != 0xf;
 }
 
+static inline bool isar_feature_aa64_pmu_8_5(const ARMISARegisters *id)
+{
+    return FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) >= 6 &&
+        FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) != 0xf;
+}
+
 static inline bool isar_feature_aa64_rcpc_8_3(const ARMISARegisters *id)
 {
     return FIELD_EX64(id->id_aa64isar1, ID_AA64ISAR1, LRCPC) != 0;
@@ -3754,6 +4109,20 @@ static inline bool isar_feature_aa64_ccidx(const ARMISARegisters *id)
     return FIELD_EX64(id->id_aa64mmfr2, ID_AA64MMFR2, CCIDX) != 0;
 }
 
+static inline bool isar_feature_aa64_scxtnum(const ARMISARegisters *id)
+{
+    int key = FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, CSV2);
+
+    if (key >= 2) {
+        return true;
+    }
+    if (key == 1) {
+        key = FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, CSV2_FRAC);
+        return key >= 2;
+    }
+    return false;
+}
+
 /*
  * Feature tests for "does this exist in either 32-bit or 64-bit?"
  */
@@ -3777,6 +4146,11 @@ static inline bool isar_feature_any_pmu_8_4(const ARMISARegisters *id)
     return isar_feature_aa64_pmu_8_4(id) || isar_feature_aa32_pmu_8_4(id);
 }
 
+static inline bool isar_feature_any_pmu_8_5(const ARMISARegisters *id)
+{
+    return isar_feature_aa64_pmu_8_5(id) || isar_feature_aa32_pmu_8_5(id);
+}
+
 static inline bool isar_feature_any_ccidx(const ARMISARegisters *id)
 {
     return isar_feature_aa64_ccidx(id) || isar_feature_aa32_ccidx(id);
diff --git a/qemu/target/arm/cpu64.c b/qemu/target/arm/cpu64.c
index 3c57a52aee..b7ecd59cb1 100644
--- a/qemu/target/arm/cpu64.c
+++ b/qemu/target/arm/cpu64.c
@@ -193,6 +193,7 @@ static void aarch64_max_initfn(struct uc_struct *uc, CPUState *obj)
 
     uint64_t t;
     uint32_t u;
+    int i;
     ARMCPU *cpu = ARM_CPU(obj);
 
     aarch64_a57_initfn(uc, obj);
@@ -244,18 +245,63 @@ static void aarch64_max_initfn(struct uc_struct *uc, CPUState *obj)
     FIELD_DP64(t, ID_AA64ISAR1, SPECRES, 1, t);
     FIELD_DP64(t, ID_AA64ISAR1, FRINTTS, 1, t);
     FIELD_DP64(t, ID_AA64ISAR1, LRCPC, 2, t); /* ARMv8.4-RCPC */
+    FIELD_DP64(t, ID_AA64ISAR1, BF16, 1, t);
+    FIELD_DP64(t, ID_AA64ISAR1, DGH, 1, t);
+    FIELD_DP64(t, ID_AA64ISAR1, I8MM, 1, t);
     cpu->isar.id_aa64isar1 = t;
 
     t = cpu->isar.id_aa64pfr0;
     FIELD_DP64(t, ID_AA64PFR0, SVE, 1, t);
     FIELD_DP64(t, ID_AA64PFR0, FP, 1, t);
     FIELD_DP64(t, ID_AA64PFR0, ADVSIMD, 1, t);
+    FIELD_DP64(t, ID_AA64PFR0, CSV2, 2, t);
+    FIELD_DP64(t, ID_AA64PFR0, CSV3, 1, t);
     cpu->isar.id_aa64pfr0 = t;
+    cpu->sve_max_vq = ARM_MAX_VQ;
+    for (i = 0; i < ARM_MAX_VQ; i++) {
+        set_bit(i, cpu->sve_vq_map);
+        set_bit(i, cpu->sve_vq_init);
+    }
+
+    t = cpu->isar.id_aa64zfr0;
+    FIELD_DP64(t, ID_AA64ZFR0, SVEVER, 1, t);
+    FIELD_DP64(t, ID_AA64ZFR0, AES, 2, t);
+    FIELD_DP64(t, ID_AA64ZFR0, BITPERM, 1, t);
+    FIELD_DP64(t, ID_AA64ZFR0, BFLOAT16, 1, t);
+    FIELD_DP64(t, ID_AA64ZFR0, SHA3, 1, t);
+    FIELD_DP64(t, ID_AA64ZFR0, SM4, 1, t);
+    FIELD_DP64(t, ID_AA64ZFR0, I8MM, 1, t);
+    FIELD_DP64(t, ID_AA64ZFR0, F32MM, 1, t);
+    FIELD_DP64(t, ID_AA64ZFR0, F64MM, 1, t);
+    cpu->isar.id_aa64zfr0 = t;
 
     t = cpu->isar.id_aa64pfr1;
     FIELD_DP64(t, ID_AA64PFR1, BT, 1, t);
+    FIELD_DP64(t, ID_AA64PFR1, MTE, 2, t);
+    FIELD_DP64(t, ID_AA64PFR1, SME, 1, t);
+    FIELD_DP64(t, ID_AA64PFR1, CSV2_FRAC, 0, t);
     cpu->isar.id_aa64pfr1 = t;
 
+    t = cpu->isar.id_aa64smfr0;
+    FIELD_DP64(t, ID_AA64SMFR0, F32F32, 1, t);
+    FIELD_DP64(t, ID_AA64SMFR0, B16F32, 1, t);
+    FIELD_DP64(t, ID_AA64SMFR0, F16F32, 1, t);
+    FIELD_DP64(t, ID_AA64SMFR0, I8I32, 0xf, t);
+    FIELD_DP64(t, ID_AA64SMFR0, F64F64, 1, t);
+    FIELD_DP64(t, ID_AA64SMFR0, I16I64, 0xf, t);
+    FIELD_DP64(t, ID_AA64SMFR0, SMEVER, 1, t);
+    FIELD_DP64(t, ID_AA64SMFR0, FA64, 1, t);
+    cpu->isar.id_aa64smfr0 = t;
+    cpu->sme_default_vq = 2;
+    for (i = 0; i < ARM_MAX_VQ; i++) {
+        uint32_t vq = i + 1;
+
+        if ((vq & (vq - 1)) == 0) {
+            set_bit(i, cpu->sme_vq_map);
+            set_bit(i, cpu->sme_vq_init);
+        }
+    }
+
     t = cpu->isar.id_aa64mmfr1;
     FIELD_DP64(t, ID_AA64MMFR1, HPDS, 1, t); /* HPD */
     FIELD_DP64(t, ID_AA64MMFR1, LO, 1, t);
@@ -298,11 +344,11 @@ static void aarch64_max_initfn(struct uc_struct *uc, CPUState *obj)
     cpu->isar.id_mmfr4 = u;
 
     u = cpu->isar.id_aa64dfr0;
-    FIELD_DP64(u, ID_AA64DFR0, PMUVER, 5, u); /* v8.4-PMU */
+    FIELD_DP64(u, ID_AA64DFR0, PMUVER, 6, u); /* PMUv3p5 */
     cpu->isar.id_aa64dfr0 = u;
 
     u = cpu->isar.id_dfr0;
-    FIELD_DP32(u, ID_DFR0, PERFMON, 5, u); /* v8.4-PMU */
+    FIELD_DP32(u, ID_DFR0, PERFMON, 6, u); /* PMUv3p5 */
     cpu->isar.id_dfr0 = u;
 }
 
diff --git a/qemu/target/arm/crypto_helper.c b/qemu/target/arm/crypto_helper.c
index 117be6f89f..99d4c82fef 100644
--- a/qemu/target/arm/crypto_helper.c
+++ b/qemu/target/arm/crypto_helper.c
@@ -13,6 +13,7 @@
 
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "tcg/tcg-gvec-desc.h"
 #include "crypto/aes.h"
 
 union CRYPTO_STATE {
@@ -29,14 +30,23 @@ union CRYPTO_STATE {
 #define CR_ST_WORD(state, i)   (state.words[i])
 #endif
 
-void HELPER(crypto_aese)(void *vd, void *vm, uint32_t decrypt)
+static void crypto_clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz)
+{
+    uint64_t *d = (uint64_t *)((char *)vd + opr_sz);
+    uintptr_t i;
+
+    for (i = opr_sz; i < max_sz; i += 8) {
+        *d++ = 0;
+    }
+}
+
+static void do_crypto_aese(uint64_t *rd, uint64_t *rn,
+                           uint64_t *rm, bool decrypt)
 {
     static uint8_t const * const sbox[2] = { AES_sbox, AES_isbox };
     static uint8_t const * const shift[2] = { AES_shifts, AES_ishifts };
-    uint64_t *rd = vd;
-    uint64_t *rm = vm;
     union CRYPTO_STATE rk = { .l = { rm[0], rm[1] } };
-    union CRYPTO_STATE st = { .l = { rd[0], rd[1] } };
+    union CRYPTO_STATE st = { .l = { rn[0], rn[1] } };
     int i;
 
     assert(decrypt < 2);
@@ -54,6 +64,25 @@ void HELPER(crypto_aese)(void *vd, void *vm, uint32_t decrypt)
     rd[1] = st.l[1];
 }
 
+void HELPER(crypto_aese)(void *vd, void *vm, uint32_t decrypt)
+{
+    do_crypto_aese(vd, vd, vm, decrypt);
+}
+
+void HELPER(crypto_sve_aese)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i;
+    intptr_t opr_sz = simd_oprsz(desc);
+    bool decrypt = simd_data(desc);
+
+    for (i = 0; i < opr_sz; i += 16) {
+        do_crypto_aese((uint64_t *)((char *)vd + i),
+                       (uint64_t *)((char *)vn + i),
+                       (uint64_t *)((char *)vm + i), decrypt);
+    }
+    crypto_clear_tail(vd, opr_sz, simd_maxsz(desc));
+}
+
 void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t decrypt)
 {
     static uint32_t const mc[][256] = { {
@@ -209,6 +238,18 @@ void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t decrypt)
     rd[1] = st.l[1];
 }
 
+void HELPER(crypto_sve_aesmc)(void *vd, void *vm, uint32_t desc)
+{
+    intptr_t i;
+    intptr_t opr_sz = simd_oprsz(desc);
+    bool decrypt = simd_data(desc);
+
+    for (i = 0; i < opr_sz; i += 16) {
+        HELPER(crypto_aesmc)((char *)vd + i, (char *)vm + i, decrypt);
+    }
+    crypto_clear_tail(vd, opr_sz, simd_maxsz(desc));
+}
+
 /*
  * SHA-1 logical functions
  */
@@ -638,12 +679,10 @@ static uint8_t const sm4_sbox[] = {
     0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48,
 };
 
-void HELPER(crypto_sm4e)(void *vd, void *vn)
+static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm)
 {
-    uint64_t *rd = vd;
-    uint64_t *rn = vn;
-    union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
-    union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
+    union CRYPTO_STATE d = { .l = { rn[0], rn[1] } };
+    union CRYPTO_STATE n = { .l = { rm[0], rm[1] } };
     uint32_t t, i;
 
     for (i = 0; i < 4; i++) {
@@ -665,11 +704,26 @@ void HELPER(crypto_sm4e)(void *vd, void *vn)
     rd[1] = d.l[1];
 }
 
-void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm)
+void HELPER(crypto_sm4e)(void *vd, void *vn)
+{
+    do_crypto_sm4e(vd, vd, vn);
+}
+
+void HELPER(crypto_sve_sm4e)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i;
+    intptr_t opr_sz = simd_oprsz(desc);
+
+    for (i = 0; i < opr_sz; i += 16) {
+        do_crypto_sm4e((uint64_t *)((char *)vd + i),
+                       (uint64_t *)((char *)vn + i),
+                       (uint64_t *)((char *)vm + i));
+    }
+    crypto_clear_tail(vd, opr_sz, simd_maxsz(desc));
+}
+
+static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm)
 {
-    uint64_t *rd = vd;
-    uint64_t *rn = vn;
-    uint64_t *rm = vm;
     union CRYPTO_STATE d;
     union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
     union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
@@ -693,3 +747,35 @@ void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm)
     rd[0] = d.l[0];
     rd[1] = d.l[1];
 }
+
+void HELPER(crypto_sm4ekey)(void *vd, void *vn, void *vm)
+{
+    do_crypto_sm4ekey(vd, vn, vm);
+}
+
+void HELPER(crypto_sve_sm4ekey)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i;
+    intptr_t opr_sz = simd_oprsz(desc);
+
+    for (i = 0; i < opr_sz; i += 16) {
+        do_crypto_sm4ekey((uint64_t *)((char *)vd + i),
+                          (uint64_t *)((char *)vn + i),
+                          (uint64_t *)((char *)vm + i));
+    }
+    crypto_clear_tail(vd, opr_sz, simd_maxsz(desc));
+}
+
+void HELPER(crypto_rax1)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i;
+    intptr_t opr_sz = simd_oprsz(desc);
+    uint64_t *d = vd;
+    uint64_t *n = vn;
+    uint64_t *m = vm;
+
+    for (i = 0; i < opr_sz / 8; i++) {
+        d[i] = n[i] ^ rol64(m[i], 1);
+    }
+    crypto_clear_tail(vd, opr_sz, simd_maxsz(desc));
+}
diff --git a/qemu/target/arm/decode-sve.inc.c b/qemu/target/arm/decode-sve.inc.c
index 9740f1aa80..bd9c2b6b9b 100644
--- a/qemu/target/arm/decode-sve.inc.c
+++ b/qemu/target/arm/decode-sve.inc.c
@@ -206,6 +206,15 @@ typedef struct {
     int s;
 } arg_ptrue;
 
+typedef struct {
+    int esz;
+    int imm;
+    int pd;
+    int pm;
+    int pn;
+    int rv;
+} arg_psel;
+
 typedef struct {
     int esz;
     int pg;
@@ -335,6 +344,15 @@ typedef struct {
     int rn;
 } arg_rprrr_esz;
 
+typedef struct {
+    int esz;
+    int ra;
+    int rd;
+    int rm;
+    int rn;
+    int rot;
+} arg_rprrr_rot_esz;
+
 typedef struct {
     int dbm;
     int rd;
@@ -374,6 +392,45 @@ typedef struct {
     int rn;
 } arg_rrri;
 
+typedef struct {
+    int esz;
+    int imm;
+    int rd;
+    int rm;
+    int rn;
+} arg_rrri_esz;
+
+typedef struct {
+    int esz;
+    int index;
+    int rd;
+    int rm;
+    int rn;
+} arg_rrx_esz;
+
+typedef struct {
+    int esz;
+    int index;
+    int ra;
+    int rd;
+    int rm;
+    int rn;
+} arg_rrxr_esz;
+
+typedef arg_rprrr_esz arg_EOR3;
+static bool trans_EOR3(DisasContext *ctx, arg_EOR3 *a);
+typedef arg_rprrr_esz arg_BSL;
+static bool trans_BSL(DisasContext *ctx, arg_BSL *a);
+typedef arg_rprrr_esz arg_BCAX;
+static bool trans_BCAX(DisasContext *ctx, arg_BCAX *a);
+typedef arg_rprrr_esz arg_BSL1N;
+static bool trans_BSL1N(DisasContext *ctx, arg_BSL1N *a);
+typedef arg_rprrr_esz arg_BSL2N;
+static bool trans_BSL2N(DisasContext *ctx, arg_BSL2N *a);
+typedef arg_rprrr_esz arg_NBSL;
+static bool trans_NBSL(DisasContext *ctx, arg_NBSL *a);
+typedef arg_rrri_esz arg_XAR;
+static bool trans_XAR(DisasContext *ctx, arg_XAR *a);
 typedef arg_rprr_esz arg_ORR_zpzz;
 static bool trans_ORR_zpzz(DisasContext *ctx, arg_ORR_zpzz *a);
 typedef arg_rprr_esz arg_EOR_zpzz;
@@ -398,6 +455,66 @@ typedef arg_rprr_esz arg_SABD_zpzz;
 static bool trans_SABD_zpzz(DisasContext *ctx, arg_SABD_zpzz *a);
 typedef arg_rprr_esz arg_UABD_zpzz;
 static bool trans_UABD_zpzz(DisasContext *ctx, arg_UABD_zpzz *a);
+typedef arg_rprr_esz arg_SADALP_zpzz;
+static bool trans_SADALP_zpzz(DisasContext *ctx, arg_SADALP_zpzz *a);
+typedef arg_rprr_esz arg_UADALP_zpzz;
+static bool trans_UADALP_zpzz(DisasContext *ctx, arg_UADALP_zpzz *a);
+typedef arg_rprr_esz arg_SHADD;
+static bool trans_SHADD(DisasContext *ctx, arg_SHADD *a);
+typedef arg_rprr_esz arg_UHADD;
+static bool trans_UHADD(DisasContext *ctx, arg_UHADD *a);
+typedef arg_rprr_esz arg_SHSUB;
+static bool trans_SHSUB(DisasContext *ctx, arg_SHSUB *a);
+typedef arg_rprr_esz arg_UHSUB;
+static bool trans_UHSUB(DisasContext *ctx, arg_UHSUB *a);
+typedef arg_rprr_esz arg_SRHADD;
+static bool trans_SRHADD(DisasContext *ctx, arg_SRHADD *a);
+typedef arg_rprr_esz arg_URHADD;
+static bool trans_URHADD(DisasContext *ctx, arg_URHADD *a);
+typedef arg_rprr_esz arg_ADDP;
+static bool trans_ADDP(DisasContext *ctx, arg_ADDP *a);
+typedef arg_rprr_esz arg_SMAXP;
+static bool trans_SMAXP(DisasContext *ctx, arg_SMAXP *a);
+typedef arg_rprr_esz arg_UMAXP;
+static bool trans_UMAXP(DisasContext *ctx, arg_UMAXP *a);
+typedef arg_rprr_esz arg_SMINP;
+static bool trans_SMINP(DisasContext *ctx, arg_SMINP *a);
+typedef arg_rprr_esz arg_UMINP;
+static bool trans_UMINP(DisasContext *ctx, arg_UMINP *a);
+typedef arg_rprr_esz arg_FADDP;
+static bool trans_FADDP(DisasContext *ctx, arg_FADDP *a);
+typedef arg_rprr_esz arg_FMAXNMP;
+static bool trans_FMAXNMP(DisasContext *ctx, arg_FMAXNMP *a);
+typedef arg_rprr_esz arg_FMINNMP;
+static bool trans_FMINNMP(DisasContext *ctx, arg_FMINNMP *a);
+typedef arg_rprr_esz arg_FMAXP;
+static bool trans_FMAXP(DisasContext *ctx, arg_FMAXP *a);
+typedef arg_rprr_esz arg_FMINP;
+static bool trans_FMINP(DisasContext *ctx, arg_FMINP *a);
+typedef arg_rprr_esz arg_SRSHL;
+static bool trans_SRSHL(DisasContext *ctx, arg_SRSHL *a);
+typedef arg_rprr_esz arg_URSHL;
+static bool trans_URSHL(DisasContext *ctx, arg_URSHL *a);
+typedef arg_rprr_esz arg_SQSHL;
+static bool trans_SQSHL(DisasContext *ctx, arg_SQSHL *a);
+typedef arg_rprr_esz arg_UQSHL;
+static bool trans_UQSHL(DisasContext *ctx, arg_UQSHL *a);
+typedef arg_rprr_esz arg_SQRSHL;
+static bool trans_SQRSHL(DisasContext *ctx, arg_SQRSHL *a);
+typedef arg_rprr_esz arg_UQRSHL;
+static bool trans_UQRSHL(DisasContext *ctx, arg_UQRSHL *a);
+typedef arg_rprr_esz arg_SQADD_zpzz;
+static bool trans_SQADD_zpzz(DisasContext *ctx, arg_SQADD_zpzz *a);
+typedef arg_rprr_esz arg_UQADD_zpzz;
+static bool trans_UQADD_zpzz(DisasContext *ctx, arg_UQADD_zpzz *a);
+typedef arg_rprr_esz arg_SQSUB_zpzz;
+static bool trans_SQSUB_zpzz(DisasContext *ctx, arg_SQSUB_zpzz *a);
+typedef arg_rprr_esz arg_UQSUB_zpzz;
+static bool trans_UQSUB_zpzz(DisasContext *ctx, arg_UQSUB_zpzz *a);
+typedef arg_rprr_esz arg_SUQADD;
+static bool trans_SUQADD(DisasContext *ctx, arg_SUQADD *a);
+typedef arg_rprr_esz arg_USQADD;
+static bool trans_USQADD(DisasContext *ctx, arg_USQADD *a);
 typedef arg_rprr_esz arg_MUL_zpzz;
 static bool trans_MUL_zpzz(DisasContext *ctx, arg_MUL_zpzz *a);
 typedef arg_rprr_esz arg_SMULH_zpzz;
@@ -468,6 +585,14 @@ typedef arg_rpr_esz arg_ABS;
 static bool trans_ABS(DisasContext *ctx, arg_ABS *a);
 typedef arg_rpr_esz arg_NEG;
 static bool trans_NEG(DisasContext *ctx, arg_NEG *a);
+typedef arg_rpr_esz arg_SQABS;
+static bool trans_SQABS(DisasContext *ctx, arg_SQABS *a);
+typedef arg_rpr_esz arg_SQNEG;
+static bool trans_SQNEG(DisasContext *ctx, arg_SQNEG *a);
+typedef arg_rpr_esz arg_URECPE;
+static bool trans_URECPE(DisasContext *ctx, arg_URECPE *a);
+typedef arg_rpr_esz arg_URSQRTE;
+static bool trans_URSQRTE(DisasContext *ctx, arg_URSQRTE *a);
 typedef arg_rpr_esz arg_SXTB;
 static bool trans_SXTB(DisasContext *ctx, arg_SXTB *a);
 typedef arg_rpr_esz arg_UXTB;
@@ -498,6 +623,102 @@ typedef arg_rprrr_esz arg_MLA;
 static bool trans_MLA(DisasContext *ctx, arg_MLA *a);
 typedef arg_rprrr_esz arg_MLS;
 static bool trans_MLS(DisasContext *ctx, arg_MLS *a);
+typedef arg_rprrr_esz arg_SQDMLALB_zzzw;
+static bool trans_SQDMLALB_zzzw(DisasContext *ctx, arg_SQDMLALB_zzzw *a);
+typedef arg_rprrr_esz arg_SQDMLALT_zzzw;
+static bool trans_SQDMLALT_zzzw(DisasContext *ctx, arg_SQDMLALT_zzzw *a);
+typedef arg_rprrr_esz arg_SQDMLALBT;
+static bool trans_SQDMLALBT(DisasContext *ctx, arg_SQDMLALBT *a);
+typedef arg_rprrr_esz arg_SQDMLSLB_zzzw;
+static bool trans_SQDMLSLB_zzzw(DisasContext *ctx, arg_SQDMLSLB_zzzw *a);
+typedef arg_rprrr_esz arg_SQDMLSLT_zzzw;
+static bool trans_SQDMLSLT_zzzw(DisasContext *ctx, arg_SQDMLSLT_zzzw *a);
+typedef arg_rprrr_esz arg_SQDMLSLBT;
+static bool trans_SQDMLSLBT(DisasContext *ctx, arg_SQDMLSLBT *a);
+typedef arg_rprrr_esz arg_SMLALB_zzzw;
+static bool trans_SMLALB_zzzw(DisasContext *ctx, arg_SMLALB_zzzw *a);
+typedef arg_rprrr_esz arg_SMLALT_zzzw;
+static bool trans_SMLALT_zzzw(DisasContext *ctx, arg_SMLALT_zzzw *a);
+typedef arg_rprrr_esz arg_UMLALB_zzzw;
+static bool trans_UMLALB_zzzw(DisasContext *ctx, arg_UMLALB_zzzw *a);
+typedef arg_rprrr_esz arg_UMLALT_zzzw;
+static bool trans_UMLALT_zzzw(DisasContext *ctx, arg_UMLALT_zzzw *a);
+typedef arg_rprrr_esz arg_SMLSLB_zzzw;
+static bool trans_SMLSLB_zzzw(DisasContext *ctx, arg_SMLSLB_zzzw *a);
+typedef arg_rprrr_esz arg_SMLSLT_zzzw;
+static bool trans_SMLSLT_zzzw(DisasContext *ctx, arg_SMLSLT_zzzw *a);
+typedef arg_rprrr_esz arg_UMLSLB_zzzw;
+static bool trans_UMLSLB_zzzw(DisasContext *ctx, arg_UMLSLB_zzzw *a);
+typedef arg_rprrr_esz arg_UMLSLT_zzzw;
+static bool trans_UMLSLT_zzzw(DisasContext *ctx, arg_UMLSLT_zzzw *a);
+typedef arg_rprrr_esz arg_SABALB;
+static bool trans_SABALB(DisasContext *ctx, arg_SABALB *a);
+typedef arg_rprrr_esz arg_SABALT;
+static bool trans_SABALT(DisasContext *ctx, arg_SABALT *a);
+typedef arg_rprrr_esz arg_UABALB;
+static bool trans_UABALB(DisasContext *ctx, arg_UABALB *a);
+typedef arg_rprrr_esz arg_UABALT;
+static bool trans_UABALT(DisasContext *ctx, arg_UABALT *a);
+typedef arg_rprrr_esz arg_SQRDMLAH_zzzz;
+static bool trans_SQRDMLAH_zzzz(DisasContext *ctx, arg_SQRDMLAH_zzzz *a);
+typedef arg_rprrr_esz arg_SQRDMLSH_zzzz;
+static bool trans_SQRDMLSH_zzzz(DisasContext *ctx, arg_SQRDMLSH_zzzz *a);
+typedef arg_rprrr_rot_esz arg_CMLA_zzzz;
+static bool trans_CMLA_zzzz(DisasContext *ctx, arg_CMLA_zzzz *a);
+typedef arg_rprrr_rot_esz arg_CDOT_zzzz;
+static bool trans_CDOT_zzzz(DisasContext *ctx, arg_CDOT_zzzz *a);
+typedef arg_rprrr_rot_esz arg_SQRDCMLAH_zzzz;
+static bool trans_SQRDCMLAH_zzzz(DisasContext *ctx, arg_SQRDCMLAH_zzzz *a);
+typedef arg_rprrr_esz arg_USDOT_zzzz;
+static bool trans_USDOT_zzzz(DisasContext *ctx, arg_USDOT_zzzz *a);
+typedef arg_rprrr_esz arg_SMMLA;
+static bool trans_SMMLA(DisasContext *ctx, arg_SMMLA *a);
+typedef arg_rprrr_esz arg_USMMLA;
+static bool trans_USMMLA(DisasContext *ctx, arg_USMMLA *a);
+typedef arg_rprrr_esz arg_UMMLA;
+static bool trans_UMMLA(DisasContext *ctx, arg_UMMLA *a);
+typedef arg_rprrr_esz arg_BFDOT_zzzz;
+static bool trans_BFDOT_zzzz(DisasContext *ctx, arg_BFDOT_zzzz *a);
+typedef arg_rrxr_esz arg_BFDOT_zzxz;
+static bool trans_BFDOT_zzxz(DisasContext *ctx, arg_BFDOT_zzxz *a);
+typedef arg_rprrr_esz arg_BFMMLA;
+static bool trans_BFMMLA(DisasContext *ctx, arg_BFMMLA *a);
+typedef arg_rprrr_esz arg_FMMLA_s;
+static bool trans_FMMLA_s(DisasContext *ctx, arg_FMMLA_s *a);
+typedef arg_rprrr_esz arg_FMMLA_d;
+static bool trans_FMMLA_d(DisasContext *ctx, arg_FMMLA_d *a);
+typedef arg_rprrr_esz arg_BFMLALB_zzzw;
+static bool trans_BFMLALB_zzzw(DisasContext *ctx, arg_BFMLALB_zzzw *a);
+typedef arg_rprrr_esz arg_BFMLALT_zzzw;
+static bool trans_BFMLALT_zzzw(DisasContext *ctx, arg_BFMLALT_zzzw *a);
+typedef arg_rrxr_esz arg_BFMLALB_zzxw;
+static bool trans_BFMLALB_zzxw(DisasContext *ctx, arg_BFMLALB_zzxw *a);
+typedef arg_rrxr_esz arg_BFMLALT_zzxw;
+static bool trans_BFMLALT_zzxw(DisasContext *ctx, arg_BFMLALT_zzxw *a);
+typedef arg_rprrr_esz arg_FMLALB_zzzw;
+static bool trans_FMLALB_zzzw(DisasContext *ctx, arg_FMLALB_zzzw *a);
+typedef arg_rprrr_esz arg_FMLALT_zzzw;
+static bool trans_FMLALT_zzzw(DisasContext *ctx, arg_FMLALT_zzzw *a);
+typedef arg_rprrr_esz arg_FMLSLB_zzzw;
+static bool trans_FMLSLB_zzzw(DisasContext *ctx, arg_FMLSLB_zzzw *a);
+typedef arg_rprrr_esz arg_FMLSLT_zzzw;
+static bool trans_FMLSLT_zzzw(DisasContext *ctx, arg_FMLSLT_zzzw *a);
+typedef arg_rrxr_esz arg_FMLALB_zzxw;
+static bool trans_FMLALB_zzxw(DisasContext *ctx, arg_FMLALB_zzxw *a);
+typedef arg_rrxr_esz arg_FMLALT_zzxw;
+static bool trans_FMLALT_zzxw(DisasContext *ctx, arg_FMLALT_zzxw *a);
+typedef arg_rrxr_esz arg_FMLSLB_zzxw;
+static bool trans_FMLSLB_zzxw(DisasContext *ctx, arg_FMLSLB_zzxw *a);
+typedef arg_rrxr_esz arg_FMLSLT_zzxw;
+static bool trans_FMLSLT_zzxw(DisasContext *ctx, arg_FMLSLT_zzxw *a);
+typedef arg_rrr_esz arg_CADD_rot90;
+static bool trans_CADD_rot90(DisasContext *ctx, arg_CADD_rot90 *a);
+typedef arg_rrr_esz arg_CADD_rot270;
+static bool trans_CADD_rot270(DisasContext *ctx, arg_CADD_rot270 *a);
+typedef arg_rrr_esz arg_SQCADD_rot90;
+static bool trans_SQCADD_rot90(DisasContext *ctx, arg_SQCADD_rot90 *a);
+typedef arg_rrr_esz arg_SQCADD_rot270;
+static bool trans_SQCADD_rot270(DisasContext *ctx, arg_SQCADD_rot270 *a);
 typedef arg_rrr_esz arg_ADD_zzz;
 static bool trans_ADD_zzz(DisasContext *ctx, arg_ADD_zzz *a);
 typedef arg_rrr_esz arg_SUB_zzz;
@@ -518,6 +739,266 @@ typedef arg_rrr_esz arg_EOR_zzz;
 static bool trans_EOR_zzz(DisasContext *ctx, arg_EOR_zzz *a);
 typedef arg_rrr_esz arg_BIC_zzz;
 static bool trans_BIC_zzz(DisasContext *ctx, arg_BIC_zzz *a);
+typedef arg_rrr_esz arg_PMULLB;
+static bool trans_PMULLB(DisasContext *ctx, arg_PMULLB *a);
+typedef arg_rrr_esz arg_PMULLT;
+static bool trans_PMULLT(DisasContext *ctx, arg_PMULLT *a);
+typedef arg_rrr_esz arg_MUL_zzz;
+static bool trans_MUL_zzz(DisasContext *ctx, arg_MUL_zzz *a);
+typedef arg_rrr_esz arg_SMULH_zzz;
+static bool trans_SMULH_zzz(DisasContext *ctx, arg_SMULH_zzz *a);
+typedef arg_rrr_esz arg_UMULH_zzz;
+static bool trans_UMULH_zzz(DisasContext *ctx, arg_UMULH_zzz *a);
+typedef arg_rrr_esz arg_PMUL_zzz;
+static bool trans_PMUL_zzz(DisasContext *ctx, arg_PMUL_zzz *a);
+typedef arg_rrr_esz arg_SQDMULH_zzz;
+static bool trans_SQDMULH_zzz(DisasContext *ctx, arg_SQDMULH_zzz *a);
+typedef arg_rrr_esz arg_SQRDMULH_zzz;
+static bool trans_SQRDMULH_zzz(DisasContext *ctx, arg_SQRDMULH_zzz *a);
+typedef arg_rrx_esz arg_MUL_zzx;
+static bool trans_MUL_zzx(DisasContext *ctx, arg_MUL_zzx *a);
+typedef arg_rrx_esz arg_SQDMULH_zzx;
+static bool trans_SQDMULH_zzx(DisasContext *ctx, arg_SQDMULH_zzx *a);
+typedef arg_rrx_esz arg_SQRDMULH_zzx;
+static bool trans_SQRDMULH_zzx(DisasContext *ctx, arg_SQRDMULH_zzx *a);
+typedef arg_rrxr_esz arg_SQRDMLAH_zzxz;
+static bool trans_SQRDMLAH_zzxz(DisasContext *ctx, arg_SQRDMLAH_zzxz *a);
+typedef arg_rrxr_esz arg_SQRDMLSH_zzxz;
+static bool trans_SQRDMLSH_zzxz(DisasContext *ctx, arg_SQRDMLSH_zzxz *a);
+typedef arg_disas_sve43 arg_CMLA_zzxz_h;
+static bool trans_CMLA_zzxz_h(DisasContext *ctx, arg_CMLA_zzxz_h *a);
+typedef arg_disas_sve43 arg_CMLA_zzxz_s;
+static bool trans_CMLA_zzxz_s(DisasContext *ctx, arg_CMLA_zzxz_s *a);
+typedef arg_disas_sve43 arg_SQRDCMLAH_zzxz_h;
+static bool trans_SQRDCMLAH_zzxz_h(DisasContext *ctx,
+                                   arg_SQRDCMLAH_zzxz_h *a);
+typedef arg_disas_sve43 arg_SQRDCMLAH_zzxz_s;
+static bool trans_SQRDCMLAH_zzxz_s(DisasContext *ctx,
+                                   arg_SQRDCMLAH_zzxz_s *a);
+typedef arg_disas_sve43 arg_CDOT_zzxw_s;
+static bool trans_CDOT_zzxw_s(DisasContext *ctx, arg_CDOT_zzxw_s *a);
+typedef arg_disas_sve43 arg_CDOT_zzxw_d;
+static bool trans_CDOT_zzxw_d(DisasContext *ctx, arg_CDOT_zzxw_d *a);
+typedef arg_rrxr_esz arg_USDOT_zzxw_s;
+static bool trans_USDOT_zzxw_s(DisasContext *ctx, arg_USDOT_zzxw_s *a);
+typedef arg_rrxr_esz arg_SUDOT_zzxw_s;
+static bool trans_SUDOT_zzxw_s(DisasContext *ctx, arg_SUDOT_zzxw_s *a);
+typedef arg_rrx_esz arg_SMULLB_zzx;
+static bool trans_SMULLB_zzx(DisasContext *ctx, arg_SMULLB_zzx *a);
+typedef arg_rrx_esz arg_SMULLT_zzx;
+static bool trans_SMULLT_zzx(DisasContext *ctx, arg_SMULLT_zzx *a);
+typedef arg_rrx_esz arg_UMULLB_zzx;
+static bool trans_UMULLB_zzx(DisasContext *ctx, arg_UMULLB_zzx *a);
+typedef arg_rrx_esz arg_UMULLT_zzx;
+static bool trans_UMULLT_zzx(DisasContext *ctx, arg_UMULLT_zzx *a);
+typedef arg_rrx_esz arg_SQDMULLB_zzx;
+static bool trans_SQDMULLB_zzx(DisasContext *ctx, arg_SQDMULLB_zzx *a);
+typedef arg_rrx_esz arg_SQDMULLT_zzx;
+static bool trans_SQDMULLT_zzx(DisasContext *ctx, arg_SQDMULLT_zzx *a);
+typedef arg_rrx_esz arg_SQDMLALB_zzxw;
+static bool trans_SQDMLALB_zzxw(DisasContext *ctx,
+                                arg_SQDMLALB_zzxw *a);
+typedef arg_rrx_esz arg_SQDMLALT_zzxw;
+static bool trans_SQDMLALT_zzxw(DisasContext *ctx,
+                                arg_SQDMLALT_zzxw *a);
+typedef arg_rrx_esz arg_SQDMLSLB_zzxw;
+static bool trans_SQDMLSLB_zzxw(DisasContext *ctx,
+                                arg_SQDMLSLB_zzxw *a);
+typedef arg_rrx_esz arg_SQDMLSLT_zzxw;
+static bool trans_SQDMLSLT_zzxw(DisasContext *ctx,
+                                arg_SQDMLSLT_zzxw *a);
+typedef arg_rrx_esz arg_SMLALB_zzxw;
+static bool trans_SMLALB_zzxw(DisasContext *ctx, arg_SMLALB_zzxw *a);
+typedef arg_rrx_esz arg_SMLALT_zzxw;
+static bool trans_SMLALT_zzxw(DisasContext *ctx, arg_SMLALT_zzxw *a);
+typedef arg_rrx_esz arg_UMLALB_zzxw;
+static bool trans_UMLALB_zzxw(DisasContext *ctx, arg_UMLALB_zzxw *a);
+typedef arg_rrx_esz arg_UMLALT_zzxw;
+static bool trans_UMLALT_zzxw(DisasContext *ctx, arg_UMLALT_zzxw *a);
+typedef arg_rrx_esz arg_SMLSLB_zzxw;
+static bool trans_SMLSLB_zzxw(DisasContext *ctx, arg_SMLSLB_zzxw *a);
+typedef arg_rrx_esz arg_SMLSLT_zzxw;
+static bool trans_SMLSLT_zzxw(DisasContext *ctx, arg_SMLSLT_zzxw *a);
+typedef arg_rrx_esz arg_UMLSLB_zzxw;
+static bool trans_UMLSLB_zzxw(DisasContext *ctx, arg_UMLSLB_zzxw *a);
+typedef arg_rrx_esz arg_UMLSLT_zzxw;
+static bool trans_UMLSLT_zzxw(DisasContext *ctx, arg_UMLSLT_zzxw *a);
+typedef arg_rrr_esz arg_SADDLB;
+static bool trans_SADDLB(DisasContext *ctx, arg_SADDLB *a);
+typedef arg_rrr_esz arg_SADDLT;
+static bool trans_SADDLT(DisasContext *ctx, arg_SADDLT *a);
+typedef arg_rrr_esz arg_UADDLB;
+static bool trans_UADDLB(DisasContext *ctx, arg_UADDLB *a);
+typedef arg_rrr_esz arg_UADDLT;
+static bool trans_UADDLT(DisasContext *ctx, arg_UADDLT *a);
+typedef arg_rrr_esz arg_SSUBLB;
+static bool trans_SSUBLB(DisasContext *ctx, arg_SSUBLB *a);
+typedef arg_rrr_esz arg_SSUBLT;
+static bool trans_SSUBLT(DisasContext *ctx, arg_SSUBLT *a);
+typedef arg_rrr_esz arg_USUBLB;
+static bool trans_USUBLB(DisasContext *ctx, arg_USUBLB *a);
+typedef arg_rrr_esz arg_USUBLT;
+static bool trans_USUBLT(DisasContext *ctx, arg_USUBLT *a);
+typedef arg_rrr_esz arg_SADDLBT;
+static bool trans_SADDLBT(DisasContext *ctx, arg_SADDLBT *a);
+typedef arg_rrr_esz arg_SSUBLBT;
+static bool trans_SSUBLBT(DisasContext *ctx, arg_SSUBLBT *a);
+typedef arg_rrr_esz arg_SSUBLTB;
+static bool trans_SSUBLTB(DisasContext *ctx, arg_SSUBLTB *a);
+typedef arg_rrr_esz arg_SABDLB;
+static bool trans_SABDLB(DisasContext *ctx, arg_SABDLB *a);
+typedef arg_rrr_esz arg_SABDLT;
+static bool trans_SABDLT(DisasContext *ctx, arg_SABDLT *a);
+typedef arg_rrr_esz arg_UABDLB;
+static bool trans_UABDLB(DisasContext *ctx, arg_UABDLB *a);
+typedef arg_rrr_esz arg_UABDLT;
+static bool trans_UABDLT(DisasContext *ctx, arg_UABDLT *a);
+typedef arg_rrr_esz arg_SABA;
+static bool trans_SABA(DisasContext *ctx, arg_SABA *a);
+typedef arg_rrr_esz arg_UABA;
+static bool trans_UABA(DisasContext *ctx, arg_UABA *a);
+typedef arg_rrr_esz arg_SMULLB_zzz;
+static bool trans_SMULLB_zzz(DisasContext *ctx, arg_SMULLB_zzz *a);
+typedef arg_rrr_esz arg_SMULLT_zzz;
+static bool trans_SMULLT_zzz(DisasContext *ctx, arg_SMULLT_zzz *a);
+typedef arg_rrr_esz arg_UMULLB_zzz;
+static bool trans_UMULLB_zzz(DisasContext *ctx, arg_UMULLB_zzz *a);
+typedef arg_rrr_esz arg_UMULLT_zzz;
+static bool trans_UMULLT_zzz(DisasContext *ctx, arg_UMULLT_zzz *a);
+typedef arg_rrr_esz arg_SQDMULLB_zzz;
+static bool trans_SQDMULLB_zzz(DisasContext *ctx, arg_SQDMULLB_zzz *a);
+typedef arg_rrr_esz arg_SQDMULLT_zzz;
+static bool trans_SQDMULLT_zzz(DisasContext *ctx, arg_SQDMULLT_zzz *a);
+typedef arg_rrr_esz arg_SADDWB;
+static bool trans_SADDWB(DisasContext *ctx, arg_SADDWB *a);
+typedef arg_rrr_esz arg_SADDWT;
+static bool trans_SADDWT(DisasContext *ctx, arg_SADDWT *a);
+typedef arg_rrr_esz arg_UADDWB;
+static bool trans_UADDWB(DisasContext *ctx, arg_UADDWB *a);
+typedef arg_rrr_esz arg_UADDWT;
+static bool trans_UADDWT(DisasContext *ctx, arg_UADDWT *a);
+typedef arg_rrr_esz arg_SSUBWB;
+static bool trans_SSUBWB(DisasContext *ctx, arg_SSUBWB *a);
+typedef arg_rrr_esz arg_SSUBWT;
+static bool trans_SSUBWT(DisasContext *ctx, arg_SSUBWT *a);
+typedef arg_rrr_esz arg_USUBWB;
+static bool trans_USUBWB(DisasContext *ctx, arg_USUBWB *a);
+typedef arg_rrr_esz arg_USUBWT;
+static bool trans_USUBWT(DisasContext *ctx, arg_USUBWT *a);
+typedef arg_rri_esz arg_SSHLLB;
+static bool trans_SSHLLB(DisasContext *ctx, arg_SSHLLB *a);
+typedef arg_rri_esz arg_SSHLLT;
+static bool trans_SSHLLT(DisasContext *ctx, arg_SSHLLT *a);
+typedef arg_rri_esz arg_USHLLB;
+static bool trans_USHLLB(DisasContext *ctx, arg_USHLLB *a);
+typedef arg_rri_esz arg_USHLLT;
+static bool trans_USHLLT(DisasContext *ctx, arg_USHLLT *a);
+typedef arg_rri_esz arg_SSRA;
+static bool trans_SSRA(DisasContext *ctx, arg_SSRA *a);
+typedef arg_rri_esz arg_USRA;
+static bool trans_USRA(DisasContext *ctx, arg_USRA *a);
+typedef arg_rri_esz arg_SRSRA;
+static bool trans_SRSRA(DisasContext *ctx, arg_SRSRA *a);
+typedef arg_rri_esz arg_URSRA;
+static bool trans_URSRA(DisasContext *ctx, arg_URSRA *a);
+typedef arg_rri_esz arg_SRI;
+static bool trans_SRI(DisasContext *ctx, arg_SRI *a);
+typedef arg_rri_esz arg_SLI;
+static bool trans_SLI(DisasContext *ctx, arg_SLI *a);
+typedef arg_rri_esz arg_SQSHRUNB;
+static bool trans_SQSHRUNB(DisasContext *ctx, arg_SQSHRUNB *a);
+typedef arg_rri_esz arg_SQSHRUNT;
+static bool trans_SQSHRUNT(DisasContext *ctx, arg_SQSHRUNT *a);
+typedef arg_rri_esz arg_SQRSHRUNB;
+static bool trans_SQRSHRUNB(DisasContext *ctx, arg_SQRSHRUNB *a);
+typedef arg_rri_esz arg_SQRSHRUNT;
+static bool trans_SQRSHRUNT(DisasContext *ctx, arg_SQRSHRUNT *a);
+typedef arg_rri_esz arg_SHRNB;
+static bool trans_SHRNB(DisasContext *ctx, arg_SHRNB *a);
+typedef arg_rri_esz arg_SHRNT;
+static bool trans_SHRNT(DisasContext *ctx, arg_SHRNT *a);
+typedef arg_rri_esz arg_RSHRNB;
+static bool trans_RSHRNB(DisasContext *ctx, arg_RSHRNB *a);
+typedef arg_rri_esz arg_RSHRNT;
+static bool trans_RSHRNT(DisasContext *ctx, arg_RSHRNT *a);
+typedef arg_rri_esz arg_SQSHRNB;
+static bool trans_SQSHRNB(DisasContext *ctx, arg_SQSHRNB *a);
+typedef arg_rri_esz arg_SQSHRNT;
+static bool trans_SQSHRNT(DisasContext *ctx, arg_SQSHRNT *a);
+typedef arg_rri_esz arg_SQRSHRNB;
+static bool trans_SQRSHRNB(DisasContext *ctx, arg_SQRSHRNB *a);
+typedef arg_rri_esz arg_SQRSHRNT;
+static bool trans_SQRSHRNT(DisasContext *ctx, arg_SQRSHRNT *a);
+typedef arg_rri_esz arg_UQSHRNB;
+static bool trans_UQSHRNB(DisasContext *ctx, arg_UQSHRNB *a);
+typedef arg_rri_esz arg_UQSHRNT;
+static bool trans_UQSHRNT(DisasContext *ctx, arg_UQSHRNT *a);
+typedef arg_rri_esz arg_UQRSHRNB;
+static bool trans_UQRSHRNB(DisasContext *ctx, arg_UQRSHRNB *a);
+typedef arg_rri_esz arg_UQRSHRNT;
+static bool trans_UQRSHRNT(DisasContext *ctx, arg_UQRSHRNT *a);
+typedef arg_rrr_esz arg_ADDHNB;
+static bool trans_ADDHNB(DisasContext *ctx, arg_ADDHNB *a);
+typedef arg_rrr_esz arg_ADDHNT;
+static bool trans_ADDHNT(DisasContext *ctx, arg_ADDHNT *a);
+typedef arg_rrr_esz arg_RADDHNB;
+static bool trans_RADDHNB(DisasContext *ctx, arg_RADDHNB *a);
+typedef arg_rrr_esz arg_RADDHNT;
+static bool trans_RADDHNT(DisasContext *ctx, arg_RADDHNT *a);
+typedef arg_rrr_esz arg_SUBHNB;
+static bool trans_SUBHNB(DisasContext *ctx, arg_SUBHNB *a);
+typedef arg_rrr_esz arg_SUBHNT;
+static bool trans_SUBHNT(DisasContext *ctx, arg_SUBHNT *a);
+typedef arg_rrr_esz arg_RSUBHNB;
+static bool trans_RSUBHNB(DisasContext *ctx, arg_RSUBHNB *a);
+typedef arg_rrr_esz arg_RSUBHNT;
+static bool trans_RSUBHNT(DisasContext *ctx, arg_RSUBHNT *a);
+typedef arg_rri_esz arg_SQXTNB;
+static bool trans_SQXTNB(DisasContext *ctx, arg_SQXTNB *a);
+typedef arg_rri_esz arg_SQXTNT;
+static bool trans_SQXTNT(DisasContext *ctx, arg_SQXTNT *a);
+typedef arg_rri_esz arg_UQXTNB;
+static bool trans_UQXTNB(DisasContext *ctx, arg_UQXTNB *a);
+typedef arg_rri_esz arg_UQXTNT;
+static bool trans_UQXTNT(DisasContext *ctx, arg_UQXTNT *a);
+typedef arg_rri_esz arg_SQXTUNB;
+static bool trans_SQXTUNB(DisasContext *ctx, arg_SQXTUNB *a);
+typedef arg_rri_esz arg_SQXTUNT;
+static bool trans_SQXTUNT(DisasContext *ctx, arg_SQXTUNT *a);
+typedef arg_rrr_esz arg_EORBT;
+static bool trans_EORBT(DisasContext *ctx, arg_EORBT *a);
+typedef arg_rrr_esz arg_EORTB;
+static bool trans_EORTB(DisasContext *ctx, arg_EORTB *a);
+typedef arg_rrr_esz arg_BEXT;
+static bool trans_BEXT(DisasContext *ctx, arg_BEXT *a);
+typedef arg_rrr_esz arg_BDEP;
+static bool trans_BDEP(DisasContext *ctx, arg_BDEP *a);
+typedef arg_rrr_esz arg_BGRP;
+static bool trans_BGRP(DisasContext *ctx, arg_BGRP *a);
+typedef arg_rprr_esz arg_MATCH;
+static bool trans_MATCH(DisasContext *ctx, arg_MATCH *a);
+typedef arg_rprr_esz arg_NMATCH;
+static bool trans_NMATCH(DisasContext *ctx, arg_NMATCH *a);
+typedef arg_rprr_esz arg_HISTCNT;
+static bool trans_HISTCNT(DisasContext *ctx, arg_HISTCNT *a);
+typedef arg_rrr_esz arg_HISTSEG;
+static bool trans_HISTSEG(DisasContext *ctx, arg_HISTSEG *a);
+typedef arg_rri arg_AESMC;
+static bool trans_AESMC(DisasContext *ctx, arg_AESMC *a);
+typedef arg_rrr_esz arg_AESE;
+static bool trans_AESE(DisasContext *ctx, arg_AESE *a);
+typedef arg_rrr_esz arg_AESD;
+static bool trans_AESD(DisasContext *ctx, arg_AESD *a);
+typedef arg_rrr_esz arg_SM4E;
+static bool trans_SM4E(DisasContext *ctx, arg_SM4E *a);
+typedef arg_rrr_esz arg_SM4EKEY;
+static bool trans_SM4EKEY(DisasContext *ctx, arg_SM4EKEY *a);
+typedef arg_rrr_esz arg_RAX1;
+static bool trans_RAX1(DisasContext *ctx, arg_RAX1 *a);
+typedef arg_rprrr_esz arg_ADCLB;
+static bool trans_ADCLB(DisasContext *ctx, arg_ADCLB *a);
+typedef arg_rprrr_esz arg_ADCLT;
+static bool trans_ADCLT(DisasContext *ctx, arg_ADCLT *a);
 typedef arg_disas_sve25 arg_INDEX_ii;
 static bool trans_INDEX_ii(DisasContext *ctx, arg_INDEX_ii *a);
 typedef arg_disas_sve26 arg_INDEX_ir;
@@ -528,10 +1009,16 @@ typedef arg_rrr_esz arg_INDEX_rr;
 static bool trans_INDEX_rr(DisasContext *ctx, arg_INDEX_rr *a);
 typedef arg_rri arg_ADDVL;
 static bool trans_ADDVL(DisasContext *ctx, arg_ADDVL *a);
+typedef arg_rri arg_ADDSVL;
+static bool trans_ADDSVL(DisasContext *ctx, arg_ADDSVL *a);
 typedef arg_rri arg_ADDPL;
 static bool trans_ADDPL(DisasContext *ctx, arg_ADDPL *a);
+typedef arg_rri arg_ADDSPL;
+static bool trans_ADDSPL(DisasContext *ctx, arg_ADDSPL *a);
 typedef arg_disas_sve27 arg_RDVL;
 static bool trans_RDVL(DisasContext *ctx, arg_RDVL *a);
+typedef arg_disas_sve27 arg_RDSVL;
+static bool trans_RDSVL(DisasContext *ctx, arg_RDSVL *a);
 typedef arg_rri_esz arg_ASR_zzi;
 static bool trans_ASR_zzi(DisasContext *ctx, arg_ASR_zzi *a);
 typedef arg_rri_esz arg_LSR_zzi;
@@ -586,6 +1073,8 @@ typedef arg_rpri_esz arg_CPY_z_i;
 static bool trans_CPY_z_i(DisasContext *ctx, arg_CPY_z_i *a);
 typedef arg_rrri arg_EXT;
 static bool trans_EXT(DisasContext *ctx, arg_EXT *a);
+typedef arg_rri arg_EXT_sve2;
+static bool trans_EXT_sve2(DisasContext *ctx, arg_EXT_sve2 *a);
 typedef arg_rr_esz arg_DUP_s;
 static bool trans_DUP_s(DisasContext *ctx, arg_DUP_s *a);
 typedef arg_rri arg_DUP_x;
@@ -598,6 +1087,10 @@ typedef arg_rr_esz arg_REV_v;
 static bool trans_REV_v(DisasContext *ctx, arg_REV_v *a);
 typedef arg_rrr_esz arg_TBL;
 static bool trans_TBL(DisasContext *ctx, arg_TBL *a);
+typedef arg_rrr_esz arg_TBL_sve2;
+static bool trans_TBL_sve2(DisasContext *ctx, arg_TBL_sve2 *a);
+typedef arg_rrr_esz arg_TBX;
+static bool trans_TBX(DisasContext *ctx, arg_TBX *a);
 typedef arg_disas_sve30 arg_UNPK;
 static bool trans_UNPK(DisasContext *ctx, arg_UNPK *a);
 typedef arg_rrr_esz arg_ZIP1_p;
@@ -622,14 +1115,26 @@ typedef arg_rrr_esz arg_ZIP1_z;
 static bool trans_ZIP1_z(DisasContext *ctx, arg_ZIP1_z *a);
 typedef arg_rrr_esz arg_ZIP2_z;
 static bool trans_ZIP2_z(DisasContext *ctx, arg_ZIP2_z *a);
+typedef arg_rrr_esz arg_ZIP1_q;
+static bool trans_ZIP1_q(DisasContext *ctx, arg_ZIP1_q *a);
+typedef arg_rrr_esz arg_ZIP2_q;
+static bool trans_ZIP2_q(DisasContext *ctx, arg_ZIP2_q *a);
 typedef arg_rrr_esz arg_UZP1_z;
 static bool trans_UZP1_z(DisasContext *ctx, arg_UZP1_z *a);
 typedef arg_rrr_esz arg_UZP2_z;
 static bool trans_UZP2_z(DisasContext *ctx, arg_UZP2_z *a);
+typedef arg_rrr_esz arg_UZP1_q;
+static bool trans_UZP1_q(DisasContext *ctx, arg_UZP1_q *a);
+typedef arg_rrr_esz arg_UZP2_q;
+static bool trans_UZP2_q(DisasContext *ctx, arg_UZP2_q *a);
 typedef arg_rrr_esz arg_TRN1_z;
 static bool trans_TRN1_z(DisasContext *ctx, arg_TRN1_z *a);
 typedef arg_rrr_esz arg_TRN2_z;
 static bool trans_TRN2_z(DisasContext *ctx, arg_TRN2_z *a);
+typedef arg_rrr_esz arg_TRN1_q;
+static bool trans_TRN1_q(DisasContext *ctx, arg_TRN1_q *a);
+typedef arg_rrr_esz arg_TRN2_q;
+static bool trans_TRN2_q(DisasContext *ctx, arg_TRN2_q *a);
 typedef arg_rpr_esz arg_COMPACT;
 static bool trans_COMPACT(DisasContext *ctx, arg_COMPACT *a);
 typedef arg_rprr_esz arg_CLASTA_z;
@@ -666,6 +1171,8 @@ typedef arg_rpr_esz arg_RBIT;
 static bool trans_RBIT(DisasContext *ctx, arg_RBIT *a);
 typedef arg_rprr_esz arg_SPLICE;
 static bool trans_SPLICE(DisasContext *ctx, arg_SPLICE *a);
+typedef arg_rpr_esz arg_SPLICE_sve2;
+static bool trans_SPLICE_sve2(DisasContext *ctx, arg_SPLICE_sve2 *a);
 typedef arg_rprr_esz arg_SEL_zpzz;
 static bool trans_SEL_zpzz(DisasContext *ctx, arg_SEL_zpzz *a);
 typedef arg_rprr_esz arg_CMPHS_ppzz;
@@ -740,6 +1247,8 @@ typedef arg_disas_sve31 arg_PTEST;
 static bool trans_PTEST(DisasContext *ctx, arg_PTEST *a);
 typedef arg_ptrue arg_PTRUE;
 static bool trans_PTRUE(DisasContext *ctx, arg_PTRUE *a);
+typedef arg_psel arg_PSEL;
+static bool trans_PSEL(DisasContext *ctx, arg_PSEL *a);
 typedef arg_disas_sve32 arg_SETFFR;
 static bool trans_SETFFR(DisasContext *ctx, arg_SETFFR *a);
 typedef arg_disas_sve33 arg_PFALSE;
@@ -926,6 +1435,24 @@ typedef arg_rpr_esz arg_FCVT_ds;
 static bool trans_FCVT_ds(DisasContext *ctx, arg_FCVT_ds *a);
 typedef arg_rpr_esz arg_FCVT_sd;
 static bool trans_FCVT_sd(DisasContext *ctx, arg_FCVT_sd *a);
+typedef arg_rpr_esz arg_BFCVT;
+static bool trans_BFCVT(DisasContext *ctx, arg_BFCVT *a);
+typedef arg_rpr_esz arg_FCVTNT_sh;
+static bool trans_FCVTNT_sh(DisasContext *ctx, arg_FCVTNT_sh *a);
+typedef arg_rpr_esz arg_FCVTNT_ds;
+static bool trans_FCVTNT_ds(DisasContext *ctx, arg_FCVTNT_ds *a);
+typedef arg_rpr_esz arg_BFCVTNT;
+static bool trans_BFCVTNT(DisasContext *ctx, arg_BFCVTNT *a);
+typedef arg_rpr_esz arg_FCVTLT_hs;
+static bool trans_FCVTLT_hs(DisasContext *ctx, arg_FCVTLT_hs *a);
+typedef arg_rpr_esz arg_FCVTLT_sd;
+static bool trans_FCVTLT_sd(DisasContext *ctx, arg_FCVTLT_sd *a);
+typedef arg_rpr_esz arg_FCVTX_ds;
+static bool trans_FCVTX_ds(DisasContext *ctx, arg_FCVTX_ds *a);
+typedef arg_rpr_esz arg_FCVTXNT_ds;
+static bool trans_FCVTXNT_ds(DisasContext *ctx, arg_FCVTXNT_ds *a);
+typedef arg_rpr_esz arg_FLOGB;
+static bool trans_FLOGB(DisasContext *ctx, arg_FLOGB *a);
 typedef arg_rpr_esz arg_FCVTZS_hh;
 static bool trans_FCVTZS_hh(DisasContext *ctx, arg_FCVTZS_hh *a);
 typedef arg_rpr_esz arg_FCVTZU_hh;
@@ -1010,6 +1537,8 @@ typedef arg_rprr_gather_load arg_LD1_zprz;
 static bool trans_LD1_zprz(DisasContext *ctx, arg_LD1_zprz *a);
 typedef arg_rpri_gather_load arg_LD1_zpiz;
 static bool trans_LD1_zpiz(DisasContext *ctx, arg_LD1_zpiz *a);
+typedef arg_rprr_gather_load arg_LDNT1_zprz;
+static bool trans_LDNT1_zprz(DisasContext *ctx, arg_LDNT1_zprz *a);
 typedef arg_rprr_load arg_LD_zprr;
 static bool trans_LD_zprr(DisasContext *ctx, arg_LD_zprr *a);
 typedef arg_rprr_load arg_LDFF1_zprr;
@@ -1022,6 +1551,10 @@ typedef arg_rprr_load arg_LD1RQ_zprr;
 static bool trans_LD1RQ_zprr(DisasContext *ctx, arg_LD1RQ_zprr *a);
 typedef arg_rpri_load arg_LD1RQ_zpri;
 static bool trans_LD1RQ_zpri(DisasContext *ctx, arg_LD1RQ_zpri *a);
+typedef arg_rprr_load arg_LD1RO_zprr;
+static bool trans_LD1RO_zprr(DisasContext *ctx, arg_LD1RO_zprr *a);
+typedef arg_rpri_load arg_LD1RO_zpri;
+static bool trans_LD1RO_zpri(DisasContext *ctx, arg_LD1RO_zpri *a);
 typedef arg_disas_sve32 arg_PRF;
 static bool trans_PRF(DisasContext *ctx, arg_PRF *a);
 typedef arg_disas_sve47 arg_PRF_rr;
@@ -1038,6 +1571,8 @@ typedef arg_rprr_scatter_store arg_ST1_zprz;
 static bool trans_ST1_zprz(DisasContext *ctx, arg_ST1_zprz *a);
 typedef arg_rpri_scatter_store arg_ST1_zpiz;
 static bool trans_ST1_zpiz(DisasContext *ctx, arg_ST1_zpiz *a);
+typedef arg_rprr_scatter_store arg_STNT1_zprz;
+static bool trans_STNT1_zprz(DisasContext *ctx, arg_STNT1_zprz *a);
 
 static void disas_sve_extract_disas_sve_Fmt_55(DisasContext *ctx, arg_disas_sve25 *a, uint32_t insn)
 {
@@ -1089,6 +1624,13 @@ static void disas_sve_extract_disas_sve_Fmt_61(DisasContext *ctx, arg_rrri *a, u
     a->imm = deposit32(extract32(insn, 10, 3), 3, 29, extract32(insn, 16, 5));
 }
 
+static void disas_sve_extract_ext_sve2(DisasContext *ctx, arg_rri *a, uint32_t insn)
+{
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->imm = deposit32(extract32(insn, 10, 3), 3, 29, extract32(insn, 16, 5));
+}
+
 static void disas_sve_extract_disas_sve_Fmt_62(DisasContext *ctx, arg_rri *a, uint32_t insn)
 {
     a->rn = extract32(insn, 5, 5);
@@ -1119,6 +1661,46 @@ static void disas_sve_extract_disas_sve_Fmt_65(DisasContext *ctx, arg_ptrue *a,
     a->rd = extract32(insn, 0, 4);
 }
 
+static bool disas_sve_extract_psel(DisasContext *ctx, arg_psel *a,
+                                   uint32_t insn)
+{
+    switch (extract32(insn, 18, 3)) {
+    case 0x1:
+    case 0x3:
+    case 0x5:
+    case 0x7:
+        a->esz = 0;
+        a->imm = (extract32(insn, 22, 2) << 2) |
+                 extract32(insn, 19, 2);
+        break;
+    case 0x2:
+    case 0x6:
+        a->esz = 1;
+        a->imm = (extract32(insn, 22, 2) << 1) |
+                 extract32(insn, 20, 1);
+        break;
+    case 0x4:
+        a->esz = 2;
+        a->imm = extract32(insn, 22, 2);
+        break;
+    case 0x0:
+        if (!extract32(insn, 22, 1)) {
+            return false;
+        }
+        a->esz = 3;
+        a->imm = extract32(insn, 23, 1);
+        break;
+    default:
+        return false;
+    }
+
+    a->pd = extract32(insn, 0, 4);
+    a->pm = extract32(insn, 5, 4);
+    a->pn = extract32(insn, 10, 4);
+    a->rv = extract32(insn, 16, 2) + 12;
+    return true;
+}
+
 static void disas_sve_extract_disas_sve_Fmt_66(DisasContext *ctx, arg_disas_sve32 *a, uint32_t insn)
 {
 }
@@ -1248,6 +1830,56 @@ static void disas_sve_extract_disas_sve_Fmt_80(DisasContext *ctx, arg_disas_sve4
     a->esz = 2;
 }
 
+static void disas_sve_extract_rrxr_rot_h(DisasContext *ctx,
+                                         arg_disas_sve43 *a, uint32_t insn)
+{
+    a->index = extract32(insn, 19, 2);
+    a->rm = extract32(insn, 16, 3);
+    a->rot = extract32(insn, 10, 2);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 1;
+}
+
+static void disas_sve_extract_rrxr_rot_s(DisasContext *ctx,
+                                         arg_disas_sve43 *a, uint32_t insn)
+{
+    a->index = extract32(insn, 20, 1);
+    a->rm = extract32(insn, 16, 4);
+    a->rot = extract32(insn, 10, 2);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 2;
+}
+
+static void disas_sve_extract_rrxr_rot_cdot_s(DisasContext *ctx,
+                                              arg_disas_sve43 *a,
+                                              uint32_t insn)
+{
+    a->index = extract32(insn, 19, 2);
+    a->rm = extract32(insn, 16, 3);
+    a->rot = extract32(insn, 10, 2);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 2;
+}
+
+static void disas_sve_extract_rrxr_rot_cdot_d(DisasContext *ctx,
+                                              arg_disas_sve43 *a,
+                                              uint32_t insn)
+{
+    a->index = extract32(insn, 20, 1);
+    a->rm = extract32(insn, 16, 4);
+    a->rot = extract32(insn, 10, 2);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 3;
+}
+
 static void disas_sve_extract_disas_sve_Fmt_81(DisasContext *ctx, arg_disas_sve44 *a, uint32_t insn)
 {
     a->rm = extract32(insn, 16, 3);
@@ -1470,6 +2102,22 @@ static void disas_sve_extract_rd_pg4_rn_rm(DisasContext *ctx, arg_rprr_esz *a, u
     a->rd = extract32(insn, 0, 5);
 }
 
+static void disas_sve_extract_rd_pg_rn_rm(DisasContext *ctx, arg_rprr_esz *a, uint32_t insn)
+{
+    a->esz = extract32(insn, 22, 2);
+    a->rm = extract32(insn, 16, 5);
+    a->pg = extract32(insn, 10, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+}
+
+static void disas_sve_extract_rd_decrypt(DisasContext *ctx, arg_rri *a, uint32_t insn)
+{
+    a->imm = extract32(insn, 10, 1);
+    a->rd = extract32(insn, 0, 5);
+    a->rn = 0;
+}
+
 static void disas_sve_extract_rd_pg_rn(DisasContext *ctx, arg_rpr_esz *a, uint32_t insn)
 {
     a->esz = extract32(insn, 22, 2);
@@ -1478,6 +2126,15 @@ static void disas_sve_extract_rd_pg_rn(DisasContext *ctx, arg_rpr_esz *a, uint32
     a->rd = extract32(insn, 0, 5);
 }
 
+static void disas_sve_extract_rd_pg_rn_esz17(DisasContext *ctx,
+                                             arg_rpr_esz *a, uint32_t insn)
+{
+    a->esz = extract32(insn, 17, 2);
+    a->pg = extract32(insn, 10, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+}
+
 static void disas_sve_extract_rd_pg_rn_e0(DisasContext *ctx, arg_rpr_esz *a, uint32_t insn)
 {
     a->pg = extract32(insn, 10, 3);
@@ -1531,6 +2188,164 @@ static void disas_sve_extract_rd_rn_rm_e0(DisasContext *ctx, arg_rrr_esz *a, uin
     a->esz = 0;
 }
 
+static void disas_sve_extract_rrx_3(DisasContext *ctx, arg_rrx_esz *a, uint32_t insn)
+{
+    a->index = (extract32(insn, 22, 1) << 2) | extract32(insn, 19, 2);
+    a->rm = extract32(insn, 16, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->esz = 1;
+}
+
+static void disas_sve_extract_rrx_2(DisasContext *ctx, arg_rrx_esz *a, uint32_t insn)
+{
+    a->index = extract32(insn, 19, 2);
+    a->rm = extract32(insn, 16, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->esz = 2;
+}
+
+static void disas_sve_extract_rrx_1(DisasContext *ctx, arg_rrx_esz *a, uint32_t insn)
+{
+    a->index = extract32(insn, 20, 1);
+    a->rm = extract32(insn, 16, 4);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->esz = 3;
+}
+
+static void disas_sve_extract_rrxr_3(DisasContext *ctx,
+                                     arg_rrxr_esz *a, uint32_t insn)
+{
+    a->index = (extract32(insn, 22, 1) << 2) | extract32(insn, 19, 2);
+    a->rm = extract32(insn, 16, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 1;
+}
+
+static void disas_sve_extract_rrxr_2(DisasContext *ctx,
+                                     arg_rrxr_esz *a, uint32_t insn)
+{
+    a->index = extract32(insn, 19, 2);
+    a->rm = extract32(insn, 16, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 2;
+}
+
+static void disas_sve_extract_rrxr_1(DisasContext *ctx,
+                                     arg_rrxr_esz *a, uint32_t insn)
+{
+    a->index = extract32(insn, 20, 1);
+    a->rm = extract32(insn, 16, 4);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 3;
+}
+
+static void disas_sve_extract_rrx_3a(DisasContext *ctx,
+                                     arg_rrx_esz *a, uint32_t insn)
+{
+    a->index = (extract32(insn, 19, 2) << 1) | extract32(insn, 11, 1);
+    a->rm = extract32(insn, 16, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->esz = 2;
+}
+
+static void disas_sve_extract_rrxr_3a(DisasContext *ctx,
+                                      arg_rrxr_esz *a, uint32_t insn)
+{
+    a->index = (extract32(insn, 19, 2) << 1) | extract32(insn, 11, 1);
+    a->rm = extract32(insn, 16, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->esz = 2;
+}
+
+static void disas_sve_extract_rrx_2a(DisasContext *ctx,
+                                     arg_rrx_esz *a, uint32_t insn)
+{
+    a->index = (extract32(insn, 20, 1) << 1) | extract32(insn, 11, 1);
+    a->rm = extract32(insn, 16, 4);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->esz = 3;
+}
+
+static void disas_sve_extract_rd_rn_tszimm_shl(DisasContext *ctx,
+                                               arg_rri_esz *a, uint32_t insn)
+{
+    int tszimm = deposit32(extract32(insn, 16, 5), 5, 27,
+                           extract32(insn, 22, 2));
+
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->esz = tszimm_esz(ctx, tszimm);
+    a->imm = tszimm_shl(ctx, tszimm);
+}
+
+static void disas_sve_extract_rd_rn_tszimm_shr(DisasContext *ctx,
+                                               arg_rri_esz *a, uint32_t insn)
+{
+    int tszimm = deposit32(extract32(insn, 16, 5), 5, 27,
+                           extract32(insn, 22, 2));
+
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->esz = tszimm_esz(ctx, tszimm);
+    a->imm = tszimm_shr(ctx, tszimm);
+}
+
+static void disas_sve_extract_rda_rn_rm(DisasContext *ctx, arg_rprrr_esz *a, uint32_t insn)
+{
+    a->esz = extract32(insn, 22, 2);
+    a->rm = extract32(insn, 16, 5);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->pg = 0;
+}
+
+static void disas_sve_extract_rda_rn_rm_e0(DisasContext *ctx,
+                                           arg_rprrr_esz *a, uint32_t insn)
+{
+    a->esz = 0;
+    a->rm = extract32(insn, 16, 5);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+    a->pg = 0;
+}
+
+static void disas_sve_extract_rda_rn_rm_rot(DisasContext *ctx,
+                                            arg_rprrr_rot_esz *a,
+                                            uint32_t insn)
+{
+    a->esz = extract32(insn, 22, 2);
+    a->rm = extract32(insn, 16, 5);
+    a->rot = extract32(insn, 10, 2);
+    a->rn = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->ra = extract32(insn, 0, 5);
+}
+
+static void disas_sve_extract_rdn_ra_rm_e0(DisasContext *ctx, arg_rprrr_esz *a, uint32_t insn)
+{
+    a->rm = extract32(insn, 16, 5);
+    a->ra = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->rn = extract32(insn, 0, 5);
+    a->pg = 0;
+    a->esz = 0;
+}
+
 static void disas_sve_extract_rd_rn_tszimm(DisasContext *ctx, arg_rri_esz *a, uint32_t insn)
 {
     a->rn = extract32(insn, 5, 5);
@@ -1538,6 +2353,17 @@ static void disas_sve_extract_rd_rn_tszimm(DisasContext *ctx, arg_rri_esz *a, ui
     a->esz = tszimm_esz(ctx, deposit32(extract32(insn, 16, 5), 5, 27, extract32(insn, 22, 2)));
 }
 
+static void disas_sve_extract_rdn_rm_tszimm16(DisasContext *ctx, arg_rrri_esz *a, uint32_t insn)
+{
+    int tszimm = deposit32(extract32(insn, 16, 5), 5, 27, extract32(insn, 22, 2));
+
+    a->rm = extract32(insn, 5, 5);
+    a->rd = extract32(insn, 0, 5);
+    a->rn = extract32(insn, 0, 5);
+    a->esz = tszimm_esz(ctx, tszimm);
+    a->imm = tszimm_shr(ctx, tszimm);
+}
+
 static void disas_sve_extract_rda_pg_rn_rm(DisasContext *ctx, arg_rprrr_esz *a, uint32_t insn)
 {
     a->esz = extract32(insn, 22, 2);
@@ -1840,6 +2666,7 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
         arg_incdec2_pred f_incdec2_pred;
         arg_incdec_cnt f_incdec_cnt;
         arg_incdec_pred f_incdec_pred;
+        arg_psel f_psel;
         arg_ptrue f_ptrue;
         arg_rpr_esz f_rpr_esz;
         arg_rpr_s f_rpr_s;
@@ -1855,12 +2682,16 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
         arg_rprr_scatter_store f_rprr_scatter_store;
         arg_rprr_store f_rprr_store;
         arg_rprrr_esz f_rprrr_esz;
+        arg_rprrr_rot_esz f_rprrr_rot_esz;
         arg_rr_dbm f_rr_dbm;
         arg_rr_esz f_rr_esz;
         arg_rri f_rri;
         arg_rri_esz f_rri_esz;
         arg_rrr_esz f_rrr_esz;
         arg_rrri f_rrri;
+        arg_rrri_esz f_rrri_esz;
+        arg_rrx_esz f_rrx_esz;
+        arg_rrxr_esz f_rrxr_esz;
     } u;
 
     switch ((insn >> 25) & 0x7f) {
@@ -2319,6 +3150,37 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                     /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:375 */
                     if (trans_BIC_zzz(ctx, &u.f_rrr_esz)) return true;
                     return false;
+                case 0x00000400:
+                case 0x00400400:
+                case 0x00800400:
+                case 0x00c00400:
+                    disas_sve_extract_rdn_rm_tszimm16(ctx, &u.f_rrri_esz, insn);
+                    if (trans_XAR(ctx, &u.f_rrri_esz)) return true;
+                    return false;
+                case 0x00000800:
+                    disas_sve_extract_rdn_ra_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                    if (trans_EOR3(ctx, &u.f_rprrr_esz)) return true;
+                    return false;
+                case 0x00000c00:
+                    disas_sve_extract_rdn_ra_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                    if (trans_BSL(ctx, &u.f_rprrr_esz)) return true;
+                    return false;
+                case 0x00400800:
+                    disas_sve_extract_rdn_ra_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                    if (trans_BCAX(ctx, &u.f_rprrr_esz)) return true;
+                    return false;
+                case 0x00400c00:
+                    disas_sve_extract_rdn_ra_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                    if (trans_BSL1N(ctx, &u.f_rprrr_esz)) return true;
+                    return false;
+                case 0x00800c00:
+                    disas_sve_extract_rdn_ra_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                    if (trans_BSL2N(ctx, &u.f_rprrr_esz)) return true;
+                    return false;
+                case 0x00c00c00:
+                    disas_sve_extract_rdn_ra_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                    if (trans_NBSL(ctx, &u.f_rprrr_esz)) return true;
+                    return false;
                 }
                 return false;
             case 0x4:
@@ -2359,12 +3221,22 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                     disas_sve_extract_rd_rn_i6(ctx, &u.f_rri, insn);
                     if (trans_ADDVL(ctx, &u.f_rri)) return true;
                     return false;
+                case 0x00000800:
+                    /* 00000100 001..... 01011... ........ */
+                    disas_sve_extract_rd_rn_i6(ctx, &u.f_rri, insn);
+                    if (trans_ADDSVL(ctx, &u.f_rri)) return true;
+                    return false;
                 case 0x00400000:
                     /* 00000100 011..... 01010... ........ */
                     /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:395 */
                     disas_sve_extract_rd_rn_i6(ctx, &u.f_rri, insn);
                     if (trans_ADDPL(ctx, &u.f_rri)) return true;
                     return false;
+                case 0x00400800:
+                    /* 00000100 011..... 01011... ........ */
+                    disas_sve_extract_rd_rn_i6(ctx, &u.f_rri, insn);
+                    if (trans_ADDSPL(ctx, &u.f_rri)) return true;
+                    return false;
                 case 0x00800000:
                     /* 00000100 101..... 01010... ........ */
                     disas_sve_extract_disas_sve_Fmt_58(ctx, &u.f_disas_sve27, insn);
@@ -2376,21 +3248,61 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                         return false;
                     }
                     return false;
+                case 0x00800800:
+                    /* 00000100 101..... 01011... ........ */
+                    disas_sve_extract_disas_sve_Fmt_58(ctx, &u.f_disas_sve27, insn);
+                    switch ((insn >> 16) & 0x1f) {
+                    case 0x1f:
+                        /* 00000100 10111111 01011... ........ */
+                        if (trans_RDSVL(ctx, &u.f_disas_sve27)) return true;
+                        return false;
+                    }
+                    return false;
                 }
                 return false;
-            case 0x8:
-                /* 00000100 ..1..... 1000.... ........ */
+            case 0x6:
+                /* 00000100 ..1..... 0110.... ........ */
                 disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
                 switch ((insn >> 10) & 0x3) {
                 case 0x0:
-                    /* 00000100 ..1..... 100000.. ........ */
-                    /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:412 */
-                    if (trans_ASR_zzw(ctx, &u.f_rrr_esz)) return true;
+                    if (trans_MUL_zzz(ctx, &u.f_rrr_esz)) return true;
                     return false;
                 case 0x1:
-                    /* 00000100 ..1..... 100001.. ........ */
-                    /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:413 */
-                    if (trans_LSR_zzw(ctx, &u.f_rrr_esz)) return true;
+                    if (trans_PMUL_zzz(ctx, &u.f_rrr_esz)) return true;
+                    return false;
+                case 0x2:
+                    if (trans_SMULH_zzz(ctx, &u.f_rrr_esz)) return true;
+                    return false;
+                case 0x3:
+                    if (trans_UMULH_zzz(ctx, &u.f_rrr_esz)) return true;
+                    return false;
+                }
+                return false;
+            case 0x7:
+                /* 00000100 ..1..... 0111.... ........ */
+                disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+                switch ((insn >> 10) & 0x3) {
+                case 0x0:
+                    if (trans_SQDMULH_zzz(ctx, &u.f_rrr_esz)) return true;
+                    return false;
+                case 0x1:
+                    if (trans_SQRDMULH_zzz(ctx, &u.f_rrr_esz)) return true;
+                    return false;
+                }
+                return false;
+            case 0x8:
+                /* 00000100 ..1..... 1000.... ........ */
+                disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+                switch ((insn >> 10) & 0x3) {
+                case 0x0:
+                    /* 00000100 ..1..... 100000.. ........ */
+                    /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:412 */
+                    if (trans_ASR_zzw(ctx, &u.f_rrr_esz)) return true;
+                    return false;
+                case 0x1:
+                    /* 00000100 ..1..... 100001.. ........ */
+                    /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:413 */
+                    if (trans_LSR_zzw(ctx, &u.f_rrr_esz)) return true;
                     return false;
                 case 0x3:
                     /* 00000100 ..1..... 100011.. ........ */
@@ -2619,6 +3531,38 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x01200000:
             /* 00000101 ..1..... ........ ........ */
+            switch (insn & 0xffe0fc00) {
+            case 0x05a00000:
+                /* 00000101 101..... 000000.. ........ */
+                disas_sve_extract_rd_rn_rm_e0(ctx, &u.f_rrr_esz, insn);
+                if (trans_ZIP1_q(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x05a00400:
+                /* 00000101 101..... 000001.. ........ */
+                disas_sve_extract_rd_rn_rm_e0(ctx, &u.f_rrr_esz, insn);
+                if (trans_ZIP2_q(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x05a00800:
+                /* 00000101 101..... 000010.. ........ */
+                disas_sve_extract_rd_rn_rm_e0(ctx, &u.f_rrr_esz, insn);
+                if (trans_UZP1_q(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x05a00c00:
+                /* 00000101 101..... 000011.. ........ */
+                disas_sve_extract_rd_rn_rm_e0(ctx, &u.f_rrr_esz, insn);
+                if (trans_UZP2_q(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x05a01800:
+                /* 00000101 101..... 000110.. ........ */
+                disas_sve_extract_rd_rn_rm_e0(ctx, &u.f_rrr_esz, insn);
+                if (trans_TRN1_q(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x05a01c00:
+                /* 00000101 101..... 000111.. ........ */
+                disas_sve_extract_rd_rn_rm_e0(ctx, &u.f_rrr_esz, insn);
+                if (trans_TRN2_q(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            }
             switch ((insn >> 14) & 0x3) {
             case 0x0:
                 /* 00000101 ..1..... 00...... ........ */
@@ -2632,6 +3576,10 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                         /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:479 */
                         if (trans_EXT(ctx, &u.f_rrri)) return true;
                         return false;
+                    case 0x1:
+                        disas_sve_extract_ext_sve2(ctx, &u.f_rri, insn);
+                        if (trans_EXT_sve2(ctx, &u.f_rri)) return true;
+                        return false;
                     }
                     return false;
                 case 0x1:
@@ -2643,6 +3591,14 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                         disas_sve_extract_disas_sve_Fmt_62(ctx, &u.f_rri, insn);
                         if (trans_DUP_x(ctx, &u.f_rri)) return true;
                         return false;
+                    case 0x2:
+                        disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+                        if (trans_TBL_sve2(ctx, &u.f_rrr_esz)) return true;
+                        return false;
+                    case 0x3:
+                        disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+                        if (trans_TBX(ctx, &u.f_rrr_esz)) return true;
+                        return false;
                     case 0x4:
                         /* 00000101 ..1..... 001100.. ........ */
                         /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:501 */
@@ -2947,6 +3903,11 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                     disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
                     if (trans_SPLICE(ctx, &u.f_rprr_esz)) return true;
                     return false;
+                case 0x000d0000:
+                    /* 00000101 ..101101 100..... ........ */
+                    disas_sve_extract_rd_pg_rn(ctx, &u.f_rpr_esz, insn);
+                    if (trans_SPLICE_sve2(ctx, &u.f_rpr_esz)) return true;
+                    return false;
                 case 0x00102000:
                     /* 00000101 ..110000 101..... ........ */
                     /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:548 */
@@ -3363,6 +4324,14 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x01200000:
             /* 00100101 ..1..... ........ ........ */
+            switch (insn & 0x0020c210) {
+            case 0x00204000:
+                if (disas_sve_extract_psel(ctx, &u.f_psel, insn) &&
+                    trans_PSEL(ctx, &u.f_psel)) {
+                    return true;
+                }
+                return false;
+            }
             switch ((insn >> 14) & 0x3) {
             case 0x0:
                 /* 00100101 ..1..... 00...... ........ */
@@ -3599,7 +4568,941 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
         return false;
     case 0x22:
         /* 0100010. ........ ........ ........ */
+        if ((insn & 0x0120e000) == 0x01208000) {
+            disas_sve_extract_pd_pg_rn_rm(ctx, &u.f_rprr_esz, insn);
+            switch ((insn >> 4) & 0x1) {
+            case 0x0:
+                if (trans_MATCH(ctx, &u.f_rprr_esz)) return true;
+                return false;
+            case 0x1:
+                if (trans_NMATCH(ctx, &u.f_rprr_esz)) return true;
+                return false;
+            }
+            return false;
+        }
+        if ((insn & 0x0120e000) == 0x0120c000) {
+            disas_sve_extract_rd_pg_rn_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_HISTCNT(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        }
+        if ((insn & 0x0120fc00) == 0x0120a000) {
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_HISTSEG(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff3fe000) {
+        case 0x4404a000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_SADALP_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x4405a000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_UADALP_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44028000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SRSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44038000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_URSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44068000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_SRSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44078000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_URSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44088000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SQSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44098000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UQSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x440a8000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SQRSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x440b8000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UQRSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x440c8000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_SQSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x440d8000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_UQSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x440e8000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_SQRSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x440f8000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_UQRSHL(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44108000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SHADD(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44118000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UHADD(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44128000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SHSUB(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44138000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UHSUB(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44148000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SRHADD(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44158000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_URHADD(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44168000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_SHSUB(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44178000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_UHSUB(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x4411a000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_ADDP(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x4414a000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SMAXP(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x4415a000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UMAXP(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x4416a000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SMINP(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x4417a000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UMINP(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44188000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SQADD_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x44198000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UQADD_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x441a8000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SQSUB_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x441b8000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_UQSUB_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x441c8000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_SUQADD(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x441d8000:
+            disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+            if (trans_USQADD(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x441e8000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_SQSUB_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x441f8000:
+            disas_sve_extract_rdm_pg_rn(ctx, &u.f_rprr_esz, insn);
+            if (trans_UQSUB_zpzz(ctx, &u.f_rprr_esz)) return true;
+            return false;
+        case 0x4400a000:
+            disas_sve_extract_rd_pg_rn(ctx, &u.f_rpr_esz, insn);
+            if (trans_URECPE(ctx, &u.f_rpr_esz)) return true;
+            return false;
+        case 0x4401a000:
+            disas_sve_extract_rd_pg_rn(ctx, &u.f_rpr_esz, insn);
+            if (trans_URSQRTE(ctx, &u.f_rpr_esz)) return true;
+            return false;
+        case 0x4408a000:
+            disas_sve_extract_rd_pg_rn(ctx, &u.f_rpr_esz, insn);
+            if (trans_SQABS(ctx, &u.f_rpr_esz)) return true;
+            return false;
+        case 0x4409a000:
+            disas_sve_extract_rd_pg_rn(ctx, &u.f_rpr_esz, insn);
+            if (trans_SQNEG(ctx, &u.f_rpr_esz)) return true;
+            return false;
+        }
+        if ((insn & 0xfffffbe0) == 0x4520e000) {
+            disas_sve_extract_rd_decrypt(ctx, &u.f_rri, insn);
+            if (trans_AESMC(ctx, &u.f_rri)) return true;
+            return false;
+        }
+        switch (insn & 0xfffffc00) {
+        case 0x4522e000:
+            disas_sve_extract_rdn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_AESE(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4522e400:
+            disas_sve_extract_rdn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_AESD(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4523e000:
+            disas_sve_extract_rdn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SM4E(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff3ffc00) {
+        case 0x4500d800:
+            disas_sve_extract_rdn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_CADD_rot90(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4500dc00:
+            disas_sve_extract_rdn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_CADD_rot270(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4501d800:
+            disas_sve_extract_rdn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SQCADD_rot90(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4501dc00:
+            disas_sve_extract_rdn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SQCADD_rot270(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xffe0fc00) {
+        case 0x4520f000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SM4EKEY(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4520f400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_RAX1(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20fc00) {
+        case 0x44000800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQDMLALBT(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44000c00:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQDMLSLBT(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44004000:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SMLALB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44004400:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SMLALT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44004800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_UMLALB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44004c00:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_UMLALT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44005000:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SMLSLB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44005400:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SMLSLT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44005800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_UMLSLB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44005c00:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_UMLSLT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44006000:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQDMLALB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44006400:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQDMLALT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44006800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQDMLSLB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44006c00:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQDMLSLT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44007000:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQRDMLAH_zzzz(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44007400:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SQRDMLSH_zzzz(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x44007800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_USDOT_zzzz(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20f000) {
+        case 0x44001000:
+            disas_sve_extract_rda_rn_rm_rot(ctx, &u.f_rprrr_rot_esz, insn);
+            if (trans_CDOT_zzzz(ctx, &u.f_rprrr_rot_esz)) return true;
+            return false;
+        case 0x44002000:
+            disas_sve_extract_rda_rn_rm_rot(ctx, &u.f_rprrr_rot_esz, insn);
+            if (trans_CMLA_zzzz(ctx, &u.f_rprrr_rot_esz)) return true;
+            return false;
+        case 0x44003000:
+            disas_sve_extract_rda_rn_rm_rot(ctx, &u.f_rprrr_rot_esz, insn);
+            if (trans_SQRDCMLAH_zzzz(ctx, &u.f_rprrr_rot_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xffa0fc00) {
+        case 0x44201000:
+            disas_sve_extract_rrxr_3(ctx, &u.f_rrxr_esz, insn);
+            if (trans_SQRDMLAH_zzxz(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x44201400:
+            disas_sve_extract_rrxr_3(ctx, &u.f_rrxr_esz, insn);
+            if (trans_SQRDMLSH_zzxz(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x4420f000:
+            disas_sve_extract_rrx_3(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMULH_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x4420f400:
+            disas_sve_extract_rrx_3(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQRDMULH_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x4420f800:
+            disas_sve_extract_rrx_3(ctx, &u.f_rrx_esz, insn);
+            if (trans_MUL_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xffe0fc00) {
+        case 0x44a01000:
+            disas_sve_extract_rrxr_2(ctx, &u.f_rrxr_esz, insn);
+            if (trans_SQRDMLAH_zzxz(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x44a01400:
+            disas_sve_extract_rrxr_2(ctx, &u.f_rrxr_esz, insn);
+            if (trans_SQRDMLSH_zzxz(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x44a01800:
+            disas_sve_extract_rrxr_2(ctx, &u.f_rrxr_esz, insn);
+            if (trans_USDOT_zzxw_s(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x44a01c00:
+            disas_sve_extract_rrxr_2(ctx, &u.f_rrxr_esz, insn);
+            if (trans_SUDOT_zzxw_s(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x44a0f000:
+            disas_sve_extract_rrx_2(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMULH_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0f400:
+            disas_sve_extract_rrx_2(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQRDMULH_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0f800:
+            disas_sve_extract_rrx_2(ctx, &u.f_rrx_esz, insn);
+            if (trans_MUL_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e01000:
+            disas_sve_extract_rrxr_1(ctx, &u.f_rrxr_esz, insn);
+            if (trans_SQRDMLAH_zzxz(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x44e01400:
+            disas_sve_extract_rrxr_1(ctx, &u.f_rrxr_esz, insn);
+            if (trans_SQRDMLSH_zzxz(ctx, &u.f_rrxr_esz)) return true;
+            return false;
+        case 0x44e0f000:
+            disas_sve_extract_rrx_1(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMULH_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0f400:
+            disas_sve_extract_rrx_1(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQRDMULH_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0f800:
+            disas_sve_extract_rrx_1(ctx, &u.f_rrx_esz, insn);
+            if (trans_MUL_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xffe0f000) {
+        case 0x44a04000:
+            disas_sve_extract_rrxr_rot_cdot_s(ctx, &u.f_disas_sve43, insn);
+            if (trans_CDOT_zzxw_s(ctx, &u.f_disas_sve43)) return true;
+            return false;
+        case 0x44e04000:
+            disas_sve_extract_rrxr_rot_cdot_d(ctx, &u.f_disas_sve43, insn);
+            if (trans_CDOT_zzxw_d(ctx, &u.f_disas_sve43)) return true;
+            return false;
+        case 0x44a06000:
+            disas_sve_extract_rrxr_rot_h(ctx, &u.f_disas_sve43, insn);
+            if (trans_CMLA_zzxz_h(ctx, &u.f_disas_sve43)) return true;
+            return false;
+        case 0x44e06000:
+            disas_sve_extract_rrxr_rot_s(ctx, &u.f_disas_sve43, insn);
+            if (trans_CMLA_zzxz_s(ctx, &u.f_disas_sve43)) return true;
+            return false;
+        case 0x44a07000:
+            disas_sve_extract_rrxr_rot_h(ctx, &u.f_disas_sve43, insn);
+            if (trans_SQRDCMLAH_zzxz_h(ctx, &u.f_disas_sve43)) return true;
+            return false;
+        case 0x44e07000:
+            disas_sve_extract_rrxr_rot_s(ctx, &u.f_disas_sve43, insn);
+            if (trans_SQRDCMLAH_zzxz_s(ctx, &u.f_disas_sve43)) return true;
+            return false;
+        }
+        switch (insn & 0xffe0f400) {
+        case 0x44a02000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLALB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a02400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLALT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a03000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLSLB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a03400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLSLT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a08000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLALB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a08400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLALT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a09000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLALB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a09400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLALT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0a000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLSLB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0a400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLSLT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0b000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLSLB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0b400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLSLT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0c000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMULLB_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0c400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMULLT_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0d000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMULLB_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0d400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMULLT_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0e000:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMULLB_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44a0e400:
+            disas_sve_extract_rrx_3a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMULLT_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e02000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLALB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e02400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLALT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e03000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLSLB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e03400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMLSLT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e08000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLALB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e08400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLALT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e09000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLALB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e09400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLALT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0a000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLSLB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0a400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMLSLT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0b000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLSLB_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0b400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMLSLT_zzxw(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0c000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMULLB_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0c400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SMULLT_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0d000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMULLB_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0d400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_UMULLT_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0e000:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMULLB_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        case 0x44e0e400:
+            disas_sve_extract_rrx_2a(ctx, &u.f_rrx_esz, insn);
+            if (trans_SQDMULLT_zzx(ctx, &u.f_rrx_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xffe0fc00) {
+        case 0x45009800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SMMLA(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x45809800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_USMMLA(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x45c09800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_UMMLA(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20fc00) {
+        case 0x45000000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SADDLB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45000400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SADDLT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45000800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UADDLB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45000c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UADDLT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45001000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SSUBLB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45001400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SSUBLT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45001800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_USUBLB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45001c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_USUBLT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45003000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SABDLB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45003400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SABDLT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45003800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UABDLB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45003c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UABDLT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45004000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SADDWB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45004400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SADDWT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45004800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UADDWB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45004c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UADDWT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45005000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SSUBWB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45005400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SSUBWT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45005800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_USUBWB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45005c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_USUBWT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45006000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SQDMULLB_zzz(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45006400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SQDMULLT_zzz(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45007000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SMULLB_zzz(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45007400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SMULLT_zzz(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45007800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UMULLB_zzz(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45007c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UMULLT_zzz(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45008000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SADDLBT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45008800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SSUBLBT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45008c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SSUBLTB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4500c000:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SABALB(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x4500c400:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_SABALT(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x4500c800:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_UABALB(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        case 0x4500cc00:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            if (trans_UABALT(ctx, &u.f_rprrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20fc00) {
+        case 0x4500a000:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_SSHLLB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500a400:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_SSHLLT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500a800:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_USHLLB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500ac00:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_USHLLT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20fc00) {
+        case 0x4500e000:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SSRA(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500e400:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_USRA(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500e800:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SRSRA(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500ec00:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_URSRA(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500f000:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SRI(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500f400:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_SLI(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x4500f800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SABA(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x4500fc00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_UABA(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20fc00) {
+        case 0x45200000:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQSHRUNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45200400:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQSHRUNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45200800:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQRSHRUNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45200c00:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQRSHRUNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45201000:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SHRNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45201400:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SHRNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45201800:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_RSHRNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45201c00:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_RSHRNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45202000:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQSHRNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45202400:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQSHRNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45202800:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQRSHRNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45202c00:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_SQRSHRNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45203000:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_UQSHRNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45203400:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_UQSHRNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45203800:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_UQRSHRNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45203c00:
+            disas_sve_extract_rd_rn_tszimm_shr(ctx, &u.f_rri_esz, insn);
+            if (trans_UQRSHRNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20fc00) {
+        case 0x45206000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_ADDHNB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45206400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_ADDHNT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45206800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_RADDHNB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45206c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_RADDHNT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45207000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SUBHNB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45207400:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_SUBHNT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45207800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_RSUBHNB(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        case 0x45207c00:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            if (trans_RSUBHNT(ctx, &u.f_rrr_esz)) return true;
+            return false;
+        }
+        switch (insn & 0xff20fc00) {
+        case 0x45204000:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_SQXTNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45204400:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_SQXTNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45204800:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_UQXTNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45204c00:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_UQXTNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45205000:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_SQXTUNB(ctx, &u.f_rri_esz)) return true;
+            return false;
+        case 0x45205400:
+            disas_sve_extract_rd_rn_tszimm_shl(ctx, &u.f_rri_esz, insn);
+            if (trans_SQXTUNT(ctx, &u.f_rri_esz)) return true;
+            return false;
+        }
         switch (insn & 0x01a0f800) {
+        case 0x01006800:
+        case 0x01806800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            switch ((insn >> 10) & 0x1) {
+            case 0x0:
+                if (trans_PMULLB(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x1:
+                if (trans_PMULLT(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            }
+            return false;
+        case 0x01009000:
+        case 0x01809000:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            switch ((insn >> 10) & 0x1) {
+            case 0x0:
+                if (trans_EORBT(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x1:
+                if (trans_EORTB(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            }
+            return false;
+        case 0x0100b000:
+        case 0x0180b000:
+        case 0x0100b800:
+        case 0x0180b800:
+            disas_sve_extract_rd_rn_rm(ctx, &u.f_rrr_esz, insn);
+            switch ((insn >> 10) & 0x3) {
+            case 0x0:
+                if (trans_BEXT(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x1:
+                if (trans_BDEP(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            case 0x2:
+                if (trans_BGRP(ctx, &u.f_rrr_esz)) return true;
+                return false;
+            }
+            return false;
+        case 0x0100d000:
+        case 0x0180d000:
+            disas_sve_extract_rda_rn_rm(ctx, &u.f_rprrr_esz, insn);
+            switch ((insn >> 10) & 0x1) {
+            case 0x0:
+                if (trans_ADCLB(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x1:
+                if (trans_ADCLT(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            }
+            return false;
         case 0x00800000:
             /* 01000100 1.0..... 00000... ........ */
             /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:730 */
@@ -3636,6 +5539,54 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x00008000:
             /* 01100100 ..0..... 1....... ........ */
+            switch (insn & 0xff3fe000) {
+            case 0x64108000:
+                disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+                if (trans_FADDP(ctx, &u.f_rprr_esz)) return true;
+                return false;
+            case 0x64148000:
+                disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+                if (trans_FMAXNMP(ctx, &u.f_rprr_esz)) return true;
+                return false;
+            case 0x64158000:
+                disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+                if (trans_FMINNMP(ctx, &u.f_rprr_esz)) return true;
+                return false;
+            case 0x64168000:
+                disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+                if (trans_FMAXP(ctx, &u.f_rprr_esz)) return true;
+                return false;
+            case 0x64178000:
+                disas_sve_extract_rdn_pg_rm(ctx, &u.f_rprr_esz, insn);
+                if (trans_FMINP(ctx, &u.f_rprr_esz)) return true;
+                return false;
+            }
+            switch (insn & 0xffffe000) {
+            case 0x640aa000:
+                disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
+                if (trans_FCVTXNT_ds(ctx, &u.f_rpr_esz)) return true;
+                return false;
+            case 0x6488a000:
+                disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
+                if (trans_FCVTNT_sh(ctx, &u.f_rpr_esz)) return true;
+                return false;
+            case 0x648aa000:
+                disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
+                if (trans_BFCVTNT(ctx, &u.f_rpr_esz)) return true;
+                return false;
+            case 0x6489a000:
+                disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
+                if (trans_FCVTLT_hs(ctx, &u.f_rpr_esz)) return true;
+                return false;
+            case 0x64caa000:
+                disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
+                if (trans_FCVTNT_ds(ctx, &u.f_rpr_esz)) return true;
+                return false;
+            case 0x64cba000:
+                disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
+                if (trans_FCVTLT_sd(ctx, &u.f_rpr_esz)) return true;
+                return false;
+            }
             disas_sve_extract_disas_sve_Fmt_77(ctx, &u.f_disas_sve41, insn);
             switch (insn & 0x001e6000) {
             case 0x00000000:
@@ -3647,6 +5598,36 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x00200000:
             /* 01100100 ..1..... 0....... ........ */
+            switch (insn & 0xffe0f400) {
+            case 0x64604000:
+                disas_sve_extract_rrxr_2(ctx, &u.f_rrxr_esz, insn);
+                if (trans_BFDOT_zzxz(ctx, &u.f_rrxr_esz)) return true;
+                return false;
+            case 0x64a04000:
+                disas_sve_extract_rrxr_3a(ctx, &u.f_rrxr_esz, insn);
+                if (trans_FMLALB_zzxw(ctx, &u.f_rrxr_esz)) return true;
+                return false;
+            case 0x64a04400:
+                disas_sve_extract_rrxr_3a(ctx, &u.f_rrxr_esz, insn);
+                if (trans_FMLALT_zzxw(ctx, &u.f_rrxr_esz)) return true;
+                return false;
+            case 0x64e04000:
+                disas_sve_extract_rrxr_3a(ctx, &u.f_rrxr_esz, insn);
+                if (trans_BFMLALB_zzxw(ctx, &u.f_rrxr_esz)) return true;
+                return false;
+            case 0x64e04400:
+                disas_sve_extract_rrxr_3a(ctx, &u.f_rrxr_esz, insn);
+                if (trans_BFMLALT_zzxw(ctx, &u.f_rrxr_esz)) return true;
+                return false;
+            case 0x64a06000:
+                disas_sve_extract_rrxr_3a(ctx, &u.f_rrxr_esz, insn);
+                if (trans_FMLSLB_zzxw(ctx, &u.f_rrxr_esz)) return true;
+                return false;
+            case 0x64a06400:
+                disas_sve_extract_rrxr_3a(ctx, &u.f_rrxr_esz, insn);
+                if (trans_FMLSLT_zzxw(ctx, &u.f_rrxr_esz)) return true;
+                return false;
+            }
             switch (insn & 0x00807000) {
             case 0x00000000:
                 /* 01100100 0.1..... 0000.... ........ */
@@ -3723,6 +5704,50 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x00208000:
+            switch (insn & 0xffe0fc00) {
+            case 0x64608000:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_BFDOT_zzzz(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x6460e400:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_BFMMLA(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64a0e400:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_FMMLA_s(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64e0e400:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_FMMLA_d(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64a08000:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_FMLALB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64a08400:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_FMLALT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64e08000:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_BFMLALB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64e08400:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_BFMLALT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64a0a000:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_FMLSLB_zzzw(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            case 0x64a0a400:
+                disas_sve_extract_rda_rn_rm_e0(ctx, &u.f_rprrr_esz, insn);
+                if (trans_FMLSLT_zzzw(ctx, &u.f_rprrr_esz)) return true;
+                return false;
+            }
+            return false;
         case 0x01000000:
             /* 01100101 ..0..... 0....... ........ */
             switch ((insn >> 13) & 0x3) {
@@ -4068,6 +6093,16 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                 return false;
             case 0x1:
                 /* 01100101 ..0..... 101..... ........ */
+                if ((insn & 0xffffe000) == 0x658aa000) {
+                    disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
+                    if (trans_BFCVT(ctx, &u.f_rpr_esz)) return true;
+                    return false;
+                }
+                if ((insn & 0xfff9e000) == 0x6518a000) {
+                    disas_sve_extract_rd_pg_rn_esz17(ctx, &u.f_rpr_esz, insn);
+                    if (trans_FLOGB(ctx, &u.f_rpr_esz)) return true;
+                    return false;
+                }
                 switch ((insn >> 16) & 0x1f) {
                 case 0x0:
                     /* 01100101 ..000000 101..... ........ */
@@ -4147,6 +6182,10 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                     /* 01100101 ..001010 101..... ........ */
                     disas_sve_extract_rd_pg_rn_e0(ctx, &u.f_rpr_esz, insn);
                     switch ((insn >> 22) & 0x3) {
+                    case 0x0:
+                        /* 01100101 00001010 101..... ........ */
+                        if (trans_FCVTX_ds(ctx, &u.f_rpr_esz)) return true;
+                        return false;
                     case 0x3:
                         /* 01100101 11001010 101..... ........ */
                         /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:860 */
@@ -4600,6 +6639,17 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                 case 0x0:
                     /* 1000010. .00..... 1....... ........ */
                     switch (insn & 0x00006010) {
+                    case 0x00002000:
+                        /* 1000010. .00..... 101..... ........ */
+                        disas_sve_extract_rprr_g_load_xs_u(ctx, &u.f_rprr_gather_load, insn);
+                        u.f_rprr_gather_load.u = extract32(insn, 13, 1);
+                        u.f_rprr_gather_load.ff = 0;
+                        u.f_rprr_gather_load.xs = 0;
+                        u.f_rprr_gather_load.esz = 2;
+                        u.f_rprr_gather_load.msz = extract32(insn, 23, 2);
+                        u.f_rprr_gather_load.scale = 0;
+                        if (trans_LDNT1_zprz(ctx, &u.f_rprr_gather_load)) return true;
+                        return false;
                     case 0x00004000:
                         /* 1000010. .00..... 110..... ...0.... */
                         /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:979 */
@@ -4646,6 +6696,11 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                 u.f_rprr_load.nreg = 0;
                 if (trans_LD1RQ_zprr(ctx, &u.f_rprr_load)) return true;
                 return false;
+            case 0x1:
+                /* 1010010. .01..... 000..... ........ */
+                u.f_rprr_load.nreg = 0;
+                if (trans_LD1RO_zprr(ctx, &u.f_rprr_load)) return true;
+                return false;
             }
             return false;
         case 0x1:
@@ -4658,6 +6713,11 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                 u.f_rpri_load.nreg = 0;
                 if (trans_LD1RQ_zpri(ctx, &u.f_rpri_load)) return true;
                 return false;
+            case 0x2:
+                /* 1010010. .010.... 001..... ........ */
+                u.f_rpri_load.nreg = 0;
+                if (trans_LD1RO_zpri(ctx, &u.f_rpri_load)) return true;
+                return false;
             }
             return false;
         case 0x2:
@@ -4786,6 +6846,15 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                     /* 1100010. .00..... 1....... ........ */
                     disas_sve_extract_disas_sve_Fmt_66(ctx, &u.f_disas_sve32, insn);
                     switch (insn & 0x00006010) {
+                    case 0x00004000:
+                        /* 1100010. .00..... 110..... ........ */
+                        disas_sve_extract_rprr_g_load_u(ctx, &u.f_rprr_gather_load, insn);
+                        u.f_rprr_gather_load.ff = 0;
+                        u.f_rprr_gather_load.esz = 3;
+                        u.f_rprr_gather_load.msz = extract32(insn, 23, 2);
+                        u.f_rprr_gather_load.scale = 0;
+                        if (trans_LDNT1_zprz(ctx, &u.f_rprr_gather_load)) return true;
+                        return false;
                     case 0x00006000:
                         /* 1100010. .00..... 111..... ...0.... */
                         /* /mnt/c/Users/me/Documents/projects/unicorn2/tmp/tmp/qemu-5.0.0/target/arm/sve.decode:1016 */
@@ -4880,6 +6949,26 @@ bool disas_sve(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x1:
+            /* 1110010. ........ 001..... ........ */
+            disas_sve_extract_rprr_scatter_store(ctx, &u.f_rprr_scatter_store, insn);
+            switch ((insn >> 21) & 0x3) {
+            case 0x0:
+                /* 1110010. .00..... 001..... ........ */
+                u.f_rprr_scatter_store.xs = 2;
+                u.f_rprr_scatter_store.esz = 3;
+                u.f_rprr_scatter_store.scale = 0;
+                if (trans_STNT1_zprz(ctx, &u.f_rprr_scatter_store)) return true;
+                return false;
+            case 0x2:
+                /* 1110010. .10..... 001..... ........ */
+                u.f_rprr_scatter_store.xs = 0;
+                u.f_rprr_scatter_store.esz = 2;
+                u.f_rprr_scatter_store.scale = 0;
+                if (trans_STNT1_zprz(ctx, &u.f_rprr_scatter_store)) return true;
+                return false;
+            }
+            return false;
         case 0x2:
             /* 1110010. ........ 010..... ........ */
             switch ((insn >> 23) & 0x3) {
diff --git a/qemu/target/arm/decode-vfp.inc.c b/qemu/target/arm/decode-vfp.inc.c
index b04ab8b08d..f662c1d89c 100644
--- a/qemu/target/arm/decode-vfp.inc.c
+++ b/qemu/target/arm/decode-vfp.inc.c
@@ -215,6 +215,8 @@ typedef arg_disas_vfp12 arg_VCVT_f32_f16;
 static bool trans_VCVT_f32_f16(DisasContext *ctx, arg_VCVT_f32_f16 *a);
 typedef arg_disas_vfp12 arg_VCVT_f64_f16;
 static bool trans_VCVT_f64_f16(DisasContext *ctx, arg_VCVT_f64_f16 *a);
+typedef arg_disas_vfp12 arg_VCVT_b16_f32;
+static bool trans_VCVT_b16_f32(DisasContext *ctx, arg_VCVT_b16_f32 *a);
 typedef arg_disas_vfp12 arg_VCVT_f16_f32;
 static bool trans_VCVT_f16_f32(DisasContext *ctx, arg_VCVT_f16_f32 *a);
 typedef arg_disas_vfp12 arg_VCVT_f16_f64;
@@ -663,6 +665,16 @@ static bool disas_vfp(DisasContext *ctx, uint32_t insn)
             return false;
         }
         return false;
+    case 0x0e000900:
+        /* ....1110 ........ ....1001 ........ */
+        switch (insn & 0x00bf0050) {
+        case 0x00b30040:
+            /* ....1110 1.110011 ....1001 .1.0.... */
+            disas_vfp_extract_disas_vfp_Fmt_27(ctx, &u.f_disas_vfp12, insn);
+            if (trans_VCVT_b16_f32(ctx, &u.f_disas_vfp12)) return true;
+            return false;
+        }
+        return false;
     case 0x0e000a00:
         /* ....1110 ........ ....1010 ........ */
         switch (insn & 0x00a00050) {
diff --git a/qemu/target/arm/helper-a64.c b/qemu/target/arm/helper-a64.c
index 12da114039..3acb0eb9c9 100644
--- a/qemu/target/arm/helper-a64.c
+++ b/qemu/target/arm/helper-a64.c
@@ -66,6 +66,38 @@ void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm)
     update_spsel(env, imm);
 }
 
+static void arm_reset_sve_state(CPUARMState *env)
+{
+    memset(env->vfp.zregs, 0, sizeof(env->vfp.zregs));
+    memset(env->vfp.pregs, 0, sizeof(env->vfp.pregs));
+    vfp_set_fpsr(env, 0x0800009f);
+}
+
+void aarch64_set_svcr(CPUARMState *env, uint64_t new, uint64_t mask)
+{
+    uint64_t change = (env->svcr ^ new) & mask;
+
+    if (change == 0) {
+        return;
+    }
+
+    env->svcr ^= change;
+
+    if (change & R_SVCR_SM_MASK) {
+        arm_reset_sve_state(env);
+    }
+    if (change & new & R_SVCR_ZA_MASK) {
+        memset(env->zarray, 0, sizeof(env->zarray));
+    }
+
+    arm_rebuild_hflags(env);
+}
+
+void HELPER(set_svcr)(CPUARMState *env, uint32_t val, uint32_t mask)
+{
+    aarch64_set_svcr(env, val, mask);
+}
+
 static void daif_check(CPUARMState *env, uint32_t op,
                        uint32_t imm, uintptr_t ra)
 {
diff --git a/qemu/target/arm/helper-a64.h b/qemu/target/arm/helper-a64.h
index 3df7c185aa..d2801fdd57 100644
--- a/qemu/target/arm/helper-a64.h
+++ b/qemu/target/arm/helper-a64.h
@@ -22,6 +22,7 @@ DEF_HELPER_FLAGS_1(rbit64, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_2(msr_i_spsel, void, env, i32)
 DEF_HELPER_2(msr_i_daifset, void, env, i32)
 DEF_HELPER_2(msr_i_daifclear, void, env, i32)
+DEF_HELPER_FLAGS_3(set_svcr, TCG_CALL_NO_RWG, void, env, i32, i32)
 DEF_HELPER_3(vfp_cmph_a64, i64, f16, f16, ptr)
 DEF_HELPER_3(vfp_cmpeh_a64, i64, f16, f16, ptr)
 DEF_HELPER_3(vfp_cmps_a64, i64, f32, f32, ptr)
@@ -97,6 +98,20 @@ DEF_HELPER_FLAGS_3(pacib, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_3(pacda, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_3(pacdb, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_3(pacga, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32)
+DEF_HELPER_FLAGS_3(ldg, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(stg, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(stg_stub, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_3(st2g, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(st2g_stub, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_3(mte_probe_data, TCG_CALL_NO_WG, void, env, i64, i32)
+DEF_HELPER_FLAGS_2(ldgm, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_3(stgm, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_3(stzgm_tags, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(dc_gva_probe, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_3(mte_check, TCG_CALL_NO_WG, i64, env, i32, i64)
+DEF_HELPER_FLAGS_3(mte_check_zva, TCG_CALL_NO_WG, i64, env, i32, i64)
 DEF_HELPER_FLAGS_3(autia, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_3(autib, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_3(autda, TCG_CALL_NO_WG, i64, env, i64, i64)
diff --git a/qemu/target/arm/helper-sme.h b/qemu/target/arm/helper-sme.h
new file mode 100644
index 0000000000..8936518389
--- /dev/null
+++ b/qemu/target/arm/helper-sme.h
@@ -0,0 +1,191 @@
+/*
+ * AArch64 SME helper definitions.
+ */
+
+DEF_HELPER_FLAGS_3(sme_zero, TCG_CALL_NO_RWG, void, env, i32, i32)
+
+DEF_HELPER_FLAGS_4(sme_mova_cz_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_zc_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_cz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_zc_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_cz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_zc_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_cz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_zc_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_cz_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sme_mova_zc_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sme_ld1b_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1b_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_5(sme_ld1b_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1b_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1h_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1s_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1d_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_ld1q_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_5(sme_st1b_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1b_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_le_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_le_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_be_h, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_be_v, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_5(sme_st1b_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1b_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1h_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1s_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1d_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_le_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_le_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_be_h_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+DEF_HELPER_FLAGS_5(sme_st1q_be_v_mte, TCG_CALL_NO_WG,
+                   void, env, ptr, ptr, tl, i32)
+
+DEF_HELPER_FLAGS_5(sme_addha_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sme_addva_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sme_addha_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sme_addva_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_7(sme_fmopa_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_FLAGS_7(sme_fmopa_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_7(sme_fmopa_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_bfmopa, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sme_smopa_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_umopa_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_sumopa_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_usmopa_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_smopa_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_umopa_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_sumopa_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sme_usmopa_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
diff --git a/qemu/target/arm/helper-sve.h b/qemu/target/arm/helper-sve.h
index 2f47279155..343affa269 100644
--- a/qemu/target/arm/helper-sve.h
+++ b/qemu/target/arm/helper-sve.h
@@ -131,6 +131,238 @@ DEF_HELPER_FLAGS_5(sve_uabd_zpzz_s, TCG_CALL_NO_RWG,
 DEF_HELPER_FLAGS_5(sve_uabd_zpzz_d, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 
+DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sadalp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uadalp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srhadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urhadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_shsub_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uhsub_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_addp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smaxp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umaxp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sminp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uminp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_faddp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxnmp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminnmp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmaxp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fminp_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_srshl_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_urshl_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqshl_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqshl_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrshl_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqrshl_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqsub_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uqsub_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_suqadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_usqadd_zpzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+
 DEF_HELPER_FLAGS_5(sve_mul_zpzz_b, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(sve_mul_zpzz_h, TCG_CALL_NO_RWG,
@@ -203,6 +435,8 @@ DEF_HELPER_FLAGS_5(sve_sel_zpzz_s, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(sve_sel_zpzz_d, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_sel_zpzz_q, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_5(sve_asr_zpzw_b, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
@@ -332,6 +566,19 @@ DEF_HELPER_FLAGS_4(sve_not_zpz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_not_zpz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_not_zpz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 
+DEF_HELPER_FLAGS_4(sve2_sqabs_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqabs_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqabs_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqabs_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqneg_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqneg_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqneg_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqneg_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_urecpe_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ursqrte_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+
 DEF_HELPER_FLAGS_4(sve_sxtb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_sxtb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_sxtb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
@@ -444,6 +691,14 @@ DEF_HELPER_FLAGS_4(sve_tbl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_tbl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_tbx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_tbx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_tbx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_tbx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_3(sve_sunpk_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
 DEF_HELPER_FLAGS_3(sve_sunpk_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
@@ -463,16 +718,19 @@ DEF_HELPER_FLAGS_4(sve_zip_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_zip_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_zip_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_zip_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_zip_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_4(sve_uzp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_uzp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_uzp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_uzp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uzp_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_4(sve_trn_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_trn_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_trn_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_trn_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_trn_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_4(sve_compact_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve_compact_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
@@ -954,6 +1212,26 @@ DEF_HELPER_FLAGS_5(sve_fcvt_hd, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(sve_fcvt_sd, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_bfcvt, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(sve2_fcvtnt_sh, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_fcvtnt_ds, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve_bfcvtnt, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_fcvtlt_hs, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_fcvtlt_sd, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(flogb_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(flogb_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(flogb_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_5(sve_fcvtzs_hh, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
@@ -1181,6 +1459,58 @@ DEF_HELPER_FLAGS_4(sve_ld1sds_le_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_ld1sdu_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_ld1sds_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 
+#define DEF_SVE_LD_MTE(NAME) \
+    DEF_HELPER_FLAGS_4(NAME##_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_SVE_LD_MTE(sve_ld1bb_r)
+DEF_SVE_LD_MTE(sve_ld1bhu_r)
+DEF_SVE_LD_MTE(sve_ld1bhs_r)
+DEF_SVE_LD_MTE(sve_ld1bsu_r)
+DEF_SVE_LD_MTE(sve_ld1bss_r)
+DEF_SVE_LD_MTE(sve_ld1bdu_r)
+DEF_SVE_LD_MTE(sve_ld1bds_r)
+DEF_SVE_LD_MTE(sve_ld1hh_le_r)
+DEF_SVE_LD_MTE(sve_ld1hh_be_r)
+DEF_SVE_LD_MTE(sve_ld1hsu_le_r)
+DEF_SVE_LD_MTE(sve_ld1hsu_be_r)
+DEF_SVE_LD_MTE(sve_ld1hss_le_r)
+DEF_SVE_LD_MTE(sve_ld1hss_be_r)
+DEF_SVE_LD_MTE(sve_ld1hdu_le_r)
+DEF_SVE_LD_MTE(sve_ld1hdu_be_r)
+DEF_SVE_LD_MTE(sve_ld1hds_le_r)
+DEF_SVE_LD_MTE(sve_ld1hds_be_r)
+DEF_SVE_LD_MTE(sve_ld1ss_le_r)
+DEF_SVE_LD_MTE(sve_ld1ss_be_r)
+DEF_SVE_LD_MTE(sve_ld1sdu_le_r)
+DEF_SVE_LD_MTE(sve_ld1sdu_be_r)
+DEF_SVE_LD_MTE(sve_ld1sds_le_r)
+DEF_SVE_LD_MTE(sve_ld1sds_be_r)
+DEF_SVE_LD_MTE(sve_ld1dd_le_r)
+DEF_SVE_LD_MTE(sve_ld1dd_be_r)
+DEF_SVE_LD_MTE(sve_ld2bb_r)
+DEF_SVE_LD_MTE(sve_ld3bb_r)
+DEF_SVE_LD_MTE(sve_ld4bb_r)
+DEF_SVE_LD_MTE(sve_ld2hh_le_r)
+DEF_SVE_LD_MTE(sve_ld2hh_be_r)
+DEF_SVE_LD_MTE(sve_ld3hh_le_r)
+DEF_SVE_LD_MTE(sve_ld3hh_be_r)
+DEF_SVE_LD_MTE(sve_ld4hh_le_r)
+DEF_SVE_LD_MTE(sve_ld4hh_be_r)
+DEF_SVE_LD_MTE(sve_ld2ss_le_r)
+DEF_SVE_LD_MTE(sve_ld2ss_be_r)
+DEF_SVE_LD_MTE(sve_ld3ss_le_r)
+DEF_SVE_LD_MTE(sve_ld3ss_be_r)
+DEF_SVE_LD_MTE(sve_ld4ss_le_r)
+DEF_SVE_LD_MTE(sve_ld4ss_be_r)
+DEF_SVE_LD_MTE(sve_ld2dd_le_r)
+DEF_SVE_LD_MTE(sve_ld2dd_be_r)
+DEF_SVE_LD_MTE(sve_ld3dd_le_r)
+DEF_SVE_LD_MTE(sve_ld3dd_be_r)
+DEF_SVE_LD_MTE(sve_ld4dd_le_r)
+DEF_SVE_LD_MTE(sve_ld4dd_be_r)
+
+#undef DEF_SVE_LD_MTE
+
 DEF_HELPER_FLAGS_4(sve_ldff1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_ldff1bhu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_ldff1bsu_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
@@ -1243,6 +1573,63 @@ DEF_HELPER_FLAGS_4(sve_ldnf1sds_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_ldnf1dd_le_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_ldnf1dd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 
+#define DEF_SVE_LDFFNF_MTE(NAME) \
+    DEF_HELPER_FLAGS_4(NAME##_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_SVE_LDFFNF_MTE(sve_ldff1bb_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1bhu_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1bhs_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1bsu_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1bss_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1bdu_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1bds_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hh_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hh_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hsu_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hsu_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hss_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hss_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hdu_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hdu_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hds_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1hds_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1ss_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1ss_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1sdu_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1sdu_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1sds_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1sds_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1dd_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldff1dd_be_r)
+
+DEF_SVE_LDFFNF_MTE(sve_ldnf1bb_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1bhu_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1bhs_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1bsu_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1bss_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1bdu_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1bds_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hh_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hh_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hsu_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hsu_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hss_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hss_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hdu_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hdu_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hds_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1hds_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1ss_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1ss_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1sdu_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1sdu_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1sds_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1sds_be_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1dd_le_r)
+DEF_SVE_LDFFNF_MTE(sve_ldnf1dd_be_r)
+
+#undef DEF_SVE_LDFFNF_MTE
+
 DEF_HELPER_FLAGS_4(sve_st1bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_st2bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_st3bb_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
@@ -1290,6 +1677,49 @@ DEF_HELPER_FLAGS_4(sve_st1hd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_st1sd_le_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 DEF_HELPER_FLAGS_4(sve_st1sd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
 
+#define DEF_SVE_ST_MTE(NAME) \
+    DEF_HELPER_FLAGS_4(NAME##_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32)
+
+DEF_SVE_ST_MTE(sve_st1bb_r)
+DEF_SVE_ST_MTE(sve_st1bh_r)
+DEF_SVE_ST_MTE(sve_st1bs_r)
+DEF_SVE_ST_MTE(sve_st1bd_r)
+DEF_SVE_ST_MTE(sve_st2bb_r)
+DEF_SVE_ST_MTE(sve_st3bb_r)
+DEF_SVE_ST_MTE(sve_st4bb_r)
+DEF_SVE_ST_MTE(sve_st1hh_le_r)
+DEF_SVE_ST_MTE(sve_st1hh_be_r)
+DEF_SVE_ST_MTE(sve_st1hs_le_r)
+DEF_SVE_ST_MTE(sve_st1hs_be_r)
+DEF_SVE_ST_MTE(sve_st1hd_le_r)
+DEF_SVE_ST_MTE(sve_st1hd_be_r)
+DEF_SVE_ST_MTE(sve_st2hh_le_r)
+DEF_SVE_ST_MTE(sve_st2hh_be_r)
+DEF_SVE_ST_MTE(sve_st3hh_le_r)
+DEF_SVE_ST_MTE(sve_st3hh_be_r)
+DEF_SVE_ST_MTE(sve_st4hh_le_r)
+DEF_SVE_ST_MTE(sve_st4hh_be_r)
+DEF_SVE_ST_MTE(sve_st1ss_le_r)
+DEF_SVE_ST_MTE(sve_st1ss_be_r)
+DEF_SVE_ST_MTE(sve_st1sd_le_r)
+DEF_SVE_ST_MTE(sve_st1sd_be_r)
+DEF_SVE_ST_MTE(sve_st2ss_le_r)
+DEF_SVE_ST_MTE(sve_st2ss_be_r)
+DEF_SVE_ST_MTE(sve_st3ss_le_r)
+DEF_SVE_ST_MTE(sve_st3ss_be_r)
+DEF_SVE_ST_MTE(sve_st4ss_le_r)
+DEF_SVE_ST_MTE(sve_st4ss_be_r)
+DEF_SVE_ST_MTE(sve_st1dd_le_r)
+DEF_SVE_ST_MTE(sve_st1dd_be_r)
+DEF_SVE_ST_MTE(sve_st2dd_le_r)
+DEF_SVE_ST_MTE(sve_st2dd_be_r)
+DEF_SVE_ST_MTE(sve_st3dd_le_r)
+DEF_SVE_ST_MTE(sve_st3dd_be_r)
+DEF_SVE_ST_MTE(sve_st4dd_le_r)
+DEF_SVE_ST_MTE(sve_st4dd_be_r)
+
+#undef DEF_SVE_ST_MTE
+
 DEF_HELPER_FLAGS_6(sve_ldbsu_zsu, TCG_CALL_NO_WG,
                    void, env, ptr, ptr, ptr, tl, i32)
 DEF_HELPER_FLAGS_6(sve_ldhsu_le_zsu, TCG_CALL_NO_WG,
@@ -1399,6 +1829,69 @@ DEF_HELPER_FLAGS_6(sve_ldsds_le_zd, TCG_CALL_NO_WG,
 DEF_HELPER_FLAGS_6(sve_ldsds_be_zd, TCG_CALL_NO_WG,
                    void, env, ptr, ptr, ptr, tl, i32)
 
+#define DEF_SVE_LD_ZPZ_MTE(NAME) \
+    DEF_HELPER_FLAGS_6(NAME##_mte, TCG_CALL_NO_WG, \
+                       void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_SVE_LD_ZPZ_MTE(sve_ldbsu_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhsu_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhsu_be_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldss_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldss_be_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldbss_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhss_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhss_be_zsu)
+
+DEF_SVE_LD_ZPZ_MTE(sve_ldbsu_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhsu_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhsu_be_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldss_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldss_be_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldbss_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhss_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhss_be_zss)
+
+DEF_SVE_LD_ZPZ_MTE(sve_ldbdu_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhdu_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhdu_be_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsdu_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsdu_be_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_lddd_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_lddd_be_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldbds_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhds_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhds_be_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsds_le_zsu)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsds_be_zsu)
+
+DEF_SVE_LD_ZPZ_MTE(sve_ldbdu_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhdu_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhdu_be_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsdu_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsdu_be_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_lddd_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_lddd_be_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldbds_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhds_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhds_be_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsds_le_zss)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsds_be_zss)
+
+DEF_SVE_LD_ZPZ_MTE(sve_ldbdu_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhdu_le_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhdu_be_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsdu_le_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsdu_be_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_lddd_le_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_lddd_be_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldbds_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhds_le_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldhds_be_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsds_le_zd)
+DEF_SVE_LD_ZPZ_MTE(sve_ldsds_be_zd)
+
+#undef DEF_SVE_LD_ZPZ_MTE
+
 DEF_HELPER_FLAGS_6(sve_ldffbsu_zsu, TCG_CALL_NO_WG,
                    void, env, ptr, ptr, ptr, tl, i32)
 DEF_HELPER_FLAGS_6(sve_ldffhsu_le_zsu, TCG_CALL_NO_WG,
@@ -1508,6 +2001,69 @@ DEF_HELPER_FLAGS_6(sve_ldffsds_le_zd, TCG_CALL_NO_WG,
 DEF_HELPER_FLAGS_6(sve_ldffsds_be_zd, TCG_CALL_NO_WG,
                    void, env, ptr, ptr, ptr, tl, i32)
 
+#define DEF_SVE_LDFF_ZPZ_MTE(NAME) \
+    DEF_HELPER_FLAGS_6(NAME##_mte, TCG_CALL_NO_WG, \
+                       void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbsu_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhsu_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhsu_be_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffss_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffss_be_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbss_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhss_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhss_be_zsu)
+
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbsu_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhsu_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhsu_be_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffss_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffss_be_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbss_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhss_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhss_be_zss)
+
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbdu_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhdu_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhdu_be_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsdu_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsdu_be_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffdd_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffdd_be_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbds_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhds_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhds_be_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsds_le_zsu)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsds_be_zsu)
+
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbdu_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhdu_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhdu_be_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsdu_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsdu_be_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffdd_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffdd_be_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbds_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhds_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhds_be_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsds_le_zss)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsds_be_zss)
+
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbdu_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhdu_le_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhdu_be_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsdu_le_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsdu_be_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffdd_le_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffdd_be_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffbds_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhds_le_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffhds_be_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsds_le_zd)
+DEF_SVE_LDFF_ZPZ_MTE(sve_ldffsds_be_zd)
+
+#undef DEF_SVE_LDFF_ZPZ_MTE
+
 DEF_HELPER_FLAGS_6(sve_stbs_zsu, TCG_CALL_NO_WG,
                    void, env, ptr, ptr, ptr, tl, i32)
 DEF_HELPER_FLAGS_6(sve_sths_le_zsu, TCG_CALL_NO_WG,
@@ -1575,4 +2131,450 @@ DEF_HELPER_FLAGS_6(sve_stdd_le_zd, TCG_CALL_NO_WG,
 DEF_HELPER_FLAGS_6(sve_stdd_be_zd, TCG_CALL_NO_WG,
                    void, env, ptr, ptr, ptr, tl, i32)
 
+#define DEF_SVE_ST_ZPZ_MTE(NAME) \
+    DEF_HELPER_FLAGS_6(NAME##_mte, TCG_CALL_NO_WG, \
+                       void, env, ptr, ptr, ptr, tl, i32)
+
+DEF_SVE_ST_ZPZ_MTE(sve_stbs_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_sths_le_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_sths_be_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_stss_le_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_stss_be_zsu)
+
+DEF_SVE_ST_ZPZ_MTE(sve_stbs_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_sths_le_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_sths_be_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_stss_le_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_stss_be_zss)
+
+DEF_SVE_ST_ZPZ_MTE(sve_stbd_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_sthd_le_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_sthd_be_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_stsd_le_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_stsd_be_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_stdd_le_zsu)
+DEF_SVE_ST_ZPZ_MTE(sve_stdd_be_zsu)
+
+DEF_SVE_ST_ZPZ_MTE(sve_stbd_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_sthd_le_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_sthd_be_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_stsd_le_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_stsd_be_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_stdd_le_zss)
+DEF_SVE_ST_ZPZ_MTE(sve_stdd_be_zss)
+
+DEF_SVE_ST_ZPZ_MTE(sve_stbd_zd)
+DEF_SVE_ST_ZPZ_MTE(sve_sthd_le_zd)
+DEF_SVE_ST_ZPZ_MTE(sve_sthd_be_zd)
+DEF_SVE_ST_ZPZ_MTE(sve_stsd_le_zd)
+DEF_SVE_ST_ZPZ_MTE(sve_stsd_be_zd)
+DEF_SVE_ST_ZPZ_MTE(sve_stdd_le_zd)
+DEF_SVE_ST_ZPZ_MTE(sve_stdd_be_zd)
+
+#undef DEF_SVE_ST_ZPZ_MTE
+
+DEF_HELPER_FLAGS_5(sve2_eor3, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_bcax, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_bsl1n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_bsl2n, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_nbsl, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_xar_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_xar_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_xar_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_xar_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(fmmla_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(fmmla_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_pmull_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_eoril_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_eoril_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_eoril_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_eoril_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bext_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bext_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bext_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bext_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bdep_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bdep_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bdep_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bdep_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bgrp_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bgrp_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bgrp_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_bgrp_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_cadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_cadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_cadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_cadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqcadd_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqcadd_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqcadd_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqcadd_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smulh_zzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smulh_zzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smulh_zzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smulh_zzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umulh_zzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umulh_zzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umulh_zzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umulh_zzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_zzzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_zzzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_mul_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_mul_idx_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_mul_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmulh_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqrdmulh_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlah_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdmlsh_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cmla_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cdot_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_cdot_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uabdl_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uabdl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uabdl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_zzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_zzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_zzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_zzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_zzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_zzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_zzz_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sabal_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sabal_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sabal_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uabal_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uabal_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_uabal_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_smull_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_umull_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_sqdmull_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_zzzw_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_zzzw_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_zzzw_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_zzzw_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_zzzw_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_zzzw_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlal_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlal_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_smlsl_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_umlsl_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlal_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_sqdmlsl_idx_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_saddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_uaddw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_ssubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubw_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubw_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_usubw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sshll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sshll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sshll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ushll_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ushll_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ushll_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ssra_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ssra_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ssra_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ssra_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_usra_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_usra_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_usra_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_usra_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_srsra_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_srsra_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_srsra_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_srsra_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ursra_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ursra_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ursra_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_ursra_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sri_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sri_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sri_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sri_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sli_b, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sli_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sli_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sli_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_shrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_rshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqrshrnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_addhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_raddhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_subhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_rsubhnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_uqxtnt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunb_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunb_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunb_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunt_h, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunt_s, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(sve2_sqxtunt_d, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_match_ppzz_b, TCG_CALL_NO_RWG,
+                   i32, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_match_ppzz_h, TCG_CALL_NO_RWG,
+                   i32, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_b, TCG_CALL_NO_RWG,
+                   i32, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_nmatch_ppzz_h, TCG_CALL_NO_RWG,
+                   i32, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_histcnt_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_histcnt_d, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(sve2_histseg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_adcl_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(sve2_adcl_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
diff --git a/qemu/target/arm/helper.c b/qemu/target/arm/helper.c
index c616f2ba9c..c6edbe17b2 100644
--- a/qemu/target/arm/helper.c
+++ b/qemu/target/arm/helper.c
@@ -738,6 +738,7 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
 /* Definitions for the PMU registers */
 #define PMCRN_MASK  0xf800
 #define PMCRN_SHIFT 11
+#define PMCRLP  0x80
 #define PMCRLC  0x40
 #define PMCRDP  0x20
 #define PMCRX   0x10
@@ -749,7 +750,7 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
  * Mask of PMCR bits writeable by guest (not including WO bits like C, P,
  * which can be written as 1 to trigger behaviour but which stay RAZ).
  */
-#define PMCR_WRITEABLE_MASK (PMCRLC | PMCRDP | PMCRX | PMCRD | PMCRE)
+#define PMCR_WRITEABLE_MASK (PMCRLP | PMCRLC | PMCRDP | PMCRX | PMCRD | PMCRE)
 
 #define PMXEVTYPER_P          0x80000000
 #define PMXEVTYPER_U          0x40000000
@@ -970,12 +971,12 @@ static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri,
      * trapping to EL2 or EL3 for other accesses.
      */
     int el = arm_current_el(env);
+    uint64_t mdcr_el2 = arm_mdcr_el2_eff(env);
 
     if (el == 0 && !(env->cp15.c9_pmuserenr & 1)) {
         return CP_ACCESS_TRAP;
     }
-    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
-        && !arm_is_secure_below_el3(env)) {
+    if (el < 2 && (mdcr_el2 & MDCR_TPM)) {
         return CP_ACCESS_TRAP_EL2;
     }
     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
@@ -1044,6 +1045,14 @@ static CPAccessResult pmreg_access_ccntr(CPUARMState *env,
     return pmreg_access(env, ri, isread);
 }
 
+/*
+ * Bits in MDCR_EL2 and MDCR_EL3 which pmu_counter_enabled() looks at.
+ * We use these to decide whether writes need pmu_op_start()/pmu_op_finish().
+ */
+#define MDCR_EL2_PMU_ENABLE_BITS \
+    (MDCR_HPME | MDCR_HPMD | MDCR_HPMN | MDCR_HCCD | MDCR_HLP)
+#define MDCR_EL3_PMU_ENABLE_BITS (MDCR_SPME | MDCR_SCCD)
+
 /* Returns true if the counter (pass 31 for PMCCNTR) should count events using
  * the current EL, security state, and register configuration.
  */
@@ -1051,36 +1060,44 @@ static bool pmu_counter_enabled(CPUARMState *env, uint8_t counter)
 {
     uint64_t filter;
     bool e, p, u, nsk, nsu, nsh, m;
-    bool enabled, prohibited, filtered;
+    bool enabled, prohibited = false, filtered;
     bool secure = arm_is_secure(env);
     int el = arm_current_el(env);
-    uint8_t hpmn = env->cp15.mdcr_el2 & MDCR_HPMN;
+    uint64_t mdcr_el2;
+    uint8_t hpmn;
 
     if (!arm_feature(env, ARM_FEATURE_PMU)) {
         return false;
     }
 
+    mdcr_el2 = arm_mdcr_el2_eff(env);
+    hpmn = mdcr_el2 & MDCR_HPMN;
+
     if (!arm_feature(env, ARM_FEATURE_EL2) ||
             (counter < hpmn || counter == 31)) {
         e = env->cp15.c9_pmcr & PMCRE;
     } else {
-        e = env->cp15.mdcr_el2 & MDCR_HPME;
+        e = mdcr_el2 & MDCR_HPME;
     }
     enabled = e && (env->cp15.c9_pmcnten & (1ULL << counter));
 
-    if (!secure) {
-        if (el == 2 && (counter < (hpmn & 0x7) || counter == 31)) {
-            prohibited = env->cp15.mdcr_el2 & MDCR_HPMD;
-        } else {
-            prohibited = false;
-        }
-    } else {
-        prohibited = arm_feature(env, ARM_FEATURE_EL3) &&
-           (env->cp15.mdcr_el3 & MDCR_SPME);
+    if (el == 2 && (counter < hpmn || counter == 31)) {
+        prohibited = mdcr_el2 & MDCR_HPMD;
+    }
+    if (secure) {
+        prohibited = prohibited || !(env->cp15.mdcr_el3 & MDCR_SPME);
     }
 
-    if (prohibited && counter == 31) {
-        prohibited = env->cp15.c9_pmcr & PMCRDP;
+    if (counter == 31) {
+        prohibited = prohibited && (env->cp15.c9_pmcr & PMCRDP);
+        if (cpu_isar_feature(any_pmu_8_5, env_archcpu(env))) {
+            if (secure) {
+                prohibited = prohibited || (env->cp15.mdcr_el3 & MDCR_SCCD);
+            }
+            if (el == 2) {
+                prohibited = prohibited || (mdcr_el2 & MDCR_HCCD);
+            }
+        }
     }
 
     if (counter == 31) {
@@ -1121,6 +1138,26 @@ static bool pmu_counter_enabled(CPUARMState *env, uint8_t counter)
     return enabled && !prohibited && !filtered;
 }
 
+static bool pmevcntr_is_64_bit(CPUARMState *env, int counter)
+{
+    assert(counter < 31);
+
+    if (!cpu_isar_feature(any_pmu_8_5, env_archcpu(env))) {
+        return false;
+    }
+
+    if (arm_feature(env, ARM_FEATURE_EL2)) {
+        bool hlp = env->cp15.mdcr_el2 & MDCR_HLP;
+        int hpmn = env->cp15.mdcr_el2 & MDCR_HPMN;
+
+        if (hpmn != 0 && counter >= hpmn) {
+            return hlp;
+        }
+    }
+
+    return env->cp15.c9_pmcr & PMCRLP;
+}
+
 /*
  * Ensure c15_ccnt is the guest-visible count so that operations such as
  * enabling/disabling the counter or filtering, modifying the count itself,
@@ -1133,7 +1170,7 @@ static void pmccntr_op_start(CPUARMState *env)
 
     if (pmu_counter_enabled(env, 31)) {
         uint64_t eff_cycles = cycles;
-        if (env->cp15.c9_pmcr & PMCRD) {
+        if ((env->cp15.c9_pmcr & (PMCRD | PMCRLC)) == PMCRD) {
             /* Increment once every 64 processor clock cycles */
             eff_cycles /= 64;
         }
@@ -1158,30 +1195,14 @@ static void pmccntr_op_start(CPUARMState *env)
  */
 static void pmccntr_op_finish(CPUARMState *env)
 {
-#if 0
     if (pmu_counter_enabled(env, 31)) {
-        /* Calculate when the counter will next overflow */
-        uint64_t remaining_cycles = -env->cp15.c15_ccnt;
-        if (!(env->cp15.c9_pmcr & PMCRLC)) {
-            remaining_cycles = (uint32_t)remaining_cycles;
-        }
-        int64_t overflow_in = cycles_ns_per(remaining_cycles);
-
-        if (overflow_in > 0) {
-            int64_t overflow_at = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                overflow_in;
-            ARMCPU *cpu = env_archcpu(env);
-            timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at);
-        }
-
         uint64_t prev_cycles = env->cp15.c15_ccnt_delta;
-        if (env->cp15.c9_pmcr & PMCRD) {
+        if ((env->cp15.c9_pmcr & (PMCRD | PMCRLC)) == PMCRD) {
             /* Increment once every 64 processor clock cycles */
             prev_cycles /= 64;
         }
         env->cp15.c15_ccnt_delta = prev_cycles - env->cp15.c15_ccnt;
     }
-#endif
 }
 
 static void pmevcntr_op_start(CPUARMState *env, uint8_t counter)
@@ -1195,9 +1216,11 @@ static void pmevcntr_op_start(CPUARMState *env, uint8_t counter)
     }
 
     if (pmu_counter_enabled(env, counter)) {
-        uint32_t new_pmevcntr = count - env->cp15.c14_pmevcntr_delta[counter];
+        uint64_t new_pmevcntr = count - env->cp15.c14_pmevcntr_delta[counter];
+        uint64_t overflow_mask = pmevcntr_is_64_bit(env, counter) ?
+                                 1ULL << 63 : 1ULL << 31;
 
-        if (env->cp15.c14_pmevcntr[counter] & ~new_pmevcntr & INT32_MIN) {
+        if (env->cp15.c14_pmevcntr[counter] & ~new_pmevcntr & overflow_mask) {
             env->cp15.c9_pmovsr |= (1ULL << counter);
         }
         env->cp15.c14_pmevcntr[counter] = new_pmevcntr;
@@ -1207,25 +1230,10 @@ static void pmevcntr_op_start(CPUARMState *env, uint8_t counter)
 
 static void pmevcntr_op_finish(CPUARMState *env, uint8_t counter)
 {
-#if 0
     if (pmu_counter_enabled(env, counter)) {
-        uint16_t event = env->cp15.c14_pmevtyper[counter] & PMXEVTYPER_EVTCOUNT;
-        uint16_t event_idx = supported_event_map[event];
-        uint64_t delta = UINT32_MAX -
-            (uint32_t)env->cp15.c14_pmevcntr[counter] + 1;
-        int64_t overflow_in = pm_events[event_idx].ns_per_count(delta);
-
-        if (overflow_in > 0) {
-            int64_t overflow_at = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                overflow_in;
-            ARMCPU *cpu = env_archcpu(env);
-            timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at);
-        }
-
         env->cp15.c14_pmevcntr_delta[counter] -=
             env->cp15.c14_pmevcntr[counter];
     }
-#endif
 }
 
 void pmu_op_start(CPUARMState *env)
@@ -1293,6 +1301,18 @@ static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     pmu_op_finish(env);
 }
 
+static uint64_t pmcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    uint64_t pmcr = env->cp15.c9_pmcr;
+
+    if (arm_current_el(env) <= 1 && arm_is_el2_enabled(env)) {
+        pmcr &= ~PMCRN_MASK;
+        pmcr |= (env->cp15.mdcr_el2 & MDCR_HPMN) << PMCRN_SHIFT;
+    }
+
+    return pmcr;
+}
+
 static void pmswinc_write(CPUARMState *env, const ARMCPRegInfo *ri,
                           uint64_t value)
 {
@@ -1310,9 +1330,11 @@ static void pmswinc_write(CPUARMState *env, const ARMCPRegInfo *ri,
              * Detect if this write causes an overflow since we can't predict
              * PMSWINC overflows like we can for other events
              */
-            uint32_t new_pmswinc = env->cp15.c14_pmevcntr[i] + 1;
+            uint64_t new_pmswinc = env->cp15.c14_pmevcntr[i] + 1;
+            uint64_t overflow_mask = pmevcntr_is_64_bit(env, i) ?
+                                     1ULL << 63 : 1ULL << 31;
 
-            if (env->cp15.c14_pmevcntr[i] & ~new_pmswinc & INT32_MIN) {
+            if (env->cp15.c14_pmevcntr[i] & ~new_pmswinc & overflow_mask) {
                 env->cp15.c9_pmovsr |= (1ULL << i);
             }
 
@@ -1386,15 +1408,19 @@ static uint64_t pmccfiltr_read_a32(CPUARMState *env, const ARMCPRegInfo *ri)
 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
                             uint64_t value)
 {
+    pmu_op_start(env);
     value &= pmu_counter_mask(env);
     env->cp15.c9_pmcnten |= value;
+    pmu_op_finish(env);
 }
 
 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                              uint64_t value)
 {
+    pmu_op_start(env);
     value &= pmu_counter_mask(env);
     env->cp15.c9_pmcnten &= ~value;
+    pmu_op_finish(env);
 }
 
 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -1512,6 +1538,10 @@ static uint64_t pmxevtyper_read(CPUARMState *env, const ARMCPRegInfo *ri)
 static void pmevcntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                              uint64_t value, uint8_t counter)
 {
+    if (!cpu_isar_feature(any_pmu_8_5, env_archcpu(env))) {
+        value &= UINT32_MAX;
+    }
+
     if (counter < pmu_num_counters(env)) {
         pmevcntr_op_start(env, counter);
         env->cp15.c14_pmevcntr[counter] = value;
@@ -1531,6 +1561,9 @@ static uint64_t pmevcntr_read(CPUARMState *env, const ARMCPRegInfo *ri,
         pmevcntr_op_start(env, counter);
         ret = env->cp15.c14_pmevcntr[counter];
         pmevcntr_op_finish(env, counter);
+        if (!cpu_isar_feature(any_pmu_8_5, env_archcpu(env))) {
+            ret &= UINT32_MAX;
+        }
         return ret;
     } else {
       /* We opt to behave as a RAZ/WI when attempts to access PM[X]EVCNTR
@@ -1649,6 +1682,12 @@ static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
     if (cpu_isar_feature(aa64_pauth, cpu)) {
         valid_mask |= SCR_API | SCR_APK;
     }
+    if (cpu_isar_feature(aa64_mte_insn_reg, cpu)) {
+        valid_mask |= SCR_ATA;
+    }
+    if (cpu_isar_feature(aa64_scxtnum, cpu)) {
+        valid_mask |= SCR_ENSCXT;
+    }
 
     /* Clear all-context RES0 bits.  */
     value &= valid_mask;
@@ -2278,52 +2317,18 @@ static uint64_t gt_get_countervalue(CPUARMState *env)
 
 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
 {
-#if 0
     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
 
     if (gt->ctl & 1) {
-        /* Timer enabled: calculate and set current ISTATUS, irq, and
-         * reset timer to when ISTATUS next has to change
-         */
         uint64_t offset = timeridx == GTIMER_VIRT ?
                                       cpu->env.cp15.cntvoff_el2 : 0;
         uint64_t count = gt_get_countervalue(&cpu->env);
-        /* Note that this must be unsigned 64 bit arithmetic: */
         int istatus = count - offset >= gt->cval;
-        uint64_t nexttick;
-        int irqstate;
 
         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
-
-        irqstate = (istatus && !(gt->ctl & 2));
-        qemu_set_irq(cpu->gt_timer_outputs[timeridx], irqstate);
-
-        if (istatus) {
-            /* Next transition is when count rolls back over to zero */
-            nexttick = UINT64_MAX;
-        } else {
-            /* Next transition is when we hit cval */
-            nexttick = gt->cval + offset;
-        }
-        /* Note that the desired next expiry time might be beyond the
-         * signed-64-bit range of a QEMUTimer -- in this case we just
-         * set the timer for as far in the future as possible. When the
-         * timer expires we will reset the timer for any remaining period.
-         */
-        if (nexttick > INT64_MAX / gt_cntfrq_period_ns(cpu)) {
-            timer_mod_ns(cpu->gt_timer[timeridx], INT64_MAX);
-        } else {
-            timer_mod(cpu->gt_timer[timeridx], nexttick);
-        }
-        trace_arm_gt_recalc(timeridx, irqstate, nexttick);
     } else {
-        /* Timer disabled: ISTATUS and timer output always clear */
         gt->ctl &= ~4;
-        qemu_set_irq(cpu->gt_timer_outputs[timeridx], 0);
-        timer_del(cpu->gt_timer[timeridx]);
-        trace_arm_gt_recalc_disabled(timeridx);
     }
-#endif
 }
 
 static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -2372,11 +2377,8 @@ static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
                           int timeridx,
                           uint64_t value)
 {
-#if 0
-    trace_arm_gt_cval_write(timeridx, value);
     env->cp15.c14_timer[timeridx].cval = value;
     gt_recalc_timer(env_archcpu(env), timeridx);
-#endif
 }
 
 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -2417,23 +2419,13 @@ static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
                          int timeridx,
                          uint64_t value)
 {
-#if 0
     ARMCPU *cpu = env_archcpu(env);
-    uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
+    uint64_t oldval = env->cp15.c14_timer[timeridx].ctl;
 
     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
     if ((oldval ^ value) & 1) {
-        /* Enable toggled */
         gt_recalc_timer(cpu, timeridx);
-    } else if ((oldval ^ value) & 2) {
-        /* IMASK toggled: don't need to recalculate,
-         * just set the interrupt line based on ISTATUS
-         */
-        int irqstate = (oldval & 4) && !(value & 2);
-
-        qemu_set_irq(cpu->gt_timer_outputs[timeridx], irqstate);
     }
-#endif
 }
 
 static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
@@ -4353,7 +4345,51 @@ static CPAccessResult fpexc32_access(CPUARMState *env, const ARMCPRegInfo *ri,
 static void sdcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                        uint64_t value)
 {
-    env->cp15.mdcr_el3 = value & SDCR_VALID_MASK;
+    uint64_t new_value = value & SDCR_VALID_MASK;
+
+    /*
+     * Some MDCR_EL3 bits affect whether PMU counters are running: if we are
+     * trying to change any of those then we must bracket this update with PMU
+     * start/finish calls.
+     */
+    bool pmu_op = (env->cp15.mdcr_el3 ^ new_value) &
+                  MDCR_EL3_PMU_ENABLE_BITS;
+
+    if (pmu_op) {
+        pmu_op_start(env);
+    }
+    env->cp15.mdcr_el3 = new_value;
+    if (pmu_op) {
+        pmu_op_finish(env);
+    }
+}
+
+static void mdcr_el3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                           uint64_t value)
+{
+    bool pmu_op = (env->cp15.mdcr_el3 ^ value) & MDCR_EL3_PMU_ENABLE_BITS;
+
+    if (pmu_op) {
+        pmu_op_start(env);
+    }
+    env->cp15.mdcr_el3 = value;
+    if (pmu_op) {
+        pmu_op_finish(env);
+    }
+}
+
+static void mdcr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                           uint64_t value)
+{
+    bool pmu_op = (env->cp15.mdcr_el2 ^ value) & MDCR_EL2_PMU_ENABLE_BITS;
+
+    if (pmu_op) {
+        pmu_op_start(env);
+    }
+    env->cp15.mdcr_el2 = value;
+    if (pmu_op) {
+        pmu_op_finish(env);
+    }
 }
 
 static const ARMCPRegInfo v8_cp_reginfo[] = {
@@ -4690,7 +4726,8 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
     { .name = "MDCR_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 3, .opc2 = 1,
       .resetvalue = 0,
-      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3) },
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3),
+      .writefn = mdcr_el3_write, .raw_writefn = raw_write },
     { .name = "SDCR", .type = ARM_CP_ALIAS,
       .cp = 15, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 1,
       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
@@ -4853,6 +4890,14 @@ static void do_hcr_write(CPUARMState *env, uint64_t value, uint64_t valid_mask)
         if (cpu_isar_feature(aa64_pauth, cpu)) {
             valid_mask |= HCR_API | HCR_APK;
         }
+        if (cpu_isar_feature(aa64_mte, cpu)) {
+            valid_mask |= HCR_ATA | HCR_DCT | HCR_TID5;
+        } else if (cpu_isar_feature(aa64_mte_insn_reg, cpu)) {
+            valid_mask |= HCR_ATA;
+        }
+        if (cpu_isar_feature(aa64_scxtnum, cpu)) {
+            valid_mask |= HCR_ENSCXT;
+        }
     }
 
     /* Clear RES0 bits.  */
@@ -4862,8 +4907,10 @@ static void do_hcr_write(CPUARMState *env, uint64_t value, uint64_t valid_mask)
      * HCR_VM enables stage 2 translation
      * HCR_PTW forbids certain page-table setups
      * HCR_DC Disables stage1 and enables stage2 translation
+     * HCR_DCT enables tagging on disabled stage1 translation
      */
-    if ((env->cp15.hcr_el2 ^ value) & (HCR_VM | HCR_PTW | HCR_DC)) {
+    if ((env->cp15.hcr_el2 ^ value) &
+        (HCR_VM | HCR_PTW | HCR_DC | HCR_DCT)) {
         tlb_flush(CPU(cpu));
     }
     env->cp15.hcr_el2 = value;
@@ -5223,15 +5270,16 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
       .resetvalue = 0,
       .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
-    /* The only field of MDCR_EL2 that has a defined architectural reset value
-     * is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N; but we
-     * don't implement any PMU event counters, so using zero as a reset
-     * value for MDCR_EL2 is okay
+    /*
+     * The only field of MDCR_EL2 that has a defined architectural reset value
+     * is MDCR_EL2.HPMN, which resets to PMCR_EL0.N. The reduced PMU surface
+     * implements four event counters.
      */
     { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
-      .access = PL2_RW, .resetvalue = 0,
-      .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2), },
+      .access = PL2_RW, .resetvalue = 4,
+      .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2),
+      .writefn = mdcr_el2_write, .raw_writefn = raw_write },
     { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
       .access = PL2_RW, .accessfn = access_el3_aa32ns,
@@ -5380,9 +5428,46 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
     REGINFO_SENTINEL
 };
 
+static bool redirect_for_e2h(CPUARMState *env)
+{
+    return arm_current_el(env) == 2 && (arm_hcr_el2_eff(env) & HCR_E2H);
+}
+
+static uint64_t el2_e2h_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    CPReadFn *readfn;
+
+    if (redirect_for_e2h(env)) {
+        ri = ri->opaque;
+        readfn = ri->readfn;
+    } else {
+        readfn = ri->orig_readfn;
+    }
+    if (readfn == NULL) {
+        readfn = raw_read;
+    }
+    return readfn(env, ri);
+}
+
+static void el2_e2h_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          uint64_t value)
+{
+    CPWriteFn *writefn;
+
+    if (redirect_for_e2h(env)) {
+        ri = ri->opaque;
+        writefn = ri->writefn;
+    } else {
+        writefn = ri->orig_writefn;
+    }
+    if (writefn == NULL) {
+        writefn = raw_write;
+    }
+    writefn(env, ri, value);
+}
+
 static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
 {
-#if 0
     struct E2HAlias {
         uint32_t src_key, dst_key, new_key;
         const char *src_name, *dst_name, *new_name;
@@ -5433,6 +5518,13 @@ static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
          */
         { K(3, 0,  1, 2, 0), K(3, 4,  1, 2, 0), K(3, 5, 1, 2, 0),
           "ZCR_EL1", "ZCR_EL2", "ZCR_EL12", isar_feature_aa64_sve },
+        { K(3, 0,  1, 2, 6), K(3, 4,  1, 2, 6), K(3, 5, 1, 2, 6),
+          "SMCR_EL1", "SMCR_EL2", "SMCR_EL12", isar_feature_aa64_sme },
+        { K(3, 0,  5, 6, 0), K(3, 4,  5, 6, 0), K(3, 5, 5, 6, 0),
+          "TFSR_EL1", "TFSR_EL2", "TFSR_EL12", isar_feature_aa64_mte },
+        { K(3, 0, 13, 0, 7), K(3, 4, 13, 0, 7), K(3, 5, 13, 0, 7),
+          "SCXTNUM_EL1", "SCXTNUM_EL2", "SCXTNUM_EL12",
+          isar_feature_aa64_scxtnum },
 
         /* TODO: ARMv8.2-SPE -- PMSCR_EL2 */
         /* TODO: ARMv8.4-Trace -- TRFCR_EL2 */
@@ -5443,7 +5535,9 @@ static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
 
     for (i = 0; i < ARRAY_SIZE(aliases); i++) {
         const struct E2HAlias *a = &aliases[i];
-        ARMCPRegInfo *src_reg, *dst_reg;
+        ARMCPRegInfo *src_reg, *dst_reg, *new_reg;
+        uint32_t *new_key;
+        bool ok;
 
         if (a->feature && !a->feature(&cpu->isar)) {
             continue;
@@ -5462,23 +5556,20 @@ static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
         g_assert(src_reg->opaque == NULL);
 
         /* Create alias before redirection so we dup the right data. */
-        if (a->new_key) {
-            ARMCPRegInfo *new_reg = g_memdup(src_reg, sizeof(ARMCPRegInfo));
-            uint32_t *new_key = g_memdup(&a->new_key, sizeof(uint32_t));
-            bool ok;
+        new_reg = g_memdup(src_reg, sizeof(ARMCPRegInfo));
+        new_key = g_memdup(&a->new_key, sizeof(uint32_t));
+        new_reg->name = g_strdup(a->new_name);
+        new_reg->type |= ARM_CP_ALIAS;
+        /* Remove PL1/PL0 access, leaving PL2/PL3 R/W in place.  */
+        new_reg->access &= PL2_RW | PL3_RW;
 
-            new_reg->name = a->new_name;
-            new_reg->type |= ARM_CP_ALIAS;
-            /* Remove PL1/PL0 access, leaving PL2/PL3 R/W in place.  */
-            new_reg->access &= PL2_RW | PL3_RW;
-
-            ok = g_hash_table_insert(cpu->cp_regs, new_key, new_reg);
-            g_assert(ok);
-        }
+        ok = g_hash_table_insert(cpu->cp_regs, new_key, new_reg);
+        g_assert(ok);
 
         src_reg->opaque = dst_reg;
-        src_reg->orig_readfn = src_reg->readfn ?: raw_read;
-        src_reg->orig_writefn = src_reg->writefn ?: raw_write;
+        src_reg->orig_readfn = src_reg->readfn ? src_reg->readfn : raw_read;
+        src_reg->orig_writefn = src_reg->writefn ? src_reg->writefn :
+            raw_write;
         if (!src_reg->raw_readfn) {
             src_reg->raw_readfn = raw_read;
         }
@@ -5488,7 +5579,6 @@ static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
         src_reg->readfn = el2_e2h_read;
         src_reg->writefn = el2_e2h_write;
     }
-#endif
 }
 
 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -5680,6 +5770,71 @@ int sve_exception_el(CPUARMState *env, int el)
     return 0;
 }
 
+#ifdef TARGET_AARCH64
+static int sme_exception_el(CPUARMState *env, int el)
+{
+    uint64_t hcr_el2 = arm_hcr_el2_eff(env);
+
+    if (el <= 1 && (hcr_el2 & (HCR_E2H | HCR_TGE)) !=
+        (HCR_E2H | HCR_TGE)) {
+        bool disabled = false;
+
+        if (!extract32(env->cp15.cpacr_el1, R_CPACR_EL1_SMEN_SHIFT, 1)) {
+            disabled = true;
+        } else if (!extract32(env->cp15.cpacr_el1,
+                              R_CPACR_EL1_SMEN_SHIFT + 1, 1)) {
+            disabled = el == 0;
+        }
+        if (disabled) {
+            return hcr_el2 & HCR_TGE ? 2 : 1;
+        }
+    }
+
+    if (el <= 2 && !arm_is_secure_below_el3(env)) {
+        if (env->cp15.hcr_el2 & HCR_E2H) {
+            switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, SMEN)) {
+            case 1:
+                if (el != 0 || !(env->cp15.hcr_el2 & HCR_TGE)) {
+                    break;
+                }
+                /* fall through */
+            case 0:
+            case 2:
+                return 2;
+            }
+        } else if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TSM)) {
+            return 2;
+        }
+    }
+
+    if (arm_feature(env, ARM_FEATURE_EL3) &&
+        !FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, ESM)) {
+        return 3;
+    }
+    return 0;
+}
+
+static bool sme_fa64(CPUARMState *env, int el)
+{
+    if (!cpu_isar_feature(aa64_sme_fa64, env_archcpu(env))) {
+        return false;
+    }
+    if (el <= 1 &&
+        !FIELD_EX64(env->vfp.smcr_el[1], SMCR, FA64)) {
+        return false;
+    }
+    if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2) &&
+        !FIELD_EX64(env->vfp.smcr_el[2], SMCR, FA64)) {
+        return false;
+    }
+    if (arm_feature(env, ARM_FEATURE_EL3) &&
+        !FIELD_EX64(env->vfp.smcr_el[3], SMCR, FA64)) {
+        return false;
+    }
+    return true;
+}
+#endif
+
 static uint32_t sve_zcr_get_valid_len(ARMCPU *cpu, uint32_t start_len)
 {
     uint32_t end_len;
@@ -5692,6 +5847,20 @@ static uint32_t sve_zcr_get_valid_len(ARMCPU *cpu, uint32_t start_len)
     return end_len;
 }
 
+#ifdef TARGET_AARCH64
+static uint32_t sme_smcr_get_valid_len(ARMCPU *cpu, uint32_t start_len)
+{
+    uint32_t end_len;
+
+    end_len = start_len &= 0xf;
+    if (!test_bit(start_len, cpu->sme_vq_map)) {
+        end_len = find_last_bit(cpu->sme_vq_map, start_len);
+        assert(end_len < start_len);
+    }
+    return end_len;
+}
+#endif
+
 /*
  * Given that SVE is enabled, return the vector length for EL.
  */
@@ -5713,6 +5882,27 @@ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el)
     return sve_zcr_get_valid_len(cpu, zcr_len);
 }
 
+#ifdef TARGET_AARCH64
+static uint32_t sme_smcr_len_for_el(CPUARMState *env, int el)
+{
+    ARMCPU *cpu = env_archcpu(env);
+    uint32_t smcr_len = find_last_bit(cpu->sme_vq_map, ARM_MAX_VQ);
+
+    assert(smcr_len < ARM_MAX_VQ);
+    if (el <= 1) {
+        smcr_len = MIN(smcr_len, 0xf & (uint32_t)env->vfp.smcr_el[1]);
+    }
+    if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) {
+        smcr_len = MIN(smcr_len, 0xf & (uint32_t)env->vfp.smcr_el[2]);
+    }
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        smcr_len = MIN(smcr_len, 0xf & (uint32_t)env->vfp.smcr_el[3]);
+    }
+
+    return sme_smcr_get_valid_len(cpu, smcr_len);
+}
+#endif
+
 static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                       uint64_t value)
 {
@@ -5765,6 +5955,59 @@ static const ARMCPRegInfo zcr_el3_reginfo = {
     .writefn = zcr_write, .raw_writefn = raw_write
 };
 
+#ifdef TARGET_AARCH64
+static void svcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                       uint64_t value)
+{
+    aarch64_set_svcr(env, value,
+                     (uint64_t)R_SVCR_SM_MASK | R_SVCR_ZA_MASK);
+}
+
+static void smcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                       uint64_t value)
+{
+    uint64_t mask = R_SMCR_LEN_MASK;
+
+    if (cpu_isar_feature(aa64_sme_fa64, env_archcpu(env))) {
+        mask |= MAKE_64BIT_MASK(R_SMCR_FA64_SHIFT, R_SMCR_FA64_LENGTH);
+    }
+    raw_write(env, ri, value & mask);
+}
+
+static const ARMCPRegInfo sme_reginfo[] = {
+    { .name = "TPIDR2_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 13, .crm = 0, .opc2 = 5,
+      .access = PL0_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidr2_el0) },
+    { .name = "SVCR", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 2,
+      .access = PL0_RW, .type = ARM_CP_SME,
+      .fieldoffset = offsetof(CPUARMState, svcr),
+      .writefn = svcr_write, .raw_writefn = raw_write },
+    { .name = "SMCR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 6,
+      .access = PL1_RW, .type = ARM_CP_SME,
+      .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[1]),
+      .writefn = smcr_write, .raw_writefn = raw_write },
+    { .name = "SMCR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 6,
+      .access = PL2_RW, .type = ARM_CP_SME,
+      .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[2]),
+      .writefn = smcr_write, .raw_writefn = raw_write },
+    { .name = "SMCR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 6,
+      .access = PL3_RW, .type = ARM_CP_SME,
+      .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[3]),
+      .writefn = smcr_write, .raw_writefn = raw_write },
+    { .name = "SMIDR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 6,
+      .access = PL1_R, .type = ARM_CP_CONST,
+      .accessfn = access_aa64_tid1,
+      .resetvalue = 0 },
+    REGINFO_SENTINEL
+};
+#endif
+
 void hw_watchpoint_update(ARMCPU *cpu, int n)
 {
     CPUARMState *env = &cpu->env;
@@ -6096,6 +6339,7 @@ static void define_pmu_regs(ARMCPU *cpu)
         .type = ARM_CP_IO | ARM_CP_ALIAS,
         .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
         .accessfn = pmreg_access, .writefn = pmcr_write,
+        .readfn = pmcr_read, .raw_readfn = raw_read,
         .raw_writefn = raw_write,
     };
     ARMCPRegInfo pmcr64 = {
@@ -6106,6 +6350,7 @@ static void define_pmu_regs(ARMCPU *cpu)
         .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
         .resetvalue = (cpu->midr & 0xff000000) | (pmcrn << PMCRN_SHIFT) |
                       PMCRLC,
+        .readfn = pmcr_read, .raw_readfn = raw_read,
         .writefn = pmcr_write, .raw_writefn = raw_write,
     };
     define_one_arm_cp_reg(cpu, &pmcr);
@@ -6120,11 +6365,11 @@ static void define_pmu_regs(ARMCPU *cpu)
               .crm = 8 | (3 & (i >> 3)), .opc1 = 0, .opc2 = i & 7,
               .access = PL0_RW, .type = ARM_CP_IO | ARM_CP_ALIAS,
               .readfn = pmevcntr_readfn, .writefn = pmevcntr_writefn,
-              .accessfn = pmreg_access },
+              .accessfn = pmreg_access_xevcntr },
             { .name = pmevcntr_el0_name, .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 8 | (3 & (i >> 3)),
-              .opc2 = i & 7, .access = PL0_RW, .accessfn = pmreg_access,
-              .type = ARM_CP_IO,
+              .opc2 = i & 7, .access = PL0_RW,
+              .accessfn = pmreg_access_xevcntr, .type = ARM_CP_IO,
               .readfn = pmevcntr_readfn, .writefn = pmevcntr_writefn,
               .raw_readfn = pmevcntr_rawread,
               .raw_writefn = pmevcntr_rawwrite },
@@ -6409,6 +6654,197 @@ static const ARMCPRegInfo dcpodp_reg[] = {
 
 #endif
 
+static CPAccessResult access_mte(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && arm_feature(env, ARM_FEATURE_EL2)) {
+        uint64_t hcr = arm_hcr_el2_eff(env);
+
+        if (!(hcr & HCR_ATA) && (!(hcr & HCR_E2H) || !(hcr & HCR_TGE))) {
+            return CP_ACCESS_TRAP_EL2;
+        }
+    }
+    if (el < 3 &&
+        arm_feature(env, ARM_FEATURE_EL3) &&
+        !(env->cp15.scr_el3 & SCR_ATA)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+static uint64_t tco_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return env->pstate & PSTATE_TCO;
+}
+
+static void tco_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
+{
+    env->pstate = (env->pstate & ~PSTATE_TCO) | (val & PSTATE_TCO);
+}
+
+static CPAccessResult access_aa64_tid5(CPUARMState *env,
+                                       const ARMCPRegInfo *ri,
+                                       bool isread)
+{
+    if ((arm_current_el(env) < 2) && (arm_hcr_el2_eff(env) & HCR_TID5)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+
+    return CP_ACCESS_OK;
+}
+
+static const ARMCPRegInfo mte_insn_reginfo[] = {
+    { .name = "TFSRE0_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 6, .opc2 = 1,
+      .access = PL1_RW, .accessfn = access_mte,
+      .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[0]) },
+    { .name = "TFSR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 6, .opc2 = 0,
+      .access = PL1_RW, .accessfn = access_mte,
+      .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[1]) },
+    { .name = "TFSR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 6, .opc2 = 0,
+      .access = PL2_RW, .accessfn = access_mte,
+      .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[2]) },
+    { .name = "TFSR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 6, .opc2 = 0,
+      .access = PL3_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[3]) },
+    { .name = "RGSR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 5,
+      .access = PL1_RW, .accessfn = access_mte,
+      .fieldoffset = offsetof(CPUARMState, cp15.rgsr_el1) },
+    { .name = "GCR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 6,
+      .access = PL1_RW, .accessfn = access_mte,
+      .fieldoffset = offsetof(CPUARMState, cp15.gcr_el1) },
+    { .name = "GMID_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 4,
+      .access = PL1_R, .accessfn = access_aa64_tid5,
+      .type = ARM_CP_CONST, .resetvalue = GMID_EL1_BS },
+    { .name = "TCO", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 7,
+      .type = ARM_CP_NO_RAW, .access = PL0_RW,
+      .readfn = tco_read, .writefn = tco_write },
+    { .name = "DC_IGVAC", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 3,
+      .type = ARM_CP_NOP, .access = PL1_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_IGSW", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 4,
+      .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
+    { .name = "DC_IGDVAC", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 5,
+      .type = ARM_CP_NOP, .access = PL1_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_IGDSW", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 6,
+      .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
+    { .name = "DC_CGSW", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 4,
+      .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
+    { .name = "DC_CGDSW", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 6,
+      .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
+    { .name = "DC_CIGSW", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 4,
+      .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
+    { .name = "DC_CIGDSW", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 6,
+      .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
+    { .name = "DC_CGVAC", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 3,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_CGDVAC", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 5,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_CGVAP", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 3,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_CGDVAP", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 5,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_CGVADP", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 3,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_CGDVADP", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 5,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_CIGVAC", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 3,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_CIGDVAC", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 5,
+      .type = ARM_CP_NOP, .access = PL0_W,
+      .accessfn = aa64_cacheop_poc_access },
+    { .name = "DC_GVA", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 3,
+      .access = PL0_W, .type = ARM_CP_DC_GVA,
+      .accessfn = aa64_zva_access },
+    { .name = "DC_GZVA", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 4,
+      .access = PL0_W, .type = ARM_CP_DC_GZVA,
+      .accessfn = aa64_zva_access },
+    REGINFO_SENTINEL
+};
+
+#ifdef TARGET_AARCH64
+static CPAccessResult access_scxtnum(CPUARMState *env,
+                                     const ARMCPRegInfo *ri, bool isread)
+{
+    uint64_t hcr = arm_hcr_el2_eff(env);
+    int el = arm_current_el(env);
+
+    if (el == 0 && !((hcr & HCR_E2H) && (hcr & HCR_TGE))) {
+        if (env->cp15.sctlr_el[1] & SCTLR_TSCXT) {
+            if (hcr & HCR_TGE) {
+                return CP_ACCESS_TRAP_EL2;
+            }
+            return CP_ACCESS_TRAP;
+        }
+    } else if (el < 2 && (env->cp15.sctlr_el[2] & SCTLR_TSCXT)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 2 && arm_is_el2_enabled(env) && !(hcr & HCR_ENSCXT)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && arm_feature(env, ARM_FEATURE_EL3) &&
+        !(env->cp15.scr_el3 & SCR_ENSCXT)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+static const ARMCPRegInfo scxtnum_reginfo[] = {
+    { .name = "SCXTNUM_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 13, .crm = 0, .opc2 = 7,
+      .access = PL0_RW, .accessfn = access_scxtnum,
+      .fieldoffset = offsetof(CPUARMState, scxtnum_el[0]) },
+    { .name = "SCXTNUM_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 7,
+      .access = PL1_RW, .accessfn = access_scxtnum,
+      .fieldoffset = offsetof(CPUARMState, scxtnum_el[1]) },
+    { .name = "SCXTNUM_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 7,
+      .access = PL2_RW, .accessfn = access_scxtnum,
+      .fieldoffset = offsetof(CPUARMState, scxtnum_el[2]) },
+    { .name = "SCXTNUM_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 7,
+      .access = PL3_RW,
+      .fieldoffset = offsetof(CPUARMState, scxtnum_el[3]) },
+    REGINFO_SENTINEL
+};
+#endif
+
 static CPAccessResult access_predinv(CPUARMState *env, const ARMCPRegInfo *ri,
                                      bool isread)
 {
@@ -6795,13 +7231,12 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
               .access = PL1_R, .type = ARM_CP_CONST,
               .accessfn = access_aa64_tid3,
-              /* At present, only SVEver == 0 is defined anyway.  */
-              .resetvalue = 0 },
-            { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .resetvalue = cpu->isar.id_aa64zfr0 },
+            { .name = "ID_AA64SMFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
               .access = PL1_R, .type = ARM_CP_CONST,
               .accessfn = access_aa64_tid3,
-              .resetvalue = 0 },
+              .resetvalue = cpu->isar.id_aa64smfr0 },
             { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -7456,6 +7891,9 @@ void register_cp_regs_for_features(ARMCPU *cpu)
     }
 
 #ifdef TARGET_AARCH64
+    if (cpu_isar_feature(aa64_sme, cpu)) {
+        define_arm_cp_regs(cpu, sme_reginfo);
+    }
     if (cpu_isar_feature(aa64_pauth, cpu)) {
         define_arm_cp_regs(cpu, pauth_reginfo);
     }
@@ -7470,6 +7908,12 @@ void register_cp_regs_for_features(ARMCPU *cpu)
             define_one_arm_cp_reg(cpu, dcpodp_reg);
         }
     }
+    if (cpu_isar_feature(aa64_mte_insn_reg, cpu)) {
+        define_arm_cp_regs(cpu, mte_insn_reginfo);
+    }
+    if (cpu_isar_feature(aa64_scxtnum, cpu)) {
+        define_arm_cp_regs(cpu, scxtnum_reginfo);
+    }
 #endif
 
     if (cpu_isar_feature(any_predinv, cpu)) {
@@ -7930,9 +8374,9 @@ void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
      *
      * In a V8 implementation, it is permitted for privileged software to
      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
-     */
+    */
     if (write_type != CPSRWriteByUnicorn &&
-         write_type != CPSRWriteRaw && 
+         write_type != CPSRWriteRaw &&
          !arm_feature(env, ARM_FEATURE_V8) &&
         arm_feature(env, ARM_FEATURE_EL3) &&
         !arm_feature(env, ARM_FEATURE_EL2) &&
@@ -9833,6 +10277,15 @@ static int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx)
     }
 }
 
+static int aa64_va_parameter_tcma(uint64_t tcr, ARMMMUIdx mmu_idx)
+{
+    if (regime_has_2_ranges(mmu_idx)) {
+        return extract64(tcr, 57, 2);
+    } else {
+        return extract32(tcr, 30, 1) * 3;
+    }
+}
+
 ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va,
                                    ARMMMUIdx mmu_idx, bool data)
 {
@@ -10492,45 +10945,46 @@ static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address,
     return !(*prot & (1 << access_type));
 }
 
+static bool v8m_is_sau_exempt(CPUARMState *env,
+                              uint32_t address, MMUAccessType access_type)
+{
+    /*
+     * The architecture specifies that certain address ranges are exempt from
+     * v8M SAU/IDAU checks.
+     */
+    return (access_type == MMU_INST_FETCH &&
+            m_is_system_region(env, address)) ||
+        (address >= 0xe0000000 && address <= 0xe0002fff) ||
+        (address >= 0xe000e000 && address <= 0xe000efff) ||
+        (address >= 0xe002e000 && address <= 0xe002efff) ||
+        (address >= 0xe0040000 && address <= 0xe0041fff) ||
+        (address >= 0xe00ff000 && address <= 0xe00fffff);
+}
+
 void v8m_security_lookup(CPUARMState *env, uint32_t address,
                                 MMUAccessType access_type, ARMMMUIdx mmu_idx,
                                 V8M_SAttributes *sattrs)
 {
-#if 0
     /* Look up the security attributes for this address. Compare the
      * pseudocode SecurityCheck() function.
      * We assume the caller has zero-initialized *sattrs.
      */
     ARMCPU *cpu = env_archcpu(env);
+    struct uc_struct *uc = env->uc;
     int r;
-    bool idau_exempt = false, idau_ns = true, idau_nsc = true;
-    int idau_region = IREGION_NOTVALID;
     uint32_t addr_page_base = address & TARGET_PAGE_MASK;
     uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1);
 
-    if (cpu->idau) {
-        IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau);
-        IDAUInterface *ii = IDAU_INTERFACE(cpu->idau);
-
-        iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns,
-                   &idau_nsc);
-    }
-
     if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) {
         /* 0xf0000000..0xffffffff is always S for insn fetches */
         return;
     }
 
-    if (idau_exempt || v8m_is_sau_exempt(env, address, access_type)) {
+    if (v8m_is_sau_exempt(env, address, access_type)) {
         sattrs->ns = !regime_is_secure(env, mmu_idx);
         return;
     }
 
-    if (idau_region != IREGION_NOTVALID) {
-        sattrs->irvalid = true;
-        sattrs->iregion = idau_region;
-    }
-
     switch (env->sau.ctrl & 3) {
     case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */
         break;
@@ -10587,18 +11041,6 @@ void v8m_security_lookup(CPUARMState *env, uint32_t address,
         }
         break;
     }
-
-    /*
-     * The IDAU will override the SAU lookup results if it specifies
-     * higher security than the SAU does.
-     */
-    if (!idau_ns) {
-        if (sattrs->ns || (!idau_nsc && sattrs->nsc)) {
-            sattrs->ns = false;
-            sattrs->nsc = idau_nsc;
-        }
-    }
-#endif
 }
 
 bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address,
@@ -10710,6 +11152,11 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address,
     } else {
         uint32_t ap = extract32(env->pmsav8.rbar[secure][matchregion], 1, 2);
         uint32_t xn = extract32(env->pmsav8.rbar[secure][matchregion], 0, 1);
+        bool pxn = false;
+
+        if (arm_feature(env, ARM_FEATURE_V8_1M)) {
+            pxn = extract32(env->pmsav8.rlar[secure][matchregion], 4, 1);
+        }
 
         if (m_is_system_region(env, address)) {
             /* System space is always execute never */
@@ -10717,7 +11164,7 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address,
         }
 
         *prot = simple_ap_to_rw_prot(env, mmu_idx, ap);
-        if (*prot && !xn) {
+        if (*prot && !xn && !(pxn && !is_user)) {
             *prot |= PAGE_EXEC;
         }
         /* We don't need to look the attribute up in the MAIR0/MAIR1
@@ -10930,6 +11377,14 @@ static ARMCacheAttrs combine_cacheattrs(ARMCacheAttrs s1, ARMCacheAttrs s2)
     uint8_t s1lo = extract32(s1.attrs, 0, 4), s2lo = extract32(s2.attrs, 0, 4);
     uint8_t s1hi = extract32(s1.attrs, 4, 4), s2hi = extract32(s2.attrs, 4, 4);
     ARMCacheAttrs ret;
+    bool tagged = false;
+
+    if (s1.attrs == 0xf0) {
+        tagged = true;
+        s1.attrs = 0xff;
+        s1lo = extract32(s1.attrs, 0, 4);
+        s1hi = extract32(s1.attrs, 4, 4);
+    }
 
     /* Combine shareability attributes (table D4-43) */
     if (s1.shareability == 2 || s2.shareability == 2) {
@@ -10977,6 +11432,10 @@ static ARMCacheAttrs combine_cacheattrs(ARMCacheAttrs s1, ARMCacheAttrs s2)
         }
     }
 
+    if (tagged && ret.attrs == 0xff) {
+        ret.attrs = 0xf0;
+    }
+
     return ret;
 }
 
@@ -11054,7 +11513,9 @@ bool get_phys_addr(CPUARMState *env, target_ulong address,
                      *  Inner Write-Back Read-Allocate Write-Allocate,
                      *  Outer Write-Back Read-Allocate Write-Allocate.
                      */
-                    cacheattrs->attrs = 0xff;
+                    if (cacheattrs->attrs != 0xf0) {
+                        cacheattrs->attrs = 0xff;
+                    }
                     cacheattrs->shareability = 0;
                 }
                 *cacheattrs = combine_cacheattrs(*cacheattrs, cacheattrs2);
@@ -11121,6 +11582,9 @@ bool get_phys_addr(CPUARMState *env, target_ulong address,
     /* Definitely a real MMU, not an MPU */
 
     if (regime_translation_disabled(env, mmu_idx)) {
+        uint8_t memattr = 0;
+        uint8_t shareability = 0;
+
         /*
          * MMU disabled.  S1 addresses within aa64 translation regimes are
          * still checked for bounds -- see AArch64.TranslateAddressS1Off.
@@ -11130,6 +11594,7 @@ bool get_phys_addr(CPUARMState *env, target_ulong address,
             if (arm_el_is_aa64(env, r_el)) {
                 int pamax = arm_pamax(env_archcpu(env));
                 uint64_t tcr = env->cp15.tcr_el[r_el].raw_tcr;
+                uint64_t hcr = arm_hcr_el2_eff(env);
                 int addrtop, tbi;
 
                 tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
@@ -11153,7 +11618,23 @@ bool get_phys_addr(CPUARMState *env, target_ulong address,
                  * the pseudocode set of addrdesc.paddress.
                  */
                 address = extract64(address, 0, 52);
+
+                if (r_el == 1 && (hcr & HCR_DC)) {
+                    memattr = (hcr & HCR_DCT) ? 0xf0 : 0xff;
+                }
+            }
+        }
+        if (memattr == 0 && access_type == MMU_INST_FETCH) {
+            if (regime_sctlr(env, mmu_idx) & SCTLR_I) {
+                memattr = 0xee;
+            } else {
+                memattr = 0x44;
             }
+            shareability = 2;
+        }
+        if (cacheattrs != NULL) {
+            cacheattrs->attrs = memattr;
+            cacheattrs->shareability = shareability;
         }
         *phys_ptr = address;
         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
@@ -11723,9 +12204,90 @@ static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el,
         }
     }
 
+    if (cpu_isar_feature(aa64_mte_insn_reg, env_archcpu(env))) {
+        bool ata = false;
+
+        if (sctlr & (el == 0 ? SCTLR_ATA0 : SCTLR_ATA)) {
+            ata = true;
+            if (el < 2 && arm_feature(env, ARM_FEATURE_EL2)) {
+                uint64_t hcr = arm_hcr_el2_eff(env);
+
+                ata = (hcr & HCR_ATA) ||
+                      ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE));
+            }
+            if (ata && el < 3 &&
+                arm_feature(env, ARM_FEATURE_EL3) &&
+                !(env->cp15.scr_el3 & SCR_ATA)) {
+                ata = false;
+            }
+            if (ata) {
+                FIELD_DP32(flags, TBFLAG_A64, ATA, 1, flags);
+                if (tbid && !(env->pstate & PSTATE_TCO) &&
+                    (sctlr & (el == 0 ? SCTLR_TCF0 : SCTLR_TCF))) {
+                    FIELD_DP32(flags, TBFLAG_A64, MTE_ACTIVE, 1, flags);
+                    if (!FIELD_EX32(flags, TBFLAG_A64, UNPRIV)) {
+                        FIELD_DP32(flags, TBFLAG_A64, MTE0_ACTIVE, 1, flags);
+                    }
+                }
+            }
+        }
+        if (FIELD_EX32(flags, TBFLAG_A64, UNPRIV) &&
+            tbid && !(env->pstate & PSTATE_TCO) &&
+            (sctlr & SCTLR_TCF0)) {
+            bool ata0 = sctlr & SCTLR_ATA0;
+
+            if (ata0 && arm_feature(env, ARM_FEATURE_EL2)) {
+                uint64_t hcr = arm_hcr_el2_eff(env);
+
+                ata0 = (hcr & HCR_ATA) ||
+                       ((hcr & (HCR_E2H | HCR_TGE)) ==
+                        (HCR_E2H | HCR_TGE));
+            }
+            if (ata0 && arm_feature(env, ARM_FEATURE_EL3) &&
+                !(env->cp15.scr_el3 & SCR_ATA)) {
+                ata0 = false;
+            }
+            if (ata0) {
+                FIELD_DP32(flags, TBFLAG_A64, MTE0_ACTIVE, 1, flags);
+            }
+        }
+        FIELD_DP32(flags, TBFLAG_A64, TCMA,
+                   aa64_va_parameter_tcma(tcr, mmu_idx), flags);
+    }
+
     return rebuild_hflags_common(env, fp_el, mmu_idx, flags);
 }
 
+#ifdef TARGET_AARCH64
+static uint32_t rebuild_hflags2_a64(CPUARMState *env, int el)
+{
+    uint32_t flags = 0;
+
+    if (cpu_isar_feature(aa64_sme, env_archcpu(env))) {
+        bool sm = FIELD_EX64(env->svcr, SVCR, SM) != 0;
+        bool za = FIELD_EX64(env->svcr, SVCR, ZA) != 0;
+        uint32_t svl = sme_smcr_len_for_el(env, el);
+        int sme_el = sme_exception_el(env, el);
+
+        FIELD_DP32(flags, TBFLAG_A64_2, SMEEXC_EL, sme_el, flags);
+        FIELD_DP32(flags, TBFLAG_A64_2, SVL, svl, flags);
+        FIELD_DP32(flags, TBFLAG_A64_2, PSTATE_SM, sm, flags);
+        FIELD_DP32(flags, TBFLAG_A64_2, PSTATE_ZA, za, flags);
+        if (sm && !sme_fa64(env, el)) {
+            FIELD_DP32(flags, TBFLAG_A64_2, SME_TRAP_NONSTREAMING, 1,
+                       flags);
+        }
+    }
+
+    return flags;
+}
+#else
+static uint32_t rebuild_hflags2_a64(CPUARMState *env, int el)
+{
+    return 0;
+}
+#endif
+
 static uint32_t rebuild_hflags_internal(CPUARMState *env)
 {
     int el = arm_current_el(env);
@@ -11744,6 +12306,8 @@ static uint32_t rebuild_hflags_internal(CPUARMState *env)
 void arm_rebuild_hflags(CPUARMState *env)
 {
     env->hflags = rebuild_hflags_internal(env);
+    env->hflags2 = is_a64(env) ? rebuild_hflags2_a64(env, arm_current_el(env))
+                               : 0;
 }
 
 /*
@@ -11756,6 +12320,7 @@ void HELPER(rebuild_hflags_m32_newel)(CPUARMState *env)
     int fp_el = fp_exception_el(env, el);
     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
     env->hflags = rebuild_hflags_m32(env, fp_el, mmu_idx);
+    env->hflags2 = 0;
 }
 
 void HELPER(rebuild_hflags_m32)(CPUARMState *env, int el)
@@ -11764,6 +12329,7 @@ void HELPER(rebuild_hflags_m32)(CPUARMState *env, int el)
     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
 
     env->hflags = rebuild_hflags_m32(env, fp_el, mmu_idx);
+    env->hflags2 = 0;
 }
 
 /*
@@ -11776,6 +12342,7 @@ void HELPER(rebuild_hflags_a32_newel)(CPUARMState *env)
     int fp_el = fp_exception_el(env, el);
     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
     env->hflags = rebuild_hflags_a32(env, fp_el, mmu_idx);
+    env->hflags2 = 0;
 }
 
 void HELPER(rebuild_hflags_a32)(CPUARMState *env, int el)
@@ -11784,6 +12351,7 @@ void HELPER(rebuild_hflags_a32)(CPUARMState *env, int el)
     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
 
     env->hflags = rebuild_hflags_a32(env, fp_el, mmu_idx);
+    env->hflags2 = 0;
 }
 
 void HELPER(rebuild_hflags_a64)(CPUARMState *env, int el)
@@ -11792,6 +12360,7 @@ void HELPER(rebuild_hflags_a64)(CPUARMState *env, int el)
     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
 
     env->hflags = rebuild_hflags_a64(env, el, fp_el, mmu_idx);
+    env->hflags2 = rebuild_hflags2_a64(env, el);
 }
 
 static inline void assert_hflags_rebuild_correctly(CPUARMState *env)
@@ -11799,22 +12368,43 @@ static inline void assert_hflags_rebuild_correctly(CPUARMState *env)
 #ifdef CONFIG_DEBUG_TCG
     uint32_t env_flags_current = env->hflags;
     uint32_t env_flags_rebuilt = rebuild_hflags_internal(env);
-
-    if (unlikely(env_flags_current != env_flags_rebuilt)) {
-        fprintf(stderr, "TCG hflags mismatch (current:0x%08x rebuilt:0x%08x)\n",
-                env_flags_current, env_flags_rebuilt);
+    uint32_t env_flags2_current = env->hflags2;
+    uint32_t env_flags2_rebuilt = is_a64(env)
+        ? rebuild_hflags2_a64(env, arm_current_el(env)) : 0;
+
+    if (unlikely(env_flags_current != env_flags_rebuilt ||
+                 env_flags2_current != env_flags2_rebuilt)) {
+        fprintf(stderr,
+                "TCG hflags mismatch (current:0x%08x/0x%08x "
+                "rebuilt:0x%08x/0x%08x)\n",
+                env_flags_current, env_flags2_current,
+                env_flags_rebuilt, env_flags2_rebuilt);
         abort();
     }
 #endif
 }
 
+static bool mve_no_pred(CPUARMState *env)
+{
+    if (!cpu_isar_feature(aa32_mve, env_archcpu(env))) {
+        return false;
+    }
+    if (env->v7m.vpr) {
+        return false;
+    }
+    if (env->v7m.ltpsize < 4) {
+        return false;
+    }
+    return true;
+}
+
 void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
                           target_ulong *cs_base, uint32_t *pflags)
 {
     uint32_t flags = env->hflags;
     uint32_t pstate_for_ss;
 
-    *cs_base = 0;
+    *cs_base = env->hflags2;
     assert_hflags_rebuild_correctly(env);
 
     if (FIELD_EX32(flags, TBFLAG_ANY, AARCH64_STATE)) {
@@ -11849,6 +12439,9 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
             if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) {
                 FIELD_DP32(flags, TBFLAG_M32, LSPACT, 1, flags);
             }
+            if (mve_no_pred(env)) {
+                FIELD_DP32(flags, TBFLAG_M32, MVE_NO_PRED, 1, flags);
+            }
         } else {
             /*
              * Note that XSCALE_CPAR shares bits with VECSTRIDE.
diff --git a/qemu/target/arm/helper.h b/qemu/target/arm/helper.h
index 616d032c84..3f8e03254b 100644
--- a/qemu/target/arm/helper.h
+++ b/qemu/target/arm/helper.h
@@ -102,6 +102,680 @@ DEF_HELPER_FLAGS_2(rebuild_hflags_a64, TCG_CALL_NO_RWG, void, env, int)
 
 DEF_HELPER_1(vfp_get_fpscr, i32, env)
 DEF_HELPER_2(vfp_set_fpscr, void, env, i32)
+DEF_HELPER_FLAGS_2(mve_vctp, TCG_CALL_NO_WG, void, env, i32)
+DEF_HELPER_FLAGS_1(mve_vpnot, TCG_CALL_NO_WG, void, env)
+DEF_HELPER_FLAGS_4(mve_vpsel, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vdup, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmovi, TCG_CALL_NO_WG, void, env, ptr, i64)
+DEF_HELPER_FLAGS_3(mve_vandi, TCG_CALL_NO_WG, void, env, ptr, i64)
+DEF_HELPER_FLAGS_3(mve_vorri, TCG_CALL_NO_WG, void, env, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vidupb, TCG_CALL_NO_WG, i32, env, ptr, i32, i32)
+DEF_HELPER_FLAGS_4(mve_viduph, TCG_CALL_NO_WG, i32, env, ptr, i32, i32)
+DEF_HELPER_FLAGS_4(mve_vidupw, TCG_CALL_NO_WG, i32, env, ptr, i32, i32)
+DEF_HELPER_FLAGS_5(mve_viwdupb, TCG_CALL_NO_WG, i32, env, ptr, i32, i32, i32)
+DEF_HELPER_FLAGS_5(mve_viwduph, TCG_CALL_NO_WG, i32, env, ptr, i32, i32, i32)
+DEF_HELPER_FLAGS_5(mve_viwdupw, TCG_CALL_NO_WG, i32, env, ptr, i32, i32, i32)
+DEF_HELPER_FLAGS_5(mve_vdwdupb, TCG_CALL_NO_WG, i32, env, ptr, i32, i32, i32)
+DEF_HELPER_FLAGS_5(mve_vdwduph, TCG_CALL_NO_WG, i32, env, ptr, i32, i32, i32)
+DEF_HELPER_FLAGS_5(mve_vdwdupw, TCG_CALL_NO_WG, i32, env, ptr, i32, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpeqb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpeqh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpeqw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpneb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpneh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpnew, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpcsb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpcsh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpcsw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmphib, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmphih, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmphiw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpgeb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpgeh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpgew, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpltb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmplth, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpltw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpgtb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpgth, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpgtw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpleb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpleh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmplew, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcmpeq_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpeq_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpeq_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpne_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpne_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpne_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpcs_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpcs_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpcs_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmphi_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmphi_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmphi_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpge_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpge_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpge_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmplt_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmplt_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmplt_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpgt_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpgt_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmpgt_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmple_scalarb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmple_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcmple_scalarw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpeqh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpeqs, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpneh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpnes, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpgeh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpges, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmplth, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmplts, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpgth, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpgts, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpleh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmples, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfcmpeq_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpeq_scalars, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpne_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpne_scalars, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpge_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpge_scalars, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmplt_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmplt_scalars, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpgt_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmpgt_scalars, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmple_scalarh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfcmple_scalars, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vfabsh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfabss, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfnegh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vfnegs, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vldrb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrb_sh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrb_uh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrb_sw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrb_uw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrh_sw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vldrh_uw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vstrb, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vstrh, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vstrw, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vstrb_h, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vstrb_w, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vstrh_w, TCG_CALL_NO_WG, void, env, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrb_sg_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrb_sg_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrh_sg_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrb_sg_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrb_sg_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrb_sg_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrh_sg_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrh_sg_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrw_sg_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrd_sg_ud, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrh_sg_os_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrh_sg_os_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrh_sg_os_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrw_sg_os_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrd_sg_os_ud, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrb_sg_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrb_sg_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrb_sg_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrh_sg_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrh_sg_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrw_sg_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrd_sg_ud, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrh_sg_os_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrh_sg_os_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrw_sg_os_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrd_sg_os_ud, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrw_sg_wb_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vldrd_sg_wb_ud, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrw_sg_wb_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vstrd_sg_wb_ud, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vld20b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld20h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld20w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld21b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld21h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld21w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld40b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld40h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld40w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld41b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld41h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld41w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld42b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld42h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld42w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld43b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld43h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vld43w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst20b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst20h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst20w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst21b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst21h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst21w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst40b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst40h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst40w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst41b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst41h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst41w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst42b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst42h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst42w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst43b, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst43h, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_vst43w, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_4(mve_vand, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vbic, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vorr, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vorn, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_veor, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vaddb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vaddh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vaddw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vadd_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vadd_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vadd_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vsubb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vsubh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vsubw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vsub_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vsub_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vsub_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmulb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmul_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmul_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmul_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmulhsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulhsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulhsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulhub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulhuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulhuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrmulhsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrmulhsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrmulhsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrmulhub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrmulhuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrmulhuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullbsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullbsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullbsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullbub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullbuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullbuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulltsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulltsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulltsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulltub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulltuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmulltuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullpbh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullpth, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullpbw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmullptw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmullbh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmullbw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmullth, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmulltw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmullb_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmullb_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmullt_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmullt_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcadd90b, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcadd90h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcadd90w, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcadd270b, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcadd270h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcadd270w, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhcadd90b, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhcadd90h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhcadd90w, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhcadd270b, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhcadd270h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhcadd270w, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vabdsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vabdsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vabdsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vabdub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vabduh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vabduw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhaddsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhaddsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhaddsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhaddub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhadduh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhadduw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhadds_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhadds_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhadds_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhaddu_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhaddu_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhaddu_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrhaddsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrhaddsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrhaddsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrhaddub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrhadduh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrhadduw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vadc, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vadci, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vsbc, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vsbci, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhsubsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhsubsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhsubsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhsubub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhsubuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhsubuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vhsubs_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhsubs_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhsubs_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhsubu_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhsubu_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vhsubu_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqaddsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqaddsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqaddsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqaddub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqadduh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqadduw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqsubsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqsubsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqsubsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqsubub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqsubuh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqsubuw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmulhb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmulhh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmulhw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmulhb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmulhh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmulhw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqadds_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqadds_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqadds_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqaddu_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqaddu_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqaddu_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqsubs_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqsubs_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqsubs_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqsubu_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqsubu_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqsubu_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmulh_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmulh_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmulh_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmulh_scalarb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmulh_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmulh_scalarw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlab, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlah, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlaw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlasb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlash, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlasw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmlahb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmlahh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmlahw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmlahb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmlahh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmlahw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmlashb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmlashh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqdmlashw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmlashb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmlashh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrdmlashw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfaddh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfadds, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfsubh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfsubs, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfmulh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfmuls, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfabdh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfabds, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxnmh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxnms, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminnmh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminnms, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxnmah, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmaxnmas, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminnmah, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vminnmas, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfcadd90h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfcadd90s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfcadd270h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfcadd270s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul0h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul0s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul90h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul90s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul180h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul180s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul270h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmul270s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla0h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla0s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla90h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla90s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla180h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla180s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla270h, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcmla270s, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfmah, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfmas, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfmsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfmss, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vfadd_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfadd_scalars, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfsub_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfsub_scalars, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfmul_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfmul_scalars, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfma_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfma_scalars, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfmas_scalarh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vfmas_scalars, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_hs, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_hu, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_sf, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_uf, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_fs, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_fu, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vcvtb_sh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcvtt_sh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcvtb_hs, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vcvtt_hs, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vcvt_rm_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_rm_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_rm_ss, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vcvt_rm_us, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrint_rm_h, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrint_rm_s, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vrintx_h, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vrintx_s, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vshlsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vshlsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vshlsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vshlub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vshluh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vshluw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrshlsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrshlsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrshlsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrshlub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrshluh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vrshluw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqshlsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqshlsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqshlsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqshlub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqshluh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqshluw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrshlsb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrshlsh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrshlsw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrshlub, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrshluh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrshluw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmladhb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmladhh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmladhw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmladhxb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmladhxh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmladhxw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmladhb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmladhh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmladhw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmladhxb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmladhxh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmladhxw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmlsdhb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmlsdhh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmlsdhw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmlsdhxb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmlsdhxh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqdmlsdhxw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmlsdhb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmlsdhh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmlsdhw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmlsdhxb, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmlsdhxh, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vqrdmlsdhxw, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vbrsrb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vbrsrh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vbrsrw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshli_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshli_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshli_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshli_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshli_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshli_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshli_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshli_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshli_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshli_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshli_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshli_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshli_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshli_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshli_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshli_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshli_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshli_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshli_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshli_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshli_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshli_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshli_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshli_uw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshlui_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshlui_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshlui_sw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshllbsb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshllbsh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshllbub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshllbuh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshlltsb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshlltsh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshlltub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshlltuh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshrnbb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshrnbh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshrntb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vshrnth, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshrnbb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshrnbh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshrntb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vrshrnth, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnb_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnb_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnt_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnt_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnb_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnb_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnt_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrnt_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrunbb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrunbh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshruntb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqshrunth, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnb_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnb_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnt_sb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnt_sh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnb_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnb_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnt_ub, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrnt_uh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrunbb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrunbh, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshruntb, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vqrshrunth, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmovnbb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vmovnbh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vmovntb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vmovnth, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovnbsb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovnbsh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovntsb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovntsh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovnbub, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovnbuh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovntub, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovntuh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovunbb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovunbh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovuntb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqmovunth, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vshlc, TCG_CALL_NO_WG, i32, env, ptr, i32, i32)
+DEF_HELPER_FLAGS_3(mve_sshrl, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_ushll, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_sqshll, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_uqshll, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_sqrshrl, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_uqrshll, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_sqrshrl48, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_uqrshll48, TCG_CALL_NO_RWG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(mve_uqshl, TCG_CALL_NO_RWG, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_sqshl, TCG_CALL_NO_RWG, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_uqrshl, TCG_CALL_NO_RWG, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(mve_sqrshr, TCG_CALL_NO_RWG, i32, env, i32, i32)
+DEF_HELPER_FLAGS_4(mve_vsrib, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vsrih, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vsriw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vslib, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vslih, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vsliw, TCG_CALL_NO_WG, void, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vclsb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vclsh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vclsw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vclzb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vclzh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vclzw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vrev16b, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vrev32b, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vrev32h, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vrev64b, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vrev64h, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vrev64w, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vmvn, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vabsb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vabsh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vabsw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vnegb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vnegh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vnegw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vmaxab, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vmaxah, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vmaxaw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vminab, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vminah, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vminaw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqabsb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqabsh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqabsw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqnegb, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqnegh, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_3(mve_vqnegw, TCG_CALL_NO_WG, void, env, ptr, ptr)
+DEF_HELPER_FLAGS_4(mve_vmlaldavsh, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlaldavsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlaldavxsh, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlaldavxsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlaldavuh, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlaldavuw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlsldavsh, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlsldavsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlsldavxsh, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmlsldavxsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vrmlaldavhsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vrmlaldavhxsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vrmlaldavhuw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vrmlsldavhsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vrmlsldavhxsw, TCG_CALL_NO_WG, i64, env, ptr, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vmladavsb, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavsh, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavsw, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavub, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavuh, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavuw, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlsdavb, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlsdavh, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlsdavw, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavsxb, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavsxh, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmladavsxw, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlsdavxb, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlsdavxh, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vmlsdavxw, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vaddvsb, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vaddvsh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vaddvsw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vaddvub, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vaddvuh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vaddvuw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxvsb, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxvsh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxvsw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxvub, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxvuh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxvuw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxavb, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxavh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxavw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminvsb, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminvsh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminvsw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminvub, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminvuh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminvuw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminavb, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminavh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminavw, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxnmvh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxnmvs, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminnmvh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminnmvs, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxnmavh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vmaxnmavs, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminnmavh, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vminnmavs, TCG_CALL_NO_WG, i32, env, ptr, i32)
+DEF_HELPER_FLAGS_3(mve_vaddlv_s, TCG_CALL_NO_WG, i64, env, ptr, i64)
+DEF_HELPER_FLAGS_3(mve_vaddlv_u, TCG_CALL_NO_WG, i64, env, ptr, i64)
+DEF_HELPER_FLAGS_4(mve_vabavsb, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vabavsh, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vabavsw, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vabavub, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vabavuh, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(mve_vabavuw, TCG_CALL_NO_WG, i32, env, ptr, ptr, i32)
 
 DEF_HELPER_3(vfp_adds, f32, f32, f32, ptr)
 DEF_HELPER_3(vfp_addd, f64, f64, f64, ptr)
@@ -161,6 +835,8 @@ DEF_HELPER_3(vfp_toshd_round_to_zero, i64, f64, i32, ptr)
 DEF_HELPER_3(vfp_tosld_round_to_zero, i64, f64, i32, ptr)
 DEF_HELPER_3(vfp_touhd_round_to_zero, i64, f64, i32, ptr)
 DEF_HELPER_3(vfp_tould_round_to_zero, i64, f64, i32, ptr)
+DEF_HELPER_3(vfp_toshh_round_to_zero, i32, f16, i32, ptr)
+DEF_HELPER_3(vfp_touhh_round_to_zero, i32, f16, i32, ptr)
 DEF_HELPER_3(vfp_touhh, i32, f16, i32, ptr)
 DEF_HELPER_3(vfp_toshh, i32, f16, i32, ptr)
 DEF_HELPER_3(vfp_toulh, i32, f16, i32, ptr)
@@ -191,6 +867,8 @@ DEF_HELPER_3(vfp_sqtod, f64, i64, i32, ptr)
 DEF_HELPER_3(vfp_uhtod, f64, i64, i32, ptr)
 DEF_HELPER_3(vfp_ultod, f64, i64, i32, ptr)
 DEF_HELPER_3(vfp_uqtod, f64, i64, i32, ptr)
+DEF_HELPER_3(vfp_shtoh, f16, i32, i32, ptr)
+DEF_HELPER_3(vfp_uhtoh, f16, i32, i32, ptr)
 DEF_HELPER_3(vfp_sltoh, f16, i32, i32, ptr)
 DEF_HELPER_3(vfp_ultoh, f16, i32, i32, ptr)
 DEF_HELPER_3(vfp_sqtoh, f16, i64, i32, ptr)
@@ -203,6 +881,8 @@ DEF_HELPER_FLAGS_3(vfp_fcvt_f16_to_f32, TCG_CALL_NO_RWG, f32, f16, ptr, i32)
 DEF_HELPER_FLAGS_3(vfp_fcvt_f32_to_f16, TCG_CALL_NO_RWG, f16, f32, ptr, i32)
 DEF_HELPER_FLAGS_3(vfp_fcvt_f16_to_f64, TCG_CALL_NO_RWG, f64, f16, ptr, i32)
 DEF_HELPER_FLAGS_3(vfp_fcvt_f64_to_f16, TCG_CALL_NO_RWG, f16, f64, ptr, i32)
+DEF_HELPER_FLAGS_2(bfcvt, TCG_CALL_NO_RWG, i32, f32, ptr)
+DEF_HELPER_FLAGS_2(bfcvt_pair, TCG_CALL_NO_RWG, i32, i64, ptr)
 
 DEF_HELPER_4(vfp_muladdd, f64, f64, f64, f64, ptr)
 DEF_HELPER_4(vfp_muladds, f32, f32, f32, f32, ptr)
@@ -224,8 +904,10 @@ DEF_HELPER_3(shr_cc, i32, env, i32, i32)
 DEF_HELPER_3(sar_cc, i32, env, i32, i32)
 DEF_HELPER_3(ror_cc, i32, env, i32, i32)
 
+DEF_HELPER_FLAGS_2(rinth_exact, TCG_CALL_NO_RWG, f16, f16, ptr)
 DEF_HELPER_FLAGS_2(rints_exact, TCG_CALL_NO_RWG, f32, f32, ptr)
 DEF_HELPER_FLAGS_2(rintd_exact, TCG_CALL_NO_RWG, f64, f64, ptr)
+DEF_HELPER_FLAGS_2(rinth, TCG_CALL_NO_RWG, f16, f16, ptr)
 DEF_HELPER_FLAGS_2(rints, TCG_CALL_NO_RWG, f32, f32, ptr)
 DEF_HELPER_FLAGS_2(rintd, TCG_CALL_NO_RWG, f64, f64, ptr)
 
@@ -540,6 +1222,8 @@ DEF_HELPER_FLAGS_2(neon_qzip32, TCG_CALL_NO_RWG, void, ptr, ptr)
 
 DEF_HELPER_FLAGS_3(crypto_aese, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
 DEF_HELPER_FLAGS_3(crypto_aesmc, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(crypto_sve_aese, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_3(crypto_sve_aesmc, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_4(crypto_sha1_3reg, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_2(crypto_sha1h, TCG_CALL_NO_RWG, void, ptr, ptr)
@@ -561,6 +1245,10 @@ DEF_HELPER_FLAGS_3(crypto_sm3partw2, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
 
 DEF_HELPER_FLAGS_2(crypto_sm4e, TCG_CALL_NO_RWG, void, ptr, ptr)
 DEF_HELPER_FLAGS_3(crypto_sm4ekey, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_4(crypto_sve_sm4e, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(crypto_sve_sm4ekey, TCG_CALL_NO_RWG, void, ptr, ptr, ptr,
+                   i32)
+DEF_HELPER_FLAGS_4(crypto_rax1, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_3(crc32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
 DEF_HELPER_FLAGS_3(crc32c, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
@@ -574,15 +1262,45 @@ DEF_HELPER_FLAGS_5(gvec_qrdmlah_s32, TCG_CALL_NO_RWG,
 DEF_HELPER_FLAGS_5(gvec_qrdmlsh_s32, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 
-DEF_HELPER_FLAGS_4(gvec_sdot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_sdot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_usdot_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
 
-DEF_HELPER_FLAGS_4(gvec_sdot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_idx_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_sdot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
-DEF_HELPER_FLAGS_4(gvec_udot_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_idx_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_idx_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sudot_idx_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_usdot_idx_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_sdot_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_udot_idx_h, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_smmla_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_ummla_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_usmmla_b, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_bfdot, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_bfdot_idx, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_bfmmla, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_bfmlal, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(gvec_bfmlal_idx, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_5(gvec_fcaddh, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
@@ -676,6 +1394,15 @@ DEF_HELPER_FLAGS_5(gvec_sqsub_s, TCG_CALL_NO_RWG,
 DEF_HELPER_FLAGS_5(gvec_sqsub_d, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 
+DEF_HELPER_FLAGS_4(gvec_saba_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_saba_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_saba_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_saba_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_uaba_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_uaba_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_uaba_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_4(gvec_uaba_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
 DEF_HELPER_FLAGS_5(gvec_fmlal_a32, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(gvec_fmlal_a64, TCG_CALL_NO_RWG,
@@ -684,6 +1411,10 @@ DEF_HELPER_FLAGS_5(gvec_fmlal_idx_a32, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(gvec_fmlal_idx_a64, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmlal_zzzw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_6(sve2_fmlal_zzxw_s, TCG_CALL_NO_RWG,
+                   void, ptr, ptr, ptr, ptr, ptr, i32)
 
 DEF_HELPER_FLAGS_2(frint32_s, TCG_CALL_NO_RWG, f32, f32, ptr)
 DEF_HELPER_FLAGS_2(frint64_s, TCG_CALL_NO_RWG, f32, f32, ptr)
@@ -703,4 +1434,5 @@ DEF_HELPER_FLAGS_4(neon_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
 #ifdef TARGET_AARCH64
 #include "helper-a64.h"
 #include "helper-sve.h"
+#include "helper-sme.h"
 #endif
diff --git a/qemu/target/arm/internals.h b/qemu/target/arm/internals.h
index 5bb1ad0e61..cee5eebf9d 100644
--- a/qemu/target/arm/internals.h
+++ b/qemu/target/arm/internals.h
@@ -26,8 +26,49 @@
 #define TARGET_ARM_INTERNALS_H
 
 #include "hw/registerfields.h"
+#include "tcg/tcg-gvec-desc.h"
 struct uc_struct;
 
+#define GMID_EL1_BS 6
+
+/* Bits within a descriptor passed to the helper_mte_check function. */
+FIELD(MTEDESC, MIDX,  0, 4)
+FIELD(MTEDESC, TBI,   4, 2)
+FIELD(MTEDESC, TCMA,  6, 2)
+FIELD(MTEDESC, WRITE, 8, 1)
+FIELD(MTEDESC, SIZEM1, 9, SIMD_DATA_BITS - 9)
+
+static inline bool tbi_check(uint32_t desc, int bit55)
+{
+    return (desc >> (R_MTEDESC_TBI_SHIFT + bit55)) & 1;
+}
+
+static inline bool tcma_check(uint32_t desc, int bit55, int ptr_tag)
+{
+    bool match = ((ptr_tag + bit55) & 0xf) == 0;
+    bool tcma = (desc >> (R_MTEDESC_TCMA_SHIFT + bit55)) & 1;
+
+    return tcma && match;
+}
+
+uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr,
+                   uintptr_t ra);
+bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr);
+
+static inline uint64_t useronly_clean_ptr(uint64_t ptr)
+{
+    return ptr;
+}
+
+/* Values for M-profile PSR.ECI for MVE insns */
+enum MVEECIState {
+    ECI_NONE = 0,     /* No completed beats */
+    ECI_A0 = 1,       /* Completed: A0 */
+    ECI_A0A1 = 2,     /* Completed: A0, A1 */
+    ECI_A0A1A2 = 4,   /* Completed: A0, A1, A2 */
+    ECI_A0A1A2B0 = 5, /* Completed: A0, A1, A2, B0 */
+};
+
 /* register banks for CPU modes */
 #define BANK_USRSYS 0
 #define BANK_SVC    1
@@ -281,6 +322,7 @@ enum arm_exception_class {
     EC_AA64_SMC               = 0x17,
     EC_SYSTEMREGISTERTRAP     = 0x18,
     EC_SVEACCESSTRAP          = 0x19,
+    EC_SMETRAP                = 0x1d,
     EC_INSNABORT              = 0x20,
     EC_INSNABORT_SAME_EL      = 0x21,
     EC_PCALIGNMENT            = 0x22,
@@ -301,6 +343,13 @@ enum arm_exception_class {
     EC_AA64_BKPT              = 0x3c,
 };
 
+typedef enum {
+    SME_ET_AccessTrap,
+    SME_ET_Streaming,
+    SME_ET_NotStreaming,
+    SME_ET_InactiveZA,
+} SMEExceptionType;
+
 #define ARM_EL_EC_SHIFT 26
 #define ARM_EL_IL_SHIFT 25
 #define ARM_EL_ISV_SHIFT 24
@@ -438,6 +487,12 @@ static inline uint32_t syn_sve_access_trap(void)
     return EC_SVEACCESSTRAP << ARM_EL_EC_SHIFT;
 }
 
+static inline uint32_t syn_smetrap(SMEExceptionType etype, bool is_16bit)
+{
+    return (EC_SMETRAP << ARM_EL_EC_SHIFT)
+        | (is_16bit ? 0 : ARM_EL_IL) | etype;
+}
+
 static inline uint32_t syn_pactrap(void)
 {
     return EC_PACTRAP << ARM_EL_EC_SHIFT;
@@ -1211,6 +1266,12 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address,
                        int *prot, bool *is_subpage,
                        ARMMMUFaultInfo *fi, uint32_t *mregion);
 
+/* Effective value of MDCR_EL2. */
+static inline uint64_t arm_mdcr_el2_eff(CPUARMState *env)
+{
+    return arm_is_el2_enabled(env) ? env->cp15.mdcr_el2 : 0;
+}
+
 /* Cacheability and shareability attributes for a memory access */
 typedef struct ARMCacheAttrs {
     unsigned int attrs:8; /* as in the MAIR register encoding */
diff --git a/qemu/target/arm/m_helper.c b/qemu/target/arm/m_helper.c
index 7fd9d21965..1259e81f8d 100644
--- a/qemu/target/arm/m_helper.c
+++ b/qemu/target/arm/m_helper.c
@@ -957,6 +957,7 @@ static void v7m_update_fpccr(CPUARMState *env, uint32_t frameptr,
 void HELPER(v7m_vlstm)(CPUARMState *env, uint32_t fptr)
 {
     /* fptr is the value of Rn, the frame pointer we store the FP regs to */
+    ARMCPU *cpu = env_archcpu(env);
     bool s = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK;
     bool lspact = env->v7m.fpccr[s] & R_V7M_FPCCR_LSPACT_MASK;
     uintptr_t ra = GETPC();
@@ -1005,9 +1006,12 @@ void HELPER(v7m_vlstm)(CPUARMState *env, uint32_t fptr)
             cpu_stl_data_ra(env, faddr + 4, shi, ra);
         }
         cpu_stl_data_ra(env, fptr + 0x40, vfp_get_fpscr(env), ra);
+        if (cpu_isar_feature(aa32_mve, cpu)) {
+            cpu_stl_data_ra(env, fptr + 0x44, env->v7m.vpr, ra);
+        }
 
         /*
-         * If TS is 0 then s0 to s15 and FPSCR are UNKNOWN; we choose to
+         * If TS is 0 then s0 to s15, FPSCR and VPR are UNKNOWN; we choose to
          * leave them unchanged, matching our choice in v7m_preserve_fp_state.
          */
         if (ts) {
@@ -1015,6 +1019,9 @@ void HELPER(v7m_vlstm)(CPUARMState *env, uint32_t fptr)
                 *aa32_vfp_dreg(env, i / 2) = 0;
             }
             vfp_set_fpscr(env, 0);
+            if (cpu_isar_feature(aa32_mve, cpu)) {
+                env->v7m.vpr = 0;
+            }
         }
     } else {
         v7m_update_fpccr(env, fptr, false);
@@ -1025,6 +1032,7 @@ void HELPER(v7m_vlstm)(CPUARMState *env, uint32_t fptr)
 
 void HELPER(v7m_vlldm)(CPUARMState *env, uint32_t fptr)
 {
+    ARMCPU *cpu = env_archcpu(env);
     uintptr_t ra = GETPC();
 
     /* fptr is the value of Rn, the frame pointer we load the FP regs from */
@@ -1057,7 +1065,7 @@ void HELPER(v7m_vlldm)(CPUARMState *env, uint32_t fptr)
             uint32_t faddr = fptr + 4 * i;
 
             if (i >= 16) {
-                faddr += 8; /* skip the slot for the FPSCR */
+                faddr += 8; /* skip the slot for the FPSCR and VPR */
             }
 
             slo = cpu_ldl_data_ra(env, faddr, ra);
@@ -1068,6 +1076,9 @@ void HELPER(v7m_vlldm)(CPUARMState *env, uint32_t fptr)
         }
         fpscr = cpu_ldl_data_ra(env, fptr + 0x40, ra);
         vfp_set_fpscr(env, fpscr);
+        if (cpu_isar_feature(aa32_mve, cpu)) {
+            env->v7m.vpr = cpu_ldl_data_ra(env, fptr + 0x44, ra);
+        }
     }
 
     env->v7m.control[M_REG_S] |= R_V7M_CONTROL_FPCA_MASK;
diff --git a/qemu/target/arm/mte_helper.c b/qemu/target/arm/mte_helper.c
new file mode 100644
index 0000000000..96324d92bc
--- /dev/null
+++ b/qemu/target/arm/mte_helper.c
@@ -0,0 +1,720 @@
+/*
+ * ARM v8.5-MemTag register-generation operations
+ *
+ * Copyright (c) 2020 Linaro, Ltd.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "internals.h"
+#include "exec/cpu-common.h"
+#include "exec/exec-all.h"
+#include "exec/cpu_ldst.h"
+#include "exec/memory.h"
+#include "exec/helper-proto.h"
+#include "qemu/guest-random.h"
+#include "uc_priv.h"
+
+static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
+{
+    if (exclude == 0xffff) {
+        return 0;
+    }
+    if (offset == 0) {
+        while (exclude & (1 << tag)) {
+            tag = (tag + 1) & 15;
+        }
+    } else {
+        do {
+            do {
+                tag = (tag + 1) & 15;
+            } while (exclude & (1 << tag));
+        } while (--offset > 0);
+    }
+    return tag;
+}
+
+static int allocation_tag_from_addr(uint64_t ptr)
+{
+    return extract64(ptr, 56, 4);
+}
+
+static uint64_t address_with_allocation_tag(uint64_t ptr, int rtag)
+{
+    return deposit64(ptr, 56, 4, rtag);
+}
+
+uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm)
+{
+    uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16);
+    int rrnd = extract32(env->cp15.gcr_el1, 16, 1);
+    int start = extract32(env->cp15.rgsr_el1, 0, 4);
+    int seed = extract32(env->cp15.rgsr_el1, 8, 16);
+    int offset, i, rtag;
+
+    if (unlikely(seed == 0) && rrnd) {
+        do {
+            uint16_t two;
+
+            if (qemu_guest_getrandom(&two, sizeof(two)) < 0) {
+                two = 1;
+            }
+            seed = two;
+        } while (seed == 0);
+    }
+
+    for (i = offset = 0; i < 4; ++i) {
+        int top = (extract32(seed, 5, 1) ^ extract32(seed, 3, 1) ^
+                   extract32(seed, 2, 1) ^ extract32(seed, 0, 1));
+
+        seed = (top << 15) | (seed >> 1);
+        offset |= top << i;
+    }
+    rtag = choose_nonexcluded_tag(start, offset, exclude);
+    env->cp15.rgsr_el1 = rtag | (seed << 8);
+
+    return address_with_allocation_tag(rn, rtag);
+}
+
+uint64_t HELPER(addsubg)(CPUARMState *env, uint64_t ptr,
+                         int32_t offset, uint32_t tag_offset)
+{
+    int start_tag = allocation_tag_from_addr(ptr);
+    uint16_t exclude = extract32(env->cp15.gcr_el1, 0, 16);
+    int rtag = choose_nonexcluded_tag(start_tag, tag_offset, exclude);
+
+    return address_with_allocation_tag(ptr + offset, rtag);
+}
+
+static uint64_t allocation_tag_clean_addr(uint64_t ptr)
+{
+    return ptr & ~MAKE_64BIT_MASK(56, 8);
+}
+
+static hwaddr memory_region_ram_addr(MemoryRegion *mr, ram_addr_t offset)
+{
+    hwaddr addr = offset;
+
+    while (mr && mr->container) {
+        addr += mr->addr;
+        mr = mr->container;
+    }
+
+    return addr;
+}
+
+static bool allocation_tag_cow(CPUARMState *env, RAMBlock *block,
+                               ram_addr_t offset, uint64_t ptr, int size,
+                               uintptr_t ra)
+{
+    struct uc_struct *uc = env->uc;
+    MemoryRegion *mr = block->mr;
+    hwaddr addr, page, end;
+    bool cowed = false;
+
+    if (!uc->snapshot_level || !mr || !mr->ram ||
+        mr->priority >= uc->snapshot_level) {
+        return false;
+    }
+
+    addr = memory_region_ram_addr(mr, offset);
+    page = addr & TARGET_PAGE_MASK;
+    end = (addr + size + ~TARGET_PAGE_MASK) & TARGET_PAGE_MASK;
+
+    while (page < end) {
+        if (!memory_cow(uc, mr, page, TARGET_PAGE_SIZE)) {
+            uc->invalid_addr = ptr;
+            uc->invalid_error = UC_ERR_NOMEM;
+            cpu_loop_exit_restore(env_cpu(env), ra);
+        }
+        cowed = true;
+        page += TARGET_PAGE_SIZE;
+    }
+
+    return cowed;
+}
+
+static bool allocation_tag_access_enabled(CPUARMState *env, uint64_t ptr,
+                                          MMUAccessType access_type,
+                                          int mmu_idx)
+{
+    uintptr_t index = tlb_index(env, mmu_idx, ptr);
+    CPUTLBEntry *entry = tlb_entry(env, mmu_idx, ptr);
+    target_ulong tlb_addr;
+
+    switch (access_type) {
+    case MMU_DATA_LOAD:
+        tlb_addr = entry->addr_read;
+        break;
+    case MMU_DATA_STORE:
+        tlb_addr = tlb_addr_write(entry);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if (!tlb_hit(env->uc, tlb_addr, ptr)) {
+        return false;
+    }
+    return env_tlb(env)->d[mmu_idx].iotlb[index].attrs.target_tlb_bit1;
+}
+
+static void allocation_tag_raise_fault(CPUARMState *env, target_ulong addr,
+                                       MMUAccessType access_type,
+                                       bool mmu_fault, bool prot,
+                                       uintptr_t ra)
+{
+    struct uc_struct *uc = env->uc;
+
+    uc->invalid_addr = addr;
+    if (access_type == MMU_DATA_STORE) {
+        uc->invalid_error = mmu_fault ? UC_ERR_MMU_WRITE :
+                            prot ? UC_ERR_WRITE_PROT :
+                            UC_ERR_WRITE_UNMAPPED;
+    } else {
+        uc->invalid_error = mmu_fault ? UC_ERR_MMU_READ :
+                            prot ? UC_ERR_READ_PROT :
+                            UC_ERR_READ_UNMAPPED;
+    }
+    cpu_exit(uc->cpu);
+    cpu_loop_exit_restore(env_cpu(env), ra);
+}
+
+static void *allocation_tag_probe_access(CPUARMState *env, target_ulong ptr,
+                                         MMUAccessType access_type, int size,
+                                         int mmu_idx, uintptr_t ra)
+{
+    struct uc_struct *uc = env->uc;
+    void *host = NULL;
+    bool first_page = true;
+
+    while (size > 0) {
+        target_ulong page_left = -(ptr | TARGET_PAGE_MASK);
+        int probe_size = MIN(size, (int)page_left);
+        target_ulong paddr;
+        void *page_host;
+        MemoryRegion *mr;
+
+        page_host = probe_access(env, ptr, probe_size, access_type, mmu_idx,
+                                 ra);
+        if (first_page) {
+            host = page_host;
+            first_page = false;
+        }
+
+        if (!tlb_vaddr_to_paddr(env, ptr, access_type, mmu_idx, &paddr)) {
+            allocation_tag_raise_fault(env, ptr, access_type, true, false,
+                                       ra);
+        }
+
+        mr = uc->memory_mapping(uc, paddr);
+        if (!mr) {
+            allocation_tag_raise_fault(env, paddr, access_type, false, false,
+                                       ra);
+        }
+        if (access_type == MMU_DATA_STORE) {
+            if (!(mr->perms & UC_PROT_WRITE)) {
+                allocation_tag_raise_fault(env, paddr, access_type, false,
+                                           true, ra);
+            }
+        } else if (!(mr->perms & UC_PROT_READ)) {
+            allocation_tag_raise_fault(env, paddr, access_type, false, true,
+                                       ra);
+        }
+
+        ptr += probe_size;
+        size -= probe_size;
+    }
+
+    return host;
+}
+
+void HELPER(dc_gva_probe)(CPUARMState *env, uint64_t ptr)
+{
+    uintptr_t ra = GETPC();
+    int mmu_idx = cpu_mmu_index(env, false);
+
+    allocation_tag_probe_access(env, ptr, MMU_DATA_STORE, 1, mmu_idx, ra);
+}
+
+void HELPER(mte_probe_data)(CPUARMState *env, uint64_t ptr, uint32_t desc)
+{
+    uintptr_t ra = GETPC();
+    int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+    MMUAccessType access_type;
+    uint32_t size;
+
+    access_type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE :
+                  MMU_DATA_LOAD;
+    size = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1;
+    allocation_tag_probe_access(env, ptr, access_type, size, mmu_idx, ra);
+}
+
+static uint8_t *allocation_tag_mem(CPUARMState *env, int mmu_idx,
+                                   uint64_t ptr, MMUAccessType access_type,
+                                   int size, bool allocate, uintptr_t ra)
+{
+    uint64_t clean_ptr = allocation_tag_clean_addr(ptr);
+    void *host;
+    RAMBlock *block;
+    ram_addr_t offset;
+
+    host = allocation_tag_probe_access(env, clean_ptr, access_type, size,
+                                       mmu_idx, ra);
+    if (!host) {
+        return NULL;
+    }
+
+    if (!allocation_tag_access_enabled(env, clean_ptr, access_type, mmu_idx)) {
+        return NULL;
+    }
+
+    block = qemu_ram_block_from_host(env->uc, host, false, &offset);
+    if (!block) {
+        return NULL;
+    }
+
+    if (allocate && access_type == MMU_DATA_STORE &&
+        allocation_tag_cow(env, block, offset, clean_ptr, size, ra)) {
+        host = allocation_tag_probe_access(env, clean_ptr, access_type, size,
+                                           mmu_idx, ra);
+        if (!host) {
+            return NULL;
+        }
+        block = qemu_ram_block_from_host(env->uc, host, false, &offset);
+        if (!block) {
+            return NULL;
+        }
+    }
+
+    if (!block->mte_tags) {
+        if (!allocate) {
+            return NULL;
+        }
+        block->mte_tags_size = DIV_ROUND_UP(block->max_length,
+                                            2 * TAG_GRANULE);
+        block->mte_tags = g_malloc0(block->mte_tags_size);
+    }
+
+    return block->mte_tags + (offset >> (LOG2_TAG_GRANULE + 1));
+}
+
+static int load_tag1(uint64_t ptr, uint8_t *mem)
+{
+    int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
+
+    return extract32(*mem, ofs, 4);
+}
+
+uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+    uint8_t *mem;
+    int rtag = 0;
+
+    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1, false,
+                             GETPC());
+    if (mem) {
+        rtag = load_tag1(ptr, mem);
+    }
+
+    return address_with_allocation_tag(xt, rtag);
+}
+
+static void check_tag_aligned(CPUARMState *env, uint64_t ptr, uintptr_t ra)
+{
+    if (unlikely(!QEMU_IS_ALIGNED(ptr, TAG_GRANULE))) {
+        arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE,
+                                    cpu_mmu_index(env, false), ra);
+        g_assert_not_reached();
+    }
+}
+
+static void store_tag1(uint64_t ptr, uint8_t *mem, int tag)
+{
+    int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
+
+    *mem = deposit32(*mem, ofs, 4, tag);
+}
+
+void HELPER(stg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    uintptr_t ra = GETPC();
+    int mmu_idx = cpu_mmu_index(env, false);
+    uint8_t *mem;
+
+    check_tag_aligned(env, ptr, ra);
+    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE,
+                             true, ra);
+    if (mem) {
+        store_tag1(ptr, mem, allocation_tag_from_addr(xt));
+    }
+}
+
+void HELPER(stg_stub)(CPUARMState *env, uint64_t ptr)
+{
+    uintptr_t ra = GETPC();
+    uint64_t clean_ptr = allocation_tag_clean_addr(ptr);
+
+    check_tag_aligned(env, ptr, ra);
+    probe_write(env, clean_ptr, TAG_GRANULE, cpu_mmu_index(env, false), ra);
+}
+
+void HELPER(st2g)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    uintptr_t ra = GETPC();
+    int mmu_idx = cpu_mmu_index(env, false);
+    int tag = allocation_tag_from_addr(xt);
+    uint8_t *mem1, *mem2;
+
+    check_tag_aligned(env, ptr, ra);
+    if (ptr & TAG_GRANULE) {
+        mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
+                                  TAG_GRANULE, true, ra);
+        mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE,
+                                  MMU_DATA_STORE, TAG_GRANULE, true, ra);
+        if (mem1) {
+            store_tag1(TAG_GRANULE, mem1, tag);
+        }
+        if (mem2) {
+            store_tag1(0, mem2, tag);
+        }
+    } else {
+        mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
+                                  2 * TAG_GRANULE, true, ra);
+        if (mem1) {
+            *mem1 = tag | (tag << 4);
+        }
+    }
+}
+
+void HELPER(st2g_stub)(CPUARMState *env, uint64_t ptr)
+{
+    struct uc_struct *uc = env->uc;
+    uintptr_t ra = GETPC();
+    uint64_t clean_ptr = allocation_tag_clean_addr(ptr);
+    int mmu_idx = cpu_mmu_index(env, false);
+    int in_page;
+
+    check_tag_aligned(env, ptr, ra);
+    in_page = -(clean_ptr | TARGET_PAGE_MASK);
+    if (likely(in_page >= 2 * TAG_GRANULE)) {
+        probe_write(env, clean_ptr, 2 * TAG_GRANULE, mmu_idx, ra);
+    } else {
+        probe_write(env, clean_ptr, TAG_GRANULE, mmu_idx, ra);
+        probe_write(env, clean_ptr + TAG_GRANULE, TAG_GRANULE, mmu_idx, ra);
+    }
+}
+
+#define LDGM_STGM_SIZE (4 << GMID_EL1_BS)
+
+uint64_t HELPER(ldgm)(CPUARMState *env, uint64_t ptr)
+{
+    uintptr_t ra = GETPC();
+    int mmu_idx = cpu_mmu_index(env, false);
+    uint8_t *mem;
+
+    ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
+    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD,
+                             LDGM_STGM_SIZE, false, ra);
+    if (!mem) {
+        return 0;
+    }
+
+    QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
+    return ldq_le_p(mem);
+}
+
+void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val)
+{
+    uintptr_t ra = GETPC();
+    int mmu_idx = cpu_mmu_index(env, false);
+    uint8_t *mem;
+
+    ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
+    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
+                             LDGM_STGM_SIZE, true, ra);
+    if (!mem) {
+        return;
+    }
+
+    QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
+    stq_le_p(mem, val);
+}
+
+void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val)
+{
+    uintptr_t ra = GETPC();
+    ARMCPU *cpu = env_archcpu(env);
+    int log2_dcz_bytes = cpu->dcz_blocksize + 2;
+    int log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
+    intptr_t dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
+    intptr_t tag_bytes = (intptr_t)1 << log2_tag_bytes;
+    int mmu_idx = cpu_mmu_index(env, false);
+    uint8_t *mem;
+
+    ptr &= -dcz_bytes;
+    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes,
+                             true, ra);
+    if (mem) {
+        int tag_pair = (val & 0xf) * 0x11;
+
+        memset(mem, tag_pair, tag_bytes);
+    }
+}
+
+static int mte_reg_el_from_mmu_idx(ARMMMUIdx mmu_idx)
+{
+    switch (mmu_idx) {
+    case ARMMMUIdx_E10_0:
+    case ARMMMUIdx_E10_1:
+    case ARMMMUIdx_E10_1_PAN:
+    case ARMMMUIdx_SE10_0:
+    case ARMMMUIdx_SE10_1:
+    case ARMMMUIdx_SE10_1_PAN:
+        return 1;
+    case ARMMMUIdx_E20_0:
+    case ARMMMUIdx_E20_2:
+    case ARMMMUIdx_E20_2_PAN:
+    case ARMMMUIdx_E2:
+        return 2;
+    case ARMMMUIdx_SE3:
+        return 3;
+    default:
+        return arm_mmu_idx_to_el(mmu_idx);
+    }
+}
+
+static void mte_sync_check_fail(CPUARMState *env, uint32_t desc,
+                                uint64_t dirty_ptr, uintptr_t ra)
+{
+    int is_write = FIELD_EX32(desc, MTEDESC, WRITE);
+    uint32_t syn;
+
+    env->exception.vaddress = dirty_ptr;
+    syn = syn_data_abort_no_iss(arm_current_el(env) != 0, 0, 0, 0,
+                                is_write, 0x11);
+    raise_exception_ra(env, EXCP_DATA_ABORT, syn, exception_target_el(env), ra);
+    g_assert_not_reached();
+}
+
+static void mte_async_check_fail(CPUARMState *env, uint64_t dirty_ptr,
+                                 ARMMMUIdx arm_mmu_idx, int el)
+{
+    int select = 0;
+
+    if (regime_has_2_ranges(arm_mmu_idx)) {
+        select = extract64(dirty_ptr, 55, 1);
+    }
+    env->cp15.tfsr_el[el] |= 1 << select;
+}
+
+static void mte_check_fail(CPUARMState *env, uint32_t desc,
+                           uint64_t dirty_ptr, uintptr_t ra)
+{
+    int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+    ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx);
+    int el, reg_el, tcf;
+    uint64_t sctlr;
+
+    reg_el = mte_reg_el_from_mmu_idx(arm_mmu_idx);
+    sctlr = env->cp15.sctlr_el[reg_el];
+
+    switch (arm_mmu_idx) {
+    case ARMMMUIdx_E10_0:
+    case ARMMMUIdx_E20_0:
+    case ARMMMUIdx_SE10_0:
+        el = 0;
+        tcf = extract64(sctlr, 38, 2);
+        break;
+    default:
+        el = reg_el;
+        tcf = extract64(sctlr, 40, 2);
+        break;
+    }
+
+    switch (tcf) {
+    case 1:
+        mte_sync_check_fail(env, desc, dirty_ptr, ra);
+        break;
+    case 2:
+        mte_async_check_fail(env, dirty_ptr, arm_mmu_idx, el);
+        break;
+    case 3:
+        if (FIELD_EX32(desc, MTEDESC, WRITE)) {
+            mte_async_check_fail(env, dirty_ptr, arm_mmu_idx, el);
+        } else {
+            mte_sync_check_fail(env, desc, dirty_ptr, ra);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static int checkN(uint8_t *mem, int odd, int cmp, int count)
+{
+    int n = 0;
+    int diff;
+
+    cmp *= 0x11;
+    diff = *mem++ ^ cmp;
+
+    if (odd) {
+        goto start_odd;
+    }
+
+    while (1) {
+        if (unlikely(diff & 0x0f)) {
+            break;
+        }
+        if (++n == count) {
+            break;
+        }
+
+    start_odd:
+        if (unlikely(diff & 0xf0)) {
+            break;
+        }
+        if (++n == count) {
+            break;
+        }
+
+        diff = *mem++ ^ cmp;
+    }
+    return n;
+}
+
+static int mte_probe_int(CPUARMState *env, uint32_t desc, uint64_t ptr,
+                         uintptr_t ra, uint64_t *fault)
+{
+    struct uc_struct *uc = env->uc;
+    int mmu_idx;
+    int ptr_tag, bit55;
+    uint64_t ptr_last, prev_page, next_page;
+    uint64_t tag_first, tag_last;
+    uint64_t tag_byte_first, tag_byte_last;
+    uint32_t sizem1, tag_count, tag_size, n, c;
+    uint8_t *mem1, *mem2;
+    MMUAccessType type;
+
+    bit55 = extract64(ptr, 55, 1);
+    *fault = ptr;
+
+    if (unlikely(!tbi_check(desc, bit55))) {
+        return -1;
+    }
+
+    ptr_tag = allocation_tag_from_addr(ptr);
+    if (tcma_check(desc, bit55, ptr_tag)) {
+        return 1;
+    }
+
+    mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+    type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
+    sizem1 = FIELD_EX32(desc, MTEDESC, SIZEM1);
+    ptr_last = ptr + sizem1;
+    tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
+    tag_last = QEMU_ALIGN_DOWN(ptr_last, TAG_GRANULE);
+    tag_count = ((tag_last - tag_first) / TAG_GRANULE) + 1;
+    tag_byte_first = QEMU_ALIGN_DOWN(ptr, 2 * TAG_GRANULE);
+    tag_byte_last = QEMU_ALIGN_DOWN(ptr_last, 2 * TAG_GRANULE);
+
+    prev_page = ptr & TARGET_PAGE_MASK;
+    next_page = prev_page + TARGET_PAGE_SIZE;
+
+    if (likely(tag_last - prev_page < TARGET_PAGE_SIZE)) {
+        mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, sizem1 + 1,
+                                  false, ra);
+        if (!mem1) {
+            return 1;
+        }
+        n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count);
+    } else {
+        tag_size = (next_page - tag_byte_first) / (2 * TAG_GRANULE);
+        mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr,
+                                  false, ra);
+
+        tag_size = ((tag_byte_last - next_page) / (2 * TAG_GRANULE)) + 1;
+        mem2 = allocation_tag_mem(env, mmu_idx, next_page, type,
+                                  ptr_last - next_page + 1, false, ra);
+
+        n = c = (next_page - tag_first) / TAG_GRANULE;
+        if (mem1) {
+            n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, c);
+        }
+        if (n == c) {
+            if (!mem2) {
+                return 1;
+            }
+            n += checkN(mem2, 0, ptr_tag, tag_count - c);
+        }
+        (void)tag_size;
+    }
+
+    if (likely(n == tag_count)) {
+        return 1;
+    }
+
+    if (n > 0) {
+        *fault = tag_first + n * TAG_GRANULE;
+    }
+    return 0;
+}
+
+uint64_t mte_check(CPUARMState *env, uint32_t desc, uint64_t ptr,
+                   uintptr_t ra)
+{
+    uint64_t fault;
+    int ret = mte_probe_int(env, desc, ptr, ra, &fault);
+
+    if (unlikely(ret == 0)) {
+        mte_check_fail(env, desc, fault, ra);
+    } else if (ret < 0) {
+        return ptr;
+    }
+    return allocation_tag_clean_addr(useronly_clean_ptr(ptr));
+}
+
+bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr)
+{
+    uint64_t fault;
+    int ret = mte_probe_int(env, desc, ptr, 0, &fault);
+
+    return ret != 0;
+}
+
+uint64_t HELPER(mte_check)(CPUARMState *env, uint32_t desc, uint64_t ptr)
+{
+    return mte_check(env, desc, ptr, GETPC());
+}
+
+uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr)
+{
+    uintptr_t ra = GETPC();
+    int log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
+    uint64_t dcz_bytes = 1ULL << log2_dcz_bytes;
+    uint64_t align_ptr = ptr & -dcz_bytes;
+    uint64_t fault;
+    int mmu_idx;
+    int ret;
+
+    FIELD_DP32(desc, MTEDESC, WRITE, 1, desc);
+    FIELD_DP32(desc, MTEDESC, SIZEM1, dcz_bytes - 1, desc);
+    mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+    (void)allocation_tag_probe_access(env, ptr, MMU_DATA_STORE, 1, mmu_idx,
+                                      ra);
+    ret = mte_probe_int(env, desc, align_ptr, ra, &fault);
+
+    if (unlikely(ret == 0)) {
+        mte_check_fail(env, desc, fault, ra);
+    } else if (ret < 0) {
+        return ptr;
+    }
+    return allocation_tag_clean_addr(useronly_clean_ptr(ptr));
+}
diff --git a/qemu/target/arm/mve_helper.c b/qemu/target/arm/mve_helper.c
new file mode 100644
index 0000000000..75db3c298d
--- /dev/null
+++ b/qemu/target/arm/mve_helper.c
@@ -0,0 +1,4221 @@
+/*
+ * M-profile MVE Operations
+ *
+ * Copyright (c) 2021 Linaro, Ltd.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "internals.h"
+#include "exec/helper-proto.h"
+#include "exec/exec-all.h"
+#include "exec/cpu_ldst.h"
+#include "fpu/softfloat.h"
+
+#define H1(x) (x)
+#define H2(x) (x)
+#define H4(x) (x)
+#define H8(x) (x)
+
+static uint64_t mve_expand_pred_b(uint8_t mask)
+{
+    uint64_t ret = 0;
+    int i;
+
+    for (i = 0; i < 8; i++) {
+        if (mask & (1U << i)) {
+            ret |= 0xffULL << (i * 8);
+        }
+    }
+
+    return ret;
+}
+
+static void mve_mergemask_ub(uint8_t *d, uint8_t r, uint8_t mask)
+{
+    if (mask & 1) {
+        *d = r;
+    }
+}
+
+static void mve_mergemask_uh(uint16_t *d, uint16_t r, uint8_t mask)
+{
+    uint16_t bmask = (uint16_t)mve_expand_pred_b(mask);
+
+    *d = (*d & ~bmask) | (r & bmask);
+}
+
+static void mve_mergemask_uw(uint32_t *d, uint32_t r, uint8_t mask)
+{
+    uint32_t bmask = (uint32_t)mve_expand_pred_b(mask);
+
+    *d = (*d & ~bmask) | (r & bmask);
+}
+
+static void mve_mergemask_uq(uint64_t *d, uint64_t r, uint8_t mask)
+{
+    uint64_t bmask = mve_expand_pred_b(mask);
+
+    *d = (*d & ~bmask) | (r & bmask);
+}
+
+static void mve_mergemask_sb(int8_t *d, int8_t r, uint8_t mask)
+{
+    mve_mergemask_ub((uint8_t *)d, (uint8_t)r, mask);
+}
+
+static void mve_mergemask_sh(int16_t *d, int16_t r, uint8_t mask)
+{
+    mve_mergemask_uh((uint16_t *)d, (uint16_t)r, mask);
+}
+
+static void mve_mergemask_sw(int32_t *d, int32_t r, uint8_t mask)
+{
+    mve_mergemask_uw((uint32_t *)d, (uint32_t)r, mask);
+}
+
+static void mve_mergemask_sq(int64_t *d, int64_t r, uint8_t mask)
+{
+    mve_mergemask_uq((uint64_t *)d, (uint64_t)r, mask);
+}
+
+static uint16_t mve_eci_mask(CPUARMState *env)
+{
+    int eci;
+
+    if ((env->condexec_bits & 0xf) != 0) {
+        return 0xffff;
+    }
+
+    eci = env->condexec_bits >> 4;
+    switch (eci) {
+    case ECI_NONE:
+        return 0xffff;
+    case ECI_A0:
+        return 0xfff0;
+    case ECI_A0A1:
+        return 0xff00;
+    case ECI_A0A1A2:
+    case ECI_A0A1A2B0:
+        return 0xf000;
+    default:
+        g_assert_not_reached();
+    }
+
+    return 0;
+}
+
+static uint16_t mve_element_mask(CPUARMState *env)
+{
+    uint16_t mask = FIELD_EX32(env->v7m.vpr, V7M_VPR, P0);
+
+    if (!(env->v7m.vpr & R_V7M_VPR_MASK01_MASK)) {
+        mask |= 0xff;
+    }
+    if (!(env->v7m.vpr & R_V7M_VPR_MASK23_MASK)) {
+        mask |= 0xff00;
+    }
+
+    if (env->v7m.ltpsize < 4 &&
+        env->regs[14] <= (1 << (4 - env->v7m.ltpsize))) {
+        int masklen = env->regs[14] << env->v7m.ltpsize;
+        uint16_t ltpmask;
+
+        assert(masklen <= 16);
+        ltpmask = masklen ? MAKE_64BIT_MASK(0, masklen) : 0;
+        mask &= ltpmask;
+    }
+
+    mask &= mve_eci_mask(env);
+    return mask;
+}
+
+static void mve_advance_vpt(CPUARMState *env)
+{
+    uint32_t vpr = env->v7m.vpr;
+    unsigned mask01;
+    unsigned mask23;
+    uint16_t inv_mask;
+    uint16_t eci_mask = mve_eci_mask(env);
+
+    if ((env->condexec_bits & 0xf) == 0) {
+        env->condexec_bits = (env->condexec_bits == (ECI_A0A1A2B0 << 4)) ?
+            (ECI_A0 << 4) : (ECI_NONE << 4);
+    }
+
+    if (!(vpr & ((uint32_t)R_V7M_VPR_MASK01_MASK |
+                 (uint32_t)R_V7M_VPR_MASK23_MASK))) {
+        return;
+    }
+
+    mask01 = FIELD_EX32(vpr, V7M_VPR, MASK01);
+    mask23 = FIELD_EX32(vpr, V7M_VPR, MASK23);
+    inv_mask = eci_mask;
+    if (mask01 <= 8) {
+        inv_mask &= ~0xff;
+    }
+    if (mask23 <= 8) {
+        inv_mask &= ~0xff00;
+    }
+    vpr ^= inv_mask;
+    if (eci_mask & 0xf0) {
+        FIELD_DP32(vpr, V7M_VPR, MASK01, mask01 << 1, vpr);
+    }
+    FIELD_DP32(vpr, V7M_VPR, MASK23, mask23 << 1, vpr);
+    env->v7m.vpr = vpr;
+}
+
+void HELPER(mve_vctp)(CPUARMState *env, uint32_t masklen)
+{
+    uint16_t mask = mve_element_mask(env);
+    uint16_t eci_mask = mve_eci_mask(env);
+    uint16_t newmask;
+
+    assert(masklen <= 16);
+    newmask = masklen ? MAKE_64BIT_MASK(0, masklen) : 0;
+    newmask &= mask;
+    env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) |
+                   (newmask & eci_mask);
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vpnot)(CPUARMState *env)
+{
+    uint16_t mask = mve_element_mask(env);
+    uint16_t eci_mask = mve_eci_mask(env);
+    uint16_t beatpred = ~env->v7m.vpr & mask;
+
+    env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) |
+                   (beatpred & eci_mask);
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vdup)(CPUARMState *env, void *vd, uint32_t val)
+{
+    uint32_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    unsigned e;
+
+    for (e = 0; e < 16 / 4; e++, mask >>= 4) {
+        mve_mergemask_uw(&d[H4(e)], val, mask);
+    }
+    mve_advance_vpt(env);
+}
+
+#define DO_VIDUP(OP, ESIZE, TYPE, MERGE, FN)                           \
+    uint32_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,        \
+                                    uint32_t offset, uint32_t imm)     \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            MERGE(&d[glue(H, ESIZE)(e)], (TYPE)offset, mask);          \
+            offset = FN(offset, imm);                                  \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return offset;                                                 \
+    }
+
+#define DO_VIWDUP(OP, ESIZE, TYPE, MERGE, FN)                          \
+    uint32_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,        \
+                                    uint32_t offset, uint32_t wrap,    \
+                                    uint32_t imm)                      \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            MERGE(&d[glue(H, ESIZE)(e)], (TYPE)offset, mask);          \
+            offset = FN(offset, wrap, imm);                            \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return offset;                                                 \
+    }
+
+static uint32_t do_add_dup(uint32_t offset, uint32_t imm)
+{
+    return offset + imm;
+}
+
+static uint32_t do_add_wrap(uint32_t offset, uint32_t wrap, uint32_t imm)
+{
+    offset += imm;
+    if (offset == wrap) {
+        offset = 0;
+    }
+    return offset;
+}
+
+static uint32_t do_sub_wrap(uint32_t offset, uint32_t wrap, uint32_t imm)
+{
+    if (offset == 0) {
+        offset = wrap;
+    }
+    offset -= imm;
+    return offset;
+}
+
+DO_VIDUP(vidupb, 1, uint8_t, mve_mergemask_ub, do_add_dup)
+DO_VIDUP(viduph, 2, uint16_t, mve_mergemask_uh, do_add_dup)
+DO_VIDUP(vidupw, 4, uint32_t, mve_mergemask_uw, do_add_dup)
+DO_VIWDUP(viwdupb, 1, uint8_t, mve_mergemask_ub, do_add_wrap)
+DO_VIWDUP(viwduph, 2, uint16_t, mve_mergemask_uh, do_add_wrap)
+DO_VIWDUP(viwdupw, 4, uint32_t, mve_mergemask_uw, do_add_wrap)
+DO_VIWDUP(vdwdupb, 1, uint8_t, mve_mergemask_ub, do_sub_wrap)
+DO_VIWDUP(vdwduph, 2, uint16_t, mve_mergemask_uh, do_sub_wrap)
+DO_VIWDUP(vdwdupw, 4, uint32_t, mve_mergemask_uw, do_sub_wrap)
+
+#define DO_VCMP(OP, ESIZE, TYPE, FN)                                    \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vn, void *vm)   \
+    {                                                                   \
+        TYPE *n = vn;                                                   \
+        TYPE *m = vm;                                                   \
+        uint16_t mask = mve_element_mask(env);                          \
+        uint16_t eci_mask = mve_eci_mask(env);                          \
+        uint16_t beatpred = 0;                                          \
+        uint16_t emask = MAKE_64BIT_MASK(0, ESIZE);                     \
+        unsigned e;                                                     \
+                                                                        \
+        for (e = 0; e < 16 / ESIZE; e++) {                              \
+            bool r = FN(n[H##ESIZE(e)], m[H##ESIZE(e)]);                \
+                                                                        \
+            beatpred |= r * emask;                                      \
+            emask <<= ESIZE;                                            \
+        }                                                               \
+        beatpred &= mask;                                               \
+        env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) |           \
+            (beatpred & eci_mask);                                      \
+        mve_advance_vpt(env);                                           \
+    }
+
+#define DO_VCMP_SCALAR(OP, ESIZE, TYPE, FN)                             \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vn,             \
+                                uint32_t rm)                            \
+    {                                                                   \
+        TYPE *n = vn;                                                   \
+        uint16_t mask = mve_element_mask(env);                          \
+        uint16_t eci_mask = mve_eci_mask(env);                          \
+        uint16_t beatpred = 0;                                          \
+        uint16_t emask = MAKE_64BIT_MASK(0, ESIZE);                     \
+        unsigned e;                                                     \
+                                                                        \
+        for (e = 0; e < 16 / ESIZE; e++) {                              \
+            bool r = FN(n[H##ESIZE(e)], (TYPE)rm);                      \
+                                                                        \
+            beatpred |= r * emask;                                      \
+            emask <<= ESIZE;                                            \
+        }                                                               \
+        beatpred &= mask;                                               \
+        env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) |           \
+            (beatpred & eci_mask);                                      \
+        mve_advance_vpt(env);                                           \
+    }
+
+#define DO_VCMP_S(OP, FN)                                               \
+    DO_VCMP(OP##b, 1, int8_t, FN)                                       \
+    DO_VCMP(OP##h, 2, int16_t, FN)                                      \
+    DO_VCMP(OP##w, 4, int32_t, FN)                                      \
+    DO_VCMP_SCALAR(OP##_scalarb, 1, int8_t, FN)                         \
+    DO_VCMP_SCALAR(OP##_scalarh, 2, int16_t, FN)                        \
+    DO_VCMP_SCALAR(OP##_scalarw, 4, int32_t, FN)
+
+#define DO_VCMP_U(OP, FN)                                               \
+    DO_VCMP(OP##b, 1, uint8_t, FN)                                      \
+    DO_VCMP(OP##h, 2, uint16_t, FN)                                     \
+    DO_VCMP(OP##w, 4, uint32_t, FN)                                     \
+    DO_VCMP_SCALAR(OP##_scalarb, 1, uint8_t, FN)                        \
+    DO_VCMP_SCALAR(OP##_scalarh, 2, uint16_t, FN)                       \
+    DO_VCMP_SCALAR(OP##_scalarw, 4, uint32_t, FN)
+
+#define DO_EQ(N, M) ((N) == (M))
+#define DO_NE(N, M) ((N) != (M))
+#define DO_GE(N, M) ((N) >= (M))
+#define DO_LT(N, M) ((N) < (M))
+#define DO_GT(N, M) ((N) > (M))
+#define DO_LE(N, M) ((N) <= (M))
+
+DO_VCMP_U(vcmpeq, DO_EQ)
+DO_VCMP_U(vcmpne, DO_NE)
+DO_VCMP_U(vcmpcs, DO_GE)
+DO_VCMP_U(vcmphi, DO_GT)
+DO_VCMP_S(vcmpge, DO_GE)
+DO_VCMP_S(vcmplt, DO_LT)
+DO_VCMP_S(vcmpgt, DO_GT)
+DO_VCMP_S(vcmple, DO_LE)
+
+#define DO_VCMP_FP(OP, ESIZE, TYPE, FN)                                \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vn, void *vm)  \
+    {                                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        uint16_t eci_mask = mve_eci_mask(env);                         \
+        uint16_t beatpred = 0;                                         \
+        uint16_t emask = MAKE_64BIT_MASK(0, ESIZE);                    \
+        unsigned e;                                                    \
+        float_status *fpst;                                            \
+        float_status scratch_fpst;                                     \
+        bool r;                                                        \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, emask <<= ESIZE) {            \
+            if ((mask & emask) == 0) {                                 \
+                continue;                                              \
+            }                                                          \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :   \
+                &env->vfp.standard_fp_status;                          \
+            if (!(mask & (1U << (e * ESIZE)))) {                       \
+                scratch_fpst = *fpst;                                  \
+                fpst = &scratch_fpst;                                  \
+            }                                                          \
+            r = FN(n[glue(H, ESIZE)(e)], m[glue(H, ESIZE)(e)], fpst);  \
+            beatpred |= r * emask;                                     \
+        }                                                              \
+        beatpred &= mask;                                              \
+        env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) |          \
+            (beatpred & eci_mask);                                     \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VCMP_FP_SCALAR(OP, ESIZE, TYPE, FN)                         \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vn,            \
+                                uint32_t rm)                           \
+    {                                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE m = (TYPE)rm;                                             \
+        uint16_t mask = mve_element_mask(env);                         \
+        uint16_t eci_mask = mve_eci_mask(env);                         \
+        uint16_t beatpred = 0;                                         \
+        uint16_t emask = MAKE_64BIT_MASK(0, ESIZE);                    \
+        unsigned e;                                                    \
+        float_status *fpst;                                            \
+        float_status scratch_fpst;                                     \
+        bool r;                                                        \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, emask <<= ESIZE) {            \
+            if ((mask & emask) == 0) {                                 \
+                continue;                                              \
+            }                                                          \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :   \
+                &env->vfp.standard_fp_status;                          \
+            if (!(mask & (1U << (e * ESIZE)))) {                       \
+                scratch_fpst = *fpst;                                  \
+                fpst = &scratch_fpst;                                  \
+            }                                                          \
+            r = FN(n[glue(H, ESIZE)(e)], m, fpst);                     \
+            beatpred |= r * emask;                                     \
+        }                                                              \
+        beatpred &= mask;                                              \
+        env->v7m.vpr = (env->v7m.vpr & ~(uint32_t)eci_mask) |          \
+            (beatpred & eci_mask);                                     \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VCMP_FP_BOTH(VOP, SOP, ESIZE, TYPE, FN)                     \
+    DO_VCMP_FP(VOP, ESIZE, TYPE, FN)                                   \
+    DO_VCMP_FP_SCALAR(SOP, ESIZE, TYPE, FN)
+
+#define DO_FEQ16(X, Y, S) (float16_compare(X, Y, S) == 0)
+#define DO_FEQ32(X, Y, S) (float32_compare(X, Y, S) == 0)
+#define DO_FLE16(X, Y, S) (float16_compare(X, Y, S) <= 0)
+#define DO_FLE32(X, Y, S) (float32_compare(X, Y, S) <= 0)
+#define DO_FLT16(X, Y, S) (float16_compare(X, Y, S) < 0)
+#define DO_FLT32(X, Y, S) (float32_compare(X, Y, S) < 0)
+#define DO_GE16(X, Y, S) DO_FLE16(Y, X, S)
+#define DO_GE32(X, Y, S) DO_FLE32(Y, X, S)
+#define DO_GT16(X, Y, S) DO_FLT16(Y, X, S)
+#define DO_GT32(X, Y, S) DO_FLT32(Y, X, S)
+
+DO_VCMP_FP_BOTH(vfcmpeqh, vfcmpeq_scalarh, 2, float16, DO_FEQ16)
+DO_VCMP_FP_BOTH(vfcmpeqs, vfcmpeq_scalars, 4, float32, DO_FEQ32)
+DO_VCMP_FP_BOTH(vfcmpneh, vfcmpne_scalarh, 2, float16, !DO_FEQ16)
+DO_VCMP_FP_BOTH(vfcmpnes, vfcmpne_scalars, 4, float32, !DO_FEQ32)
+DO_VCMP_FP_BOTH(vfcmpgeh, vfcmpge_scalarh, 2, float16, DO_GE16)
+DO_VCMP_FP_BOTH(vfcmpges, vfcmpge_scalars, 4, float32, DO_GE32)
+DO_VCMP_FP_BOTH(vfcmplth, vfcmplt_scalarh, 2, float16, !DO_GE16)
+DO_VCMP_FP_BOTH(vfcmplts, vfcmplt_scalars, 4, float32, !DO_GE32)
+DO_VCMP_FP_BOTH(vfcmpgth, vfcmpgt_scalarh, 2, float16, DO_GT16)
+DO_VCMP_FP_BOTH(vfcmpgts, vfcmpgt_scalars, 4, float32, DO_GT32)
+DO_VCMP_FP_BOTH(vfcmpleh, vfcmple_scalarh, 2, float16, !DO_GT16)
+DO_VCMP_FP_BOTH(vfcmples, vfcmple_scalars, 4, float32, !DO_GT32)
+
+#define DO_1OP_IMM(OP, FN)                                             \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                uint64_t imm)                          \
+    {                                                                  \
+        uint64_t *d = vd;                                              \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / 8; e++, mask >>= 8) {                     \
+            mve_mergemask_uq(&d[H8(e)], FN(d[H8(e)], imm), mask);      \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_MOVI(N, I) (I)
+#define DO_ANDI(N, I) ((N) & (I))
+#define DO_ORRI(N, I) ((N) | (I))
+
+DO_1OP_IMM(vmovi, DO_MOVI)
+DO_1OP_IMM(vandi, DO_ANDI)
+DO_1OP_IMM(vorri, DO_ORRI)
+
+#define DO_2OP(OP, ESIZE, TYPE, MERGE, FN)                             \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            TYPE r = FN(n[glue(H, ESIZE)(e)], m[glue(H, ESIZE)(e)]);   \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_2OP_SCALAR(OP, ESIZE, TYPE, MERGE, FN)                      \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, uint32_t rm)                 \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE m = (TYPE)rm;                                             \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            TYPE r = FN(n[glue(H, ESIZE)(e)], m);                      \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_2OP_L(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, MERGE, FN)        \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        LTYPE *d = vd;                                                 \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned le;                                                   \
+                                                                       \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {        \
+            LTYPE r = FN((LTYPE)n[glue(H, ESIZE)(le * 2 + TOP)],       \
+                         m[glue(H, ESIZE)(le * 2 + TOP)]);             \
+                                                                       \
+            MERGE(&d[glue(H, LESIZE)(le)], r, mask);                   \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_AND(N, M) ((N) & (M))
+#define DO_BIC(N, M) ((N) & ~(M))
+#define DO_ORR(N, M) ((N) | (M))
+#define DO_ORN(N, M) ((N) | ~(M))
+#define DO_EOR(N, M) ((N) ^ (M))
+#define DO_ADD(N, M) ((N) + (M))
+#define DO_SUB(N, M) ((N) - (M))
+#define DO_MUL(N, M) ((N) * (M))
+#define DO_MAX(N, M) ((N) >= (M) ? (N) : (M))
+#define DO_MIN(N, M) ((N) >= (M) ? (M) : (N))
+#define DO_ABD(N, M) ((N) >= (M) ? (N) - (M) : (M) - (N))
+
+DO_2OP(vand, 8, uint64_t, mve_mergemask_uq, DO_AND)
+DO_2OP(vbic, 8, uint64_t, mve_mergemask_uq, DO_BIC)
+DO_2OP(vorr, 8, uint64_t, mve_mergemask_uq, DO_ORR)
+DO_2OP(vorn, 8, uint64_t, mve_mergemask_uq, DO_ORN)
+DO_2OP(veor, 8, uint64_t, mve_mergemask_uq, DO_EOR)
+
+DO_2OP(vaddb, 1, uint8_t, mve_mergemask_ub, DO_ADD)
+DO_2OP(vaddh, 2, uint16_t, mve_mergemask_uh, DO_ADD)
+DO_2OP(vaddw, 4, uint32_t, mve_mergemask_uw, DO_ADD)
+DO_2OP_SCALAR(vadd_scalarb, 1, uint8_t, mve_mergemask_ub, DO_ADD)
+DO_2OP_SCALAR(vadd_scalarh, 2, uint16_t, mve_mergemask_uh, DO_ADD)
+DO_2OP_SCALAR(vadd_scalarw, 4, uint32_t, mve_mergemask_uw, DO_ADD)
+DO_2OP(vsubb, 1, uint8_t, mve_mergemask_ub, DO_SUB)
+DO_2OP(vsubh, 2, uint16_t, mve_mergemask_uh, DO_SUB)
+DO_2OP(vsubw, 4, uint32_t, mve_mergemask_uw, DO_SUB)
+DO_2OP_SCALAR(vsub_scalarb, 1, uint8_t, mve_mergemask_ub, DO_SUB)
+DO_2OP_SCALAR(vsub_scalarh, 2, uint16_t, mve_mergemask_uh, DO_SUB)
+DO_2OP_SCALAR(vsub_scalarw, 4, uint32_t, mve_mergemask_uw, DO_SUB)
+DO_2OP(vmulb, 1, uint8_t, mve_mergemask_ub, DO_MUL)
+DO_2OP(vmulh, 2, uint16_t, mve_mergemask_uh, DO_MUL)
+DO_2OP(vmulw, 4, uint32_t, mve_mergemask_uw, DO_MUL)
+DO_2OP_SCALAR(vmul_scalarb, 1, uint8_t, mve_mergemask_ub, DO_MUL)
+DO_2OP_SCALAR(vmul_scalarh, 2, uint16_t, mve_mergemask_uh, DO_MUL)
+DO_2OP_SCALAR(vmul_scalarw, 4, uint32_t, mve_mergemask_uw, DO_MUL)
+
+static inline int8_t do_mulh_s_b(int8_t n, int8_t m)
+{
+    return ((int32_t)n * m) >> 8;
+}
+
+static inline int16_t do_mulh_s_h(int16_t n, int16_t m)
+{
+    return ((int32_t)n * m) >> 16;
+}
+
+static inline int32_t do_mulh_s_w(int32_t n, int32_t m)
+{
+    return ((int64_t)n * m) >> 32;
+}
+
+static inline uint8_t do_mulh_u_b(uint8_t n, uint8_t m)
+{
+    return ((uint32_t)n * m) >> 8;
+}
+
+static inline uint16_t do_mulh_u_h(uint16_t n, uint16_t m)
+{
+    return ((uint32_t)n * m) >> 16;
+}
+
+static inline uint32_t do_mulh_u_w(uint32_t n, uint32_t m)
+{
+    return ((uint64_t)n * m) >> 32;
+}
+
+static inline int8_t do_rmulh_s_b(int8_t n, int8_t m)
+{
+    return (((int32_t)n * m) + (1U << 7)) >> 8;
+}
+
+static inline int16_t do_rmulh_s_h(int16_t n, int16_t m)
+{
+    return (((int32_t)n * m) + (1U << 15)) >> 16;
+}
+
+static inline int32_t do_rmulh_s_w(int32_t n, int32_t m)
+{
+    return (((int64_t)n * m) + (1U << 31)) >> 32;
+}
+
+static inline uint8_t do_rmulh_u_b(uint8_t n, uint8_t m)
+{
+    return (((uint32_t)n * m) + (1U << 7)) >> 8;
+}
+
+static inline uint16_t do_rmulh_u_h(uint16_t n, uint16_t m)
+{
+    return (((uint32_t)n * m) + (1U << 15)) >> 16;
+}
+
+static inline uint32_t do_rmulh_u_w(uint32_t n, uint32_t m)
+{
+    return (((uint64_t)n * m) + (1U << 31)) >> 32;
+}
+
+DO_2OP(vmulhsb, 1, int8_t, mve_mergemask_sb, do_mulh_s_b)
+DO_2OP(vmulhsh, 2, int16_t, mve_mergemask_sh, do_mulh_s_h)
+DO_2OP(vmulhsw, 4, int32_t, mve_mergemask_sw, do_mulh_s_w)
+DO_2OP(vmulhub, 1, uint8_t, mve_mergemask_ub, do_mulh_u_b)
+DO_2OP(vmulhuh, 2, uint16_t, mve_mergemask_uh, do_mulh_u_h)
+DO_2OP(vmulhuw, 4, uint32_t, mve_mergemask_uw, do_mulh_u_w)
+DO_2OP(vrmulhsb, 1, int8_t, mve_mergemask_sb, do_rmulh_s_b)
+DO_2OP(vrmulhsh, 2, int16_t, mve_mergemask_sh, do_rmulh_s_h)
+DO_2OP(vrmulhsw, 4, int32_t, mve_mergemask_sw, do_rmulh_s_w)
+DO_2OP(vrmulhub, 1, uint8_t, mve_mergemask_ub, do_rmulh_u_b)
+DO_2OP(vrmulhuh, 2, uint16_t, mve_mergemask_uh, do_rmulh_u_h)
+DO_2OP(vrmulhuw, 4, uint32_t, mve_mergemask_uw, do_rmulh_u_w)
+
+DO_2OP_L(vmullbsb, 0, 1, int8_t, 2, int16_t, mve_mergemask_sh, DO_MUL)
+DO_2OP_L(vmullbsh, 0, 2, int16_t, 4, int32_t, mve_mergemask_sw, DO_MUL)
+DO_2OP_L(vmullbsw, 0, 4, int32_t, 8, int64_t, mve_mergemask_sq, DO_MUL)
+DO_2OP_L(vmullbub, 0, 1, uint8_t, 2, uint16_t, mve_mergemask_uh, DO_MUL)
+DO_2OP_L(vmullbuh, 0, 2, uint16_t, 4, uint32_t, mve_mergemask_uw, DO_MUL)
+DO_2OP_L(vmullbuw, 0, 4, uint32_t, 8, uint64_t, mve_mergemask_uq, DO_MUL)
+DO_2OP_L(vmulltsb, 1, 1, int8_t, 2, int16_t, mve_mergemask_sh, DO_MUL)
+DO_2OP_L(vmulltsh, 1, 2, int16_t, 4, int32_t, mve_mergemask_sw, DO_MUL)
+DO_2OP_L(vmulltsw, 1, 4, int32_t, 8, int64_t, mve_mergemask_sq, DO_MUL)
+DO_2OP_L(vmulltub, 1, 1, uint8_t, 2, uint16_t, mve_mergemask_uh, DO_MUL)
+DO_2OP_L(vmulltuh, 1, 2, uint16_t, 4, uint32_t, mve_mergemask_uw, DO_MUL)
+DO_2OP_L(vmulltuw, 1, 4, uint32_t, 8, uint64_t, mve_mergemask_uq, DO_MUL)
+
+static uint64_t mve_pmull_h(uint64_t op1, uint64_t op2)
+{
+    uint64_t result = 0;
+    int i;
+
+    for (i = 0; i < 8; i++) {
+        uint64_t mask = (op1 & 0x0001000100010001ULL) * 0xffff;
+
+        result ^= op2 & mask;
+        op1 >>= 1;
+        op2 <<= 1;
+    }
+    return result;
+}
+
+static uint64_t mve_pmull_w(uint64_t op1, uint64_t op2)
+{
+    uint64_t result = 0;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        uint64_t mask = (op1 & 0x0000000100000001ULL) * 0xffffffff;
+
+        result ^= op2 & mask;
+        op1 >>= 1;
+        op2 <<= 1;
+    }
+    return result;
+}
+
+#define VMULLPH_MASK 0x00ff00ff00ff00ffULL
+#define VMULLPW_MASK 0x0000ffff0000ffffULL
+#define DO_VMULLPBH(N, M) \
+    mve_pmull_h((N) & VMULLPH_MASK, (M) & VMULLPH_MASK)
+#define DO_VMULLPTH(N, M) DO_VMULLPBH((N) >> 8, (M) >> 8)
+#define DO_VMULLPBW(N, M) \
+    mve_pmull_w((N) & VMULLPW_MASK, (M) & VMULLPW_MASK)
+#define DO_VMULLPTW(N, M) DO_VMULLPBW((N) >> 16, (M) >> 16)
+
+DO_2OP(vmullpbh, 8, uint64_t, mve_mergemask_uq, DO_VMULLPBH)
+DO_2OP(vmullpth, 8, uint64_t, mve_mergemask_uq, DO_VMULLPTH)
+DO_2OP(vmullpbw, 8, uint64_t, mve_mergemask_uq, DO_VMULLPBW)
+DO_2OP(vmullptw, 8, uint64_t, mve_mergemask_uq, DO_VMULLPTW)
+
+DO_2OP(vmaxsb, 1, int8_t, mve_mergemask_sb, DO_MAX)
+DO_2OP(vmaxsh, 2, int16_t, mve_mergemask_sh, DO_MAX)
+DO_2OP(vmaxsw, 4, int32_t, mve_mergemask_sw, DO_MAX)
+DO_2OP(vmaxub, 1, uint8_t, mve_mergemask_ub, DO_MAX)
+DO_2OP(vmaxuh, 2, uint16_t, mve_mergemask_uh, DO_MAX)
+DO_2OP(vmaxuw, 4, uint32_t, mve_mergemask_uw, DO_MAX)
+DO_2OP(vminsb, 1, int8_t, mve_mergemask_sb, DO_MIN)
+DO_2OP(vminsh, 2, int16_t, mve_mergemask_sh, DO_MIN)
+DO_2OP(vminsw, 4, int32_t, mve_mergemask_sw, DO_MIN)
+DO_2OP(vminub, 1, uint8_t, mve_mergemask_ub, DO_MIN)
+DO_2OP(vminuh, 2, uint16_t, mve_mergemask_uh, DO_MIN)
+DO_2OP(vminuw, 4, uint32_t, mve_mergemask_uw, DO_MIN)
+DO_2OP(vabdsb, 1, int8_t, mve_mergemask_sb, DO_ABD)
+DO_2OP(vabdsh, 2, int16_t, mve_mergemask_sh, DO_ABD)
+DO_2OP(vabdsw, 4, int32_t, mve_mergemask_sw, DO_ABD)
+DO_2OP(vabdub, 1, uint8_t, mve_mergemask_ub, DO_ABD)
+DO_2OP(vabduh, 2, uint16_t, mve_mergemask_uh, DO_ABD)
+DO_2OP(vabduw, 4, uint32_t, mve_mergemask_uw, DO_ABD)
+
+static inline uint32_t do_vhadd_u(uint32_t n, uint32_t m)
+{
+    return ((uint64_t)n + m) >> 1;
+}
+
+static inline int32_t do_vhadd_s(int32_t n, int32_t m)
+{
+    return ((int64_t)n + m) >> 1;
+}
+
+static inline uint32_t do_vrhadd_u(uint32_t n, uint32_t m)
+{
+    return ((uint64_t)n + m + 1) >> 1;
+}
+
+static inline int32_t do_vrhadd_s(int32_t n, int32_t m)
+{
+    return ((int64_t)n + m + 1) >> 1;
+}
+
+static inline uint32_t do_vhsub_u(uint32_t n, uint32_t m)
+{
+    return ((uint64_t)n - m) >> 1;
+}
+
+static inline int32_t do_vhsub_s(int32_t n, int32_t m)
+{
+    return ((int64_t)n - m) >> 1;
+}
+
+DO_2OP(vhaddsb, 1, int8_t, mve_mergemask_sb, do_vhadd_s)
+DO_2OP(vhaddsh, 2, int16_t, mve_mergemask_sh, do_vhadd_s)
+DO_2OP(vhaddsw, 4, int32_t, mve_mergemask_sw, do_vhadd_s)
+DO_2OP(vhaddub, 1, uint8_t, mve_mergemask_ub, do_vhadd_u)
+DO_2OP(vhadduh, 2, uint16_t, mve_mergemask_uh, do_vhadd_u)
+DO_2OP(vhadduw, 4, uint32_t, mve_mergemask_uw, do_vhadd_u)
+DO_2OP_SCALAR(vhadds_scalarb, 1, int8_t, mve_mergemask_sb, do_vhadd_s)
+DO_2OP_SCALAR(vhadds_scalarh, 2, int16_t, mve_mergemask_sh, do_vhadd_s)
+DO_2OP_SCALAR(vhadds_scalarw, 4, int32_t, mve_mergemask_sw, do_vhadd_s)
+DO_2OP_SCALAR(vhaddu_scalarb, 1, uint8_t, mve_mergemask_ub, do_vhadd_u)
+DO_2OP_SCALAR(vhaddu_scalarh, 2, uint16_t, mve_mergemask_uh, do_vhadd_u)
+DO_2OP_SCALAR(vhaddu_scalarw, 4, uint32_t, mve_mergemask_uw, do_vhadd_u)
+DO_2OP(vrhaddsb, 1, int8_t, mve_mergemask_sb, do_vrhadd_s)
+DO_2OP(vrhaddsh, 2, int16_t, mve_mergemask_sh, do_vrhadd_s)
+DO_2OP(vrhaddsw, 4, int32_t, mve_mergemask_sw, do_vrhadd_s)
+DO_2OP(vrhaddub, 1, uint8_t, mve_mergemask_ub, do_vrhadd_u)
+DO_2OP(vrhadduh, 2, uint16_t, mve_mergemask_uh, do_vrhadd_u)
+DO_2OP(vrhadduw, 4, uint32_t, mve_mergemask_uw, do_vrhadd_u)
+DO_2OP(vhsubsb, 1, int8_t, mve_mergemask_sb, do_vhsub_s)
+DO_2OP(vhsubsh, 2, int16_t, mve_mergemask_sh, do_vhsub_s)
+DO_2OP(vhsubsw, 4, int32_t, mve_mergemask_sw, do_vhsub_s)
+DO_2OP(vhsubub, 1, uint8_t, mve_mergemask_ub, do_vhsub_u)
+DO_2OP(vhsubuh, 2, uint16_t, mve_mergemask_uh, do_vhsub_u)
+DO_2OP(vhsubuw, 4, uint32_t, mve_mergemask_uw, do_vhsub_u)
+DO_2OP_SCALAR(vhsubs_scalarb, 1, int8_t, mve_mergemask_sb, do_vhsub_s)
+DO_2OP_SCALAR(vhsubs_scalarh, 2, int16_t, mve_mergemask_sh, do_vhsub_s)
+DO_2OP_SCALAR(vhsubs_scalarw, 4, int32_t, mve_mergemask_sw, do_vhsub_s)
+DO_2OP_SCALAR(vhsubu_scalarb, 1, uint8_t, mve_mergemask_ub, do_vhsub_u)
+DO_2OP_SCALAR(vhsubu_scalarh, 2, uint16_t, mve_mergemask_uh, do_vhsub_u)
+DO_2OP_SCALAR(vhsubu_scalarw, 4, uint32_t, mve_mergemask_uw, do_vhsub_u)
+
+static void do_vadc(CPUARMState *env, uint32_t *d, uint32_t *n,
+                    uint32_t *m, uint32_t inv, uint32_t carry_in,
+                    bool update_flags)
+{
+    uint16_t mask = mve_element_mask(env);
+    unsigned e;
+
+    if (mask & 0x1111) {
+        update_flags = true;
+    }
+
+    for (e = 0; e < 16 / 4; e++, mask >>= 4) {
+        uint64_t r = carry_in;
+
+        r += n[H4(e)];
+        r += m[H4(e)] ^ inv;
+        if (mask & 1) {
+            carry_in = (uint32_t)(r >> 32);
+        }
+        mve_mergemask_uw(&d[H4(e)], (uint32_t)r, mask);
+    }
+
+    if (update_flags) {
+        env->vfp.xregs[ARM_VFP_FPSCR] &= ~FPCR_NZCV_MASK;
+        env->vfp.xregs[ARM_VFP_FPSCR] |= carry_in ? FPCR_C : 0;
+    }
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vadc)(CPUARMState *env, void *vd, void *vn, void *vm)
+{
+    uint32_t carry_in =
+        (env->vfp.xregs[ARM_VFP_FPSCR] & FPCR_C) != 0;
+
+    do_vadc(env, vd, vn, vm, 0, carry_in, false);
+}
+
+void HELPER(mve_vadci)(CPUARMState *env, void *vd, void *vn, void *vm)
+{
+    do_vadc(env, vd, vn, vm, 0, 0, true);
+}
+
+void HELPER(mve_vsbc)(CPUARMState *env, void *vd, void *vn, void *vm)
+{
+    uint32_t carry_in =
+        (env->vfp.xregs[ARM_VFP_FPSCR] & FPCR_C) != 0;
+
+    do_vadc(env, vd, vn, vm, UINT32_MAX, carry_in, false);
+}
+
+void HELPER(mve_vsbci)(CPUARMState *env, void *vd, void *vn, void *vm)
+{
+    do_vadc(env, vd, vn, vm, UINT32_MAX, 1, true);
+}
+
+#define DO_VCADD(OP, ESIZE, TYPE, MERGE, FN0, FN1)                     \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        TYPE r[16 / ESIZE];                                            \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++) {                             \
+            if (!(e & 1)) {                                            \
+                r[e] = FN0(n[glue(H, ESIZE)(e)],                       \
+                           m[glue(H, ESIZE)(e + 1)]);                  \
+            } else {                                                   \
+                r[e] = FN1(n[glue(H, ESIZE)(e)],                       \
+                           m[glue(H, ESIZE)(e - 1)]);                  \
+            }                                                          \
+        }                                                              \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            MERGE(&d[glue(H, ESIZE)(e)], r[e], mask);                  \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VCADD_ALL(OP, FN0, FN1)                                     \
+    DO_VCADD(OP##b, 1, int8_t, mve_mergemask_sb, FN0, FN1)             \
+    DO_VCADD(OP##h, 2, int16_t, mve_mergemask_sh, FN0, FN1)            \
+    DO_VCADD(OP##w, 4, int32_t, mve_mergemask_sw, FN0, FN1)
+
+DO_VCADD_ALL(vcadd90, DO_SUB, DO_ADD)
+DO_VCADD_ALL(vcadd270, DO_ADD, DO_SUB)
+DO_VCADD_ALL(vhcadd90, do_vhsub_s, do_vhadd_s)
+DO_VCADD_ALL(vhcadd270, do_vhadd_s, do_vhsub_s)
+
+#define DO_1OP(OP, ESIZE, TYPE, MERGE, FN)                             \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd, void *vm)  \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            TYPE r = FN(m[glue(H, ESIZE)(e)]);                         \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_CLS_B(N) (clrsb32(N) - 24)
+#define DO_CLS_H(N) (clrsb32(N) - 16)
+#define DO_CLZ_B(N) (clz32(N) - 24)
+#define DO_CLZ_H(N) (clz32(N) - 16)
+#define DO_NOT(N) (~(N))
+#define DO_ABS(N) ((N) < 0 ? -(N) : (N))
+#define DO_NEG(N) (-(N))
+#define DO_FABSH(N) ((N) & dup_const(MO_16, 0x7fff))
+#define DO_FABSS(N) ((N) & dup_const(MO_32, 0x7fffffff))
+#define DO_FNEGH(N) ((N) ^ dup_const(MO_16, 0x8000))
+#define DO_FNEGS(N) ((N) ^ dup_const(MO_32, 0x80000000))
+
+static inline uint32_t mve_hswap32(uint32_t h)
+{
+    return (h << 16) | (h >> 16);
+}
+
+static inline uint64_t mve_hswap64(uint64_t h)
+{
+    uint64_t m = 0x0000ffff0000ffffull;
+
+    h = (h << 32) | (h >> 32);
+    return ((h & m) << 16) | ((h >> 16) & m);
+}
+
+static inline uint64_t mve_wswap64(uint64_t h)
+{
+    return (h << 32) | (h >> 32);
+}
+
+DO_1OP(vclsb, 1, int8_t, mve_mergemask_sb, DO_CLS_B)
+DO_1OP(vclsh, 2, int16_t, mve_mergemask_sh, DO_CLS_H)
+DO_1OP(vclsw, 4, int32_t, mve_mergemask_sw, clrsb32)
+DO_1OP(vclzb, 1, uint8_t, mve_mergemask_ub, DO_CLZ_B)
+DO_1OP(vclzh, 2, uint16_t, mve_mergemask_uh, DO_CLZ_H)
+DO_1OP(vclzw, 4, uint32_t, mve_mergemask_uw, clz32)
+DO_1OP(vrev16b, 2, uint16_t, mve_mergemask_uh, bswap16)
+DO_1OP(vrev32b, 4, uint32_t, mve_mergemask_uw, bswap32)
+DO_1OP(vrev32h, 4, uint32_t, mve_mergemask_uw, mve_hswap32)
+DO_1OP(vrev64b, 8, uint64_t, mve_mergemask_uq, bswap64)
+DO_1OP(vrev64h, 8, uint64_t, mve_mergemask_uq, mve_hswap64)
+DO_1OP(vrev64w, 8, uint64_t, mve_mergemask_uq, mve_wswap64)
+DO_1OP(vmvn, 8, uint64_t, mve_mergemask_uq, DO_NOT)
+DO_1OP(vabsb, 1, int8_t, mve_mergemask_sb, DO_ABS)
+DO_1OP(vabsh, 2, int16_t, mve_mergemask_sh, DO_ABS)
+DO_1OP(vabsw, 4, int32_t, mve_mergemask_sw, DO_ABS)
+DO_1OP(vfabsh, 8, uint64_t, mve_mergemask_uq, DO_FABSH)
+DO_1OP(vfabss, 8, uint64_t, mve_mergemask_uq, DO_FABSS)
+DO_1OP(vnegb, 1, int8_t, mve_mergemask_sb, DO_NEG)
+DO_1OP(vnegh, 2, int16_t, mve_mergemask_sh, DO_NEG)
+DO_1OP(vnegw, 4, int32_t, mve_mergemask_sw, DO_NEG)
+DO_1OP(vfnegh, 8, uint64_t, mve_mergemask_uq, DO_FNEGH)
+DO_1OP(vfnegs, 8, uint64_t, mve_mergemask_uq, DO_FNEGS)
+
+#define DO_VMAXMINA(OP, ESIZE, STYPE, UTYPE, MERGE, FN)                \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd, void *vm)  \
+    {                                                                  \
+        UTYPE *d = vd;                                                 \
+        STYPE *m = vm;                                                 \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            UTYPE r = DO_ABS(m[glue(H, ESIZE)(e)]);                    \
+                                                                       \
+            r = FN(d[glue(H, ESIZE)(e)], r);                           \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VMAXMINA(vmaxab, 1, int8_t, uint8_t, mve_mergemask_ub, DO_MAX)
+DO_VMAXMINA(vmaxah, 2, int16_t, uint16_t, mve_mergemask_uh, DO_MAX)
+DO_VMAXMINA(vmaxaw, 4, int32_t, uint32_t, mve_mergemask_uw, DO_MAX)
+DO_VMAXMINA(vminab, 1, int8_t, uint8_t, mve_mergemask_ub, DO_MIN)
+DO_VMAXMINA(vminah, 2, int16_t, uint16_t, mve_mergemask_uh, DO_MIN)
+DO_VMAXMINA(vminaw, 4, int32_t, uint32_t, mve_mergemask_uw, DO_MIN)
+
+#define DO_LDAV(OP, ESIZE, TYPE, XCHG, EVENACC, ODDACC)                \
+    uint64_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vn,        \
+                                    void *vm, uint64_t ra)             \
+    {                                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if (mask & 1) {                                            \
+                if (e & 1) {                                           \
+                    ra ODDACC                                          \
+                        (int64_t)n[glue(H, ESIZE)(e - 1 * XCHG)] *     \
+                        m[glue(H, ESIZE)(e)];                          \
+                } else {                                               \
+                    ra EVENACC                                         \
+                        (int64_t)n[glue(H, ESIZE)(e + 1 * XCHG)] *     \
+                        m[glue(H, ESIZE)(e)];                          \
+                }                                                      \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+DO_LDAV(vmlaldavsh, 2, int16_t, false, +=, +=)
+DO_LDAV(vmlaldavxsh, 2, int16_t, true, +=, +=)
+DO_LDAV(vmlaldavsw, 4, int32_t, false, +=, +=)
+DO_LDAV(vmlaldavxsw, 4, int32_t, true, +=, +=)
+DO_LDAV(vmlaldavuh, 2, uint16_t, false, +=, +=)
+DO_LDAV(vmlaldavuw, 4, uint32_t, false, +=, +=)
+DO_LDAV(vmlsldavsh, 2, int16_t, false, +=, -=)
+DO_LDAV(vmlsldavxsh, 2, int16_t, true, +=, -=)
+DO_LDAV(vmlsldavsw, 4, int32_t, false, +=, -=)
+DO_LDAV(vmlsldavxsw, 4, int32_t, true, +=, -=)
+
+#define DO_LDAVH(OP, TYPE, LTYPE, XCHG, SUB)                           \
+    uint64_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vn,        \
+                                    void *vm, uint64_t ra)             \
+    {                                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / 4; e++, mask >>= 4) {                     \
+            if (mask & 1) {                                            \
+                LTYPE product;                                         \
+                                                                       \
+                if (e & 1) {                                           \
+                    product = (LTYPE)n[H4(e - 1 * XCHG)] * m[H4(e)];   \
+                    if (SUB) {                                         \
+                        product = -product;                            \
+                    }                                                  \
+                } else {                                               \
+                    product = (LTYPE)n[H4(e + 1 * XCHG)] * m[H4(e)];   \
+                }                                                      \
+                product = (product >> 8) + ((product >> 7) & 1);       \
+                ra += product;                                         \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+DO_LDAVH(vrmlaldavhsw, int32_t, int64_t, false, false)
+DO_LDAVH(vrmlaldavhxsw, int32_t, int64_t, true, false)
+DO_LDAVH(vrmlaldavhuw, uint32_t, uint64_t, false, false)
+DO_LDAVH(vrmlsldavhsw, int32_t, int64_t, false, true)
+DO_LDAVH(vrmlsldavhxsw, int32_t, int64_t, true, true)
+
+#define DO_DAV(OP, ESIZE, TYPE, XCHG, EVENACC, ODDACC)                 \
+    uint32_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vn,        \
+                                    void *vm, uint32_t ra)             \
+    {                                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if (mask & 1) {                                            \
+                if (e & 1) {                                           \
+                    ra ODDACC n[glue(H, ESIZE)(e - 1 * XCHG)] *        \
+                        m[glue(H, ESIZE)(e)];                          \
+                } else {                                               \
+                    ra EVENACC n[glue(H, ESIZE)(e + 1 * XCHG)] *       \
+                        m[glue(H, ESIZE)(e)];                          \
+                }                                                      \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+#define DO_DAV_S(INSN, XCHG, EVENACC, ODDACC)                          \
+    DO_DAV(INSN##b, 1, int8_t, XCHG, EVENACC, ODDACC)                  \
+    DO_DAV(INSN##h, 2, int16_t, XCHG, EVENACC, ODDACC)                 \
+    DO_DAV(INSN##w, 4, int32_t, XCHG, EVENACC, ODDACC)
+
+#define DO_DAV_U(INSN, XCHG, EVENACC, ODDACC)                          \
+    DO_DAV(INSN##b, 1, uint8_t, XCHG, EVENACC, ODDACC)                 \
+    DO_DAV(INSN##h, 2, uint16_t, XCHG, EVENACC, ODDACC)                \
+    DO_DAV(INSN##w, 4, uint32_t, XCHG, EVENACC, ODDACC)
+
+DO_DAV_S(vmladavs, false, +=, +=)
+DO_DAV_U(vmladavu, false, +=, +=)
+DO_DAV_S(vmlsdav, false, +=, -=)
+DO_DAV_S(vmladavsx, true, +=, +=)
+DO_DAV_S(vmlsdavx, true, +=, -=)
+
+#define DO_VADDV(OP, ESIZE, TYPE)                                      \
+    uint32_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vm,        \
+                                    uint32_t ra)                       \
+    {                                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if (mask & 1) {                                            \
+                ra += m[glue(H, ESIZE)(e)];                            \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+DO_VADDV(vaddvsb, 1, int8_t)
+DO_VADDV(vaddvsh, 2, int16_t)
+DO_VADDV(vaddvsw, 4, int32_t)
+DO_VADDV(vaddvub, 1, uint8_t)
+DO_VADDV(vaddvuh, 2, uint16_t)
+DO_VADDV(vaddvuw, 4, uint32_t)
+
+#define DO_VMAXMINV(OP, ESIZE, TYPE, RATYPE, FN)                       \
+    uint32_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vm,        \
+                                    uint32_t ra_in)                    \
+    {                                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        int64_t ra = (RATYPE)ra_in;                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if (mask & 1) {                                            \
+                ra = FN(ra, m[glue(H, ESIZE)(e)]);                     \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+#define DO_VMAXMINV_U(OP, FN)                                          \
+    DO_VMAXMINV(OP##b, 1, uint8_t, uint8_t, FN)                        \
+    DO_VMAXMINV(OP##h, 2, uint16_t, uint16_t, FN)                      \
+    DO_VMAXMINV(OP##w, 4, uint32_t, uint32_t, FN)
+#define DO_VMAXMINV_S(OP, FN)                                          \
+    DO_VMAXMINV(OP##b, 1, int8_t, int8_t, FN)                          \
+    DO_VMAXMINV(OP##h, 2, int16_t, int16_t, FN)                        \
+    DO_VMAXMINV(OP##w, 4, int32_t, int32_t, FN)
+
+static int64_t do_maxa(int64_t n, int64_t m)
+{
+    if (m < 0) {
+        m = -m;
+    }
+    return DO_MAX(n, m);
+}
+
+static int64_t do_mina(int64_t n, int64_t m)
+{
+    if (m < 0) {
+        m = -m;
+    }
+    return DO_MIN(n, m);
+}
+
+DO_VMAXMINV_S(vmaxvs, DO_MAX)
+DO_VMAXMINV_U(vmaxvu, DO_MAX)
+DO_VMAXMINV_S(vminvs, DO_MIN)
+DO_VMAXMINV_U(vminvu, DO_MIN)
+DO_VMAXMINV(vmaxavb, 1, int8_t, uint8_t, do_maxa)
+DO_VMAXMINV(vmaxavh, 2, int16_t, uint16_t, do_maxa)
+DO_VMAXMINV(vmaxavw, 4, int32_t, uint32_t, do_maxa)
+DO_VMAXMINV(vminavb, 1, int8_t, uint8_t, do_mina)
+DO_VMAXMINV(vminavh, 2, int16_t, uint16_t, do_mina)
+DO_VMAXMINV(vminavw, 4, int32_t, uint32_t, do_mina)
+
+#define DO_FP_VMAXMINV(OP, ESIZE, TYPE, ABS, FN)                       \
+    uint32_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vm,        \
+                                    uint32_t ra_in)                    \
+    {                                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        TYPE ra = (TYPE)ra_in;                                         \
+        float_status *fpst = (ESIZE == 2) ?                            \
+            &env->vfp.standard_fp_status_f16 :                         \
+            &env->vfp.standard_fp_status;                              \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if (mask & 1) {                                            \
+                TYPE v = m[glue(H, ESIZE)(e)];                         \
+                                                                       \
+                if (TYPE##_is_signaling_nan(ra, fpst)) {               \
+                    ra = TYPE##_silence_nan(ra, fpst);                 \
+                    fpst->float_exception_flags |= float_flag_invalid;  \
+                }                                                      \
+                if (TYPE##_is_signaling_nan(v, fpst)) {                \
+                    v = TYPE##_silence_nan(v, fpst);                   \
+                    fpst->float_exception_flags |= float_flag_invalid;  \
+                }                                                      \
+                if (ABS) {                                             \
+                    v &= MAKE_64BIT_MASK(0, ESIZE * 8 - 1);            \
+                }                                                      \
+                ra = FN(ra, v, fpst);                                  \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+DO_FP_VMAXMINV(vmaxnmvh, 2, float16, false, float16_maxnum)
+DO_FP_VMAXMINV(vmaxnmvs, 4, float32, false, float32_maxnum)
+DO_FP_VMAXMINV(vminnmvh, 2, float16, false, float16_minnum)
+DO_FP_VMAXMINV(vminnmvs, 4, float32, false, float32_minnum)
+DO_FP_VMAXMINV(vmaxnmavh, 2, float16, true, float16_maxnum)
+DO_FP_VMAXMINV(vmaxnmavs, 4, float32, true, float32_maxnum)
+DO_FP_VMAXMINV(vminnmavh, 2, float16, true, float16_minnum)
+DO_FP_VMAXMINV(vminnmavs, 4, float32, true, float32_minnum)
+
+#define DO_VADDLV(OP, TYPE, LTYPE)                                     \
+    uint64_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vm,        \
+                                    uint64_t ra)                       \
+    {                                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / 4; e++, mask >>= 4) {                     \
+            if (mask & 1) {                                            \
+                ra += (LTYPE)m[H4(e)];                                 \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+DO_VADDLV(vaddlv_s, int32_t, int64_t)
+DO_VADDLV(vaddlv_u, uint32_t, uint64_t)
+
+#define DO_VABAV(OP, ESIZE, TYPE)                                      \
+    uint32_t HELPER(glue(mve_, OP))(CPUARMState *env, void *vn,        \
+                                    void *vm, uint32_t ra)             \
+    {                                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if (mask & 1) {                                            \
+                int64_t n0 = n[glue(H, ESIZE)(e)];                     \
+                int64_t m0 = m[glue(H, ESIZE)(e)];                     \
+                uint32_t r = n0 >= m0 ? n0 - m0 : m0 - n0;             \
+                                                                       \
+                ra += r;                                               \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+        return ra;                                                     \
+    }
+
+DO_VABAV(vabavsb, 1, int8_t)
+DO_VABAV(vabavsh, 2, int16_t)
+DO_VABAV(vabavsw, 4, int32_t)
+DO_VABAV(vabavub, 1, uint8_t)
+DO_VABAV(vabavuh, 2, uint16_t)
+DO_VABAV(vabavuw, 4, uint32_t)
+
+static inline int32_t mve_do_sat_bhs(int64_t val, int64_t min, int64_t max,
+                                     bool *satp)
+{
+    if (val > max) {
+        *satp = true;
+        return max;
+    } else if (val < min) {
+        *satp = true;
+        return min;
+    }
+    return val;
+}
+
+static inline uint32_t mve_do_usat_bhs(uint64_t val, uint64_t max,
+                                       bool *satp)
+{
+    if (val > max) {
+        *satp = true;
+        return (uint32_t)max;
+    }
+    return val;
+}
+
+#define DO_2OP_SAT(OP, ESIZE, TYPE, MERGE, FN)                         \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        bool qc = false;                                               \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            bool sat = false;                                          \
+            TYPE r = FN(n[glue(H, ESIZE)(e)],                          \
+                        m[glue(H, ESIZE)(e)], &sat);                   \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+            qc |= sat && (mask & 1);                                   \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_2OP_SCALAR_SAT(OP, ESIZE, TYPE, MERGE, FN)                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, uint32_t rm)                 \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE m = (TYPE)rm;                                             \
+        uint16_t mask = mve_element_mask(env);                         \
+        bool qc = false;                                               \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            bool sat = false;                                          \
+            TYPE r = FN(n[glue(H, ESIZE)(e)], m, &sat);                \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+            qc |= sat && (mask & 1);                                   \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_2OP_SCALAR_ACC(OP, ESIZE, TYPE, MERGE, FN)                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, uint32_t rm)                 \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE m = (TYPE)rm;                                             \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            TYPE r = FN(d[glue(H, ESIZE)(e)], n[glue(H, ESIZE)(e)], m);\
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_2OP_SCALAR_SAT_ACC(OP, ESIZE, TYPE, MERGE, FN)              \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, uint32_t rm)                 \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE m = (TYPE)rm;                                             \
+        uint16_t mask = mve_element_mask(env);                         \
+        bool qc = false;                                               \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            bool sat = false;                                          \
+            TYPE r = FN(d[glue(H, ESIZE)(e)], n[glue(H, ESIZE)(e)],    \
+                        m, &sat);                                      \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+            qc |= sat && (mask & 1);                                   \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_SQADD_B(N, M, SATP) \
+    mve_do_sat_bhs((int64_t)(N) + (M), INT8_MIN, INT8_MAX, SATP)
+#define DO_SQADD_H(N, M, SATP) \
+    mve_do_sat_bhs((int64_t)(N) + (M), INT16_MIN, INT16_MAX, SATP)
+#define DO_SQADD_W(N, M, SATP) \
+    mve_do_sat_bhs((int64_t)(N) + (M), INT32_MIN, INT32_MAX, SATP)
+#define DO_UQADD_B(N, M, SATP) \
+    mve_do_usat_bhs((uint64_t)(N) + (M), UINT8_MAX, SATP)
+#define DO_UQADD_H(N, M, SATP) \
+    mve_do_usat_bhs((uint64_t)(N) + (M), UINT16_MAX, SATP)
+#define DO_UQADD_W(N, M, SATP) \
+    mve_do_usat_bhs((uint64_t)(N) + (M), UINT32_MAX, SATP)
+#define DO_SQSUB_B(N, M, SATP) \
+    mve_do_sat_bhs((int64_t)(N) - (M), INT8_MIN, INT8_MAX, SATP)
+#define DO_SQSUB_H(N, M, SATP) \
+    mve_do_sat_bhs((int64_t)(N) - (M), INT16_MIN, INT16_MAX, SATP)
+#define DO_SQSUB_W(N, M, SATP) \
+    mve_do_sat_bhs((int64_t)(N) - (M), INT32_MIN, INT32_MAX, SATP)
+#define DO_UQSUB_B(N, M, SATP) \
+    mve_do_usat_bhs((uint64_t)(N) - (M), (N) >= (M) ? UINT8_MAX : 0, SATP)
+#define DO_UQSUB_H(N, M, SATP) \
+    mve_do_usat_bhs((uint64_t)(N) - (M), (N) >= (M) ? UINT16_MAX : 0, SATP)
+#define DO_UQSUB_W(N, M, SATP) \
+    mve_do_usat_bhs((uint64_t)(N) - (M), (N) >= (M) ? UINT32_MAX : 0, SATP)
+#define DO_QDMULH_B(N, M, SATP) \
+    mve_do_sat_bhs(((int64_t)(N) * (M)) >> 7, INT8_MIN, INT8_MAX, SATP)
+#define DO_QDMULH_H(N, M, SATP) \
+    mve_do_sat_bhs(((int64_t)(N) * (M)) >> 15, INT16_MIN, INT16_MAX, SATP)
+#define DO_QDMULH_W(N, M, SATP) \
+    mve_do_sat_bhs(((int64_t)(N) * (M)) >> 31, INT32_MIN, INT32_MAX, SATP)
+#define DO_QRDMULH_B(N, M, SATP) \
+    mve_do_sat_bhs((((int64_t)(N) * (M)) + (1 << 6)) >> 7, \
+                   INT8_MIN, INT8_MAX, SATP)
+#define DO_QRDMULH_H(N, M, SATP) \
+    mve_do_sat_bhs((((int64_t)(N) * (M)) + (1 << 14)) >> 15, \
+                   INT16_MIN, INT16_MAX, SATP)
+#define DO_QRDMULH_W(N, M, SATP) \
+    mve_do_sat_bhs((((int64_t)(N) * (M)) + (1 << 30)) >> 31, \
+                   INT32_MIN, INT32_MAX, SATP)
+
+#define DO_2OP_SAT_L(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, MERGE, FN, SATMASK) \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        LTYPE *d = vd;                                                 \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        bool qc = false;                                               \
+        unsigned le;                                                   \
+                                                                       \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {        \
+            bool sat = false;                                          \
+            LTYPE r = FN(n[glue(H, ESIZE)(le * 2 + TOP)],              \
+                         m[glue(H, ESIZE)(le * 2 + TOP)], &sat);       \
+                                                                       \
+            MERGE(&d[glue(H, LESIZE)(le)], r, mask);                   \
+            qc |= sat && (mask & SATMASK);                             \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_2OP_SCALAR_SAT_L(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE,       \
+                            MERGE, FN, SATMASK)                        \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, uint32_t rm)                 \
+    {                                                                  \
+        LTYPE *d = vd;                                                 \
+        TYPE *n = vn;                                                  \
+        TYPE m = (TYPE)rm;                                             \
+        uint16_t mask = mve_element_mask(env);                         \
+        bool qc = false;                                               \
+        unsigned le;                                                   \
+                                                                       \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {        \
+            bool sat = false;                                          \
+            LTYPE r = FN(n[glue(H, ESIZE)(le * 2 + TOP)], m, &sat);    \
+                                                                       \
+            MERGE(&d[glue(H, LESIZE)(le)], r, mask);                   \
+            qc |= sat && (mask & SATMASK);                             \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+static inline int32_t do_qdmull_h(int16_t n, int16_t m, bool *sat)
+{
+    int64_t r = ((int64_t)n * m) * 2;
+
+    return mve_do_sat_bhs(r, INT32_MIN, INT32_MAX, sat);
+}
+
+static inline int64_t do_qdmull_w(int32_t n, int32_t m, bool *sat)
+{
+    int64_t r = (int64_t)n * m;
+
+    if (r > INT64_MAX / 2) {
+        *sat = true;
+        return INT64_MAX;
+    } else if (r < INT64_MIN / 2) {
+        *sat = true;
+        return INT64_MIN;
+    }
+    return r * 2;
+}
+
+#define SATMASK16B 1
+#define SATMASK16T (1 << 2)
+#define SATMASK32 ((1 << 4) | 1)
+
+DO_2OP_SAT(vqaddsb, 1, int8_t, mve_mergemask_sb, DO_SQADD_B)
+DO_2OP_SAT(vqaddsh, 2, int16_t, mve_mergemask_sh, DO_SQADD_H)
+DO_2OP_SAT(vqaddsw, 4, int32_t, mve_mergemask_sw, DO_SQADD_W)
+DO_2OP_SAT(vqaddub, 1, uint8_t, mve_mergemask_ub, DO_UQADD_B)
+DO_2OP_SAT(vqadduh, 2, uint16_t, mve_mergemask_uh, DO_UQADD_H)
+DO_2OP_SAT(vqadduw, 4, uint32_t, mve_mergemask_uw, DO_UQADD_W)
+DO_2OP_SAT(vqsubsb, 1, int8_t, mve_mergemask_sb, DO_SQSUB_B)
+DO_2OP_SAT(vqsubsh, 2, int16_t, mve_mergemask_sh, DO_SQSUB_H)
+DO_2OP_SAT(vqsubsw, 4, int32_t, mve_mergemask_sw, DO_SQSUB_W)
+DO_2OP_SAT(vqsubub, 1, uint8_t, mve_mergemask_ub, DO_UQSUB_B)
+DO_2OP_SAT(vqsubuh, 2, uint16_t, mve_mergemask_uh, DO_UQSUB_H)
+DO_2OP_SAT(vqsubuw, 4, uint32_t, mve_mergemask_uw, DO_UQSUB_W)
+DO_2OP_SAT(vqdmulhb, 1, int8_t, mve_mergemask_sb, DO_QDMULH_B)
+DO_2OP_SAT(vqdmulhh, 2, int16_t, mve_mergemask_sh, DO_QDMULH_H)
+DO_2OP_SAT(vqdmulhw, 4, int32_t, mve_mergemask_sw, DO_QDMULH_W)
+DO_2OP_SAT(vqrdmulhb, 1, int8_t, mve_mergemask_sb, DO_QRDMULH_B)
+DO_2OP_SAT(vqrdmulhh, 2, int16_t, mve_mergemask_sh, DO_QRDMULH_H)
+DO_2OP_SAT(vqrdmulhw, 4, int32_t, mve_mergemask_sw, DO_QRDMULH_W)
+DO_2OP_SCALAR_SAT(vqadds_scalarb, 1, int8_t, mve_mergemask_sb, DO_SQADD_B)
+DO_2OP_SCALAR_SAT(vqadds_scalarh, 2, int16_t, mve_mergemask_sh, DO_SQADD_H)
+DO_2OP_SCALAR_SAT(vqadds_scalarw, 4, int32_t, mve_mergemask_sw, DO_SQADD_W)
+DO_2OP_SCALAR_SAT(vqaddu_scalarb, 1, uint8_t, mve_mergemask_ub, DO_UQADD_B)
+DO_2OP_SCALAR_SAT(vqaddu_scalarh, 2, uint16_t, mve_mergemask_uh, DO_UQADD_H)
+DO_2OP_SCALAR_SAT(vqaddu_scalarw, 4, uint32_t, mve_mergemask_uw, DO_UQADD_W)
+DO_2OP_SCALAR_SAT(vqsubs_scalarb, 1, int8_t, mve_mergemask_sb, DO_SQSUB_B)
+DO_2OP_SCALAR_SAT(vqsubs_scalarh, 2, int16_t, mve_mergemask_sh, DO_SQSUB_H)
+DO_2OP_SCALAR_SAT(vqsubs_scalarw, 4, int32_t, mve_mergemask_sw, DO_SQSUB_W)
+DO_2OP_SCALAR_SAT(vqsubu_scalarb, 1, uint8_t, mve_mergemask_ub, DO_UQSUB_B)
+DO_2OP_SCALAR_SAT(vqsubu_scalarh, 2, uint16_t, mve_mergemask_uh, DO_UQSUB_H)
+DO_2OP_SCALAR_SAT(vqsubu_scalarw, 4, uint32_t, mve_mergemask_uw, DO_UQSUB_W)
+DO_2OP_SCALAR_SAT(vqdmulh_scalarb, 1, int8_t, mve_mergemask_sb, DO_QDMULH_B)
+DO_2OP_SCALAR_SAT(vqdmulh_scalarh, 2, int16_t, mve_mergemask_sh, DO_QDMULH_H)
+DO_2OP_SCALAR_SAT(vqdmulh_scalarw, 4, int32_t, mve_mergemask_sw, DO_QDMULH_W)
+DO_2OP_SCALAR_SAT(vqrdmulh_scalarb, 1, int8_t, mve_mergemask_sb,
+                  DO_QRDMULH_B)
+DO_2OP_SCALAR_SAT(vqrdmulh_scalarh, 2, int16_t, mve_mergemask_sh,
+                  DO_QRDMULH_H)
+DO_2OP_SCALAR_SAT(vqrdmulh_scalarw, 4, int32_t, mve_mergemask_sw,
+                  DO_QRDMULH_W)
+
+#define DO_VMLA(D, N, M) ((N) * (M) + (D))
+#define DO_VMLAS(D, N, M) ((N) * (D) + (M))
+
+DO_2OP_SCALAR_ACC(vmlab, 1, uint8_t, mve_mergemask_ub, DO_VMLA)
+DO_2OP_SCALAR_ACC(vmlah, 2, uint16_t, mve_mergemask_uh, DO_VMLA)
+DO_2OP_SCALAR_ACC(vmlaw, 4, uint32_t, mve_mergemask_uw, DO_VMLA)
+DO_2OP_SCALAR_ACC(vmlasb, 1, uint8_t, mve_mergemask_ub, DO_VMLAS)
+DO_2OP_SCALAR_ACC(vmlash, 2, uint16_t, mve_mergemask_uh, DO_VMLAS)
+DO_2OP_SCALAR_ACC(vmlasw, 4, uint32_t, mve_mergemask_uw, DO_VMLAS)
+
+static int8_t do_vqdmlah_b(int8_t d, int8_t n, int8_t m, int round,
+                           bool *sat)
+{
+    int64_t r = (int64_t)n * m * 2 + ((int64_t)d << 8) + (round << 7);
+
+    return mve_do_sat_bhs(r, INT16_MIN, INT16_MAX, sat) >> 8;
+}
+
+static int16_t do_vqdmlah_h(int16_t d, int16_t n, int16_t m, int round,
+                            bool *sat)
+{
+    int64_t r = (int64_t)n * m * 2 + ((int64_t)d << 16) + (round << 15);
+
+    return mve_do_sat_bhs(r, INT32_MIN, INT32_MAX, sat) >> 16;
+}
+
+static int32_t do_vqdmlah_w(int32_t d, int32_t n, int32_t m, int round,
+                            bool *sat)
+{
+    int64_t m1 = (int64_t)n * m;
+    int64_t m2 = (int64_t)d << 31;
+    int64_t r;
+
+    if (sadd64_overflow(m1, m2, &r) ||
+        sadd64_overflow(r, (int64_t)round << 30, &r) ||
+        sadd64_overflow(r, r, &r)) {
+        *sat = true;
+        return r < 0 ? INT32_MAX : INT32_MIN;
+    }
+    return r >> 32;
+}
+
+#define DO_VQDMLAH_B(D, N, M, S) do_vqdmlah_b(D, N, M, 0, S)
+#define DO_VQDMLAH_H(D, N, M, S) do_vqdmlah_h(D, N, M, 0, S)
+#define DO_VQDMLAH_W(D, N, M, S) do_vqdmlah_w(D, N, M, 0, S)
+#define DO_VQRDMLAH_B(D, N, M, S) do_vqdmlah_b(D, N, M, 1, S)
+#define DO_VQRDMLAH_H(D, N, M, S) do_vqdmlah_h(D, N, M, 1, S)
+#define DO_VQRDMLAH_W(D, N, M, S) do_vqdmlah_w(D, N, M, 1, S)
+#define DO_VQDMLASH_B(D, N, M, S) do_vqdmlah_b(M, N, D, 0, S)
+#define DO_VQDMLASH_H(D, N, M, S) do_vqdmlah_h(M, N, D, 0, S)
+#define DO_VQDMLASH_W(D, N, M, S) do_vqdmlah_w(M, N, D, 0, S)
+#define DO_VQRDMLASH_B(D, N, M, S) do_vqdmlah_b(M, N, D, 1, S)
+#define DO_VQRDMLASH_H(D, N, M, S) do_vqdmlah_h(M, N, D, 1, S)
+#define DO_VQRDMLASH_W(D, N, M, S) do_vqdmlah_w(M, N, D, 1, S)
+
+DO_2OP_SCALAR_SAT_ACC(vqdmlahb, 1, int8_t, mve_mergemask_sb, DO_VQDMLAH_B)
+DO_2OP_SCALAR_SAT_ACC(vqdmlahh, 2, int16_t, mve_mergemask_sh, DO_VQDMLAH_H)
+DO_2OP_SCALAR_SAT_ACC(vqdmlahw, 4, int32_t, mve_mergemask_sw, DO_VQDMLAH_W)
+DO_2OP_SCALAR_SAT_ACC(vqrdmlahb, 1, int8_t, mve_mergemask_sb, DO_VQRDMLAH_B)
+DO_2OP_SCALAR_SAT_ACC(vqrdmlahh, 2, int16_t, mve_mergemask_sh, DO_VQRDMLAH_H)
+DO_2OP_SCALAR_SAT_ACC(vqrdmlahw, 4, int32_t, mve_mergemask_sw, DO_VQRDMLAH_W)
+DO_2OP_SCALAR_SAT_ACC(vqdmlashb, 1, int8_t, mve_mergemask_sb,
+                      DO_VQDMLASH_B)
+DO_2OP_SCALAR_SAT_ACC(vqdmlashh, 2, int16_t, mve_mergemask_sh,
+                      DO_VQDMLASH_H)
+DO_2OP_SCALAR_SAT_ACC(vqdmlashw, 4, int32_t, mve_mergemask_sw,
+                      DO_VQDMLASH_W)
+DO_2OP_SCALAR_SAT_ACC(vqrdmlashb, 1, int8_t, mve_mergemask_sb,
+                      DO_VQRDMLASH_B)
+DO_2OP_SCALAR_SAT_ACC(vqrdmlashh, 2, int16_t, mve_mergemask_sh,
+                      DO_VQRDMLASH_H)
+DO_2OP_SCALAR_SAT_ACC(vqrdmlashw, 4, int32_t, mve_mergemask_sw,
+                      DO_VQRDMLASH_W)
+
+static uint32_t do_vbrsrb(uint32_t n, uint32_t m)
+{
+    m &= 0xff;
+    if (m == 0) {
+        return 0;
+    }
+    n = revbit8(n);
+    if (m < 8) {
+        n >>= 8 - m;
+    }
+    return n;
+}
+
+static uint32_t do_vbrsrh(uint32_t n, uint32_t m)
+{
+    m &= 0xff;
+    if (m == 0) {
+        return 0;
+    }
+    n = revbit16(n);
+    if (m < 16) {
+        n >>= 16 - m;
+    }
+    return n;
+}
+
+static uint32_t do_vbrsrw(uint32_t n, uint32_t m)
+{
+    m &= 0xff;
+    if (m == 0) {
+        return 0;
+    }
+    n = revbit32(n);
+    if (m < 32) {
+        n >>= 32 - m;
+    }
+    return n;
+}
+
+DO_2OP_SCALAR(vbrsrb, 1, uint8_t, mve_mergemask_ub, do_vbrsrb)
+DO_2OP_SCALAR(vbrsrh, 2, uint16_t, mve_mergemask_uh, do_vbrsrh)
+DO_2OP_SCALAR(vbrsrw, 4, uint32_t, mve_mergemask_uw, do_vbrsrw)
+DO_2OP_SAT_L(vqdmullbh, 0, 2, int16_t, 4, int32_t, mve_mergemask_sw,
+             do_qdmull_h, SATMASK16B)
+DO_2OP_SAT_L(vqdmullbw, 0, 4, int32_t, 8, int64_t, mve_mergemask_sq,
+             do_qdmull_w, SATMASK32)
+DO_2OP_SAT_L(vqdmullth, 1, 2, int16_t, 4, int32_t, mve_mergemask_sw,
+             do_qdmull_h, SATMASK16T)
+DO_2OP_SAT_L(vqdmulltw, 1, 4, int32_t, 8, int64_t, mve_mergemask_sq,
+             do_qdmull_w, SATMASK32)
+DO_2OP_SCALAR_SAT_L(vqdmullb_scalarh, 0, 2, int16_t, 4, int32_t,
+                    mve_mergemask_sw, do_qdmull_h, SATMASK16B)
+DO_2OP_SCALAR_SAT_L(vqdmullb_scalarw, 0, 4, int32_t, 8, int64_t,
+                    mve_mergemask_sq, do_qdmull_w, SATMASK32)
+DO_2OP_SCALAR_SAT_L(vqdmullt_scalarh, 1, 2, int16_t, 4, int32_t,
+                    mve_mergemask_sw, do_qdmull_h, SATMASK16T)
+DO_2OP_SCALAR_SAT_L(vqdmullt_scalarw, 1, 4, int32_t, 8, int64_t,
+                    mve_mergemask_sq, do_qdmull_w, SATMASK32)
+
+#define DO_VCVT_FIXED(OP, ESIZE, TYPE, MERGE, FN)                      \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vm, uint32_t shift)              \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *m = vm;                                                  \
+        TYPE r;                                                        \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        float_status *fpst;                                            \
+        float_status scratch_fpst;                                     \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) {             \
+                continue;                                              \
+            }                                                          \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :   \
+                &env->vfp.standard_fp_status;                          \
+            if (!(mask & 1)) {                                         \
+                scratch_fpst = *fpst;                                  \
+                fpst = &scratch_fpst;                                  \
+            }                                                          \
+            r = FN(m[glue(H, ESIZE)(e)], shift, fpst);                 \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VCVT_FIXED(vcvt_sh, 2, int16_t, mve_mergemask_sh, helper_vfp_shtoh)
+DO_VCVT_FIXED(vcvt_uh, 2, uint16_t, mve_mergemask_uh, helper_vfp_uhtoh)
+DO_VCVT_FIXED(vcvt_hs, 2, int16_t, mve_mergemask_sh,
+              helper_vfp_toshh_round_to_zero)
+DO_VCVT_FIXED(vcvt_hu, 2, uint16_t, mve_mergemask_uh,
+              helper_vfp_touhh_round_to_zero)
+DO_VCVT_FIXED(vcvt_sf, 4, int32_t, mve_mergemask_sw, helper_vfp_sltos)
+DO_VCVT_FIXED(vcvt_uf, 4, uint32_t, mve_mergemask_uw, helper_vfp_ultos)
+DO_VCVT_FIXED(vcvt_fs, 4, int32_t, mve_mergemask_sw,
+              helper_vfp_tosls_round_to_zero)
+DO_VCVT_FIXED(vcvt_fu, 4, uint32_t, mve_mergemask_uw,
+              helper_vfp_touls_round_to_zero)
+
+#define DO_VCVT_RMODE(OP, ESIZE, TYPE, MERGE, FN)                      \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vm, uint32_t rmode)              \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *m = vm;                                                  \
+        TYPE r;                                                        \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        float_status *fpst;                                            \
+        float_status scratch_fpst;                                     \
+        float_status *base_fpst = (ESIZE == 2) ?                       \
+            &env->vfp.standard_fp_status_f16 :                         \
+            &env->vfp.standard_fp_status;                              \
+        uint32_t prev_rmode = get_float_rounding_mode(base_fpst);      \
+                                                                       \
+        set_float_rounding_mode(rmode, base_fpst);                     \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) {             \
+                continue;                                              \
+            }                                                          \
+            fpst = base_fpst;                                          \
+            if (!(mask & 1)) {                                         \
+                scratch_fpst = *fpst;                                  \
+                fpst = &scratch_fpst;                                  \
+            }                                                          \
+            r = FN(m[glue(H, ESIZE)(e)], 0, fpst);                     \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        set_float_rounding_mode(prev_rmode, base_fpst);                \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VCVT_RMODE(vcvt_rm_sh, 2, uint16_t, mve_mergemask_uh, helper_vfp_toshh)
+DO_VCVT_RMODE(vcvt_rm_uh, 2, uint16_t, mve_mergemask_uh, helper_vfp_touhh)
+DO_VCVT_RMODE(vcvt_rm_ss, 4, uint32_t, mve_mergemask_uw, helper_vfp_tosls)
+DO_VCVT_RMODE(vcvt_rm_us, 4, uint32_t, mve_mergemask_uw, helper_vfp_touls)
+
+static inline uint16_t mve_vrint_rm_h(uint16_t m, uint32_t ignored,
+                                      float_status *fpst)
+{
+    (void)ignored;
+    return helper_rinth(m, fpst);
+}
+
+static inline uint32_t mve_vrint_rm_s(uint32_t m, uint32_t ignored,
+                                      float_status *fpst)
+{
+    (void)ignored;
+    return helper_rints(m, fpst);
+}
+
+DO_VCVT_RMODE(vrint_rm_h, 2, uint16_t, mve_mergemask_uh, mve_vrint_rm_h)
+DO_VCVT_RMODE(vrint_rm_s, 4, uint32_t, mve_mergemask_uw, mve_vrint_rm_s)
+
+static void do_vcvt_sh(CPUARMState *env, void *vd, void *vm, int top)
+{
+    uint16_t *d = vd;
+    uint32_t *m = vm;
+    uint16_t r;
+    uint16_t mask = mve_element_mask(env);
+    bool ieee = !(env->vfp.xregs[ARM_VFP_FPSCR] & FPCR_AHP);
+    unsigned e;
+    float_status *fpst;
+    float_status scratch_fpst;
+    float_status *base_fpst = &env->vfp.standard_fp_status;
+    bool old_fz = get_flush_to_zero(base_fpst);
+
+    set_flush_to_zero(false, base_fpst);
+    for (e = 0; e < 16 / 4; e++, mask >>= 4) {
+        if ((mask & MAKE_64BIT_MASK(0, 4)) == 0) {
+            continue;
+        }
+        fpst = base_fpst;
+        if (!(mask & 1)) {
+            scratch_fpst = *fpst;
+            fpst = &scratch_fpst;
+        }
+        r = float32_to_float16(m[H4(e)], ieee, fpst);
+        mve_mergemask_uh(&d[H2(e * 2 + top)], r, mask >> (top * 2));
+    }
+    set_flush_to_zero(old_fz, base_fpst);
+    mve_advance_vpt(env);
+}
+
+static void do_vcvt_hs(CPUARMState *env, void *vd, void *vm, int top)
+{
+    uint32_t *d = vd;
+    uint16_t *m = vm;
+    uint32_t r;
+    uint16_t mask = mve_element_mask(env);
+    bool ieee = !(env->vfp.xregs[ARM_VFP_FPSCR] & FPCR_AHP);
+    unsigned e;
+    float_status *fpst;
+    float_status scratch_fpst;
+    float_status *base_fpst = &env->vfp.standard_fp_status;
+    bool old_fiz = get_flush_inputs_to_zero(base_fpst);
+
+    set_flush_inputs_to_zero(false, base_fpst);
+    for (e = 0; e < 16 / 4; e++, mask >>= 4) {
+        if ((mask & MAKE_64BIT_MASK(0, 4)) == 0) {
+            continue;
+        }
+        fpst = base_fpst;
+        if (!(mask & (1 << (top * 2)))) {
+            scratch_fpst = *fpst;
+            fpst = &scratch_fpst;
+        }
+        r = float16_to_float32(m[H2(e * 2 + top)], ieee, fpst);
+        mve_mergemask_uw(&d[H4(e)], r, mask);
+    }
+    set_flush_inputs_to_zero(old_fiz, base_fpst);
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vcvtb_sh)(CPUARMState *env, void *vd, void *vm)
+{
+    do_vcvt_sh(env, vd, vm, 0);
+}
+
+void HELPER(mve_vcvtt_sh)(CPUARMState *env, void *vd, void *vm)
+{
+    do_vcvt_sh(env, vd, vm, 1);
+}
+
+void HELPER(mve_vcvtb_hs)(CPUARMState *env, void *vd, void *vm)
+{
+    do_vcvt_hs(env, vd, vm, 0);
+}
+
+void HELPER(mve_vcvtt_hs)(CPUARMState *env, void *vd, void *vm)
+{
+    do_vcvt_hs(env, vd, vm, 1);
+}
+
+#define DO_1OP_FP(OP, ESIZE, TYPE, MERGE, FN)                          \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd, void *vm)  \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *m = vm;                                                  \
+        TYPE r;                                                        \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        float_status *fpst;                                            \
+        float_status scratch_fpst;                                     \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) {             \
+                continue;                                              \
+            }                                                          \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :   \
+                &env->vfp.standard_fp_status;                          \
+            if (!(mask & 1)) {                                         \
+                scratch_fpst = *fpst;                                  \
+                fpst = &scratch_fpst;                                  \
+            }                                                          \
+            r = FN(m[glue(H, ESIZE)(e)], fpst);                        \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_1OP_FP(vrintx_h, 2, uint16_t, mve_mergemask_uh, float16_round_to_int)
+DO_1OP_FP(vrintx_s, 4, uint32_t, mve_mergemask_uw, float32_round_to_int)
+
+#define DO_2OP_FP(OP, ESIZE, TYPE, MERGE, FN)                          \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        float_status *fpst;                                            \
+        float_status scratch_fpst;                                     \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            TYPE r;                                                    \
+                                                                       \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) {             \
+                continue;                                              \
+            }                                                          \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :   \
+                &env->vfp.standard_fp_status;                          \
+            if (!(mask & 1)) {                                         \
+                scratch_fpst = *fpst;                                  \
+                fpst = &scratch_fpst;                                  \
+            }                                                          \
+            r = FN(n[glue(H, ESIZE)(e)], m[glue(H, ESIZE)(e)], fpst);  \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_2OP_FP(vfaddh, 2, float16, mve_mergemask_uh, float16_add)
+DO_2OP_FP(vfadds, 4, float32, mve_mergemask_uw, float32_add)
+DO_2OP_FP(vfsubh, 2, float16, mve_mergemask_uh, float16_sub)
+DO_2OP_FP(vfsubs, 4, float32, mve_mergemask_uw, float32_sub)
+DO_2OP_FP(vfmulh, 2, float16, mve_mergemask_uh, float16_mul)
+DO_2OP_FP(vfmuls, 4, float32, mve_mergemask_uw, float32_mul)
+
+static inline float16 float16_abd(float16 a, float16 b, float_status *s)
+{
+    return make_float16(float16_val(float16_sub(a, b, s)) & 0x7fff);
+}
+
+static inline float32 float32_abd(float32 a, float32 b, float_status *s)
+{
+    return make_float32(float32_val(float32_sub(a, b, s)) & 0x7fffffff);
+}
+
+DO_2OP_FP(vfabdh, 2, float16, mve_mergemask_uh, float16_abd)
+DO_2OP_FP(vfabds, 4, float32, mve_mergemask_uw, float32_abd)
+DO_2OP_FP(vmaxnmh, 2, float16, mve_mergemask_uh, float16_maxnum)
+DO_2OP_FP(vmaxnms, 4, float32, mve_mergemask_uw, float32_maxnum)
+DO_2OP_FP(vminnmh, 2, float16, mve_mergemask_uh, float16_minnum)
+DO_2OP_FP(vminnms, 4, float32, mve_mergemask_uw, float32_minnum)
+
+static inline float16 float16_maxnuma(float16 a, float16 b, float_status *s)
+{
+    return float16_maxnum(float16_abs(a), float16_abs(b), s);
+}
+
+static inline float32 float32_maxnuma(float32 a, float32 b, float_status *s)
+{
+    return float32_maxnum(float32_abs(a), float32_abs(b), s);
+}
+
+static inline float16 float16_minnuma(float16 a, float16 b, float_status *s)
+{
+    return float16_minnum(float16_abs(a), float16_abs(b), s);
+}
+
+static inline float32 float32_minnuma(float32 a, float32 b, float_status *s)
+{
+    return float32_minnum(float32_abs(a), float32_abs(b), s);
+}
+
+DO_2OP_FP(vmaxnmah, 2, float16, mve_mergemask_uh, float16_maxnuma)
+DO_2OP_FP(vmaxnmas, 4, float32, mve_mergemask_uw, float32_maxnuma)
+DO_2OP_FP(vminnmah, 2, float16, mve_mergemask_uh, float16_minnuma)
+DO_2OP_FP(vminnmas, 4, float32, mve_mergemask_uw, float32_minnuma)
+
+#define DO_VCADD_FP(OP, ESIZE, TYPE, MERGE, FN0, FN1)                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,             \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        TYPE r[16 / ESIZE];                                            \
+        uint16_t tm;                                                   \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        float_status *fpst;                                            \
+        float_status scratch_fpst;                                     \
+                                                                       \
+        for (e = 0, tm = mask; e < 16 / ESIZE; e++, tm >>= ESIZE) {    \
+            if ((tm & MAKE_64BIT_MASK(0, ESIZE)) == 0) {               \
+                r[e] = 0;                                              \
+                continue;                                              \
+            }                                                          \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :   \
+                &env->vfp.standard_fp_status;                          \
+            if (!(tm & 1)) {                                           \
+                scratch_fpst = *fpst;                                  \
+                fpst = &scratch_fpst;                                  \
+            }                                                          \
+            if (!(e & 1)) {                                            \
+                r[e] = FN0(n[glue(H, ESIZE)(e)],                       \
+                           m[glue(H, ESIZE)(e + 1)], fpst);            \
+            } else {                                                   \
+                r[e] = FN1(n[glue(H, ESIZE)(e)],                       \
+                           m[glue(H, ESIZE)(e - 1)], fpst);            \
+            }                                                          \
+        }                                                              \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            MERGE(&d[glue(H, ESIZE)(e)], r[e], mask);                  \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VCADD_FP(vfcadd90h, 2, float16, mve_mergemask_uh, float16_sub,
+            float16_add)
+DO_VCADD_FP(vfcadd90s, 4, float32, mve_mergemask_uw, float32_sub,
+            float32_add)
+DO_VCADD_FP(vfcadd270h, 2, float16, mve_mergemask_uh, float16_add,
+            float16_sub)
+DO_VCADD_FP(vfcadd270s, 4, float32, mve_mergemask_uw, float32_add,
+            float32_sub)
+
+static inline float16 mve_float16_chs(float16 a)
+{
+    return make_float16(float16_val(a) ^ 0x8000);
+}
+
+static inline float32 mve_float32_chs(float32 a)
+{
+    return make_float32(float32_val(a) ^ 0x80000000);
+}
+
+static inline float16 mve_float16_fma(float16 n, float16 m, float16 d,
+                                      float_status *s)
+{
+    return float16_muladd(n, m, d, 0, s);
+}
+
+static inline float32 mve_float32_fma(float32 n, float32 m, float32 d,
+                                      float_status *s)
+{
+    return float32_muladd(n, m, d, 0, s);
+}
+
+static inline float16 mve_float16_fms(float16 n, float16 m, float16 d,
+                                      float_status *s)
+{
+    return float16_muladd(mve_float16_chs(n), m, d, 0, s);
+}
+
+static inline float32 mve_float32_fms(float32 n, float32 m, float32 d,
+                                      float_status *s)
+{
+    return float32_muladd(mve_float32_chs(n), m, d, 0, s);
+}
+
+#define DO_VCMUL_FP(OP, ESIZE, TYPE, MERGE, CHS, MUL, ROT)             \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,             \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        TYPE e1;                                                       \
+        TYPE e2;                                                       \
+        TYPE e3;                                                       \
+        TYPE e4;                                                       \
+        TYPE r0;                                                       \
+        TYPE r1;                                                       \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        float_status *fpst0;                                           \
+        float_status *fpst1;                                           \
+        float_status scratch_fpst0;                                    \
+        float_status scratch_fpst1;                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e += 2, mask >>= ESIZE * 2) {      \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE * 2)) == 0) {         \
+                continue;                                              \
+            }                                                          \
+            fpst0 = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :  \
+                &env->vfp.standard_fp_status;                          \
+            fpst1 = fpst0;                                             \
+            if (!(mask & 1)) {                                         \
+                scratch_fpst0 = *fpst0;                                \
+                fpst0 = &scratch_fpst0;                                \
+            }                                                          \
+            if (!(mask & (1 << ESIZE))) {                              \
+                scratch_fpst1 = *fpst1;                                \
+                fpst1 = &scratch_fpst1;                                \
+            }                                                          \
+            switch (ROT) {                                             \
+            case 0:                                                    \
+                e1 = m[glue(H, ESIZE)(e)];                             \
+                e2 = n[glue(H, ESIZE)(e)];                             \
+                e3 = m[glue(H, ESIZE)(e + 1)];                         \
+                e4 = n[glue(H, ESIZE)(e)];                             \
+                break;                                                 \
+            case 1:                                                    \
+                e1 = CHS(m[glue(H, ESIZE)(e + 1)]);                    \
+                e2 = n[glue(H, ESIZE)(e + 1)];                         \
+                e3 = m[glue(H, ESIZE)(e)];                             \
+                e4 = n[glue(H, ESIZE)(e + 1)];                         \
+                break;                                                 \
+            case 2:                                                    \
+                e1 = CHS(m[glue(H, ESIZE)(e)]);                        \
+                e2 = n[glue(H, ESIZE)(e)];                             \
+                e3 = CHS(m[glue(H, ESIZE)(e + 1)]);                    \
+                e4 = n[glue(H, ESIZE)(e)];                             \
+                break;                                                 \
+            case 3:                                                    \
+                e1 = m[glue(H, ESIZE)(e + 1)];                         \
+                e2 = n[glue(H, ESIZE)(e + 1)];                         \
+                e3 = CHS(m[glue(H, ESIZE)(e)]);                        \
+                e4 = n[glue(H, ESIZE)(e + 1)];                         \
+                break;                                                 \
+            default:                                                   \
+                g_assert_not_reached();                                \
+            }                                                          \
+            r0 = MUL(e2, e1, fpst0);                                   \
+            r1 = MUL(e4, e3, fpst1);                                   \
+            MERGE(&d[glue(H, ESIZE)(e)], r0, mask);                    \
+            MERGE(&d[glue(H, ESIZE)(e + 1)], r1, mask >> ESIZE);       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VCMUL_FP(vcmul0h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            float16_mul, 0)
+DO_VCMUL_FP(vcmul0s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            float32_mul, 0)
+DO_VCMUL_FP(vcmul90h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            float16_mul, 1)
+DO_VCMUL_FP(vcmul90s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            float32_mul, 1)
+DO_VCMUL_FP(vcmul180h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            float16_mul, 2)
+DO_VCMUL_FP(vcmul180s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            float32_mul, 2)
+DO_VCMUL_FP(vcmul270h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            float16_mul, 3)
+DO_VCMUL_FP(vcmul270s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            float32_mul, 3)
+
+#define DO_VCMLA_FP(OP, ESIZE, TYPE, MERGE, CHS, MULADD, ROT)          \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,             \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        TYPE e1;                                                       \
+        TYPE e2;                                                       \
+        TYPE e3;                                                       \
+        TYPE e4;                                                       \
+        TYPE r0;                                                       \
+        TYPE r1;                                                       \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+        float_status *fpst0;                                           \
+        float_status *fpst1;                                           \
+        float_status scratch_fpst0;                                    \
+        float_status scratch_fpst1;                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e += 2, mask >>= ESIZE * 2) {      \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE * 2)) == 0) {         \
+                continue;                                              \
+            }                                                          \
+            fpst0 = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :  \
+                &env->vfp.standard_fp_status;                          \
+            fpst1 = fpst0;                                             \
+            if (!(mask & 1)) {                                         \
+                scratch_fpst0 = *fpst0;                                \
+                fpst0 = &scratch_fpst0;                                \
+            }                                                          \
+            if (!(mask & (1 << ESIZE))) {                              \
+                scratch_fpst1 = *fpst1;                                \
+                fpst1 = &scratch_fpst1;                                \
+            }                                                          \
+            switch (ROT) {                                             \
+            case 0:                                                    \
+                e1 = m[glue(H, ESIZE)(e)];                             \
+                e2 = n[glue(H, ESIZE)(e)];                             \
+                e3 = m[glue(H, ESIZE)(e + 1)];                         \
+                e4 = n[glue(H, ESIZE)(e)];                             \
+                break;                                                 \
+            case 1:                                                    \
+                e1 = CHS(m[glue(H, ESIZE)(e + 1)]);                    \
+                e2 = n[glue(H, ESIZE)(e + 1)];                         \
+                e3 = m[glue(H, ESIZE)(e)];                             \
+                e4 = n[glue(H, ESIZE)(e + 1)];                         \
+                break;                                                 \
+            case 2:                                                    \
+                e1 = CHS(m[glue(H, ESIZE)(e)]);                        \
+                e2 = n[glue(H, ESIZE)(e)];                             \
+                e3 = CHS(m[glue(H, ESIZE)(e + 1)]);                    \
+                e4 = n[glue(H, ESIZE)(e)];                             \
+                break;                                                 \
+            case 3:                                                    \
+                e1 = m[glue(H, ESIZE)(e + 1)];                         \
+                e2 = n[glue(H, ESIZE)(e + 1)];                         \
+                e3 = CHS(m[glue(H, ESIZE)(e)]);                        \
+                e4 = n[glue(H, ESIZE)(e + 1)];                         \
+                break;                                                 \
+            default:                                                   \
+                g_assert_not_reached();                                \
+            }                                                          \
+            r0 = MULADD(e2, e1, d[glue(H, ESIZE)(e)], fpst0);          \
+            r1 = MULADD(e4, e3, d[glue(H, ESIZE)(e + 1)], fpst1);      \
+            MERGE(&d[glue(H, ESIZE)(e)], r0, mask);                    \
+            MERGE(&d[glue(H, ESIZE)(e + 1)], r1, mask >> ESIZE);       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VCMLA_FP(vcmla0h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            mve_float16_fma, 0)
+DO_VCMLA_FP(vcmla0s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            mve_float32_fma, 0)
+DO_VCMLA_FP(vcmla90h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            mve_float16_fma, 1)
+DO_VCMLA_FP(vcmla90s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            mve_float32_fma, 1)
+DO_VCMLA_FP(vcmla180h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            mve_float16_fma, 2)
+DO_VCMLA_FP(vcmla180s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            mve_float32_fma, 2)
+DO_VCMLA_FP(vcmla270h, 2, float16, mve_mergemask_uh, mve_float16_chs,
+            mve_float16_fma, 3)
+DO_VCMLA_FP(vcmla270s, 4, float32, mve_mergemask_uw, mve_float32_chs,
+            mve_float32_fma, 3)
+
+#define DO_2OP_FP_ACC(OP, ESIZE, TYPE, MERGE, FN)                       \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,             \
+                                void *vn, void *vm)                     \
+    {                                                                   \
+        TYPE *d = vd;                                                   \
+        TYPE *n = vn;                                                   \
+        TYPE *m = vm;                                                   \
+        uint16_t mask = mve_element_mask(env);                          \
+        unsigned e;                                                     \
+        float_status *fpst;                                             \
+        float_status scratch_fpst;                                      \
+                                                                        \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {              \
+            TYPE r;                                                     \
+                                                                        \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) {              \
+                continue;                                               \
+            }                                                           \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :    \
+                &env->vfp.standard_fp_status;                           \
+            if (!(mask & 1)) {                                          \
+                scratch_fpst = *fpst;                                   \
+                fpst = &scratch_fpst;                                   \
+            }                                                           \
+            r = FN(n[glue(H, ESIZE)(e)], m[glue(H, ESIZE)(e)],          \
+                   d[glue(H, ESIZE)(e)], fpst);                         \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                      \
+        }                                                               \
+        mve_advance_vpt(env);                                           \
+    }
+
+DO_2OP_FP_ACC(vfmah, 2, float16, mve_mergemask_uh, mve_float16_fma)
+DO_2OP_FP_ACC(vfmas, 4, float32, mve_mergemask_uw, mve_float32_fma)
+DO_2OP_FP_ACC(vfmsh, 2, float16, mve_mergemask_uh, mve_float16_fms)
+DO_2OP_FP_ACC(vfmss, 4, float32, mve_mergemask_uw, mve_float32_fms)
+
+#define DO_2OP_FP_SCALAR(OP, ESIZE, TYPE, MERGE, FN)                    \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,             \
+                                void *vn, uint32_t rm)                  \
+    {                                                                   \
+        TYPE *d = vd;                                                   \
+        TYPE *n = vn;                                                   \
+        TYPE m = (TYPE)rm;                                              \
+        uint16_t mask = mve_element_mask(env);                          \
+        unsigned e;                                                     \
+        float_status *fpst;                                             \
+        float_status scratch_fpst;                                      \
+                                                                        \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {              \
+            TYPE r;                                                     \
+                                                                        \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) {              \
+                continue;                                               \
+            }                                                           \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :    \
+                &env->vfp.standard_fp_status;                           \
+            if (!(mask & 1)) {                                          \
+                scratch_fpst = *fpst;                                   \
+                fpst = &scratch_fpst;                                   \
+            }                                                           \
+            r = FN(n[glue(H, ESIZE)(e)], m, fpst);                      \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                      \
+        }                                                               \
+        mve_advance_vpt(env);                                           \
+    }
+
+DO_2OP_FP_SCALAR(vfadd_scalarh, 2, float16, mve_mergemask_uh, float16_add)
+DO_2OP_FP_SCALAR(vfadd_scalars, 4, float32, mve_mergemask_uw, float32_add)
+DO_2OP_FP_SCALAR(vfsub_scalarh, 2, float16, mve_mergemask_uh, float16_sub)
+DO_2OP_FP_SCALAR(vfsub_scalars, 4, float32, mve_mergemask_uw, float32_sub)
+DO_2OP_FP_SCALAR(vfmul_scalarh, 2, float16, mve_mergemask_uh, float16_mul)
+DO_2OP_FP_SCALAR(vfmul_scalars, 4, float32, mve_mergemask_uw, float32_mul)
+
+#define DO_2OP_FP_ACC_SCALAR(OP, ESIZE, TYPE, MERGE, FN)                \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,             \
+                                void *vn, uint32_t rm)                  \
+    {                                                                   \
+        TYPE *d = vd;                                                   \
+        TYPE *n = vn;                                                   \
+        TYPE m = (TYPE)rm;                                              \
+        uint16_t mask = mve_element_mask(env);                          \
+        unsigned e;                                                     \
+        float_status *fpst;                                             \
+        float_status scratch_fpst;                                      \
+                                                                        \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {              \
+            TYPE r;                                                     \
+                                                                        \
+            if ((mask & MAKE_64BIT_MASK(0, ESIZE)) == 0) {              \
+                continue;                                               \
+            }                                                           \
+            fpst = (ESIZE == 2) ? &env->vfp.standard_fp_status_f16 :    \
+                &env->vfp.standard_fp_status;                           \
+            if (!(mask & 1)) {                                          \
+                scratch_fpst = *fpst;                                   \
+                fpst = &scratch_fpst;                                   \
+            }                                                           \
+            r = FN(n[glue(H, ESIZE)(e)], m, d[glue(H, ESIZE)(e)],       \
+                   0, fpst);                                           \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                      \
+        }                                                               \
+        mve_advance_vpt(env);                                           \
+    }
+
+#define DO_VFMAS_SCALARH(N, M, D, F, S) float16_muladd(N, D, M, F, S)
+#define DO_VFMAS_SCALARS(N, M, D, F, S) float32_muladd(N, D, M, F, S)
+
+DO_2OP_FP_ACC_SCALAR(vfma_scalarh, 2, float16, mve_mergemask_uh,
+                     float16_muladd)
+DO_2OP_FP_ACC_SCALAR(vfma_scalars, 4, float32, mve_mergemask_uw,
+                     float32_muladd)
+DO_2OP_FP_ACC_SCALAR(vfmas_scalarh, 2, float16, mve_mergemask_uh,
+                     DO_VFMAS_SCALARH)
+DO_2OP_FP_ACC_SCALAR(vfmas_scalars, 4, float32, mve_mergemask_uw,
+                     DO_VFMAS_SCALARS)
+
+static inline int32_t do_vshl_s(int32_t n, int32_t m, unsigned bits,
+                                bool rounded)
+{
+    int8_t shift = (int8_t)m;
+
+    if (shift <= -(int)bits) {
+        return rounded ? 0 : n >> 31;
+    } else if (shift < 0) {
+        if (rounded) {
+            n >>= -shift - 1;
+            return (n >> 1) + (n & 1);
+        }
+        return n >> -shift;
+    } else if (shift < (int)bits) {
+        uint32_t val = (uint32_t)n << shift;
+
+        if (bits == 32) {
+            return val;
+        }
+        return sextract32(val, 0, bits);
+    }
+    return 0;
+}
+
+static inline uint32_t do_vshl_u(uint32_t n, uint32_t m, unsigned bits,
+                                 bool rounded)
+{
+    int8_t shift = (int8_t)m;
+
+    if (shift <= -((int)bits + rounded)) {
+        return 0;
+    } else if (shift < 0) {
+        if (rounded) {
+            n >>= -shift - 1;
+            return (n >> 1) + (n & 1);
+        }
+        return n >> -shift;
+    } else if (shift < (int)bits) {
+        uint32_t val = n << shift;
+
+        if (bits == 32) {
+            return val;
+        }
+        return extract32(val, 0, bits);
+    }
+    return 0;
+}
+
+static inline int8_t do_vshl_s_b(int8_t n, int8_t m)
+{
+    return do_vshl_s(n, m, 8, false);
+}
+
+static inline int16_t do_vshl_s_h(int16_t n, int16_t m)
+{
+    return do_vshl_s(n, m, 16, false);
+}
+
+static inline int32_t do_vshl_s_w(int32_t n, int32_t m)
+{
+    return do_vshl_s(n, m, 32, false);
+}
+
+static inline uint8_t do_vshl_u_b(uint8_t n, uint8_t m)
+{
+    return do_vshl_u(n, m, 8, false);
+}
+
+static inline uint16_t do_vshl_u_h(uint16_t n, uint16_t m)
+{
+    return do_vshl_u(n, m, 16, false);
+}
+
+static inline uint32_t do_vshl_u_w(uint32_t n, uint32_t m)
+{
+    return do_vshl_u(n, m, 32, false);
+}
+
+static inline int8_t do_vrshl_s_b(int8_t n, int8_t m)
+{
+    return do_vshl_s(n, m, 8, true);
+}
+
+static inline int16_t do_vrshl_s_h(int16_t n, int16_t m)
+{
+    return do_vshl_s(n, m, 16, true);
+}
+
+static inline int32_t do_vrshl_s_w(int32_t n, int32_t m)
+{
+    return do_vshl_s(n, m, 32, true);
+}
+
+static inline uint8_t do_vrshl_u_b(uint8_t n, uint8_t m)
+{
+    return do_vshl_u(n, m, 8, true);
+}
+
+static inline uint16_t do_vrshl_u_h(uint16_t n, uint16_t m)
+{
+    return do_vshl_u(n, m, 16, true);
+}
+
+static inline uint32_t do_vrshl_u_w(uint32_t n, uint32_t m)
+{
+    return do_vshl_u(n, m, 32, true);
+}
+
+DO_2OP(vshlsb, 1, int8_t, mve_mergemask_sb, do_vshl_s_b)
+DO_2OP(vshlsh, 2, int16_t, mve_mergemask_sh, do_vshl_s_h)
+DO_2OP(vshlsw, 4, int32_t, mve_mergemask_sw, do_vshl_s_w)
+DO_2OP(vshlub, 1, uint8_t, mve_mergemask_ub, do_vshl_u_b)
+DO_2OP(vshluh, 2, uint16_t, mve_mergemask_uh, do_vshl_u_h)
+DO_2OP(vshluw, 4, uint32_t, mve_mergemask_uw, do_vshl_u_w)
+DO_2OP(vrshlsb, 1, int8_t, mve_mergemask_sb, do_vrshl_s_b)
+DO_2OP(vrshlsh, 2, int16_t, mve_mergemask_sh, do_vrshl_s_h)
+DO_2OP(vrshlsw, 4, int32_t, mve_mergemask_sw, do_vrshl_s_w)
+DO_2OP(vrshlub, 1, uint8_t, mve_mergemask_ub, do_vrshl_u_b)
+DO_2OP(vrshluh, 2, uint16_t, mve_mergemask_uh, do_vrshl_u_h)
+DO_2OP(vrshluw, 4, uint32_t, mve_mergemask_uw, do_vrshl_u_w)
+
+#define DO_2SHIFT_IMM(OP, ESIZE, TYPE, MERGE, FN)                      \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vm, uint32_t shift)              \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            TYPE r = FN(m[glue(H, ESIZE)(e)], shift);                  \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_2SHIFT_IMM_U(OP, FN)                                        \
+    DO_2SHIFT_IMM(OP##b, 1, uint8_t, mve_mergemask_ub, FN##_b)         \
+    DO_2SHIFT_IMM(OP##h, 2, uint16_t, mve_mergemask_uh, FN##_h)        \
+    DO_2SHIFT_IMM(OP##w, 4, uint32_t, mve_mergemask_uw, FN##_w)
+
+#define DO_2SHIFT_IMM_S(OP, FN)                                        \
+    DO_2SHIFT_IMM(OP##b, 1, int8_t, mve_mergemask_sb, FN##_b)          \
+    DO_2SHIFT_IMM(OP##h, 2, int16_t, mve_mergemask_sh, FN##_h)         \
+    DO_2SHIFT_IMM(OP##w, 4, int32_t, mve_mergemask_sw, FN##_w)
+
+DO_2SHIFT_IMM_U(vshli_u, do_vshl_u)
+DO_2SHIFT_IMM_S(vshli_s, do_vshl_s)
+DO_2SHIFT_IMM_U(vrshli_u, do_vrshl_u)
+DO_2SHIFT_IMM_S(vrshli_s, do_vrshl_s)
+
+#define DO_2SHIFT_IMM_SAT(OP, ESIZE, TYPE, MERGE, FN)                 \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,           \
+                                void *vm, uint32_t shift)             \
+    {                                                                 \
+        TYPE *d = vd;                                                 \
+        TYPE *m = vm;                                                 \
+        uint16_t mask = mve_element_mask(env);                        \
+        bool qc = false;                                              \
+        unsigned e;                                                   \
+                                                                      \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {            \
+            bool sat = false;                                         \
+            TYPE r = FN(m[glue(H, ESIZE)(e)], shift, &sat);           \
+                                                                      \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                    \
+            qc |= sat && (mask & 1);                                  \
+        }                                                             \
+        if (qc) {                                                     \
+            env->vfp.qc[0] = qc;                                      \
+        }                                                             \
+        mve_advance_vpt(env);                                         \
+    }
+
+#define DO_2SHIFT_IMM_SAT_U(OP, FN)                                   \
+    DO_2SHIFT_IMM_SAT(OP##b, 1, uint8_t, mve_mergemask_ub, FN##_B)    \
+    DO_2SHIFT_IMM_SAT(OP##h, 2, uint16_t, mve_mergemask_uh, FN##_H)   \
+    DO_2SHIFT_IMM_SAT(OP##w, 4, uint32_t, mve_mergemask_uw, FN##_W)
+
+#define DO_2SHIFT_IMM_SAT_S(OP, FN)                                   \
+    DO_2SHIFT_IMM_SAT(OP##b, 1, int8_t, mve_mergemask_sb, FN##_B)     \
+    DO_2SHIFT_IMM_SAT(OP##h, 2, int16_t, mve_mergemask_sh, FN##_H)    \
+    DO_2SHIFT_IMM_SAT(OP##w, 4, int32_t, mve_mergemask_sw, FN##_W)
+
+#define DO_SQSHLI_B(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs((int64_t)(N) * (1LL << (SHIFT)),                   \
+                   INT8_MIN, INT8_MAX, SATP)
+#define DO_SQSHLI_H(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs((int64_t)(N) * (1LL << (SHIFT)),                   \
+                   INT16_MIN, INT16_MAX, SATP)
+#define DO_SQSHLI_W(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs((int64_t)(N) * (1LL << (SHIFT)),                   \
+                   INT32_MIN, INT32_MAX, SATP)
+#define DO_UQSHLI_B(N, SHIFT, SATP)                                   \
+    mve_do_usat_bhs((uint64_t)(N) << (SHIFT), UINT8_MAX, SATP)
+#define DO_UQSHLI_H(N, SHIFT, SATP)                                   \
+    mve_do_usat_bhs((uint64_t)(N) << (SHIFT), UINT16_MAX, SATP)
+#define DO_UQSHLI_W(N, SHIFT, SATP)                                   \
+    mve_do_usat_bhs((uint64_t)(N) << (SHIFT), UINT32_MAX, SATP)
+
+static inline int32_t mve_do_sqrshl_bhs(int32_t src, int32_t shift,
+                                        int bits, bool round, bool *satp)
+{
+    if (shift <= -bits) {
+        return round ? 0 : src >> 31;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < bits) {
+        uint32_t val_u = (uint32_t)src << shift;
+        int32_t val = (int32_t)val_u;
+
+        if (bits == 32) {
+            if (val >> shift == src) {
+                return val;
+            }
+        } else {
+            int32_t extval = sextract32(val, 0, bits);
+
+            if (val == extval) {
+                return extval;
+            }
+        }
+    } else if (src == 0) {
+        return 0;
+    }
+
+    *satp = true;
+    return (1u << (bits - 1)) - (src >= 0);
+}
+
+static inline uint32_t mve_do_uqrshl_bhs(uint32_t src, int32_t shift,
+                                         int bits, bool round, bool *satp)
+{
+    if (shift <= -(bits + round)) {
+        return 0;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < bits) {
+        uint32_t val = src << shift;
+
+        if (bits == 32) {
+            if (val >> shift == src) {
+                return val;
+            }
+        } else {
+            uint32_t extval = extract32(val, 0, bits);
+
+            if (val == extval) {
+                return extval;
+            }
+        }
+    } else if (src == 0) {
+        return 0;
+    }
+
+    *satp = true;
+    return (uint32_t)MAKE_64BIT_MASK(0, bits);
+}
+
+static inline uint32_t mve_do_suqrshl_bhs(int32_t src, int32_t shift,
+                                          int bits, bool round, bool *satp)
+{
+    if (src < 0) {
+        *satp = true;
+        return 0;
+    }
+    return mve_do_uqrshl_bhs((uint32_t)src, shift, bits, round, satp);
+}
+
+static inline int64_t mve_do_sqrshl_d(int64_t src, int64_t shift,
+                                      bool round, bool *satp)
+{
+    if (shift <= -64) {
+        return round ? 0 : src >> 63;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < 64) {
+        uint64_t val_u = (uint64_t)src << shift;
+        int64_t val = (int64_t)val_u;
+
+        if (!satp || val >> shift == src) {
+            return val;
+        }
+    } else if (!satp || src == 0) {
+        return 0;
+    }
+
+    *satp = true;
+    return src >= 0 ? INT64_MAX : INT64_MIN;
+}
+
+static inline uint64_t mve_do_uqrshl_d(uint64_t src, int64_t shift,
+                                       bool round, bool *satp)
+{
+    if (shift <= -(64 + round)) {
+        return 0;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < 64) {
+        uint64_t val = src << shift;
+
+        if (!satp || val >> shift == src) {
+            return val;
+        }
+    } else if (!satp || src == 0) {
+        return 0;
+    }
+
+    *satp = true;
+    return UINT64_MAX;
+}
+
+static inline int64_t mve_do_sqrshl48_d(int64_t src, int64_t shift,
+                                        bool round, bool *satp)
+{
+    int64_t val;
+    int64_t extval;
+
+    if (shift <= -48) {
+        return round ? 0 : src >> 63;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            val = (src >> 1) + (src & 1);
+        } else {
+            val = src >> -shift;
+        }
+        extval = sextract64(val, 0, 48);
+        if (val == extval) {
+            return extval;
+        }
+    } else if (shift < 48) {
+        extval = sextract64((uint64_t)src << shift, 0, 48);
+        if (src == (extval >> shift)) {
+            return extval;
+        }
+    } else if (src == 0) {
+        return 0;
+    }
+
+    if (satp) {
+        *satp = true;
+    }
+    return src >= 0 ? MAKE_64BIT_MASK(0, 47) : MAKE_64BIT_MASK(47, 17);
+}
+
+static inline uint64_t mve_do_uqrshl48_d(uint64_t src, int64_t shift,
+                                         bool round, bool *satp)
+{
+    uint64_t val;
+    uint64_t extval;
+
+    if (shift <= -(48 + round)) {
+        return 0;
+    } else if (shift < 0) {
+        if (round) {
+            val = src >> (-shift - 1);
+            val = (val >> 1) + (val & 1);
+        } else {
+            val = src >> -shift;
+        }
+        extval = extract64(val, 0, 48);
+        if (val == extval) {
+            return extval;
+        }
+    } else if (shift < 48) {
+        extval = extract64(src << shift, 0, 48);
+        if (src == (extval >> shift)) {
+            return extval;
+        }
+    } else if (src == 0) {
+        return 0;
+    }
+
+    if (satp) {
+        *satp = true;
+    }
+    return MAKE_64BIT_MASK(0, 48);
+}
+
+uint64_t HELPER(mve_sshrl)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    return mve_do_sqrshl_d(n, -(int8_t)shift, false, NULL);
+}
+
+uint64_t HELPER(mve_ushll)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    return mve_do_uqrshl_d(n, (int8_t)shift, false, NULL);
+}
+
+uint64_t HELPER(mve_sqshll)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint64_t ret = mve_do_sqrshl_d(n, (int8_t)shift, false, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint64_t HELPER(mve_uqshll)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint64_t ret = mve_do_uqrshl_d(n, (int8_t)shift, false, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint64_t HELPER(mve_sqrshrl)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint64_t ret = mve_do_sqrshl_d(n, -(int8_t)shift, true, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint64_t HELPER(mve_uqrshll)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint64_t ret = mve_do_uqrshl_d(n, (int8_t)shift, true, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint64_t HELPER(mve_sqrshrl48)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint64_t ret = mve_do_sqrshl48_d(n, -(int8_t)shift, true, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint64_t HELPER(mve_uqrshll48)(CPUARMState *env, uint64_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint64_t ret = mve_do_uqrshl48_d(n, (int8_t)shift, true, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint32_t HELPER(mve_uqshl)(CPUARMState *env, uint32_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint32_t ret = mve_do_uqrshl_bhs(n, (int8_t)shift, 32, false, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint32_t HELPER(mve_sqshl)(CPUARMState *env, uint32_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint32_t ret = mve_do_sqrshl_bhs(n, (int8_t)shift, 32, false, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint32_t HELPER(mve_uqrshl)(CPUARMState *env, uint32_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint32_t ret = mve_do_uqrshl_bhs(n, (int8_t)shift, 32, true, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+uint32_t HELPER(mve_sqrshr)(CPUARMState *env, uint32_t n, uint32_t shift)
+{
+    bool sat = false;
+    uint32_t ret = mve_do_sqrshl_bhs(n, -(int8_t)shift, 32, true, &sat);
+
+    if (sat) {
+        env->vfp.qc[0] = 1;
+    }
+    return ret;
+}
+
+#undef DO_SQSHLI_B
+#undef DO_SQSHLI_H
+#undef DO_SQSHLI_W
+#undef DO_UQSHLI_B
+#undef DO_UQSHLI_H
+#undef DO_UQSHLI_W
+
+#define DO_SQSHLI_B(N, SHIFT, SATP)                                   \
+    mve_do_sqrshl_bhs((int8_t)(N), (int8_t)(SHIFT), 8, false, SATP)
+#define DO_SQSHLI_H(N, SHIFT, SATP)                                   \
+    mve_do_sqrshl_bhs((int16_t)(N), (int8_t)(SHIFT), 16, false, SATP)
+#define DO_SQSHLI_W(N, SHIFT, SATP)                                   \
+    mve_do_sqrshl_bhs((int32_t)(N), (int8_t)(SHIFT), 32, false, SATP)
+#define DO_UQSHLI_B(N, SHIFT, SATP)                                   \
+    mve_do_uqrshl_bhs((uint8_t)(N), (int8_t)(SHIFT), 8, false, SATP)
+#define DO_UQSHLI_H(N, SHIFT, SATP)                                   \
+    mve_do_uqrshl_bhs((uint16_t)(N), (int8_t)(SHIFT), 16, false, SATP)
+#define DO_UQSHLI_W(N, SHIFT, SATP)                                   \
+    mve_do_uqrshl_bhs((uint32_t)(N), (int8_t)(SHIFT), 32, false, SATP)
+#define DO_SQRSHLI_B(N, SHIFT, SATP)                                  \
+    mve_do_sqrshl_bhs((int8_t)(N), (int8_t)(SHIFT), 8, true, SATP)
+#define DO_SQRSHLI_H(N, SHIFT, SATP)                                  \
+    mve_do_sqrshl_bhs((int16_t)(N), (int8_t)(SHIFT), 16, true, SATP)
+#define DO_SQRSHLI_W(N, SHIFT, SATP)                                  \
+    mve_do_sqrshl_bhs((int32_t)(N), (int8_t)(SHIFT), 32, true, SATP)
+#define DO_UQRSHLI_B(N, SHIFT, SATP)                                  \
+    mve_do_uqrshl_bhs((uint8_t)(N), (int8_t)(SHIFT), 8, true, SATP)
+#define DO_UQRSHLI_H(N, SHIFT, SATP)                                  \
+    mve_do_uqrshl_bhs((uint16_t)(N), (int8_t)(SHIFT), 16, true, SATP)
+#define DO_UQRSHLI_W(N, SHIFT, SATP)                                  \
+    mve_do_uqrshl_bhs((uint32_t)(N), (int8_t)(SHIFT), 32, true, SATP)
+
+#define DO_SUQSHLI_B(N, SHIFT, SATP)                                  \
+    mve_do_suqrshl_bhs((int8_t)(N), (int8_t)(SHIFT), 8, false, SATP)
+#define DO_SUQSHLI_H(N, SHIFT, SATP)                                  \
+    mve_do_suqrshl_bhs((int16_t)(N), (int8_t)(SHIFT), 16, false, SATP)
+#define DO_SUQSHLI_W(N, SHIFT, SATP)                                  \
+    mve_do_suqrshl_bhs((int32_t)(N), (int8_t)(SHIFT), 32, false, SATP)
+
+DO_2SHIFT_IMM_SAT_S(vqshli_s, DO_SQSHLI)
+DO_2SHIFT_IMM_SAT_U(vqshli_u, DO_UQSHLI)
+DO_2SHIFT_IMM_SAT_S(vqrshli_s, DO_SQRSHLI)
+DO_2SHIFT_IMM_SAT_U(vqrshli_u, DO_UQRSHLI)
+DO_2SHIFT_IMM_SAT_S(vqshlui_s, DO_SUQSHLI)
+
+DO_2OP_SAT(vqshlsb, 1, int8_t, mve_mergemask_sb, DO_SQSHLI_B)
+DO_2OP_SAT(vqshlsh, 2, int16_t, mve_mergemask_sh, DO_SQSHLI_H)
+DO_2OP_SAT(vqshlsw, 4, int32_t, mve_mergemask_sw, DO_SQSHLI_W)
+DO_2OP_SAT(vqshlub, 1, uint8_t, mve_mergemask_ub, DO_UQSHLI_B)
+DO_2OP_SAT(vqshluh, 2, uint16_t, mve_mergemask_uh, DO_UQSHLI_H)
+DO_2OP_SAT(vqshluw, 4, uint32_t, mve_mergemask_uw, DO_UQSHLI_W)
+DO_2OP_SAT(vqrshlsb, 1, int8_t, mve_mergemask_sb, DO_SQRSHLI_B)
+DO_2OP_SAT(vqrshlsh, 2, int16_t, mve_mergemask_sh, DO_SQRSHLI_H)
+DO_2OP_SAT(vqrshlsw, 4, int32_t, mve_mergemask_sw, DO_SQRSHLI_W)
+DO_2OP_SAT(vqrshlub, 1, uint8_t, mve_mergemask_ub, DO_UQRSHLI_B)
+DO_2OP_SAT(vqrshluh, 2, uint16_t, mve_mergemask_uh, DO_UQRSHLI_H)
+DO_2OP_SAT(vqrshluw, 4, uint32_t, mve_mergemask_uw, DO_UQRSHLI_W)
+
+#define DO_VQDMLADH_OP(OP, ESIZE, TYPE, MERGE, XCHG, ROUND, FN)        \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vn, void *vm)                    \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *n = vn;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        bool qc = false;                                               \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            bool sat = false;                                          \
+                                                                       \
+            if ((e & 1) == XCHG) {                                     \
+                TYPE r = FN(n[glue(H, ESIZE)(e)],                      \
+                            m[glue(H, ESIZE)(e - XCHG)],               \
+                            n[glue(H, ESIZE)(e + (1 - 2 * XCHG))],     \
+                            m[glue(H, ESIZE)(e + (1 - XCHG))],         \
+                            ROUND, &sat);                              \
+                                                                       \
+                MERGE(&d[glue(H, ESIZE)(e)], r, mask);                 \
+                qc |= sat && (mask & 1);                               \
+            }                                                          \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+static int8_t do_vqdmladh_b(int8_t a, int8_t b, int8_t c, int8_t d,
+                            int round, bool *sat)
+{
+    int64_t r = ((int64_t)a * b + (int64_t)c * d) * 2 +
+        ((int64_t)round << 7);
+
+    return mve_do_sat_bhs(r, INT16_MIN, INT16_MAX, sat) >> 8;
+}
+
+static int16_t do_vqdmladh_h(int16_t a, int16_t b, int16_t c, int16_t d,
+                             int round, bool *sat)
+{
+    int64_t r = ((int64_t)a * b + (int64_t)c * d) * 2 +
+        ((int64_t)round << 15);
+
+    return mve_do_sat_bhs(r, INT32_MIN, INT32_MAX, sat) >> 16;
+}
+
+static int32_t do_vqdmladh_w(int32_t a, int32_t b, int32_t c, int32_t d,
+                             int round, bool *sat)
+{
+    int64_t m1 = (int64_t)a * b;
+    int64_t m2 = (int64_t)c * d;
+    int64_t r;
+
+    if (sadd64_overflow(m1, m2, &r) ||
+        sadd64_overflow(r, (int64_t)round << 30, &r) ||
+        sadd64_overflow(r, r, &r)) {
+        *sat = true;
+        return r < 0 ? INT32_MAX : INT32_MIN;
+    }
+    return r >> 32;
+}
+
+static int8_t do_vqdmlsdh_b(int8_t a, int8_t b, int8_t c, int8_t d,
+                            int round, bool *sat)
+{
+    int64_t r = ((int64_t)a * b - (int64_t)c * d) * 2 +
+        ((int64_t)round << 7);
+
+    return mve_do_sat_bhs(r, INT16_MIN, INT16_MAX, sat) >> 8;
+}
+
+static int16_t do_vqdmlsdh_h(int16_t a, int16_t b, int16_t c, int16_t d,
+                             int round, bool *sat)
+{
+    int64_t r = ((int64_t)a * b - (int64_t)c * d) * 2 +
+        ((int64_t)round << 15);
+
+    return mve_do_sat_bhs(r, INT32_MIN, INT32_MAX, sat) >> 16;
+}
+
+static int32_t do_vqdmlsdh_w(int32_t a, int32_t b, int32_t c, int32_t d,
+                             int round, bool *sat)
+{
+    int64_t m1 = (int64_t)a * b;
+    int64_t m2 = (int64_t)c * d;
+    int64_t r;
+
+    if (ssub64_overflow(m1, m2, &r) ||
+        sadd64_overflow(r, (int64_t)round << 30, &r) ||
+        sadd64_overflow(r, r, &r)) {
+        *sat = true;
+        return r < 0 ? INT32_MAX : INT32_MIN;
+    }
+    return r >> 32;
+}
+
+DO_VQDMLADH_OP(vqdmladhb, 1, int8_t, mve_mergemask_sb, 0, 0, do_vqdmladh_b)
+DO_VQDMLADH_OP(vqdmladhh, 2, int16_t, mve_mergemask_sh, 0, 0,
+               do_vqdmladh_h)
+DO_VQDMLADH_OP(vqdmladhw, 4, int32_t, mve_mergemask_sw, 0, 0,
+               do_vqdmladh_w)
+DO_VQDMLADH_OP(vqdmladhxb, 1, int8_t, mve_mergemask_sb, 1, 0,
+               do_vqdmladh_b)
+DO_VQDMLADH_OP(vqdmladhxh, 2, int16_t, mve_mergemask_sh, 1, 0,
+               do_vqdmladh_h)
+DO_VQDMLADH_OP(vqdmladhxw, 4, int32_t, mve_mergemask_sw, 1, 0,
+               do_vqdmladh_w)
+DO_VQDMLADH_OP(vqrdmladhb, 1, int8_t, mve_mergemask_sb, 0, 1,
+               do_vqdmladh_b)
+DO_VQDMLADH_OP(vqrdmladhh, 2, int16_t, mve_mergemask_sh, 0, 1,
+               do_vqdmladh_h)
+DO_VQDMLADH_OP(vqrdmladhw, 4, int32_t, mve_mergemask_sw, 0, 1,
+               do_vqdmladh_w)
+DO_VQDMLADH_OP(vqrdmladhxb, 1, int8_t, mve_mergemask_sb, 1, 1,
+               do_vqdmladh_b)
+DO_VQDMLADH_OP(vqrdmladhxh, 2, int16_t, mve_mergemask_sh, 1, 1,
+               do_vqdmladh_h)
+DO_VQDMLADH_OP(vqrdmladhxw, 4, int32_t, mve_mergemask_sw, 1, 1,
+               do_vqdmladh_w)
+DO_VQDMLADH_OP(vqdmlsdhb, 1, int8_t, mve_mergemask_sb, 0, 0, do_vqdmlsdh_b)
+DO_VQDMLADH_OP(vqdmlsdhh, 2, int16_t, mve_mergemask_sh, 0, 0,
+               do_vqdmlsdh_h)
+DO_VQDMLADH_OP(vqdmlsdhw, 4, int32_t, mve_mergemask_sw, 0, 0,
+               do_vqdmlsdh_w)
+DO_VQDMLADH_OP(vqdmlsdhxb, 1, int8_t, mve_mergemask_sb, 1, 0,
+               do_vqdmlsdh_b)
+DO_VQDMLADH_OP(vqdmlsdhxh, 2, int16_t, mve_mergemask_sh, 1, 0,
+               do_vqdmlsdh_h)
+DO_VQDMLADH_OP(vqdmlsdhxw, 4, int32_t, mve_mergemask_sw, 1, 0,
+               do_vqdmlsdh_w)
+DO_VQDMLADH_OP(vqrdmlsdhb, 1, int8_t, mve_mergemask_sb, 0, 1,
+               do_vqdmlsdh_b)
+DO_VQDMLADH_OP(vqrdmlsdhh, 2, int16_t, mve_mergemask_sh, 0, 1,
+               do_vqdmlsdh_h)
+DO_VQDMLADH_OP(vqrdmlsdhw, 4, int32_t, mve_mergemask_sw, 0, 1,
+               do_vqdmlsdh_w)
+DO_VQDMLADH_OP(vqrdmlsdhxb, 1, int8_t, mve_mergemask_sb, 1, 1,
+               do_vqdmlsdh_b)
+DO_VQDMLADH_OP(vqrdmlsdhxh, 2, int16_t, mve_mergemask_sh, 1, 1,
+               do_vqdmlsdh_h)
+DO_VQDMLADH_OP(vqrdmlsdhxw, 4, int32_t, mve_mergemask_sw, 1, 1,
+               do_vqdmlsdh_w)
+
+#define DO_VSHLL(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, MERGE, FN)      \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,           \
+                                void *vm, uint32_t shift)             \
+    {                                                                 \
+        LTYPE *d = vd;                                                \
+        TYPE *m = vm;                                                 \
+        uint16_t mask = mve_element_mask(env);                        \
+        unsigned le;                                                  \
+                                                                      \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {       \
+            LTYPE r = (LTYPE)FN(m[glue(H, ESIZE)(le * 2 + TOP)],      \
+                                shift);                               \
+                                                                      \
+            MERGE(&d[glue(H, LESIZE)(le)], r, mask);                  \
+        }                                                             \
+        mve_advance_vpt(env);                                         \
+    }
+
+#define DO_SHLL_S(N, SHIFT) ((int64_t)(N) * (1LL << (SHIFT)))
+#define DO_SHLL_U(N, SHIFT) ((uint64_t)(N) << (SHIFT))
+
+DO_VSHLL(vshllbsb, 0, 1, int8_t, 2, int16_t, mve_mergemask_sh,
+         DO_SHLL_S)
+DO_VSHLL(vshllbsh, 0, 2, int16_t, 4, int32_t, mve_mergemask_sw,
+         DO_SHLL_S)
+DO_VSHLL(vshllbub, 0, 1, uint8_t, 2, uint16_t, mve_mergemask_uh,
+         DO_SHLL_U)
+DO_VSHLL(vshllbuh, 0, 2, uint16_t, 4, uint32_t, mve_mergemask_uw,
+         DO_SHLL_U)
+DO_VSHLL(vshlltsb, 1, 1, int8_t, 2, int16_t, mve_mergemask_sh,
+         DO_SHLL_S)
+DO_VSHLL(vshlltsh, 1, 2, int16_t, 4, int32_t, mve_mergemask_sw,
+         DO_SHLL_S)
+DO_VSHLL(vshlltub, 1, 1, uint8_t, 2, uint16_t, mve_mergemask_uh,
+         DO_SHLL_U)
+DO_VSHLL(vshlltuh, 1, 2, uint16_t, 4, uint32_t, mve_mergemask_uw,
+         DO_SHLL_U)
+
+#define DO_SHRN(N, SHIFT) ((N) >> (SHIFT))
+
+#define DO_VSHRN(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, MERGE, FN)       \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,           \
+                                void *vm, uint32_t shift)             \
+    {                                                                 \
+        TYPE *d = vd;                                                 \
+        LTYPE *m = vm;                                                \
+        uint16_t mask = mve_element_mask(env) >> (ESIZE * TOP);       \
+        unsigned le;                                                  \
+                                                                      \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {       \
+            TYPE r = (TYPE)FN(m[glue(H, LESIZE)(le)], shift);         \
+                                                                      \
+            MERGE(&d[glue(H, ESIZE)(le * 2 + TOP)], r, mask);         \
+        }                                                             \
+        mve_advance_vpt(env);                                         \
+    }
+
+static inline uint64_t do_urshr(uint64_t n, uint32_t shift)
+{
+    return (n >> shift) + ((n >> (shift - 1)) & 1);
+}
+
+DO_VSHRN(vshrnbb, 0, 1, uint8_t, 2, uint16_t, mve_mergemask_ub, DO_SHRN)
+DO_VSHRN(vshrnbh, 0, 2, uint16_t, 4, uint32_t, mve_mergemask_uh, DO_SHRN)
+DO_VSHRN(vshrntb, 1, 1, uint8_t, 2, uint16_t, mve_mergemask_ub, DO_SHRN)
+DO_VSHRN(vshrnth, 1, 2, uint16_t, 4, uint32_t, mve_mergemask_uh, DO_SHRN)
+DO_VSHRN(vrshrnbb, 0, 1, uint8_t, 2, uint16_t, mve_mergemask_ub,
+         do_urshr)
+DO_VSHRN(vrshrnbh, 0, 2, uint16_t, 4, uint32_t, mve_mergemask_uh,
+         do_urshr)
+DO_VSHRN(vrshrntb, 1, 1, uint8_t, 2, uint16_t, mve_mergemask_ub,
+         do_urshr)
+DO_VSHRN(vrshrnth, 1, 2, uint16_t, 4, uint32_t, mve_mergemask_uh,
+         do_urshr)
+
+static inline int64_t do_srshr(int64_t n, uint32_t shift)
+{
+    return (n >> shift) + ((n >> (shift - 1)) & 1);
+}
+
+#define DO_VSHRN_SAT(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, MERGE, FN)  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,           \
+                                void *vm, uint32_t shift)             \
+    {                                                                 \
+        TYPE *d = vd;                                                 \
+        LTYPE *m = vm;                                                \
+        uint16_t mask = mve_element_mask(env) >> (ESIZE * TOP);       \
+        bool qc = false;                                              \
+        unsigned le;                                                  \
+                                                                      \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {       \
+            bool sat = false;                                         \
+            TYPE r = (TYPE)FN(m[glue(H, LESIZE)(le)], shift, &sat);   \
+                                                                      \
+            MERGE(&d[glue(H, ESIZE)(le * 2 + TOP)], r, mask);         \
+            qc |= sat && (mask & 1);                                  \
+        }                                                             \
+        if (qc) {                                                     \
+            env->vfp.qc[0] = qc;                                      \
+        }                                                             \
+        mve_advance_vpt(env);                                         \
+    }
+
+#define DO_SHRN_SB(N, SHIFT, SATP)                                    \
+    mve_do_sat_bhs((int64_t)(N) >> (SHIFT), INT8_MIN, INT8_MAX, SATP)
+#define DO_SHRN_SH(N, SHIFT, SATP)                                    \
+    mve_do_sat_bhs((int64_t)(N) >> (SHIFT), INT16_MIN, INT16_MAX, SATP)
+#define DO_SHRN_UB(N, SHIFT, SATP)                                    \
+    mve_do_sat_bhs((uint64_t)(N) >> (SHIFT), 0, UINT8_MAX, SATP)
+#define DO_SHRN_UH(N, SHIFT, SATP)                                    \
+    mve_do_sat_bhs((uint64_t)(N) >> (SHIFT), 0, UINT16_MAX, SATP)
+#define DO_SHRUN_B(N, SHIFT, SATP)                                    \
+    mve_do_sat_bhs((int64_t)(N) >> (SHIFT), 0, UINT8_MAX, SATP)
+#define DO_SHRUN_H(N, SHIFT, SATP)                                    \
+    mve_do_sat_bhs((int64_t)(N) >> (SHIFT), 0, UINT16_MAX, SATP)
+#define DO_RSHRN_SB(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs(do_srshr(N, SHIFT), INT8_MIN, INT8_MAX, SATP)
+#define DO_RSHRN_SH(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs(do_srshr(N, SHIFT), INT16_MIN, INT16_MAX, SATP)
+#define DO_RSHRN_UB(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs(do_urshr(N, SHIFT), 0, UINT8_MAX, SATP)
+#define DO_RSHRN_UH(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs(do_urshr(N, SHIFT), 0, UINT16_MAX, SATP)
+#define DO_RSHRUN_B(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs(do_srshr(N, SHIFT), 0, UINT8_MAX, SATP)
+#define DO_RSHRUN_H(N, SHIFT, SATP)                                   \
+    mve_do_sat_bhs(do_srshr(N, SHIFT), 0, UINT16_MAX, SATP)
+
+#define DO_VSHRN_SAT_B(OPB, OPT, TYPE, LTYPE, MERGE, FN)              \
+    DO_VSHRN_SAT(OPB, 0, 1, TYPE, 2, LTYPE, MERGE, FN)                \
+    DO_VSHRN_SAT(OPT, 1, 1, TYPE, 2, LTYPE, MERGE, FN)
+
+#define DO_VSHRN_SAT_H(OPB, OPT, TYPE, LTYPE, MERGE, FN)              \
+    DO_VSHRN_SAT(OPB, 0, 2, TYPE, 4, LTYPE, MERGE, FN)                \
+    DO_VSHRN_SAT(OPT, 1, 2, TYPE, 4, LTYPE, MERGE, FN)
+
+DO_VSHRN_SAT_B(vqshrnb_sb, vqshrnt_sb, int8_t, int16_t,
+               mve_mergemask_sb, DO_SHRN_SB)
+DO_VSHRN_SAT_H(vqshrnb_sh, vqshrnt_sh, int16_t, int32_t,
+               mve_mergemask_sh, DO_SHRN_SH)
+DO_VSHRN_SAT_B(vqshrnb_ub, vqshrnt_ub, uint8_t, uint16_t,
+               mve_mergemask_ub, DO_SHRN_UB)
+DO_VSHRN_SAT_H(vqshrnb_uh, vqshrnt_uh, uint16_t, uint32_t,
+               mve_mergemask_uh, DO_SHRN_UH)
+DO_VSHRN_SAT_B(vqshrunbb, vqshruntb, uint8_t, int16_t,
+               mve_mergemask_ub, DO_SHRUN_B)
+DO_VSHRN_SAT_H(vqshrunbh, vqshrunth, uint16_t, int32_t,
+               mve_mergemask_uh, DO_SHRUN_H)
+DO_VSHRN_SAT_B(vqrshrnb_sb, vqrshrnt_sb, int8_t, int16_t,
+               mve_mergemask_sb, DO_RSHRN_SB)
+DO_VSHRN_SAT_H(vqrshrnb_sh, vqrshrnt_sh, int16_t, int32_t,
+               mve_mergemask_sh, DO_RSHRN_SH)
+DO_VSHRN_SAT_B(vqrshrnb_ub, vqrshrnt_ub, uint8_t, uint16_t,
+               mve_mergemask_ub, DO_RSHRN_UB)
+DO_VSHRN_SAT_H(vqrshrnb_uh, vqrshrnt_uh, uint16_t, uint32_t,
+               mve_mergemask_uh, DO_RSHRN_UH)
+DO_VSHRN_SAT_B(vqrshrunbb, vqrshruntb, uint8_t, int16_t,
+               mve_mergemask_ub, DO_RSHRUN_B)
+DO_VSHRN_SAT_H(vqrshrunbh, vqrshrunth, uint16_t, int32_t,
+               mve_mergemask_uh, DO_RSHRUN_H)
+
+#define DO_VMOVN(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, MERGE)           \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd, void *vm)  \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        LTYPE *m = vm;                                                 \
+        uint16_t mask = mve_element_mask(env) >> (ESIZE * TOP);        \
+        unsigned le;                                                   \
+                                                                       \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {        \
+            TYPE r = (TYPE)m[glue(H, LESIZE)(le)];                     \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(le * 2 + TOP)], r, mask);          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VMOVN(vmovnbb, 0, 1, uint8_t, 2, uint16_t, mve_mergemask_ub)
+DO_VMOVN(vmovnbh, 0, 2, uint16_t, 4, uint32_t, mve_mergemask_uh)
+DO_VMOVN(vmovntb, 1, 1, uint8_t, 2, uint16_t, mve_mergemask_ub)
+DO_VMOVN(vmovnth, 1, 2, uint16_t, 4, uint32_t, mve_mergemask_uh)
+
+#define DO_VMOVN_SAT(OP, TOP, ESIZE, TYPE, LESIZE, LTYPE, MERGE, FN)   \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd, void *vm)  \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        LTYPE *m = vm;                                                 \
+        uint16_t mask = mve_element_mask(env) >> (ESIZE * TOP);        \
+        bool qc = false;                                               \
+        unsigned le;                                                   \
+                                                                       \
+        for (le = 0; le < 16 / LESIZE; le++, mask >>= LESIZE) {        \
+            bool sat = false;                                          \
+            TYPE r = (TYPE)FN(m[glue(H, LESIZE)(le)], &sat);           \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(le * 2 + TOP)], r, mask);          \
+            qc |= sat && (mask & 1);                                   \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_MOVN_SB(N, SATP)                                            \
+    mve_do_sat_bhs((int64_t)(N), INT8_MIN, INT8_MAX, SATP)
+#define DO_MOVN_SH(N, SATP)                                            \
+    mve_do_sat_bhs((int64_t)(N), INT16_MIN, INT16_MAX, SATP)
+#define DO_MOVN_UB(N, SATP)                                            \
+    mve_do_sat_bhs((uint64_t)(N), 0, UINT8_MAX, SATP)
+#define DO_MOVN_UH(N, SATP)                                            \
+    mve_do_sat_bhs((uint64_t)(N), 0, UINT16_MAX, SATP)
+#define DO_MOVUN_B(N, SATP)                                            \
+    mve_do_sat_bhs((int64_t)(N), 0, UINT8_MAX, SATP)
+#define DO_MOVUN_H(N, SATP)                                            \
+    mve_do_sat_bhs((int64_t)(N), 0, UINT16_MAX, SATP)
+
+#define DO_VMOVN_SAT_B(OPB, OPT, TYPE, LTYPE, MERGE, FN)               \
+    DO_VMOVN_SAT(OPB, 0, 1, TYPE, 2, LTYPE, MERGE, FN)                 \
+    DO_VMOVN_SAT(OPT, 1, 1, TYPE, 2, LTYPE, MERGE, FN)
+
+#define DO_VMOVN_SAT_H(OPB, OPT, TYPE, LTYPE, MERGE, FN)               \
+    DO_VMOVN_SAT(OPB, 0, 2, TYPE, 4, LTYPE, MERGE, FN)                 \
+    DO_VMOVN_SAT(OPT, 1, 2, TYPE, 4, LTYPE, MERGE, FN)
+
+DO_VMOVN_SAT_B(vqmovnbsb, vqmovntsb, int8_t, int16_t,
+               mve_mergemask_sb, DO_MOVN_SB)
+DO_VMOVN_SAT_H(vqmovnbsh, vqmovntsh, int16_t, int32_t,
+               mve_mergemask_sh, DO_MOVN_SH)
+DO_VMOVN_SAT_B(vqmovnbub, vqmovntub, uint8_t, uint16_t,
+               mve_mergemask_ub, DO_MOVN_UB)
+DO_VMOVN_SAT_H(vqmovnbuh, vqmovntuh, uint16_t, uint32_t,
+               mve_mergemask_uh, DO_MOVN_UH)
+DO_VMOVN_SAT_B(vqmovunbb, vqmovuntb, uint8_t, int16_t,
+               mve_mergemask_ub, DO_MOVUN_B)
+DO_VMOVN_SAT_H(vqmovunbh, vqmovunth, uint16_t, int32_t,
+               mve_mergemask_uh, DO_MOVUN_H)
+
+uint32_t HELPER(mve_vshlc)(CPUARMState *env, void *vd, uint32_t rdm,
+                           uint32_t shift)
+{
+    uint32_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    unsigned e;
+    uint32_t r;
+
+    if (shift == 0) {
+        for (e = 0; e < 16 / 4; e++, mask >>= 4) {
+            r = rdm;
+            if (mask & 1) {
+                rdm = d[H4(e)];
+            }
+            mve_mergemask_uw(&d[H4(e)], r, mask);
+        }
+    } else {
+        uint32_t shiftmask = MAKE_64BIT_MASK(0, shift);
+
+        for (e = 0; e < 16 / 4; e++, mask >>= 4) {
+            r = (d[H4(e)] << shift) | (rdm & shiftmask);
+            if (mask & 1) {
+                rdm = d[H4(e)] >> (32 - shift);
+            }
+            mve_mergemask_uw(&d[H4(e)], r, mask);
+        }
+    }
+    mve_advance_vpt(env);
+    return rdm;
+}
+
+#define DO_2SHIFT_INSERT(OP, ESIZE, SHIFTFN, MASKFN)                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,           \
+                                void *vm, uint32_t shift)             \
+    {                                                                 \
+        uint64_t *d = vd;                                             \
+        uint64_t *m = vm;                                             \
+        uint16_t mask;                                                \
+        uint64_t shiftmask;                                           \
+        unsigned e;                                                   \
+                                                                      \
+        if (shift == ESIZE * 8) {                                     \
+            mve_advance_vpt(env);                                     \
+            return;                                                   \
+        }                                                             \
+        mask = mve_element_mask(env);                                 \
+        shiftmask = dup_const(ESIZE / 2, MASKFN(ESIZE * 8, shift));   \
+        for (e = 0; e < 16 / 8; e++, mask >>= 8) {                    \
+            uint64_t r = (SHIFTFN(m[H8(e)], shift) & shiftmask) |     \
+                         (d[H8(e)] & ~shiftmask);                    \
+                                                                      \
+            mve_mergemask_uq(&d[H8(e)], r, mask);                    \
+        }                                                             \
+        mve_advance_vpt(env);                                         \
+    }
+
+#define DO_SHL(N, SHIFT) ((N) << (SHIFT))
+#define DO_SHR(N, SHIFT) ((N) >> (SHIFT))
+#define SHL_MASK(EBITS, SHIFT) MAKE_64BIT_MASK((SHIFT), (EBITS) - (SHIFT))
+#define SHR_MASK(EBITS, SHIFT) MAKE_64BIT_MASK(0, (EBITS) - (SHIFT))
+
+DO_2SHIFT_INSERT(vsrib, 1, DO_SHR, SHR_MASK)
+DO_2SHIFT_INSERT(vsrih, 2, DO_SHR, SHR_MASK)
+DO_2SHIFT_INSERT(vsriw, 4, DO_SHR, SHR_MASK)
+DO_2SHIFT_INSERT(vslib, 1, DO_SHL, SHL_MASK)
+DO_2SHIFT_INSERT(vslih, 2, DO_SHL, SHL_MASK)
+DO_2SHIFT_INSERT(vsliw, 4, DO_SHL, SHL_MASK)
+
+#define DO_1OP_SAT(OP, ESIZE, TYPE, MERGE, FN)                         \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd, void *vm)  \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        bool qc = false;                                               \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE) {             \
+            bool sat = false;                                          \
+            TYPE r = FN(m[glue(H, ESIZE)(e)], &sat);                   \
+                                                                       \
+            MERGE(&d[glue(H, ESIZE)(e)], r, mask);                     \
+            qc |= sat && (mask & 1);                                   \
+        }                                                              \
+        if (qc) {                                                      \
+            env->vfp.qc[0] = qc;                                       \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VQABS_B(N, SATP) \
+    mve_do_sat_bhs(DO_ABS((int64_t)N), INT8_MIN, INT8_MAX, SATP)
+#define DO_VQABS_H(N, SATP) \
+    mve_do_sat_bhs(DO_ABS((int64_t)N), INT16_MIN, INT16_MAX, SATP)
+#define DO_VQABS_W(N, SATP) \
+    mve_do_sat_bhs(DO_ABS((int64_t)N), INT32_MIN, INT32_MAX, SATP)
+#define DO_VQNEG_B(N, SATP) \
+    mve_do_sat_bhs(-(int64_t)N, INT8_MIN, INT8_MAX, SATP)
+#define DO_VQNEG_H(N, SATP) \
+    mve_do_sat_bhs(-(int64_t)N, INT16_MIN, INT16_MAX, SATP)
+#define DO_VQNEG_W(N, SATP) \
+    mve_do_sat_bhs(-(int64_t)N, INT32_MIN, INT32_MAX, SATP)
+
+DO_1OP_SAT(vqabsb, 1, int8_t, mve_mergemask_sb, DO_VQABS_B)
+DO_1OP_SAT(vqabsh, 2, int16_t, mve_mergemask_sh, DO_VQABS_H)
+DO_1OP_SAT(vqabsw, 4, int32_t, mve_mergemask_sw, DO_VQABS_W)
+DO_1OP_SAT(vqnegb, 1, int8_t, mve_mergemask_sb, DO_VQNEG_B)
+DO_1OP_SAT(vqnegh, 2, int16_t, mve_mergemask_sh, DO_VQNEG_H)
+DO_1OP_SAT(vqnegw, 4, int32_t, mve_mergemask_sw, DO_VQNEG_W)
+
+#undef DO_1OP
+#undef DO_CLS_B
+#undef DO_CLS_H
+#undef DO_CLZ_B
+#undef DO_CLZ_H
+#undef DO_NOT
+#undef DO_ABS
+#undef DO_NEG
+#undef DO_VMAXMINA
+#undef DO_LDAV
+#undef DO_LDAVH
+#undef DO_DAV
+#undef DO_DAV_S
+#undef DO_DAV_U
+#undef DO_VADDV
+#undef DO_VMAXMINV
+#undef DO_VMAXMINV_U
+#undef DO_VMAXMINV_S
+#undef DO_FP_VMAXMINV
+#undef DO_VADDLV
+#undef DO_VABAV
+#undef DO_VCADD
+#undef DO_VCADD_ALL
+#undef DO_2OP_SAT
+#undef DO_VQDMLADH_OP
+#undef DO_2OP_SCALAR_ACC
+#undef DO_2OP_SCALAR_SAT_ACC
+#undef DO_2OP_SAT_L
+#undef DO_2OP_SCALAR_SAT_L
+#undef DO_VMLA
+#undef DO_VMLAS
+#undef DO_VQDMLAH_B
+#undef DO_VQDMLAH_H
+#undef DO_VQDMLAH_W
+#undef DO_VQRDMLAH_B
+#undef DO_VQRDMLAH_H
+#undef DO_VQRDMLAH_W
+#undef DO_VQDMLASH_B
+#undef DO_VQDMLASH_H
+#undef DO_VQDMLASH_W
+#undef DO_VQRDMLASH_B
+#undef DO_VQRDMLASH_H
+#undef DO_VQRDMLASH_W
+#undef DO_SQADD_B
+#undef DO_SQADD_H
+#undef DO_SQADD_W
+#undef DO_UQADD_B
+#undef DO_UQADD_H
+#undef DO_UQADD_W
+#undef DO_SQSUB_B
+#undef DO_SQSUB_H
+#undef DO_SQSUB_W
+#undef DO_UQSUB_B
+#undef DO_UQSUB_H
+#undef DO_UQSUB_W
+#undef DO_QDMULH_B
+#undef DO_QDMULH_H
+#undef DO_QDMULH_W
+#undef DO_QRDMULH_B
+#undef DO_QRDMULH_H
+#undef DO_QRDMULH_W
+#undef SATMASK16B
+#undef SATMASK16T
+#undef SATMASK32
+#undef DO_1OP_SAT
+#undef DO_VQABS_B
+#undef DO_VQABS_H
+#undef DO_VQABS_W
+#undef DO_VQNEG_B
+#undef DO_VQNEG_H
+#undef DO_VQNEG_W
+#undef DO_2SHIFT_IMM
+#undef DO_2SHIFT_IMM_U
+#undef DO_2SHIFT_IMM_S
+#undef DO_2SHIFT_IMM_SAT
+#undef DO_2SHIFT_IMM_SAT_U
+#undef DO_2SHIFT_IMM_SAT_S
+#undef DO_SQSHLI_B
+#undef DO_SQSHLI_H
+#undef DO_SQSHLI_W
+#undef DO_UQSHLI_B
+#undef DO_UQSHLI_H
+#undef DO_UQSHLI_W
+#undef DO_SQRSHLI_B
+#undef DO_SQRSHLI_H
+#undef DO_SQRSHLI_W
+#undef DO_UQRSHLI_B
+#undef DO_UQRSHLI_H
+#undef DO_UQRSHLI_W
+#undef DO_SUQSHLI_B
+#undef DO_SUQSHLI_H
+#undef DO_SUQSHLI_W
+#undef DO_VSHLL
+#undef DO_SHLL_S
+#undef DO_SHLL_U
+#undef DO_VSHRN
+#undef DO_SHRN
+#undef DO_VSHRN_SAT
+#undef DO_SHRN_SB
+#undef DO_SHRN_SH
+#undef DO_SHRN_UB
+#undef DO_SHRN_UH
+#undef DO_SHRUN_B
+#undef DO_SHRUN_H
+#undef DO_RSHRN_SB
+#undef DO_RSHRN_SH
+#undef DO_RSHRN_UB
+#undef DO_RSHRN_UH
+#undef DO_RSHRUN_B
+#undef DO_RSHRUN_H
+#undef DO_VSHRN_SAT_B
+#undef DO_VSHRN_SAT_H
+#undef DO_VMOVN
+#undef DO_VMOVN_SAT
+#undef DO_MOVN_SB
+#undef DO_MOVN_SH
+#undef DO_MOVN_UB
+#undef DO_MOVN_UH
+#undef DO_MOVUN_B
+#undef DO_MOVUN_H
+#undef DO_VMOVN_SAT_B
+#undef DO_VMOVN_SAT_H
+#undef DO_2SHIFT_INSERT
+#undef DO_SHL
+#undef DO_SHR
+#undef SHL_MASK
+#undef SHR_MASK
+#undef DO_VCMP
+#undef DO_VCMP_SCALAR
+#undef DO_VCMP_S
+#undef DO_VCMP_U
+#undef DO_EQ
+#undef DO_NE
+#undef DO_GE
+#undef DO_LT
+#undef DO_GT
+#undef DO_LE
+#undef DO_1OP_IMM
+#undef DO_MOVI
+#undef DO_ANDI
+#undef DO_ORRI
+
+void HELPER(mve_vldrb)(CPUARMState *env, void *vd, uint32_t addr)
+{
+    uint8_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    uint16_t eci_mask = mve_eci_mask(env);
+    unsigned b;
+
+    for (b = 0; b < 16; b++) {
+        if (eci_mask & (1U << b)) {
+            d[H1(b)] = (mask & (1U << b)) ?
+                cpu_ldub_data_ra(env, addr, GETPC()) : 0;
+        }
+        addr++;
+    }
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vldrh)(CPUARMState *env, void *vd, uint32_t addr)
+{
+    uint16_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    uint16_t eci_mask = mve_eci_mask(env);
+    unsigned b;
+    unsigned e;
+
+    for (b = 0, e = 0; b < 16; b += 2, e++) {
+        if (eci_mask & (1U << b)) {
+            d[H2(e)] = (mask & (1U << b)) ?
+                cpu_lduw_data_ra(env, addr, GETPC()) : 0;
+        }
+        addr += 2;
+    }
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vldrw)(CPUARMState *env, void *vd, uint32_t addr)
+{
+    uint32_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    uint16_t eci_mask = mve_eci_mask(env);
+    unsigned b;
+    unsigned e;
+
+    for (b = 0, e = 0; b < 16; b += 4, e++) {
+        if (eci_mask & (1U << b)) {
+            d[H4(e)] = (mask & (1U << b)) ?
+                cpu_ldl_data_ra(env, addr, GETPC()) : 0;
+        }
+        addr += 4;
+    }
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vstrb)(CPUARMState *env, void *vd, uint32_t addr)
+{
+    uint8_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    unsigned b;
+
+    for (b = 0; b < 16; b++) {
+        if (mask & (1U << b)) {
+            cpu_stb_data_ra(env, addr, d[H1(b)], GETPC());
+        }
+        addr++;
+    }
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vstrh)(CPUARMState *env, void *vd, uint32_t addr)
+{
+    uint16_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    unsigned b;
+    unsigned e;
+
+    for (b = 0, e = 0; b < 16; b += 2, e++) {
+        if (mask & (1U << b)) {
+            cpu_stw_data_ra(env, addr, d[H2(e)], GETPC());
+        }
+        addr += 2;
+    }
+    mve_advance_vpt(env);
+}
+
+void HELPER(mve_vstrw)(CPUARMState *env, void *vd, uint32_t addr)
+{
+    uint32_t *d = vd;
+    uint16_t mask = mve_element_mask(env);
+    unsigned b;
+    unsigned e;
+
+    for (b = 0, e = 0; b < 16; b += 4, e++) {
+        if (mask & (1U << b)) {
+            cpu_stl_data_ra(env, addr, d[H4(e)], GETPC());
+        }
+        addr += 4;
+    }
+    mve_advance_vpt(env);
+}
+
+#define DO_VLDR_WIDE(OP, MSIZE, LDTYPE, ESIZE, TYPE)                   \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                uint32_t addr)                         \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        uint16_t eci_mask = mve_eci_mask(env);                         \
+        unsigned b;                                                    \
+        unsigned e;                                                    \
+                                                                       \
+        for (b = 0, e = 0; b < 16; b += ESIZE, e++) {                  \
+            if (eci_mask & (1U << b)) {                                \
+                d[glue(H, ESIZE)(e)] = (mask & (1U << b)) ?            \
+                    cpu_##LDTYPE##_data_ra(env, addr, GETPC()) : 0;    \
+            }                                                          \
+            addr += MSIZE;                                             \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VSTR_NARROW(OP, MSIZE, STTYPE, ESIZE, TYPE)                 \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                uint32_t addr)                         \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        unsigned b;                                                    \
+        unsigned e;                                                    \
+                                                                       \
+        for (b = 0, e = 0; b < 16; b += ESIZE, e++) {                  \
+            if (mask & (1U << b)) {                                    \
+                cpu_##STTYPE##_data_ra(env, addr,                      \
+                                       d[glue(H, ESIZE)(e)], GETPC()); \
+            }                                                          \
+            addr += MSIZE;                                             \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VLDR_WIDE(vldrb_sh, 1, ldsb, 2, int16_t)
+DO_VLDR_WIDE(vldrb_uh, 1, ldub, 2, uint16_t)
+DO_VLDR_WIDE(vldrb_sw, 1, ldsb, 4, int32_t)
+DO_VLDR_WIDE(vldrb_uw, 1, ldub, 4, uint32_t)
+DO_VLDR_WIDE(vldrh_sw, 2, ldsw, 4, int32_t)
+DO_VLDR_WIDE(vldrh_uw, 2, lduw, 4, uint32_t)
+
+DO_VSTR_NARROW(vstrb_h, 1, stb, 2, int16_t)
+DO_VSTR_NARROW(vstrb_w, 1, stb, 4, int32_t)
+DO_VSTR_NARROW(vstrh_w, 2, stw, 4, int32_t)
+
+#undef DO_VLDR_WIDE
+#undef DO_VSTR_NARROW
+
+#define ADDR_ADD(BASE, OFFSET) ((BASE) + (OFFSET))
+#define ADDR_ADD_OSH(BASE, OFFSET) ((BASE) + ((OFFSET) << 1))
+#define ADDR_ADD_OSW(BASE, OFFSET) ((BASE) + ((OFFSET) << 2))
+#define ADDR_ADD_OSD(BASE, OFFSET) ((BASE) + ((OFFSET) << 3))
+
+#define DO_VLDR_SG(OP, LDTYPE, ESIZE, TYPE, OFFTYPE, ADDRFN, WB)       \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vm, uint32_t base)               \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        OFFTYPE *m = vm;                                               \
+        uint16_t mask = mve_element_mask(env);                         \
+        uint16_t eci_mask = mve_eci_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE,               \
+             eci_mask >>= ESIZE) {                                     \
+            uint32_t addr;                                             \
+                                                                       \
+            if (!(eci_mask & 1)) {                                     \
+                continue;                                              \
+            }                                                          \
+            addr = ADDRFN(base, m[glue(H, ESIZE)(e)]);                 \
+            d[glue(H, ESIZE)(e)] = (mask & 1) ?                        \
+                cpu_##LDTYPE##_data_ra(env, addr, GETPC()) : 0;        \
+            if (WB) {                                                  \
+                m[glue(H, ESIZE)(e)] = addr;                           \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VSTR_SG(OP, STTYPE, ESIZE, TYPE, ADDRFN, WB)                \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vm, uint32_t base)               \
+    {                                                                  \
+        TYPE *d = vd;                                                  \
+        TYPE *m = vm;                                                  \
+        uint16_t mask = mve_element_mask(env);                         \
+        uint16_t eci_mask = mve_eci_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / ESIZE; e++, mask >>= ESIZE,               \
+             eci_mask >>= ESIZE) {                                     \
+            uint32_t addr;                                             \
+                                                                       \
+            if (!(eci_mask & 1)) {                                     \
+                continue;                                              \
+            }                                                          \
+            addr = ADDRFN(base, m[glue(H, ESIZE)(e)]);                 \
+            if (mask & 1) {                                            \
+                cpu_##STTYPE##_data_ra(env, addr,                      \
+                                       d[glue(H, ESIZE)(e)], GETPC()); \
+            }                                                          \
+            if (WB) {                                                  \
+                m[glue(H, ESIZE)(e)] = addr;                           \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VLDR64_SG(OP, ADDRFN, WB)                                   \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vm, uint32_t base)               \
+    {                                                                  \
+        uint32_t *d = vd;                                              \
+        uint32_t *m = vm;                                              \
+        uint16_t mask = mve_element_mask(env);                         \
+        uint16_t eci_mask = mve_eci_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / 4; e++, mask >>= 4, eci_mask >>= 4) {     \
+            uint32_t addr;                                             \
+                                                                       \
+            if (!(eci_mask & 1)) {                                     \
+                continue;                                              \
+            }                                                          \
+            addr = ADDRFN(base, m[H4(e & ~1)]);                        \
+            addr += 4 * (e & 1);                                       \
+            d[H4(e)] = (mask & 1) ?                                    \
+                cpu_ldl_data_ra(env, addr, GETPC()) : 0;               \
+            if (WB && (e & 1)) {                                       \
+                m[H4(e & ~1)] = addr - 4;                              \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+#define DO_VSTR64_SG(OP, ADDRFN, WB)                                   \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, void *vd,            \
+                                void *vm, uint32_t base)               \
+    {                                                                  \
+        uint32_t *d = vd;                                              \
+        uint32_t *m = vm;                                              \
+        uint16_t mask = mve_element_mask(env);                         \
+        uint16_t eci_mask = mve_eci_mask(env);                         \
+        unsigned e;                                                    \
+                                                                       \
+        for (e = 0; e < 16 / 4; e++, mask >>= 4, eci_mask >>= 4) {     \
+            uint32_t addr;                                             \
+                                                                       \
+            if (!(eci_mask & 1)) {                                     \
+                continue;                                              \
+            }                                                          \
+            addr = ADDRFN(base, m[H4(e & ~1)]);                        \
+            addr += 4 * (e & 1);                                       \
+            if (mask & 1) {                                            \
+                cpu_stl_data_ra(env, addr, d[H4(e)], GETPC());         \
+            }                                                          \
+            if (WB && (e & 1)) {                                       \
+                m[H4(e & ~1)] = addr - 4;                              \
+            }                                                          \
+        }                                                              \
+        mve_advance_vpt(env);                                          \
+    }
+
+DO_VLDR_SG(vldrb_sg_sh, ldsb, 2, int16_t, uint16_t, ADDR_ADD, false)
+DO_VLDR_SG(vldrb_sg_sw, ldsb, 4, int32_t, uint32_t, ADDR_ADD, false)
+DO_VLDR_SG(vldrh_sg_sw, ldsw, 4, int32_t, uint32_t, ADDR_ADD, false)
+
+DO_VLDR_SG(vldrb_sg_ub, ldub, 1, uint8_t, uint8_t, ADDR_ADD, false)
+DO_VLDR_SG(vldrb_sg_uh, ldub, 2, uint16_t, uint16_t, ADDR_ADD, false)
+DO_VLDR_SG(vldrb_sg_uw, ldub, 4, uint32_t, uint32_t, ADDR_ADD, false)
+DO_VLDR_SG(vldrh_sg_uh, lduw, 2, uint16_t, uint16_t, ADDR_ADD, false)
+DO_VLDR_SG(vldrh_sg_uw, lduw, 4, uint32_t, uint32_t, ADDR_ADD, false)
+DO_VLDR_SG(vldrw_sg_uw, ldl, 4, uint32_t, uint32_t, ADDR_ADD, false)
+DO_VLDR64_SG(vldrd_sg_ud, ADDR_ADD, false)
+
+DO_VLDR_SG(vldrh_sg_os_sw, ldsw, 4, int32_t, uint32_t, ADDR_ADD_OSH, false)
+DO_VLDR_SG(vldrh_sg_os_uh, lduw, 2, uint16_t, uint16_t, ADDR_ADD_OSH, false)
+DO_VLDR_SG(vldrh_sg_os_uw, lduw, 4, uint32_t, uint32_t, ADDR_ADD_OSH, false)
+DO_VLDR_SG(vldrw_sg_os_uw, ldl, 4, uint32_t, uint32_t, ADDR_ADD_OSW, false)
+DO_VLDR64_SG(vldrd_sg_os_ud, ADDR_ADD_OSD, false)
+
+DO_VSTR_SG(vstrb_sg_ub, stb, 1, uint8_t, ADDR_ADD, false)
+DO_VSTR_SG(vstrb_sg_uh, stb, 2, uint16_t, ADDR_ADD, false)
+DO_VSTR_SG(vstrb_sg_uw, stb, 4, uint32_t, ADDR_ADD, false)
+DO_VSTR_SG(vstrh_sg_uh, stw, 2, uint16_t, ADDR_ADD, false)
+DO_VSTR_SG(vstrh_sg_uw, stw, 4, uint32_t, ADDR_ADD, false)
+DO_VSTR_SG(vstrw_sg_uw, stl, 4, uint32_t, ADDR_ADD, false)
+DO_VSTR64_SG(vstrd_sg_ud, ADDR_ADD, false)
+
+DO_VSTR_SG(vstrh_sg_os_uh, stw, 2, uint16_t, ADDR_ADD_OSH, false)
+DO_VSTR_SG(vstrh_sg_os_uw, stw, 4, uint32_t, ADDR_ADD_OSH, false)
+DO_VSTR_SG(vstrw_sg_os_uw, stl, 4, uint32_t, ADDR_ADD_OSW, false)
+DO_VSTR64_SG(vstrd_sg_os_ud, ADDR_ADD_OSD, false)
+
+DO_VLDR_SG(vldrw_sg_wb_uw, ldl, 4, uint32_t, uint32_t, ADDR_ADD, true)
+DO_VLDR64_SG(vldrd_sg_wb_ud, ADDR_ADD, true)
+DO_VSTR_SG(vstrw_sg_wb_uw, stl, 4, uint32_t, ADDR_ADD, true)
+DO_VSTR64_SG(vstrd_sg_wb_ud, ADDR_ADD, true)
+
+#undef DO_VLDR_SG
+#undef DO_VSTR_SG
+#undef DO_VLDR64_SG
+#undef DO_VSTR64_SG
+#undef ADDR_ADD
+#undef ADDR_ADD_OSH
+#undef ADDR_ADD_OSW
+#undef ADDR_ADD_OSD
+
+static uint32_t mve_ldl_le_data_ra(CPUARMState *env, uint32_t addr,
+                                   uintptr_t ra)
+{
+    uint32_t data = cpu_ldl_data_ra(env, addr, ra);
+
+    return arm_cpu_data_is_big_endian(env) ? bswap32(data) : data;
+}
+
+static void mve_stl_le_data_ra(CPUARMState *env, uint32_t addr,
+                               uint32_t data, uintptr_t ra)
+{
+    if (arm_cpu_data_is_big_endian(env)) {
+        data = bswap32(data);
+    }
+    cpu_stl_data_ra(env, addr, data, ra);
+}
+
+/*
+ * Deinterleaving loads/interleaving stores.
+ *
+ * For these helpers we are passed the index of the first Qreg.
+ * VLD2/VST2 also access Qn+1; VLD4/VST4 access Qn..Qn+3.
+ * The helpers are specialized for pattern and element size, so
+ * vld42h is VLD4 with pattern 2 and halfword elements.
+ *
+ * These instructions are beatwise but not predicated, so they honour ECI
+ * but do not use mve_element_mask().
+ */
+#define DO_VLD4B(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat, e;                                                   \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 4;                               \
+            data = mve_ldl_le_data_ra(env, addr, GETPC());             \
+            for (e = 0; e < 4; e++, data >>= 8) {                     \
+                uint8_t *qd = (uint8_t *)aa32_vfp_qreg(env, qnidx + e);\
+                                                                      \
+                qd[H1(off[beat])] = data;                              \
+            }                                                         \
+        }                                                             \
+    }
+
+#define DO_VLD4H(OP, O1, O2)                                          \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O1, O2, O2 };              \
+        uint32_t addr, data;                                           \
+        int y;                                                         \
+        uint16_t *qd;                                                  \
+                                                                      \
+        for (beat = 0, y = 0; beat < 4; beat++, mask >>= 4, y ^= 2) {  \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 8 + (beat & 1) * 4;              \
+            data = mve_ldl_le_data_ra(env, addr, GETPC());             \
+            qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y);            \
+            qd[H2(off[beat])] = data;                                  \
+            data >>= 16;                                               \
+            qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y + 1);        \
+            qd[H2(off[beat])] = data;                                  \
+        }                                                             \
+    }
+
+#define DO_VLD4W(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        uint32_t *qd;                                                  \
+        int y;                                                         \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 4;                               \
+            data = mve_ldl_le_data_ra(env, addr, GETPC());             \
+            y = (beat + (O1 & 2)) & 3;                                 \
+            qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + y);            \
+            qd[H4(off[beat] >> 2)] = data;                             \
+        }                                                             \
+    }
+
+DO_VLD4B(vld40b, 0, 1, 10, 11)
+DO_VLD4B(vld41b, 2, 3, 12, 13)
+DO_VLD4B(vld42b, 4, 5, 14, 15)
+DO_VLD4B(vld43b, 6, 7, 8, 9)
+
+DO_VLD4H(vld40h, 0, 5)
+DO_VLD4H(vld41h, 1, 6)
+DO_VLD4H(vld42h, 2, 7)
+DO_VLD4H(vld43h, 3, 4)
+
+DO_VLD4W(vld40w, 0, 1, 10, 11)
+DO_VLD4W(vld41w, 2, 3, 12, 13)
+DO_VLD4W(vld42w, 4, 5, 14, 15)
+DO_VLD4W(vld43w, 6, 7, 8, 9)
+
+#define DO_VLD2B(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat, e;                                                   \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        uint8_t *qd;                                                   \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 2;                               \
+            data = mve_ldl_le_data_ra(env, addr, GETPC());             \
+            for (e = 0; e < 4; e++, data >>= 8) {                     \
+                qd = (uint8_t *)aa32_vfp_qreg(env, qnidx + (e & 1));   \
+                qd[H1(off[beat] + (e >> 1))] = data;                   \
+            }                                                         \
+        }                                                             \
+    }
+
+#define DO_VLD2H(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        int e;                                                         \
+        uint16_t *qd;                                                  \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 4;                               \
+            data = mve_ldl_le_data_ra(env, addr, GETPC());             \
+            for (e = 0; e < 2; e++, data >>= 16) {                    \
+                qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + e);        \
+                qd[H2(off[beat])] = data;                              \
+            }                                                         \
+        }                                                             \
+    }
+
+#define DO_VLD2W(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        uint32_t *qd;                                                  \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat];                                   \
+            data = mve_ldl_le_data_ra(env, addr, GETPC());             \
+            qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + (beat & 1));   \
+            qd[H4(off[beat] >> 3)] = data;                             \
+        }                                                             \
+    }
+
+DO_VLD2B(vld20b, 0, 2, 12, 14)
+DO_VLD2B(vld21b, 4, 6, 8, 10)
+
+DO_VLD2H(vld20h, 0, 1, 6, 7)
+DO_VLD2H(vld21h, 2, 3, 4, 5)
+
+DO_VLD2W(vld20w, 0, 4, 24, 28)
+DO_VLD2W(vld21w, 8, 12, 16, 20)
+
+#define DO_VST4B(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat, e;                                                   \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 4;                               \
+            data = 0;                                                  \
+            for (e = 3; e >= 0; e--) {                                 \
+                uint8_t *qd = (uint8_t *)aa32_vfp_qreg(env, qnidx + e);\
+                                                                      \
+                data = (data << 8) | qd[H1(off[beat])];                \
+            }                                                         \
+            mve_stl_le_data_ra(env, addr, data, GETPC());              \
+        }                                                             \
+    }
+
+#define DO_VST4H(OP, O1, O2)                                          \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O1, O2, O2 };              \
+        uint32_t addr, data;                                           \
+        int y;                                                         \
+        uint16_t *qd;                                                  \
+                                                                      \
+        for (beat = 0, y = 0; beat < 4; beat++, mask >>= 4, y ^= 2) {  \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 8 + (beat & 1) * 4;              \
+            qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y);            \
+            data = qd[H2(off[beat])];                                  \
+            qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + y + 1);        \
+            data |= qd[H2(off[beat])] << 16;                           \
+            mve_stl_le_data_ra(env, addr, data, GETPC());              \
+        }                                                             \
+    }
+
+#define DO_VST4W(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        uint32_t *qd;                                                  \
+        int y;                                                         \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 4;                               \
+            y = (beat + (O1 & 2)) & 3;                                 \
+            qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + y);            \
+            data = qd[H4(off[beat] >> 2)];                             \
+            mve_stl_le_data_ra(env, addr, data, GETPC());              \
+        }                                                             \
+    }
+
+DO_VST4B(vst40b, 0, 1, 10, 11)
+DO_VST4B(vst41b, 2, 3, 12, 13)
+DO_VST4B(vst42b, 4, 5, 14, 15)
+DO_VST4B(vst43b, 6, 7, 8, 9)
+
+DO_VST4H(vst40h, 0, 5)
+DO_VST4H(vst41h, 1, 6)
+DO_VST4H(vst42h, 2, 7)
+DO_VST4H(vst43h, 3, 4)
+
+DO_VST4W(vst40w, 0, 1, 10, 11)
+DO_VST4W(vst41w, 2, 3, 12, 13)
+DO_VST4W(vst42w, 4, 5, 14, 15)
+DO_VST4W(vst43w, 6, 7, 8, 9)
+
+#define DO_VST2B(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat, e;                                                   \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        uint8_t *qd;                                                   \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 2;                               \
+            data = 0;                                                  \
+            for (e = 3; e >= 0; e--) {                                 \
+                qd = (uint8_t *)aa32_vfp_qreg(env, qnidx + (e & 1));   \
+                data = (data << 8) | qd[H1(off[beat] + (e >> 1))];     \
+            }                                                         \
+            mve_stl_le_data_ra(env, addr, data, GETPC());              \
+        }                                                             \
+    }
+
+#define DO_VST2H(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        int e;                                                         \
+        uint16_t *qd;                                                  \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat] * 4;                               \
+            data = 0;                                                  \
+            for (e = 1; e >= 0; e--) {                                 \
+                qd = (uint16_t *)aa32_vfp_qreg(env, qnidx + e);        \
+                data = (data << 16) | qd[H2(off[beat])];               \
+            }                                                         \
+            mve_stl_le_data_ra(env, addr, data, GETPC());              \
+        }                                                             \
+    }
+
+#define DO_VST2W(OP, O1, O2, O3, O4)                                  \
+    void HELPER(glue(mve_, OP))(CPUARMState *env, uint32_t qnidx,     \
+                                uint32_t base)                        \
+    {                                                                 \
+        int beat;                                                      \
+        uint16_t mask = mve_eci_mask(env);                             \
+        static const uint8_t off[4] = { O1, O2, O3, O4 };              \
+        uint32_t addr, data;                                           \
+        uint32_t *qd;                                                  \
+                                                                      \
+        for (beat = 0; beat < 4; beat++, mask >>= 4) {                \
+            if ((mask & 1) == 0) {                                     \
+                continue;                                             \
+            }                                                         \
+            addr = base + off[beat];                                   \
+            qd = (uint32_t *)aa32_vfp_qreg(env, qnidx + (beat & 1));   \
+            data = qd[H4(off[beat] >> 3)];                             \
+            mve_stl_le_data_ra(env, addr, data, GETPC());              \
+        }                                                             \
+    }
+
+DO_VST2B(vst20b, 0, 2, 12, 14)
+DO_VST2B(vst21b, 4, 6, 8, 10)
+
+DO_VST2H(vst20h, 0, 1, 6, 7)
+DO_VST2H(vst21h, 2, 3, 4, 5)
+
+DO_VST2W(vst20w, 0, 4, 24, 28)
+DO_VST2W(vst21w, 8, 12, 16, 20)
+
+#undef DO_VLD4B
+#undef DO_VLD4H
+#undef DO_VLD4W
+#undef DO_VLD2B
+#undef DO_VLD2H
+#undef DO_VLD2W
+#undef DO_VST4B
+#undef DO_VST4H
+#undef DO_VST4W
+#undef DO_VST2B
+#undef DO_VST2H
+#undef DO_VST2W
+
+void HELPER(mve_vpsel)(CPUARMState *env, void *vd, void *vn, void *vm)
+{
+    uint64_t *d = vd;
+    uint64_t *n = vn;
+    uint64_t *m = vm;
+    uint16_t mask = mve_element_mask(env);
+    uint16_t p0 = FIELD_EX32(env->v7m.vpr, V7M_VPR, P0);
+    unsigned e;
+
+    for (e = 0; e < 16 / 8; e++, mask >>= 8, p0 >>= 8) {
+        uint64_t r = m[H8(e)];
+
+        mve_mergemask_uq(&r, n[H8(e)], p0);
+        mve_mergemask_uq(&d[H8(e)], r, mask);
+    }
+    mve_advance_vpt(env);
+}
diff --git a/qemu/target/arm/sme_helper.c b/qemu/target/arm/sme_helper.c
new file mode 100644
index 0000000000..7b8d1e658f
--- /dev/null
+++ b/qemu/target/arm/sme_helper.c
@@ -0,0 +1,1010 @@
+/*
+ * ARM SME operations.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "internals.h"
+#include "tcg/tcg-gvec-desc.h"
+#include "exec/helper-proto.h"
+#include "exec/exec-all.h"
+#include "exec/cpu_ldst.h"
+#include "qemu/int128.h"
+#include "fpu/softfloat.h"
+
+#ifdef HOST_WORDS_BIGENDIAN
+#define H1(x)   ((x) ^ 7)
+#define H1_2(x) ((x) ^ 6)
+#define H1_4(x) ((x) ^ 4)
+#define H2(x)   ((x) ^ 3)
+#else
+#define H1(x)   (x)
+#define H1_2(x) (x)
+#define H1_4(x) (x)
+#define H2(x)   (x)
+#endif
+
+void HELPER(sme_zero)(CPUARMState *env, uint32_t imm, uint32_t svl)
+{
+    uint32_t i;
+
+    if (imm == 0xff) {
+        memset(env->zarray, 0, sizeof(env->zarray));
+        return;
+    }
+
+    for (i = 0; i < svl; i++) {
+        if (imm & (1 << (i % 8))) {
+            memset(&env->zarray[i], 0, svl);
+        }
+    }
+}
+
+#define tile_vslice_index(i) ((i) * sizeof(ARMVectorReg))
+#define tile_vslice_offset(byteoff) ((byteoff) * sizeof(ARMVectorReg))
+
+#define DO_MOVA_C(NAME, TYPE, H)                                      \
+void HELPER(NAME)(void *za, void *vn, void *vg, uint32_t desc)        \
+{                                                                     \
+    int i, oprsz = simd_oprsz(desc);                                  \
+    char *za_b = za;                                                  \
+    char *vn_b = vn;                                                  \
+    char *vg_b = vg;                                                  \
+                                                                      \
+    for (i = 0; i < oprsz; ) {                                        \
+        uint16_t pg = *(uint16_t *)(vg_b + H1_2(i >> 3));             \
+        do {                                                          \
+            if (pg & 1) {                                             \
+                *(TYPE *)(za_b + tile_vslice_offset(i)) =             \
+                    *(TYPE *)(vn_b + H(i));                           \
+            }                                                         \
+            i += sizeof(TYPE);                                        \
+            pg >>= sizeof(TYPE);                                      \
+        } while (i & 15);                                             \
+    }                                                                 \
+}
+
+DO_MOVA_C(sme_mova_cz_b, uint8_t, H1)
+DO_MOVA_C(sme_mova_cz_h, uint16_t, H1_2)
+DO_MOVA_C(sme_mova_cz_s, uint32_t, H1_4)
+
+void HELPER(sme_mova_cz_d)(void *za, void *vn, void *vg, uint32_t desc)
+{
+    int i, oprsz = simd_oprsz(desc) / 8;
+    uint8_t *pg = vg;
+    uint64_t *n = vn;
+    char *za_b = za;
+
+    for (i = 0; i < oprsz; i++) {
+        if (pg[H1(i)] & 1) {
+            *(uint64_t *)(za_b + tile_vslice_index(i) * sizeof(uint64_t)) =
+                n[i];
+        }
+    }
+}
+
+void HELPER(sme_mova_cz_q)(void *za, void *vn, void *vg, uint32_t desc)
+{
+    int i, oprsz = simd_oprsz(desc) / 16;
+    uint16_t *pg = vg;
+    Int128 *n = vn;
+    char *za_b = za;
+
+    for (i = 0; i < oprsz; i++) {
+        if (pg[H2(i)] & 1) {
+            *(Int128 *)(za_b + tile_vslice_index(i) * sizeof(Int128)) = n[i];
+        }
+    }
+}
+
+#undef DO_MOVA_C
+
+#define DO_MOVA_Z(NAME, TYPE, H)                                      \
+void HELPER(NAME)(void *vd, void *za, void *vg, uint32_t desc)        \
+{                                                                     \
+    int i, oprsz = simd_oprsz(desc);                                  \
+    char *vd_b = vd;                                                  \
+    char *za_b = za;                                                  \
+    char *vg_b = vg;                                                  \
+                                                                      \
+    for (i = 0; i < oprsz; ) {                                        \
+        uint16_t pg = *(uint16_t *)(vg_b + H1_2(i >> 3));             \
+        do {                                                          \
+            if (pg & 1) {                                             \
+                *(TYPE *)(vd_b + H(i)) =                              \
+                    *(TYPE *)(za_b + tile_vslice_offset(i));          \
+            }                                                         \
+            i += sizeof(TYPE);                                        \
+            pg >>= sizeof(TYPE);                                      \
+        } while (i & 15);                                             \
+    }                                                                 \
+}
+
+DO_MOVA_Z(sme_mova_zc_b, uint8_t, H1)
+DO_MOVA_Z(sme_mova_zc_h, uint16_t, H1_2)
+DO_MOVA_Z(sme_mova_zc_s, uint32_t, H1_4)
+
+void HELPER(sme_mova_zc_d)(void *vd, void *za, void *vg, uint32_t desc)
+{
+    int i, oprsz = simd_oprsz(desc) / 8;
+    uint8_t *pg = vg;
+    uint64_t *d = vd;
+    char *za_b = za;
+
+    for (i = 0; i < oprsz; i++) {
+        if (pg[H1(i)] & 1) {
+            d[i] =
+                *(uint64_t *)(za_b + tile_vslice_index(i) * sizeof(uint64_t));
+        }
+    }
+}
+
+void HELPER(sme_mova_zc_q)(void *vd, void *za, void *vg, uint32_t desc)
+{
+    int i, oprsz = simd_oprsz(desc) / 16;
+    uint16_t *pg = vg;
+    Int128 *d = vd;
+    char *za_b = za;
+
+    for (i = 0; i < oprsz; i++) {
+        if (pg[H2(i)] & 1) {
+            d[i] = *(Int128 *)(za_b + tile_vslice_index(i) * sizeof(Int128));
+        }
+    }
+}
+
+static inline void *sme_ldst1_za_ptr(void *za, intptr_t off, int esz,
+                                     bool vertical)
+{
+    if (vertical) {
+        return (char *)za + tile_vslice_offset(off);
+    }
+
+    switch (esz) {
+    case MO_8:
+        return (char *)za + H1(off);
+    case MO_16:
+        return (char *)za + H1_2(off);
+    case MO_32:
+        return (char *)za + H1_4(off);
+    default:
+        return (char *)za + off;
+    }
+}
+
+static uint64_t sme_ldst1_load(CPUARMState *env, target_ulong addr,
+                               int size, bool be)
+{
+    uint64_t ret = 0;
+    int i;
+
+    if (be) {
+        for (i = 0; i < size; i++) {
+            ret = (ret << 8) | cpu_ldub_data_ra(env, addr + i, GETPC());
+        }
+    } else {
+        for (i = 0; i < size; i++) {
+            ret |= (uint64_t)cpu_ldub_data_ra(env, addr + i, GETPC()) <<
+                   (i * 8);
+        }
+    }
+    return ret;
+}
+
+static void sme_ldst1_store(CPUARMState *env, target_ulong addr,
+                            uint64_t value, int size, bool be)
+{
+    int i;
+
+    if (be) {
+        for (i = 0; i < size; i++) {
+            int shift = (size - 1 - i) * 8;
+
+            cpu_stb_data_ra(env, addr + i, value >> shift, GETPC());
+        }
+    } else {
+        for (i = 0; i < size; i++) {
+            cpu_stb_data_ra(env, addr + i, value >> (i * 8), GETPC());
+        }
+    }
+}
+
+static void sme_ldst1_clear(void *za, intptr_t off, int esz, bool vertical)
+{
+    memset(sme_ldst1_za_ptr(za, off, esz, vertical), 0, 1 << esz);
+}
+
+static void sme_ld1_element(CPUARMState *env, void *za, target_ulong addr,
+                            intptr_t off, int esz, bool be, bool vertical)
+{
+    void *ptr = sme_ldst1_za_ptr(za, off, esz, vertical);
+
+    switch (esz) {
+    case MO_8:
+        *(uint8_t *)ptr = sme_ldst1_load(env, addr, 1, be);
+        break;
+    case MO_16:
+        *(uint16_t *)ptr = sme_ldst1_load(env, addr, 2, be);
+        break;
+    case MO_32:
+        *(uint32_t *)ptr = sme_ldst1_load(env, addr, 4, be);
+        break;
+    case MO_64:
+        *(uint64_t *)ptr = sme_ldst1_load(env, addr, 8, be);
+        break;
+    case MO_128:
+    {
+        uint64_t val0 = sme_ldst1_load(env, addr, 8, be);
+        uint64_t val1 = sme_ldst1_load(env, addr + 8, 8, be);
+        uint64_t *dst = ptr;
+
+        dst[0] = be ? val1 : val0;
+        dst[1] = be ? val0 : val1;
+        break;
+    }
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static void sme_st1_element(CPUARMState *env, void *za, target_ulong addr,
+                            intptr_t off, int esz, bool be, bool vertical)
+{
+    void *ptr = sme_ldst1_za_ptr(za, off, esz, vertical);
+    uint64_t val;
+
+    switch (esz) {
+    case MO_8:
+        val = *(uint8_t *)ptr;
+        sme_ldst1_store(env, addr, val, 1, be);
+        break;
+    case MO_16:
+        val = *(uint16_t *)ptr;
+        sme_ldst1_store(env, addr, val, 2, be);
+        break;
+    case MO_32:
+        val = *(uint32_t *)ptr;
+        sme_ldst1_store(env, addr, val, 4, be);
+        break;
+    case MO_64:
+        val = *(uint64_t *)ptr;
+        sme_ldst1_store(env, addr, val, 8, be);
+        break;
+    case MO_128:
+    {
+        uint64_t *src = ptr;
+
+        sme_ldst1_store(env, addr, src[be], 8, be);
+        sme_ldst1_store(env, addr + 8, src[!be], 8, be);
+        break;
+    }
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static bool sme_ldst1_active(uint64_t *pg, intptr_t off)
+{
+    return (pg[off >> 6] >> (off & 63)) & 1;
+}
+
+static void sme_ldst1_probe_addr(CPUARMState *env, target_ulong addr,
+                                 int size, MMUAccessType access_type,
+                                 int mmu_idx, uintptr_t ra)
+{
+    struct uc_struct *uc = env->uc;
+
+    while (size > 0) {
+        target_ulong page_left = -(addr | TARGET_PAGE_MASK);
+        int probe_size = MIN(size, (int)page_left);
+        target_ulong paddr;
+        MemoryRegion *mr;
+
+        if (!tlb_vaddr_to_paddr(env, addr, access_type, mmu_idx, &paddr)) {
+            uc->invalid_addr = addr;
+            uc->invalid_error = access_type == MMU_DATA_STORE ?
+                                UC_ERR_MMU_WRITE : UC_ERR_MMU_READ;
+            cpu_exit(uc->cpu);
+            cpu_loop_exit_restore(env_cpu(env), ra);
+        }
+
+        mr = uc->memory_mapping(uc, paddr);
+        if (mr == NULL) {
+            uc->invalid_addr = paddr;
+            uc->invalid_error = access_type == MMU_DATA_STORE ?
+                                UC_ERR_WRITE_UNMAPPED :
+                                UC_ERR_READ_UNMAPPED;
+            cpu_exit(uc->cpu);
+            cpu_loop_exit_restore(env_cpu(env), ra);
+        }
+        if (access_type == MMU_DATA_STORE) {
+            if (!(mr->perms & UC_PROT_WRITE)) {
+                uc->invalid_addr = paddr;
+                uc->invalid_error = UC_ERR_WRITE_PROT;
+                cpu_exit(uc->cpu);
+                cpu_loop_exit_restore(env_cpu(env), ra);
+            }
+        } else if (!(mr->perms & UC_PROT_READ)) {
+            uc->invalid_addr = paddr;
+            uc->invalid_error = UC_ERR_READ_PROT;
+            cpu_exit(uc->cpu);
+            cpu_loop_exit_restore(env_cpu(env), ra);
+        }
+
+        probe_access(env, addr, probe_size, access_type, mmu_idx, ra);
+        addr += probe_size;
+        size -= probe_size;
+    }
+}
+
+static int sme_allocation_tag_from_addr(target_ulong ptr)
+{
+    return extract64(ptr, 56, 4);
+}
+
+static target_ulong sme_allocation_tag_clean_addr(target_ulong ptr)
+{
+    return ptr & ~MAKE_64BIT_MASK(56, 8);
+}
+
+static target_ulong sme_ldst1_mte_addr(CPUARMState *env, target_ulong addr,
+                                       uint32_t mtedesc)
+{
+    int bit55 = extract64(addr, 55, 1);
+
+    if (!tbi_check(mtedesc, bit55)) {
+        return addr;
+    }
+    if (tcma_check(mtedesc, bit55, sme_allocation_tag_from_addr(addr))) {
+        return sme_allocation_tag_clean_addr(useronly_clean_ptr(addr));
+    }
+    return mte_check(env, mtedesc, addr, GETPC());
+}
+
+static target_ulong sme_ldst1_clean_addr(target_ulong addr, uint32_t mtedesc)
+{
+    int bit55 = extract64(addr, 55, 1);
+
+    if (!mtedesc || !tbi_check(mtedesc, bit55)) {
+        return addr;
+    }
+    return sme_allocation_tag_clean_addr(useronly_clean_ptr(addr));
+}
+
+static void do_sme_ldst1(CPUARMState *env, void *za, void *vg,
+                         target_ulong addr, uint32_t desc, int esz,
+                         bool be, bool vertical, bool store, bool mte)
+{
+    intptr_t off, oprsz = simd_oprsz(desc);
+    uint64_t *pg = vg;
+    int esize = 1 << esz;
+    uint32_t mtedesc = mte ? extract32(desc, SIMD_DATA_SHIFT,
+                                       SIMD_DATA_BITS) : 0;
+    int mmu_idx = mte ? FIELD_EX32(mtedesc, MTEDESC, MIDX) :
+                        extract32(desc, SIMD_DATA_SHIFT,
+                                  R_MTEDESC_MIDX_LENGTH);
+    MMUAccessType access_type = store ? MMU_DATA_STORE : MMU_DATA_LOAD;
+
+    for (off = 0; off < oprsz; off += esize) {
+        if (sme_ldst1_active(pg, off)) {
+            target_ulong elem_addr = sme_ldst1_clean_addr(addr + off,
+                                                          mtedesc);
+
+            sme_ldst1_probe_addr(env, elem_addr, esize, access_type, mmu_idx,
+                                 GETPC());
+        }
+    }
+
+    if (mtedesc) {
+        for (off = 0; off < oprsz; off += esize) {
+            if (sme_ldst1_active(pg, off)) {
+                (void)sme_ldst1_mte_addr(env, addr + off, mtedesc);
+            }
+        }
+    }
+
+    for (off = 0; off < oprsz; off += esize) {
+        if (store) {
+            if (sme_ldst1_active(pg, off)) {
+                target_ulong elem_addr = sme_ldst1_clean_addr(addr + off,
+                                                              mtedesc);
+
+                sme_st1_element(env, za, elem_addr, off, esz, be, vertical);
+            }
+        } else {
+            sme_ldst1_clear(za, off, esz, vertical);
+            if (sme_ldst1_active(pg, off)) {
+                target_ulong elem_addr = sme_ldst1_clean_addr(addr + off,
+                                                              mtedesc);
+
+                sme_ld1_element(env, za, elem_addr, off, esz, be, vertical);
+            }
+        }
+    }
+}
+
+#define DO_SME_LDST1(NAME, ESZ, BE, VERTICAL, STORE, MTE)               \
+void HELPER(NAME)(CPUARMState *env, void *za, void *vg,                 \
+                  target_ulong addr, uint32_t desc)                     \
+{                                                                       \
+    do_sme_ldst1(env, za, vg, addr, desc, ESZ, BE, VERTICAL, STORE,     \
+                 MTE);                                                  \
+}
+
+DO_SME_LDST1(sme_ld1b_h, MO_8, false, false, false, false)
+DO_SME_LDST1(sme_ld1b_v, MO_8, false, true, false, false)
+DO_SME_LDST1(sme_ld1h_le_h, MO_16, false, false, false, false)
+DO_SME_LDST1(sme_ld1h_le_v, MO_16, false, true, false, false)
+DO_SME_LDST1(sme_ld1h_be_h, MO_16, true, false, false, false)
+DO_SME_LDST1(sme_ld1h_be_v, MO_16, true, true, false, false)
+DO_SME_LDST1(sme_ld1s_le_h, MO_32, false, false, false, false)
+DO_SME_LDST1(sme_ld1s_le_v, MO_32, false, true, false, false)
+DO_SME_LDST1(sme_ld1s_be_h, MO_32, true, false, false, false)
+DO_SME_LDST1(sme_ld1s_be_v, MO_32, true, true, false, false)
+DO_SME_LDST1(sme_ld1d_le_h, MO_64, false, false, false, false)
+DO_SME_LDST1(sme_ld1d_le_v, MO_64, false, true, false, false)
+DO_SME_LDST1(sme_ld1d_be_h, MO_64, true, false, false, false)
+DO_SME_LDST1(sme_ld1d_be_v, MO_64, true, true, false, false)
+DO_SME_LDST1(sme_ld1q_le_h, MO_128, false, false, false, false)
+DO_SME_LDST1(sme_ld1q_le_v, MO_128, false, true, false, false)
+DO_SME_LDST1(sme_ld1q_be_h, MO_128, true, false, false, false)
+DO_SME_LDST1(sme_ld1q_be_v, MO_128, true, true, false, false)
+
+DO_SME_LDST1(sme_ld1b_h_mte, MO_8, false, false, false, true)
+DO_SME_LDST1(sme_ld1b_v_mte, MO_8, false, true, false, true)
+DO_SME_LDST1(sme_ld1h_le_h_mte, MO_16, false, false, false, true)
+DO_SME_LDST1(sme_ld1h_le_v_mte, MO_16, false, true, false, true)
+DO_SME_LDST1(sme_ld1h_be_h_mte, MO_16, true, false, false, true)
+DO_SME_LDST1(sme_ld1h_be_v_mte, MO_16, true, true, false, true)
+DO_SME_LDST1(sme_ld1s_le_h_mte, MO_32, false, false, false, true)
+DO_SME_LDST1(sme_ld1s_le_v_mte, MO_32, false, true, false, true)
+DO_SME_LDST1(sme_ld1s_be_h_mte, MO_32, true, false, false, true)
+DO_SME_LDST1(sme_ld1s_be_v_mte, MO_32, true, true, false, true)
+DO_SME_LDST1(sme_ld1d_le_h_mte, MO_64, false, false, false, true)
+DO_SME_LDST1(sme_ld1d_le_v_mte, MO_64, false, true, false, true)
+DO_SME_LDST1(sme_ld1d_be_h_mte, MO_64, true, false, false, true)
+DO_SME_LDST1(sme_ld1d_be_v_mte, MO_64, true, true, false, true)
+DO_SME_LDST1(sme_ld1q_le_h_mte, MO_128, false, false, false, true)
+DO_SME_LDST1(sme_ld1q_le_v_mte, MO_128, false, true, false, true)
+DO_SME_LDST1(sme_ld1q_be_h_mte, MO_128, true, false, false, true)
+DO_SME_LDST1(sme_ld1q_be_v_mte, MO_128, true, true, false, true)
+
+DO_SME_LDST1(sme_st1b_h, MO_8, false, false, true, false)
+DO_SME_LDST1(sme_st1b_v, MO_8, false, true, true, false)
+DO_SME_LDST1(sme_st1h_le_h, MO_16, false, false, true, false)
+DO_SME_LDST1(sme_st1h_le_v, MO_16, false, true, true, false)
+DO_SME_LDST1(sme_st1h_be_h, MO_16, true, false, true, false)
+DO_SME_LDST1(sme_st1h_be_v, MO_16, true, true, true, false)
+DO_SME_LDST1(sme_st1s_le_h, MO_32, false, false, true, false)
+DO_SME_LDST1(sme_st1s_le_v, MO_32, false, true, true, false)
+DO_SME_LDST1(sme_st1s_be_h, MO_32, true, false, true, false)
+DO_SME_LDST1(sme_st1s_be_v, MO_32, true, true, true, false)
+DO_SME_LDST1(sme_st1d_le_h, MO_64, false, false, true, false)
+DO_SME_LDST1(sme_st1d_le_v, MO_64, false, true, true, false)
+DO_SME_LDST1(sme_st1d_be_h, MO_64, true, false, true, false)
+DO_SME_LDST1(sme_st1d_be_v, MO_64, true, true, true, false)
+DO_SME_LDST1(sme_st1q_le_h, MO_128, false, false, true, false)
+DO_SME_LDST1(sme_st1q_le_v, MO_128, false, true, true, false)
+DO_SME_LDST1(sme_st1q_be_h, MO_128, true, false, true, false)
+DO_SME_LDST1(sme_st1q_be_v, MO_128, true, true, true, false)
+
+DO_SME_LDST1(sme_st1b_h_mte, MO_8, false, false, true, true)
+DO_SME_LDST1(sme_st1b_v_mte, MO_8, false, true, true, true)
+DO_SME_LDST1(sme_st1h_le_h_mte, MO_16, false, false, true, true)
+DO_SME_LDST1(sme_st1h_le_v_mte, MO_16, false, true, true, true)
+DO_SME_LDST1(sme_st1h_be_h_mte, MO_16, true, false, true, true)
+DO_SME_LDST1(sme_st1h_be_v_mte, MO_16, true, true, true, true)
+DO_SME_LDST1(sme_st1s_le_h_mte, MO_32, false, false, true, true)
+DO_SME_LDST1(sme_st1s_le_v_mte, MO_32, false, true, true, true)
+DO_SME_LDST1(sme_st1s_be_h_mte, MO_32, true, false, true, true)
+DO_SME_LDST1(sme_st1s_be_v_mte, MO_32, true, true, true, true)
+DO_SME_LDST1(sme_st1d_le_h_mte, MO_64, false, false, true, true)
+DO_SME_LDST1(sme_st1d_le_v_mte, MO_64, false, true, true, true)
+DO_SME_LDST1(sme_st1d_be_h_mte, MO_64, true, false, true, true)
+DO_SME_LDST1(sme_st1d_be_v_mte, MO_64, true, true, true, true)
+DO_SME_LDST1(sme_st1q_le_h_mte, MO_128, false, false, true, true)
+DO_SME_LDST1(sme_st1q_le_v_mte, MO_128, false, true, true, true)
+DO_SME_LDST1(sme_st1q_be_h_mte, MO_128, true, false, true, true)
+DO_SME_LDST1(sme_st1q_be_v_mte, MO_128, true, true, true, true)
+
+#undef DO_SME_LDST1
+
+void HELPER(sme_addha_s)(void *vza, void *vzn, void *vpn, void *vpm,
+                         uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_oprsz(desc) / sizeof(uint32_t);
+    char *za = vza;
+    char *zn = vzn;
+    uint64_t *pn = vpn;
+    uint64_t *pm = vpm;
+
+    for (row = 0; row < oprsz; ) {
+        uint64_t pa = pn[row >> 4];
+        do {
+            if (pa & 1) {
+                char *za_row = za + tile_vslice_offset(row * sizeof(uint32_t));
+
+                for (col = 0; col < oprsz; ) {
+                    uint64_t pb = pm[col >> 4];
+                    do {
+                        if (pb & 1) {
+                            uint32_t *cell =
+                                (uint32_t *)(za_row +
+                                             H1_4(col * sizeof(uint32_t)));
+                            uint32_t *src =
+                                (uint32_t *)(zn +
+                                             H1_4(col * sizeof(uint32_t)));
+
+                            *cell += *src;
+                        }
+                        pb >>= 4;
+                    } while (++col & 15);
+                }
+            }
+            pa >>= 4;
+        } while (++row & 15);
+    }
+}
+
+void HELPER(sme_addha_d)(void *vza, void *vzn, void *vpn, void *vpm,
+                         uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_oprsz(desc) / sizeof(uint64_t);
+    char *za = vza;
+    uint64_t *zn = vzn;
+    uint8_t *pn = vpn;
+    uint8_t *pm = vpm;
+
+    for (row = 0; row < oprsz; row++) {
+        if (pn[H1(row)] & 1) {
+            char *za_row = za + tile_vslice_offset(row * sizeof(uint64_t));
+
+            for (col = 0; col < oprsz; col++) {
+                if (pm[H1(col)] & 1) {
+                    uint64_t *cell =
+                        (uint64_t *)(za_row + col * sizeof(uint64_t));
+
+                    *cell += zn[col];
+                }
+            }
+        }
+    }
+}
+
+void HELPER(sme_addva_s)(void *vza, void *vzn, void *vpn, void *vpm,
+                         uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_oprsz(desc) / sizeof(uint32_t);
+    char *za = vza;
+    char *zn = vzn;
+    uint64_t *pn = vpn;
+    uint64_t *pm = vpm;
+
+    for (row = 0; row < oprsz; ) {
+        uint64_t pa = pn[row >> 4];
+        do {
+            if (pa & 1) {
+                char *za_row = za + tile_vslice_offset(row * sizeof(uint32_t));
+                uint32_t src =
+                    *(uint32_t *)(zn + H1_4(row * sizeof(uint32_t)));
+
+                for (col = 0; col < oprsz; ) {
+                    uint64_t pb = pm[col >> 4];
+                    do {
+                        if (pb & 1) {
+                            uint32_t *cell =
+                                (uint32_t *)(za_row +
+                                             H1_4(col * sizeof(uint32_t)));
+
+                            *cell += src;
+                        }
+                        pb >>= 4;
+                    } while (++col & 15);
+                }
+            }
+            pa >>= 4;
+        } while (++row & 15);
+    }
+}
+
+void HELPER(sme_addva_d)(void *vza, void *vzn, void *vpn, void *vpm,
+                         uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_oprsz(desc) / sizeof(uint64_t);
+    char *za = vza;
+    uint64_t *zn = vzn;
+    uint8_t *pn = vpn;
+    uint8_t *pm = vpm;
+
+    for (row = 0; row < oprsz; row++) {
+        if (pn[H1(row)] & 1) {
+            char *za_row = za + tile_vslice_offset(row * sizeof(uint64_t));
+            uint64_t src = zn[row];
+
+            for (col = 0; col < oprsz; col++) {
+                if (pm[H1(col)] & 1) {
+                    uint64_t *cell =
+                        (uint64_t *)(za_row + col * sizeof(uint64_t));
+
+                    *cell += src;
+                }
+            }
+        }
+    }
+}
+
+void HELPER(sme_fmopa_s)(void *vza, void *vzn, void *vzm, void *vpn,
+                         void *vpm, void *vst, uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_maxsz(desc);
+    uint32_t neg = simd_data(desc) << 31;
+    uint16_t *pn = vpn;
+    uint16_t *pm = vpm;
+    char *za = vza;
+    char *zn = vzn;
+    char *zm = vzm;
+    float_status fpst = *(float_status *)vst;
+
+    set_default_nan_mode(true, &fpst);
+
+    for (row = 0; row < oprsz;) {
+        uint16_t pa = pn[H2(row >> 4)];
+
+        do {
+            if (pa & 1) {
+                char *za_row = za + tile_vslice_offset(row);
+                uint32_t n = *(uint32_t *)(zn + H1_4(row)) ^ neg;
+
+                for (col = 0; col < oprsz;) {
+                    uint16_t pb = pm[H2(col >> 4)];
+
+                    do {
+                        if (pb & 1) {
+                            uint32_t *a = (uint32_t *)(za_row + H1_4(col));
+                            uint32_t *m = (uint32_t *)(zm + H1_4(col));
+
+                            *a = float32_muladd(n, *m, *a, 0, &fpst);
+                        }
+                        col += sizeof(uint32_t);
+                        pb >>= sizeof(uint32_t);
+                    } while (col & 15);
+                }
+            }
+            row += sizeof(uint32_t);
+            pa >>= sizeof(uint32_t);
+        } while (row & 15);
+    }
+}
+
+void HELPER(sme_fmopa_d)(void *vza, void *vzn, void *vzm, void *vpn,
+                         void *vpm, void *vst, uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_oprsz(desc) / sizeof(uint64_t);
+    uint64_t neg = (uint64_t)simd_data(desc) << 63;
+    uint64_t *za = vza;
+    uint64_t *zn = vzn;
+    uint64_t *zm = vzm;
+    uint8_t *pn = vpn;
+    uint8_t *pm = vpm;
+    float_status fpst = *(float_status *)vst;
+
+    set_default_nan_mode(true, &fpst);
+
+    for (row = 0; row < oprsz; row++) {
+        if (pn[H1(row)] & 1) {
+            uint64_t *za_row = &za[tile_vslice_index(row)];
+            uint64_t n = zn[row] ^ neg;
+
+            for (col = 0; col < oprsz; col++) {
+                if (pm[H1(col)] & 1) {
+                    uint64_t *a = &za_row[col];
+
+                    *a = float64_muladd(n, zm[col], *a, 0, &fpst);
+                }
+            }
+        }
+    }
+}
+
+static uint32_t sme_f16mop_adj_pair(uint32_t pair, uint32_t pg, uint32_t neg)
+{
+    pair ^= neg;
+    if (!(pg & 1)) {
+        pair &= 0xffff0000u;
+    }
+    if (!(pg & 4)) {
+        pair &= 0x0000ffffu;
+    }
+    return pair;
+}
+
+static float32 sme_f16_dotadd(float32 sum, uint32_t e1, uint32_t e2,
+                              float_status *s_f16, float_status *s_std,
+                              float_status *s_odd)
+{
+    float16 h1r = e1 & 0xffff;
+    float16 h1c = e1 >> 16;
+    float16 h2r = e2 & 0xffff;
+    float16 h2c = e2 >> 16;
+    float32 t32;
+
+    if (float16_is_any_nan(h1r) || float16_is_any_nan(h1c) ||
+        float16_is_any_nan(h2r) || float16_is_any_nan(h2c)) {
+        float16 t16;
+
+        if (float16_is_signaling_nan(h1r, s_f16)) {
+            t16 = h1r;
+        } else if (float16_is_signaling_nan(h1c, s_f16)) {
+            t16 = h1c;
+        } else if (float16_is_signaling_nan(h2r, s_f16)) {
+            t16 = h2r;
+        } else if (float16_is_signaling_nan(h2c, s_f16)) {
+            t16 = h2c;
+        } else if (float16_is_any_nan(h1r)) {
+            t16 = h1r;
+        } else if (float16_is_any_nan(h1c)) {
+            t16 = h1c;
+        } else if (float16_is_any_nan(h2r)) {
+            t16 = h2r;
+        } else {
+            t16 = h2c;
+        }
+        t32 = float16_to_float32(t16, true, s_f16);
+    } else {
+        float64 e1r = float16_to_float64(h1r, true, s_f16);
+        float64 e1c = float16_to_float64(h1c, true, s_f16);
+        float64 e2r = float16_to_float64(h2r, true, s_f16);
+        float64 e2c = float16_to_float64(h2c, true, s_f16);
+        float64 t64;
+
+        t64 = float64_mul(e1r, e2r, s_odd);
+        t64 = float64r32_muladd(e1c, e2c, t64, 0, s_std);
+        t32 = float64_to_float32(t64, s_std);
+    }
+
+    return float32_add(sum, t32, s_std);
+}
+
+void HELPER(sme_fmopa_h)(void *vza, void *vzn, void *vzm, void *vpn,
+                         void *vpm, CPUARMState *env, uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_maxsz(desc);
+    uint32_t neg = simd_data(desc) * 0x80008000u;
+    uint16_t *pn = vpn;
+    uint16_t *pm = vpm;
+    char *za = vza;
+    char *zn = vzn;
+    char *zm = vzm;
+    float_status fpst_odd;
+    float_status fpst_std;
+    float_status fpst_f16;
+
+    fpst_f16 = env->vfp.fp_status_f16;
+    fpst_std = env->vfp.fp_status;
+    set_default_nan_mode(true, &fpst_std);
+    set_default_nan_mode(true, &fpst_f16);
+    fpst_odd = fpst_std;
+    set_float_rounding_mode(float_round_to_odd, &fpst_odd);
+
+    for (row = 0; row < oprsz;) {
+        uint16_t prow = pn[H2(row >> 4)];
+
+        do {
+            char *za_row = za + tile_vslice_offset(row);
+            uint32_t n = *(uint32_t *)(zn + H1_4(row));
+
+            n = sme_f16mop_adj_pair(n, prow, neg);
+
+            for (col = 0; col < oprsz;) {
+                uint16_t pcol = pm[H2(col >> 4)];
+
+                do {
+                    if (prow & pcol & 0x5) {
+                        uint32_t *a = (uint32_t *)(za_row + H1_4(col));
+                        uint32_t m = *(uint32_t *)(zm + H1_4(col));
+
+                        m = sme_f16mop_adj_pair(m, pcol, 0);
+                        *a = sme_f16_dotadd(*a, n, m,
+                                            &fpst_f16, &fpst_std,
+                                            &fpst_odd);
+                    }
+                    col += sizeof(uint32_t);
+                    pcol >>= sizeof(uint32_t);
+                } while (col & 15);
+            }
+            row += sizeof(uint32_t);
+            prow >>= sizeof(uint32_t);
+        } while (row & 15);
+    }
+}
+
+static float32 sme_bfdotadd(float32 sum, uint32_t e1, uint32_t e2)
+{
+    float_status bf_status = {
+        .tininess_before_rounding = float_tininess_before_rounding,
+        .float_rounding_mode = float_round_to_odd_inf,
+        .flush_to_zero = true,
+        .flush_inputs_to_zero = true,
+        .default_nan_mode = true,
+    };
+    float32 t1;
+    float32 t2;
+
+    t1 = float32_mul(e1 << 16, e2 << 16, &bf_status);
+    t2 = float32_mul(e1 & 0xffff0000u, e2 & 0xffff0000u, &bf_status);
+    t1 = float32_add(t1, t2, &bf_status);
+    return float32_add(sum, t1, &bf_status);
+}
+
+void HELPER(sme_bfmopa)(void *vza, void *vzn, void *vzm, void *vpn,
+                        void *vpm, uint32_t desc)
+{
+    intptr_t row, col, oprsz = simd_maxsz(desc);
+    uint32_t neg = simd_data(desc) * 0x80008000u;
+    uint16_t *pn = vpn;
+    uint16_t *pm = vpm;
+    char *za = vza;
+    char *zn = vzn;
+    char *zm = vzm;
+
+    for (row = 0; row < oprsz;) {
+        uint16_t prow = pn[H2(row >> 4)];
+
+        do {
+            char *za_row = za + tile_vslice_offset(row);
+            uint32_t n = *(uint32_t *)(zn + H1_4(row));
+
+            n = sme_f16mop_adj_pair(n, prow, neg);
+
+            for (col = 0; col < oprsz;) {
+                uint16_t pcol = pm[H2(col >> 4)];
+
+                do {
+                    if (prow & pcol & 0x5) {
+                        uint32_t *a = (uint32_t *)(za_row + H1_4(col));
+                        uint32_t m = *(uint32_t *)(zm + H1_4(col));
+
+                        m = sme_f16mop_adj_pair(m, pcol, 0);
+                        *a = sme_bfdotadd(*a, n, m);
+                    }
+                    col += sizeof(uint32_t);
+                    pcol >>= sizeof(uint32_t);
+                } while (col & 15);
+            }
+            row += sizeof(uint32_t);
+            prow >>= sizeof(uint32_t);
+        } while (row & 15);
+    }
+}
+
+static uint64_t sme_expand_pred_b(uint8_t pred)
+{
+    uint64_t ret = 0;
+    int i;
+
+    for (i = 0; i < 8; i++) {
+        if (pred & (1U << i)) {
+            ret |= 0xffULL << (i * 8);
+        }
+    }
+    return ret;
+}
+
+static uint64_t sme_expand_pred_h(uint8_t pred)
+{
+    uint64_t ret = 0;
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        if (pred & (1U << (i * 2))) {
+            ret |= 0xffffULL << (i * 16);
+        }
+    }
+    return ret;
+}
+
+typedef uint64_t SMEIntOuterProductFn(uint64_t n, uint64_t m, uint64_t a,
+                                      uint8_t pred, bool subtract);
+
+static void do_sme_int_outer_product(void *vza, void *vzn, void *vzm,
+                                     void *vpn, void *vpm, uint32_t desc,
+                                     SMEIntOuterProductFn *fn)
+{
+    intptr_t row, col, oprsz = simd_oprsz(desc) / sizeof(uint64_t);
+    bool subtract = simd_data(desc);
+    uint64_t *za = vza;
+    uint64_t *zn = vzn;
+    uint64_t *zm = vzm;
+    uint8_t *pn = vpn;
+    uint8_t *pm = vpm;
+
+    for (row = 0; row < oprsz; row++) {
+        uint8_t pa = pn[H1(row)];
+        uint64_t *za_row = &za[tile_vslice_index(row)];
+        uint64_t n = zn[row];
+
+        for (col = 0; col < oprsz; col++) {
+            uint8_t pb = pm[H1(col)];
+            uint64_t *cell = &za_row[col];
+
+            *cell = fn(n, zm[col], *cell, pa & pb, subtract);
+        }
+    }
+}
+
+#define DO_SME_IMOPA_S(NAME, NTYPE, MTYPE)                              \
+static uint64_t NAME(uint64_t n, uint64_t m, uint64_t a,                \
+                     uint8_t pred, bool subtract)                       \
+{                                                                       \
+    uint32_t sum0 = 0;                                                  \
+    uint32_t sum1 = 0;                                                  \
+                                                                        \
+    n &= sme_expand_pred_b(pred);                                       \
+    sum0 += (NTYPE)(n >> 0) * (MTYPE)(m >> 0);                          \
+    sum0 += (NTYPE)(n >> 8) * (MTYPE)(m >> 8);                          \
+    sum0 += (NTYPE)(n >> 16) * (MTYPE)(m >> 16);                        \
+    sum0 += (NTYPE)(n >> 24) * (MTYPE)(m >> 24);                        \
+    sum1 += (NTYPE)(n >> 32) * (MTYPE)(m >> 32);                        \
+    sum1 += (NTYPE)(n >> 40) * (MTYPE)(m >> 40);                        \
+    sum1 += (NTYPE)(n >> 48) * (MTYPE)(m >> 48);                        \
+    sum1 += (NTYPE)(n >> 56) * (MTYPE)(m >> 56);                        \
+    if (subtract) {                                                     \
+        sum0 = (uint32_t)a - sum0;                                      \
+        sum1 = (uint32_t)(a >> 32) - sum1;                              \
+    } else {                                                            \
+        sum0 = (uint32_t)a + sum0;                                      \
+        sum1 = (uint32_t)(a >> 32) + sum1;                              \
+    }                                                                   \
+    return ((uint64_t)sum1 << 32) | sum0;                               \
+}
+
+#define DO_SME_IMOPA_D(NAME, NTYPE, MTYPE)                              \
+static uint64_t NAME(uint64_t n, uint64_t m, uint64_t a,                \
+                     uint8_t pred, bool subtract)                       \
+{                                                                       \
+    uint64_t sum = 0;                                                   \
+                                                                        \
+    n &= sme_expand_pred_h(pred);                                       \
+    sum += (int64_t)(NTYPE)(n >> 0) * (MTYPE)(m >> 0);                  \
+    sum += (int64_t)(NTYPE)(n >> 16) * (MTYPE)(m >> 16);                \
+    sum += (int64_t)(NTYPE)(n >> 32) * (MTYPE)(m >> 32);                \
+    sum += (int64_t)(NTYPE)(n >> 48) * (MTYPE)(m >> 48);                \
+    return subtract ? a - sum : a + sum;                                \
+}
+
+DO_SME_IMOPA_S(sme_smopa_s_op, int8_t, int8_t)
+DO_SME_IMOPA_S(sme_umopa_s_op, uint8_t, uint8_t)
+DO_SME_IMOPA_S(sme_sumopa_s_op, int8_t, uint8_t)
+DO_SME_IMOPA_S(sme_usmopa_s_op, uint8_t, int8_t)
+DO_SME_IMOPA_D(sme_smopa_d_op, int16_t, int16_t)
+DO_SME_IMOPA_D(sme_umopa_d_op, uint16_t, uint16_t)
+DO_SME_IMOPA_D(sme_sumopa_d_op, int16_t, uint16_t)
+DO_SME_IMOPA_D(sme_usmopa_d_op, uint16_t, int16_t)
+
+#define DO_SME_IMOPA_HELPER(NAME)                                       \
+void HELPER(NAME)(void *vza, void *vzn, void *vzm, void *vpn,           \
+                  void *vpm, uint32_t desc)                             \
+{                                                                       \
+    do_sme_int_outer_product(vza, vzn, vzm, vpn, vpm, desc, NAME##_op); \
+}
+
+DO_SME_IMOPA_HELPER(sme_smopa_s)
+DO_SME_IMOPA_HELPER(sme_umopa_s)
+DO_SME_IMOPA_HELPER(sme_sumopa_s)
+DO_SME_IMOPA_HELPER(sme_usmopa_s)
+DO_SME_IMOPA_HELPER(sme_smopa_d)
+DO_SME_IMOPA_HELPER(sme_umopa_d)
+DO_SME_IMOPA_HELPER(sme_sumopa_d)
+DO_SME_IMOPA_HELPER(sme_usmopa_d)
+
+#undef DO_SME_IMOPA_HELPER
+#undef DO_SME_IMOPA_D
+#undef DO_SME_IMOPA_S
+
+#undef DO_MOVA_Z
+#undef tile_vslice_index
+#undef tile_vslice_offset
+#undef H1
+#undef H1_2
+#undef H1_4
+#undef H2
diff --git a/qemu/target/arm/sve_helper.c b/qemu/target/arm/sve_helper.c
index 2abbeba57b..890c827b2e 100644
--- a/qemu/target/arm/sve_helper.c
+++ b/qemu/target/arm/sve_helper.c
@@ -43,6 +43,8 @@
 #define H2(x)   (x)
 #define H4(x)   (x)
 #endif
+#define H1_8(x) (x)
+#define H8(x)   (x)
 
 /* Return a value for NZCV as per the ARM PredTest pseudofunction.
  *
@@ -451,4829 +453,8504 @@ DO_ZPZZ(sve_uabd_zpzz_h, uint16_t, H1_2, DO_ABD)
 DO_ZPZZ(sve_uabd_zpzz_s, uint32_t, H1_4, DO_ABD)
 DO_ZPZZ_D(sve_uabd_zpzz_d, uint64_t, DO_ABD)
 
-/* Because the computation type is at least twice as large as required,
-   these work for both signed and unsigned source types.  */
-static inline uint8_t do_mulh_b(int32_t n, int32_t m)
+static inline uint16_t do_sadalp_h(int16_t n, int16_t m)
 {
-    return (n * m) >> 8;
+    int8_t n1 = n;
+    int8_t n2 = n >> 8;
+
+    return m + n1 + n2;
 }
 
-static inline uint16_t do_mulh_h(int32_t n, int32_t m)
+static inline uint32_t do_sadalp_s(int32_t n, int32_t m)
 {
-    return (n * m) >> 16;
+    int16_t n1 = n;
+    int16_t n2 = n >> 16;
+
+    return m + n1 + n2;
 }
 
-static inline uint32_t do_mulh_s(int64_t n, int64_t m)
+static inline uint64_t do_sadalp_d(int64_t n, int64_t m)
 {
-    return (n * m) >> 32;
+    int32_t n1 = n;
+    int32_t n2 = n >> 32;
+
+    return m + n1 + n2;
 }
 
-static inline uint64_t do_smulh_d(uint64_t n, uint64_t m)
+DO_ZPZZ(sve2_sadalp_zpzz_h, int16_t, H1_2, do_sadalp_h)
+DO_ZPZZ(sve2_sadalp_zpzz_s, int32_t, H1_4, do_sadalp_s)
+DO_ZPZZ_D(sve2_sadalp_zpzz_d, int64_t, do_sadalp_d)
+
+static inline uint16_t do_uadalp_h(uint16_t n, uint16_t m)
 {
-    uint64_t lo, hi;
-    muls64(&lo, &hi, n, m);
-    return hi;
+    uint8_t n1 = n;
+    uint8_t n2 = n >> 8;
+
+    return m + n1 + n2;
 }
 
-static inline uint64_t do_umulh_d(uint64_t n, uint64_t m)
+static inline uint32_t do_uadalp_s(uint32_t n, uint32_t m)
 {
-    uint64_t lo, hi;
-    mulu64(&lo, &hi, n, m);
-    return hi;
+    uint16_t n1 = n;
+    uint16_t n2 = n >> 16;
+
+    return m + n1 + n2;
 }
 
-DO_ZPZZ(sve_mul_zpzz_b, uint8_t, H1, DO_MUL)
-DO_ZPZZ(sve_mul_zpzz_h, uint16_t, H1_2, DO_MUL)
-DO_ZPZZ(sve_mul_zpzz_s, uint32_t, H1_4, DO_MUL)
-DO_ZPZZ_D(sve_mul_zpzz_d, uint64_t, DO_MUL)
+static inline uint64_t do_uadalp_d(uint64_t n, uint64_t m)
+{
+    uint32_t n1 = n;
+    uint32_t n2 = n >> 32;
 
-DO_ZPZZ(sve_smulh_zpzz_b, int8_t, H1, do_mulh_b)
-DO_ZPZZ(sve_smulh_zpzz_h, int16_t, H1_2, do_mulh_h)
-DO_ZPZZ(sve_smulh_zpzz_s, int32_t, H1_4, do_mulh_s)
-DO_ZPZZ_D(sve_smulh_zpzz_d, uint64_t, do_smulh_d)
+    return m + n1 + n2;
+}
 
-DO_ZPZZ(sve_umulh_zpzz_b, uint8_t, H1, do_mulh_b)
-DO_ZPZZ(sve_umulh_zpzz_h, uint16_t, H1_2, do_mulh_h)
-DO_ZPZZ(sve_umulh_zpzz_s, uint32_t, H1_4, do_mulh_s)
-DO_ZPZZ_D(sve_umulh_zpzz_d, uint64_t, do_umulh_d)
+DO_ZPZZ(sve2_uadalp_zpzz_h, uint16_t, H1_2, do_uadalp_h)
+DO_ZPZZ(sve2_uadalp_zpzz_s, uint32_t, H1_4, do_uadalp_s)
+DO_ZPZZ_D(sve2_uadalp_zpzz_d, uint64_t, do_uadalp_d)
 
-DO_ZPZZ(sve_sdiv_zpzz_s, int32_t, H1_4, DO_SDIV)
-DO_ZPZZ_D(sve_sdiv_zpzz_d, int64_t, DO_SDIV)
+static inline int32_t do_sve2_sqrshl_bhs(int32_t src, int32_t shift,
+                                         int bits, bool round,
+                                         uint32_t *sat)
+{
+    if (shift <= -bits) {
+        if (round) {
+            return 0;
+        }
+        return src >> 31;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < bits) {
+        uint32_t val_u = (uint32_t)src << shift;
+        int32_t val = (int32_t)val_u;
+
+        if (bits == 32) {
+            if (!sat || val >> shift == src) {
+                return val;
+            }
+        } else {
+            int32_t extval = sextract32(val, 0, bits);
 
-DO_ZPZZ(sve_udiv_zpzz_s, uint32_t, H1_4, DO_UDIV)
-DO_ZPZZ_D(sve_udiv_zpzz_d, uint64_t, DO_UDIV)
+            if (!sat || val == extval) {
+                return extval;
+            }
+        }
+    } else if (!sat || src == 0) {
+        return 0;
+    }
 
-/* Note that all bits of the shift are significant
-   and not modulo the element size.  */
-#define DO_ASR(N, M)  (N >> MIN(M, sizeof(N) * 8 - 1))
-#define DO_LSR(N, M)  (M < sizeof(N) * 8 ? N >> M : 0)
-#define DO_LSL(N, M)  (M < sizeof(N) * 8 ? N << M : 0)
+    *sat = 1;
+    return (1u << (bits - 1)) - (src >= 0);
+}
 
-DO_ZPZZ(sve_asr_zpzz_b, int8_t, H1, DO_ASR)
-DO_ZPZZ(sve_lsr_zpzz_b, uint8_t, H1_2, DO_LSR)
-DO_ZPZZ(sve_lsl_zpzz_b, uint8_t, H1_4, DO_LSL)
+static inline uint32_t do_sve2_uqrshl_bhs(uint32_t src, int32_t shift,
+                                          int bits, bool round,
+                                          uint32_t *sat)
+{
+    if (shift <= -(bits + round)) {
+        return 0;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < bits) {
+        uint32_t val = src << shift;
 
-DO_ZPZZ(sve_asr_zpzz_h, int16_t, H1, DO_ASR)
-DO_ZPZZ(sve_lsr_zpzz_h, uint16_t, H1_2, DO_LSR)
-DO_ZPZZ(sve_lsl_zpzz_h, uint16_t, H1_4, DO_LSL)
+        if (bits == 32) {
+            if (!sat || val >> shift == src) {
+                return val;
+            }
+        } else {
+            uint32_t extval = extract32(val, 0, bits);
 
-DO_ZPZZ(sve_asr_zpzz_s, int32_t, H1, DO_ASR)
-DO_ZPZZ(sve_lsr_zpzz_s, uint32_t, H1_2, DO_LSR)
-DO_ZPZZ(sve_lsl_zpzz_s, uint32_t, H1_4, DO_LSL)
+            if (!sat || val == extval) {
+                return extval;
+            }
+        }
+    } else if (!sat || src == 0) {
+        return 0;
+    }
 
-DO_ZPZZ_D(sve_asr_zpzz_d, int64_t, DO_ASR)
-DO_ZPZZ_D(sve_lsr_zpzz_d, uint64_t, DO_LSR)
-DO_ZPZZ_D(sve_lsl_zpzz_d, uint64_t, DO_LSL)
+    *sat = 1;
+    return MAKE_64BIT_MASK(0, bits);
+}
 
-#undef DO_ZPZZ
-#undef DO_ZPZZ_D
+static inline int64_t do_sve2_sqrshl_d_raw(int64_t src, int64_t shift,
+                                           bool round, uint32_t *sat)
+{
+    if (shift <= -64) {
+        if (round) {
+            return 0;
+        }
+        return src >> 63;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < 64) {
+        uint64_t val_u = (uint64_t)src << shift;
+        int64_t val = (int64_t)val_u;
 
-/* Three-operand expander, controlled by a predicate, in which the
- * third operand is "wide".  That is, for D = N op M, the same 64-bit
- * value of M is used with all of the narrower values of N.
- */
-#define DO_ZPZW(NAME, TYPE, TYPEW, H, OP)                               \
-void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
-{                                                                       \
-    intptr_t i, opr_sz = simd_oprsz(desc);                              \
-    for (i = 0; i < opr_sz; ) {                                         \
-        uint8_t pg = *(uint8_t *)((char *)vg + H1(i >> 3));                     \
-        TYPEW mm = *(TYPEW *)((char *)vm + i);                                  \
-        do {                                                            \
-            if (pg & 1) {                                               \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                         \
-                *(TYPE *)((char *)vd + H(i)) = OP(nn, mm);                      \
-            }                                                           \
-            i += sizeof(TYPE), pg >>= sizeof(TYPE);                     \
-        } while (i & 7);                                                \
-    }                                                                   \
+        if (!sat || val >> shift == src) {
+            return val;
+        }
+    } else if (!sat || src == 0) {
+        return 0;
+    }
+
+    *sat = 1;
+    return src < 0 ? INT64_MIN : INT64_MAX;
 }
 
-DO_ZPZW(sve_asr_zpzw_b, int8_t, uint64_t, H1, DO_ASR)
-DO_ZPZW(sve_lsr_zpzw_b, uint8_t, uint64_t, H1, DO_LSR)
-DO_ZPZW(sve_lsl_zpzw_b, uint8_t, uint64_t, H1, DO_LSL)
+static inline uint64_t do_sve2_uqrshl_d_raw(uint64_t src, int64_t shift,
+                                            bool round, uint32_t *sat)
+{
+    if (shift <= -(64 + round)) {
+        return 0;
+    } else if (shift < 0) {
+        if (round) {
+            src >>= -shift - 1;
+            return (src >> 1) + (src & 1);
+        }
+        return src >> -shift;
+    } else if (shift < 64) {
+        uint64_t val = src << shift;
 
-DO_ZPZW(sve_asr_zpzw_h, int16_t, uint64_t, H1_2, DO_ASR)
-DO_ZPZW(sve_lsr_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
-DO_ZPZW(sve_lsl_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSL)
+        if (!sat || val >> shift == src) {
+            return val;
+        }
+    } else if (!sat || src == 0) {
+        return 0;
+    }
 
-DO_ZPZW(sve_asr_zpzw_s, int32_t, uint64_t, H1_4, DO_ASR)
-DO_ZPZW(sve_lsr_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
-DO_ZPZW(sve_lsl_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSL)
+    *sat = 1;
+    return UINT64_MAX;
+}
 
-#undef DO_ZPZW
+static inline int8_t do_sve2_srshl_b(int8_t n, int8_t m)
+{
+    return do_sve2_sqrshl_bhs(n, m, 8, true, NULL);
+}
 
-/* Fully general two-operand expander, controlled by a predicate.
- */
-#define DO_ZPZ(NAME, TYPE, H, OP)                               \
-void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
-{                                                               \
-    intptr_t i, opr_sz = simd_oprsz(desc);                      \
-    for (i = 0; i < opr_sz; ) {                                 \
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));         \
-        do {                                                    \
-            if (pg & 1) {                                       \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                 \
-                *(TYPE *)((char *)vd + H(i)) = OP(nn);                  \
-            }                                                   \
-            i += sizeof(TYPE), pg >>= sizeof(TYPE);             \
-        } while (i & 15);                                       \
-    }                                                           \
+static inline int16_t do_sve2_srshl_h(int16_t n, int16_t m)
+{
+    return do_sve2_sqrshl_bhs(n, m, 16, true, NULL);
 }
 
-/* Similarly, specialized for 64-bit operands.  */
-#define DO_ZPZ_D(NAME, TYPE, OP)                                \
-void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
-{                                                               \
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                  \
-    TYPE *d = vd, *n = vn;                                      \
-    uint8_t *pg = vg;                                           \
-    for (i = 0; i < opr_sz; i += 1) {                           \
-        if (pg[H1(i)] & 1) {                                    \
-            TYPE nn = n[i];                                     \
-            d[i] = OP(nn);                                      \
-        }                                                       \
-    }                                                           \
+static inline int32_t do_sve2_srshl_s(int32_t n, int32_t m)
+{
+    return do_sve2_sqrshl_bhs(n, m, 32, true, NULL);
 }
 
-#define DO_CLS_B(N)   (clrsb32(N) - 24)
-#define DO_CLS_H(N)   (clrsb32(N) - 16)
+static inline int64_t do_sve2_srshl_d(int64_t n, int64_t m)
+{
+    return do_sve2_sqrshl_d_raw(n, m, true, NULL);
+}
 
-DO_ZPZ(sve_cls_b, int8_t, H1, DO_CLS_B)
-DO_ZPZ(sve_cls_h, int16_t, H1_2, DO_CLS_H)
-DO_ZPZ(sve_cls_s, int32_t, H1_4, clrsb32)
-DO_ZPZ_D(sve_cls_d, int64_t, clrsb64)
+DO_ZPZZ(sve2_srshl_zpzz_b, int8_t, H1, do_sve2_srshl_b)
+DO_ZPZZ(sve2_srshl_zpzz_h, int16_t, H1_2, do_sve2_srshl_h)
+DO_ZPZZ(sve2_srshl_zpzz_s, int32_t, H1_4, do_sve2_srshl_s)
+DO_ZPZZ_D(sve2_srshl_zpzz_d, int64_t, do_sve2_srshl_d)
 
-#define DO_CLZ_B(N)   (clz32(N) - 24)
-#define DO_CLZ_H(N)   (clz32(N) - 16)
+static inline uint8_t do_sve2_urshl_b(uint8_t n, uint8_t m)
+{
+    return do_sve2_uqrshl_bhs(n, (int8_t)m, 8, true, NULL);
+}
 
-DO_ZPZ(sve_clz_b, uint8_t, H1, DO_CLZ_B)
-DO_ZPZ(sve_clz_h, uint16_t, H1_2, DO_CLZ_H)
-DO_ZPZ(sve_clz_s, uint32_t, H1_4, clz32)
-DO_ZPZ_D(sve_clz_d, uint64_t, clz64)
+static inline uint16_t do_sve2_urshl_h(uint16_t n, uint16_t m)
+{
+    return do_sve2_uqrshl_bhs(n, (int16_t)m, 16, true, NULL);
+}
 
-DO_ZPZ(sve_cnt_zpz_b, uint8_t, H1, ctpop8)
-DO_ZPZ(sve_cnt_zpz_h, uint16_t, H1_2, ctpop16)
-DO_ZPZ(sve_cnt_zpz_s, uint32_t, H1_4, ctpop32)
-DO_ZPZ_D(sve_cnt_zpz_d, uint64_t, ctpop64)
+static inline uint32_t do_sve2_urshl_s(uint32_t n, uint32_t m)
+{
+    return do_sve2_uqrshl_bhs(n, (int32_t)m, 32, true, NULL);
+}
 
-#define DO_CNOT(N)    (N == 0)
+static inline uint64_t do_sve2_urshl_d(uint64_t n, uint64_t m)
+{
+    return do_sve2_uqrshl_d_raw(n, (int64_t)m, true, NULL);
+}
 
-DO_ZPZ(sve_cnot_b, uint8_t, H1, DO_CNOT)
-DO_ZPZ(sve_cnot_h, uint16_t, H1_2, DO_CNOT)
-DO_ZPZ(sve_cnot_s, uint32_t, H1_4, DO_CNOT)
-DO_ZPZ_D(sve_cnot_d, uint64_t, DO_CNOT)
+DO_ZPZZ(sve2_urshl_zpzz_b, uint8_t, H1, do_sve2_urshl_b)
+DO_ZPZZ(sve2_urshl_zpzz_h, uint16_t, H1_2, do_sve2_urshl_h)
+DO_ZPZZ(sve2_urshl_zpzz_s, uint32_t, H1_4, do_sve2_urshl_s)
+DO_ZPZZ_D(sve2_urshl_zpzz_d, uint64_t, do_sve2_urshl_d)
 
-#ifdef _MSC_VER
-#define DO_FABS16(N)    (N & ((uint16_t)-1 >> 1))
-#define DO_FABS32(N)    (N & ((uint32_t)-1 >> 1))
-#define DO_FABS64(N)    (N & ((uint64_t)-1 >> 1))
+static inline int8_t do_sve2_sqshl_b(int8_t n, int8_t m)
+{
+    uint32_t discard;
 
-DO_ZPZ(sve_fabs_h, uint16_t, H1_2, DO_FABS16)
-DO_ZPZ(sve_fabs_s, uint32_t, H1_4, DO_FABS32)
-DO_ZPZ_D(sve_fabs_d, uint64_t, DO_FABS64)
-#else
-#define DO_FABS(N)    (N & ((__typeof(N))-1 >> 1))
+    return do_sve2_sqrshl_bhs(n, m, 8, false, &discard);
+}
 
-DO_ZPZ(sve_fabs_h, uint16_t, H1_2, DO_FABS)
-DO_ZPZ(sve_fabs_s, uint32_t, H1_4, DO_FABS)
-DO_ZPZ_D(sve_fabs_d, uint64_t, DO_FABS)
-#endif
+static inline int16_t do_sve2_sqshl_h(int16_t n, int16_t m)
+{
+    uint32_t discard;
 
-#ifdef _MSC_VER
-#define DO_FNEG16(N)    (N ^ ~((uint16_t)-1 >> 1))
-#define DO_FNEG32(N)    (N ^ ~((uint32_t)-1 >> 1))
-#define DO_FNEG64(N)    (N ^ ~((uint64_t)-1 >> 1))
+    return do_sve2_sqrshl_bhs(n, m, 16, false, &discard);
+}
 
-DO_ZPZ(sve_fneg_h, uint16_t, H1_2, DO_FNEG16)
-DO_ZPZ(sve_fneg_s, uint32_t, H1_4, DO_FNEG32)
-DO_ZPZ_D(sve_fneg_d, uint64_t, DO_FNEG64)
-#else
-#define DO_FNEG(N)    (N ^ ~((__typeof(N))-1 >> 1))
+static inline int32_t do_sve2_sqshl_s(int32_t n, int32_t m)
+{
+    uint32_t discard;
 
-DO_ZPZ(sve_fneg_h, uint16_t, H1_2, DO_FNEG)
-DO_ZPZ(sve_fneg_s, uint32_t, H1_4, DO_FNEG)
-DO_ZPZ_D(sve_fneg_d, uint64_t, DO_FNEG)
-#endif
+    return do_sve2_sqrshl_bhs(n, m, 32, false, &discard);
+}
 
-#define DO_NOT(N)    (~N)
+static inline int64_t do_sve2_sqshl_d(int64_t n, int64_t m)
+{
+    uint32_t discard;
 
-DO_ZPZ(sve_not_zpz_b, uint8_t, H1, DO_NOT)
-DO_ZPZ(sve_not_zpz_h, uint16_t, H1_2, DO_NOT)
-DO_ZPZ(sve_not_zpz_s, uint32_t, H1_4, DO_NOT)
-DO_ZPZ_D(sve_not_zpz_d, uint64_t, DO_NOT)
+    return do_sve2_sqrshl_d_raw(n, m, false, &discard);
+}
 
-#define DO_SXTB(N)    ((int8_t)N)
-#define DO_SXTH(N)    ((int16_t)N)
-#define DO_SXTS(N)    ((int32_t)N)
-#define DO_UXTB(N)    ((uint8_t)N)
-#define DO_UXTH(N)    ((uint16_t)N)
-#define DO_UXTS(N)    ((uint32_t)N)
+DO_ZPZZ(sve2_sqshl_zpzz_b, int8_t, H1, do_sve2_sqshl_b)
+DO_ZPZZ(sve2_sqshl_zpzz_h, int16_t, H1_2, do_sve2_sqshl_h)
+DO_ZPZZ(sve2_sqshl_zpzz_s, int32_t, H1_4, do_sve2_sqshl_s)
+DO_ZPZZ_D(sve2_sqshl_zpzz_d, int64_t, do_sve2_sqshl_d)
 
-DO_ZPZ(sve_sxtb_h, uint16_t, H1_2, DO_SXTB)
-DO_ZPZ(sve_sxtb_s, uint32_t, H1_4, DO_SXTB)
-DO_ZPZ(sve_sxth_s, uint32_t, H1_4, DO_SXTH)
-DO_ZPZ_D(sve_sxtb_d, uint64_t, DO_SXTB)
-DO_ZPZ_D(sve_sxth_d, uint64_t, DO_SXTH)
-DO_ZPZ_D(sve_sxtw_d, uint64_t, DO_SXTS)
+static inline uint8_t do_sve2_uqshl_b(uint8_t n, uint8_t m)
+{
+    uint32_t discard;
 
-DO_ZPZ(sve_uxtb_h, uint16_t, H1_2, DO_UXTB)
-DO_ZPZ(sve_uxtb_s, uint32_t, H1_4, DO_UXTB)
-DO_ZPZ(sve_uxth_s, uint32_t, H1_4, DO_UXTH)
-DO_ZPZ_D(sve_uxtb_d, uint64_t, DO_UXTB)
-DO_ZPZ_D(sve_uxth_d, uint64_t, DO_UXTH)
-DO_ZPZ_D(sve_uxtw_d, uint64_t, DO_UXTS)
+    return do_sve2_uqrshl_bhs(n, (int8_t)m, 8, false, &discard);
+}
 
-#ifdef _MSC_VER
-#define DO_ABS(N)    (N < 0 ? (0 - N) : N)
-#else
-#define DO_ABS(N)    (N < 0 ? -N : N)
-#endif
+static inline uint16_t do_sve2_uqshl_h(uint16_t n, uint16_t m)
+{
+    uint32_t discard;
 
-DO_ZPZ(sve_abs_b, int8_t, H1, DO_ABS)
-DO_ZPZ(sve_abs_h, int16_t, H1_2, DO_ABS)
-DO_ZPZ(sve_abs_s, int32_t, H1_4, DO_ABS)
-DO_ZPZ_D(sve_abs_d, int64_t, DO_ABS)
+    return do_sve2_uqrshl_bhs(n, (int16_t)m, 16, false, &discard);
+}
 
-#ifdef _MSC_VER
-#define DO_NEG(N)    (0 - N)
-#else
-#define DO_NEG(N)    (-N)
-#endif
+static inline uint32_t do_sve2_uqshl_s(uint32_t n, uint32_t m)
+{
+    uint32_t discard;
 
-DO_ZPZ(sve_neg_b, uint8_t, H1, DO_NEG)
-DO_ZPZ(sve_neg_h, uint16_t, H1_2, DO_NEG)
-DO_ZPZ(sve_neg_s, uint32_t, H1_4, DO_NEG)
-DO_ZPZ_D(sve_neg_d, uint64_t, DO_NEG)
+    return do_sve2_uqrshl_bhs(n, (int32_t)m, 32, false, &discard);
+}
 
-DO_ZPZ(sve_revb_h, uint16_t, H1_2, bswap16)
-DO_ZPZ(sve_revb_s, uint32_t, H1_4, bswap32)
-DO_ZPZ_D(sve_revb_d, uint64_t, bswap64)
+static inline uint64_t do_sve2_uqshl_d(uint64_t n, uint64_t m)
+{
+    uint32_t discard;
 
-DO_ZPZ(sve_revh_s, uint32_t, H1_4, hswap32)
-DO_ZPZ_D(sve_revh_d, uint64_t, hswap64)
+    return do_sve2_uqrshl_d_raw(n, (int64_t)m, false, &discard);
+}
 
-DO_ZPZ_D(sve_revw_d, uint64_t, wswap64)
+DO_ZPZZ(sve2_uqshl_zpzz_b, uint8_t, H1, do_sve2_uqshl_b)
+DO_ZPZZ(sve2_uqshl_zpzz_h, uint16_t, H1_2, do_sve2_uqshl_h)
+DO_ZPZZ(sve2_uqshl_zpzz_s, uint32_t, H1_4, do_sve2_uqshl_s)
+DO_ZPZZ_D(sve2_uqshl_zpzz_d, uint64_t, do_sve2_uqshl_d)
 
-DO_ZPZ(sve_rbit_b, uint8_t, H1, revbit8)
-DO_ZPZ(sve_rbit_h, uint16_t, H1_2, revbit16)
-DO_ZPZ(sve_rbit_s, uint32_t, H1_4, revbit32)
-DO_ZPZ_D(sve_rbit_d, uint64_t, revbit64)
+static inline int8_t do_sve2_sqrshl_b(int8_t n, int8_t m)
+{
+    uint32_t discard;
 
-/* Three-operand expander, unpredicated, in which the third operand is "wide".
- */
-#define DO_ZZW(NAME, TYPE, TYPEW, H, OP)                       \
-void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
-{                                                              \
-    intptr_t i, opr_sz = simd_oprsz(desc);                     \
-    for (i = 0; i < opr_sz; ) {                                \
-        TYPEW mm = *(TYPEW *)((char *)vm + i);                         \
-        do {                                                   \
-            TYPE nn = *(TYPE *)((char *)vn + H(i));                    \
-            *(TYPE *)((char *)vd + H(i)) = OP(nn, mm);                 \
-            i += sizeof(TYPE);                                 \
-        } while (i & 7);                                       \
-    }                                                          \
+    return do_sve2_sqrshl_bhs(n, m, 8, true, &discard);
 }
 
-DO_ZZW(sve_asr_zzw_b, int8_t, uint64_t, H1, DO_ASR)
-DO_ZZW(sve_lsr_zzw_b, uint8_t, uint64_t, H1, DO_LSR)
-DO_ZZW(sve_lsl_zzw_b, uint8_t, uint64_t, H1, DO_LSL)
+static inline int16_t do_sve2_sqrshl_h(int16_t n, int16_t m)
+{
+    uint32_t discard;
 
-DO_ZZW(sve_asr_zzw_h, int16_t, uint64_t, H1_2, DO_ASR)
-DO_ZZW(sve_lsr_zzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
-DO_ZZW(sve_lsl_zzw_h, uint16_t, uint64_t, H1_2, DO_LSL)
+    return do_sve2_sqrshl_bhs(n, m, 16, true, &discard);
+}
 
-DO_ZZW(sve_asr_zzw_s, int32_t, uint64_t, H1_4, DO_ASR)
-DO_ZZW(sve_lsr_zzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
-DO_ZZW(sve_lsl_zzw_s, uint32_t, uint64_t, H1_4, DO_LSL)
+static inline int32_t do_sve2_sqrshl_s(int32_t n, int32_t m)
+{
+    uint32_t discard;
 
-#undef DO_ZZW
+    return do_sve2_sqrshl_bhs(n, m, 32, true, &discard);
+}
 
-#undef DO_CLS_B
-#undef DO_CLS_H
-#undef DO_CLZ_B
-#undef DO_CLZ_H
-#undef DO_CNOT
-#undef DO_FABS
-#undef DO_FNEG
-#undef DO_ABS
-#undef DO_NEG
-#undef DO_ZPZ
-#undef DO_ZPZ_D
+static inline int64_t do_sve2_sqrshl_d(int64_t n, int64_t m)
+{
+    uint32_t discard;
 
-/* Two-operand reduction expander, controlled by a predicate.
- * The difference between TYPERED and TYPERET has to do with
- * sign-extension.  E.g. for SMAX, TYPERED must be signed,
- * but TYPERET must be unsigned so that e.g. a 32-bit value
- * is not sign-extended to the ABI uint64_t return type.
- */
-/* ??? If we were to vectorize this by hand the reduction ordering
- * would change.  For integer operands, this is perfectly fine.
- */
-#define DO_VPZ(NAME, TYPEELT, TYPERED, TYPERET, H, INIT, OP) \
-uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc)   \
-{                                                          \
-    intptr_t i, opr_sz = simd_oprsz(desc);                 \
-    TYPERED ret = INIT;                                    \
-    for (i = 0; i < opr_sz; ) {                            \
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));    \
-        do {                                               \
-            if (pg & 1) {                                  \
-                TYPEELT nn = *(TYPEELT *)((char *)vn + H(i));      \
-                ret = OP(ret, nn);                         \
-            }                                              \
-            i += sizeof(TYPEELT), pg >>= sizeof(TYPEELT);  \
-        } while (i & 15);                                  \
-    }                                                      \
-    return (TYPERET)ret;                                   \
+    return do_sve2_sqrshl_d_raw(n, m, true, &discard);
 }
 
-#define DO_VPZ_D(NAME, TYPEE, TYPER, INIT, OP)             \
-uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc)   \
-{                                                          \
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;             \
-    TYPEE *n = vn;                                         \
-    uint8_t *pg = vg;                                      \
-    TYPER ret = INIT;                                      \
-    for (i = 0; i < opr_sz; i += 1) {                      \
-        if (pg[H1(i)] & 1) {                               \
-            TYPEE nn = n[i];                               \
-            ret = OP(ret, nn);                             \
-        }                                                  \
-    }                                                      \
-    return ret;                                            \
+DO_ZPZZ(sve2_sqrshl_zpzz_b, int8_t, H1, do_sve2_sqrshl_b)
+DO_ZPZZ(sve2_sqrshl_zpzz_h, int16_t, H1_2, do_sve2_sqrshl_h)
+DO_ZPZZ(sve2_sqrshl_zpzz_s, int32_t, H1_4, do_sve2_sqrshl_s)
+DO_ZPZZ_D(sve2_sqrshl_zpzz_d, int64_t, do_sve2_sqrshl_d)
+
+static inline uint8_t do_sve2_uqrshl_b(uint8_t n, uint8_t m)
+{
+    uint32_t discard;
+
+    return do_sve2_uqrshl_bhs(n, (int8_t)m, 8, true, &discard);
 }
 
-DO_VPZ(sve_orv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_ORR)
-DO_VPZ(sve_orv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_ORR)
-DO_VPZ(sve_orv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_ORR)
-DO_VPZ_D(sve_orv_d, uint64_t, uint64_t, 0, DO_ORR)
+static inline uint16_t do_sve2_uqrshl_h(uint16_t n, uint16_t m)
+{
+    uint32_t discard;
 
-DO_VPZ(sve_eorv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_EOR)
-DO_VPZ(sve_eorv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_EOR)
-DO_VPZ(sve_eorv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_EOR)
-DO_VPZ_D(sve_eorv_d, uint64_t, uint64_t, 0, DO_EOR)
+    return do_sve2_uqrshl_bhs(n, (int16_t)m, 16, true, &discard);
+}
 
-DO_VPZ(sve_andv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_AND)
-DO_VPZ(sve_andv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_AND)
-DO_VPZ(sve_andv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_AND)
-DO_VPZ_D(sve_andv_d, uint64_t, uint64_t, -1, DO_AND)
+static inline uint32_t do_sve2_uqrshl_s(uint32_t n, uint32_t m)
+{
+    uint32_t discard;
 
-DO_VPZ(sve_saddv_b, int8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
-DO_VPZ(sve_saddv_h, int16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
-DO_VPZ(sve_saddv_s, int32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)
+    return do_sve2_uqrshl_bhs(n, (int32_t)m, 32, true, &discard);
+}
 
-DO_VPZ(sve_uaddv_b, uint8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
-DO_VPZ(sve_uaddv_h, uint16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
-DO_VPZ(sve_uaddv_s, uint32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)
-DO_VPZ_D(sve_uaddv_d, uint64_t, uint64_t, 0, DO_ADD)
+static inline uint64_t do_sve2_uqrshl_d(uint64_t n, uint64_t m)
+{
+    uint32_t discard;
 
-DO_VPZ(sve_smaxv_b, int8_t, int8_t, uint8_t, H1, INT8_MIN, DO_MAX)
-DO_VPZ(sve_smaxv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MIN, DO_MAX)
-DO_VPZ(sve_smaxv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MIN, DO_MAX)
-DO_VPZ_D(sve_smaxv_d, int64_t, int64_t, INT64_MIN, DO_MAX)
+    return do_sve2_uqrshl_d_raw(n, (int64_t)m, true, &discard);
+}
 
-DO_VPZ(sve_umaxv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_MAX)
-DO_VPZ(sve_umaxv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_MAX)
-DO_VPZ(sve_umaxv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_MAX)
-DO_VPZ_D(sve_umaxv_d, uint64_t, uint64_t, 0, DO_MAX)
+DO_ZPZZ(sve2_uqrshl_zpzz_b, uint8_t, H1, do_sve2_uqrshl_b)
+DO_ZPZZ(sve2_uqrshl_zpzz_h, uint16_t, H1_2, do_sve2_uqrshl_h)
+DO_ZPZZ(sve2_uqrshl_zpzz_s, uint32_t, H1_4, do_sve2_uqrshl_s)
+DO_ZPZZ_D(sve2_uqrshl_zpzz_d, uint64_t, do_sve2_uqrshl_d)
 
-DO_VPZ(sve_sminv_b, int8_t, int8_t, uint8_t, H1, INT8_MAX, DO_MIN)
-DO_VPZ(sve_sminv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MAX, DO_MIN)
-DO_VPZ(sve_sminv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MAX, DO_MIN)
-DO_VPZ_D(sve_sminv_d, int64_t, int64_t, INT64_MAX, DO_MIN)
+#define DO_HADD_BHS(N, M)  (((int64_t)(N) + (M)) >> 1)
+#define DO_HADD_D(N, M)    (((N) >> 1) + ((M) >> 1) + ((N) & (M) & 1))
 
-DO_VPZ(sve_uminv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_MIN)
-DO_VPZ(sve_uminv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_MIN)
-DO_VPZ(sve_uminv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_MIN)
-DO_VPZ_D(sve_uminv_d, uint64_t, uint64_t, -1, DO_MIN)
+DO_ZPZZ(sve2_shadd_zpzz_b, int8_t, H1, DO_HADD_BHS)
+DO_ZPZZ(sve2_shadd_zpzz_h, int16_t, H1_2, DO_HADD_BHS)
+DO_ZPZZ(sve2_shadd_zpzz_s, int32_t, H1_4, DO_HADD_BHS)
+DO_ZPZZ_D(sve2_shadd_zpzz_d, int64_t, DO_HADD_D)
 
-#undef DO_VPZ
-#undef DO_VPZ_D
+DO_ZPZZ(sve2_uhadd_zpzz_b, uint8_t, H1, DO_HADD_BHS)
+DO_ZPZZ(sve2_uhadd_zpzz_h, uint16_t, H1_2, DO_HADD_BHS)
+DO_ZPZZ(sve2_uhadd_zpzz_s, uint32_t, H1_4, DO_HADD_BHS)
+DO_ZPZZ_D(sve2_uhadd_zpzz_d, uint64_t, DO_HADD_D)
 
-/* Two vector operand, one scalar operand, unpredicated.  */
-#define DO_ZZI(NAME, TYPE, OP)                                       \
-void HELPER(NAME)(void *vd, void *vn, uint64_t s64, uint32_t desc)   \
-{                                                                    \
-    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE);            \
-    TYPE s = s64, *d = vd, *n = vn;                                  \
-    for (i = 0; i < opr_sz; ++i) {                                   \
-        d[i] = OP(n[i], s);                                          \
-    }                                                                \
-}
+#define DO_RHADD_BHS(N, M)  (((int64_t)(N) + (M) + 1) >> 1)
+#define DO_RHADD_D(N, M)    (((N) >> 1) + ((M) >> 1) + (((N) | (M)) & 1))
 
-#define DO_SUBR(X, Y)   (Y - X)
+DO_ZPZZ(sve2_srhadd_zpzz_b, int8_t, H1, DO_RHADD_BHS)
+DO_ZPZZ(sve2_srhadd_zpzz_h, int16_t, H1_2, DO_RHADD_BHS)
+DO_ZPZZ(sve2_srhadd_zpzz_s, int32_t, H1_4, DO_RHADD_BHS)
+DO_ZPZZ_D(sve2_srhadd_zpzz_d, int64_t, DO_RHADD_D)
 
-DO_ZZI(sve_subri_b, uint8_t, DO_SUBR)
-DO_ZZI(sve_subri_h, uint16_t, DO_SUBR)
-DO_ZZI(sve_subri_s, uint32_t, DO_SUBR)
-DO_ZZI(sve_subri_d, uint64_t, DO_SUBR)
+DO_ZPZZ(sve2_urhadd_zpzz_b, uint8_t, H1, DO_RHADD_BHS)
+DO_ZPZZ(sve2_urhadd_zpzz_h, uint16_t, H1_2, DO_RHADD_BHS)
+DO_ZPZZ(sve2_urhadd_zpzz_s, uint32_t, H1_4, DO_RHADD_BHS)
+DO_ZPZZ_D(sve2_urhadd_zpzz_d, uint64_t, DO_RHADD_D)
 
-DO_ZZI(sve_smaxi_b, int8_t, DO_MAX)
-DO_ZZI(sve_smaxi_h, int16_t, DO_MAX)
-DO_ZZI(sve_smaxi_s, int32_t, DO_MAX)
-DO_ZZI(sve_smaxi_d, int64_t, DO_MAX)
+#define DO_HSUB_BHS(N, M)  (((int64_t)(N) - (M)) >> 1)
+#define DO_HSUB_D(N, M) \
+    (((N) >> 1) - ((M) >> 1) - (~(N) & (M) & 1))
 
-DO_ZZI(sve_smini_b, int8_t, DO_MIN)
-DO_ZZI(sve_smini_h, int16_t, DO_MIN)
-DO_ZZI(sve_smini_s, int32_t, DO_MIN)
-DO_ZZI(sve_smini_d, int64_t, DO_MIN)
+DO_ZPZZ(sve2_shsub_zpzz_b, int8_t, H1, DO_HSUB_BHS)
+DO_ZPZZ(sve2_shsub_zpzz_h, int16_t, H1_2, DO_HSUB_BHS)
+DO_ZPZZ(sve2_shsub_zpzz_s, int32_t, H1_4, DO_HSUB_BHS)
+DO_ZPZZ_D(sve2_shsub_zpzz_d, int64_t, DO_HSUB_D)
 
-DO_ZZI(sve_umaxi_b, uint8_t, DO_MAX)
-DO_ZZI(sve_umaxi_h, uint16_t, DO_MAX)
-DO_ZZI(sve_umaxi_s, uint32_t, DO_MAX)
-DO_ZZI(sve_umaxi_d, uint64_t, DO_MAX)
+DO_ZPZZ(sve2_uhsub_zpzz_b, uint8_t, H1, DO_HSUB_BHS)
+DO_ZPZZ(sve2_uhsub_zpzz_h, uint16_t, H1_2, DO_HSUB_BHS)
+DO_ZPZZ(sve2_uhsub_zpzz_s, uint32_t, H1_4, DO_HSUB_BHS)
+DO_ZPZZ_D(sve2_uhsub_zpzz_d, uint64_t, DO_HSUB_D)
 
-DO_ZZI(sve_umini_b, uint8_t, DO_MIN)
-DO_ZZI(sve_umini_h, uint16_t, DO_MIN)
-DO_ZZI(sve_umini_s, uint32_t, DO_MIN)
-DO_ZZI(sve_umini_d, uint64_t, DO_MIN)
+#undef DO_HADD_BHS
+#undef DO_HADD_D
+#undef DO_RHADD_BHS
+#undef DO_RHADD_D
+#undef DO_HSUB_BHS
+#undef DO_HSUB_D
 
-#undef DO_ZZI
+#define DO_SAT_BHS(VAL, MIN, MAX) \
+    ((VAL) > (MAX) ? (MAX) : (VAL) < (MIN) ? (MIN) : (VAL))
 
-#undef DO_AND
-#undef DO_ORR
-#undef DO_EOR
-#undef DO_BIC
-#undef DO_ADD
-#undef DO_SUB
-#undef DO_MAX
-#undef DO_MIN
-#undef DO_ABD
-#undef DO_MUL
-#undef DO_DIV
-#undef DO_ASR
-#undef DO_LSR
-#undef DO_LSL
-#undef DO_SUBR
+#define DO_SQADD_B(N, M) DO_SAT_BHS((int64_t)(N) + (M), INT8_MIN, INT8_MAX)
+#define DO_SQADD_H(N, M) DO_SAT_BHS((int64_t)(N) + (M), INT16_MIN, INT16_MAX)
+#define DO_SQADD_S(N, M) DO_SAT_BHS((int64_t)(N) + (M), INT32_MIN, INT32_MAX)
 
-/* Similar to the ARM LastActiveElement pseudocode function, except the
-   result is multiplied by the element size.  This includes the not found
-   indication; e.g. not found for esz=3 is -8.  */
-static intptr_t last_active_element(uint64_t *g, intptr_t words, intptr_t esz)
+static inline int64_t do_sve2_sqadd_d(int64_t n, int64_t m)
 {
-    uint64_t mask = pred_esz_masks[esz];
-    intptr_t i = words;
+    int64_t r = (int64_t)((uint64_t)n + (uint64_t)m);
 
-    do {
-        uint64_t this_g = g[--i] & mask;
-        if (this_g) {
-            return i * 64 + (63 - clz64(this_g));
-        }
-    } while (i > 0);
-    return (intptr_t)-1 << esz;
+    if (m > 0 && r < n) {
+        return INT64_MAX;
+    }
+    if (m < 0 && r > n) {
+        return INT64_MIN;
+    }
+    return r;
 }
 
-uint32_t HELPER(sve_pfirst)(void *vd, void *vg, uint32_t words)
+DO_ZPZZ(sve2_sqadd_zpzz_b, int8_t, H1, DO_SQADD_B)
+DO_ZPZZ(sve2_sqadd_zpzz_h, int16_t, H1_2, DO_SQADD_H)
+DO_ZPZZ(sve2_sqadd_zpzz_s, int32_t, H1_4, DO_SQADD_S)
+DO_ZPZZ_D(sve2_sqadd_zpzz_d, int64_t, do_sve2_sqadd_d)
+
+#define DO_UQADD_B(N, M) DO_SAT_BHS((int64_t)(N) + (M), 0, UINT8_MAX)
+#define DO_UQADD_H(N, M) DO_SAT_BHS((int64_t)(N) + (M), 0, UINT16_MAX)
+#define DO_UQADD_S(N, M) DO_SAT_BHS((int64_t)(N) + (M), 0, UINT32_MAX)
+
+static inline uint64_t do_sve2_uqadd_d(uint64_t n, uint64_t m)
 {
-    uint32_t flags = PREDTEST_INIT;
-    uint64_t *d = vd, *g = vg;
-    intptr_t i = 0;
+    uint64_t r = n + m;
 
-    do {
-        uint64_t this_d = d[i];
-        uint64_t this_g = g[i];
+    return r < n ? UINT64_MAX : r;
+}
 
-        if (this_g) {
-            if (!(flags & 4)) {
-                /* Set in D the first bit of G.  */
-#ifdef _MSC_VER
-                this_d |= this_g & (0 - this_g);
-#else
-                this_d |= this_g & -this_g;
-#endif
-                d[i] = this_d;
-            }
-            flags = iter_predtest_fwd(this_d, this_g, flags);
-        }
-    } while (++i < words);
+DO_ZPZZ(sve2_uqadd_zpzz_b, uint8_t, H1, DO_UQADD_B)
+DO_ZPZZ(sve2_uqadd_zpzz_h, uint16_t, H1_2, DO_UQADD_H)
+DO_ZPZZ(sve2_uqadd_zpzz_s, uint32_t, H1_4, DO_UQADD_S)
+DO_ZPZZ_D(sve2_uqadd_zpzz_d, uint64_t, do_sve2_uqadd_d)
 
-    return flags;
+#define DO_SQSUB_B(N, M) DO_SAT_BHS((int64_t)(N) - (M), INT8_MIN, INT8_MAX)
+#define DO_SQSUB_H(N, M) DO_SAT_BHS((int64_t)(N) - (M), INT16_MIN, INT16_MAX)
+#define DO_SQSUB_S(N, M) DO_SAT_BHS((int64_t)(N) - (M), INT32_MIN, INT32_MAX)
+
+static inline int64_t do_sve2_sqsub_d(int64_t n, int64_t m)
+{
+    int64_t r = (int64_t)((uint64_t)n - (uint64_t)m);
+
+    if (m > 0 && r > n) {
+        return INT64_MIN;
+    }
+    if (m < 0 && r < n) {
+        return INT64_MAX;
+    }
+    return r;
 }
 
-uint32_t HELPER(sve_pnext)(void *vd, void *vg, uint32_t pred_desc)
+DO_ZPZZ(sve2_sqsub_zpzz_b, int8_t, H1, DO_SQSUB_B)
+DO_ZPZZ(sve2_sqsub_zpzz_h, int16_t, H1_2, DO_SQSUB_H)
+DO_ZPZZ(sve2_sqsub_zpzz_s, int32_t, H1_4, DO_SQSUB_S)
+DO_ZPZZ_D(sve2_sqsub_zpzz_d, int64_t, do_sve2_sqsub_d)
+
+#define DO_UQSUB_B(N, M) DO_SAT_BHS((int64_t)(N) - (M), 0, UINT8_MAX)
+#define DO_UQSUB_H(N, M) DO_SAT_BHS((int64_t)(N) - (M), 0, UINT16_MAX)
+#define DO_UQSUB_S(N, M) DO_SAT_BHS((int64_t)(N) - (M), 0, UINT32_MAX)
+
+static inline uint64_t do_sve2_uqsub_d(uint64_t n, uint64_t m)
 {
-    intptr_t words = extract32(pred_desc, 0, SIMD_OPRSZ_BITS);
-    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
-    uint32_t flags = PREDTEST_INIT;
-    uint64_t *d = vd, *g = vg, esz_mask;
-    intptr_t i, next;
+    return n > m ? n - m : 0;
+}
 
-    next = last_active_element(vd, words, esz) + (1ULL << esz);
-    esz_mask = pred_esz_masks[esz];
+DO_ZPZZ(sve2_uqsub_zpzz_b, uint8_t, H1, DO_UQSUB_B)
+DO_ZPZZ(sve2_uqsub_zpzz_h, uint16_t, H1_2, DO_UQSUB_H)
+DO_ZPZZ(sve2_uqsub_zpzz_s, uint32_t, H1_4, DO_UQSUB_S)
+DO_ZPZZ_D(sve2_uqsub_zpzz_d, uint64_t, do_sve2_uqsub_d)
 
-    /* Similar to the pseudocode for pnext, but scaled by ESZ
-       so that we find the correct bit.  */
-    if (next < words * 64) {
-        uint64_t mask = -1;
+#define DO_SUQADD_B(N, M) \
+    DO_SAT_BHS((int64_t)(int8_t)(N) + (M), INT8_MIN, INT8_MAX)
+#define DO_SUQADD_H(N, M) \
+    DO_SAT_BHS((int64_t)(int16_t)(N) + (M), INT16_MIN, INT16_MAX)
+#define DO_SUQADD_S(N, M) \
+    DO_SAT_BHS((int64_t)(int32_t)(N) + (M), INT32_MIN, INT32_MAX)
 
-        if (next & 63) {
-            mask = ~((1ull << (next & 63)) - 1);
-            next &= -64;
+static inline int64_t do_sve2_suqadd_d(int64_t n, uint64_t m)
+{
+    if (n < 0) {
+        uint64_t abs_n = ~((uint64_t)n) + 1;
+
+        if (m >= abs_n && m - abs_n > INT64_MAX) {
+            return INT64_MAX;
         }
-        do {
-            uint64_t this_g = g[next / 64] & esz_mask & mask;
-            if (this_g != 0) {
-                next = (next & -64) + ctz64(this_g);
-                break;
-            }
-            next += 64;
-            mask = -1;
-        } while (next < words * 64);
+        return (int64_t)((uint64_t)n + m);
+    }
+    if (m > (uint64_t)INT64_MAX - (uint64_t)n) {
+        return INT64_MAX;
     }
+    return (int64_t)((uint64_t)n + m);
+}
 
-    i = 0;
-    do {
-        uint64_t this_d = 0;
-        if (i == next / 64) {
-            this_d = 1ull << (next & 63);
-        }
-        d[i] = this_d;
-        flags = iter_predtest_fwd(this_d, g[i] & esz_mask, flags);
-    } while (++i < words);
+DO_ZPZZ(sve2_suqadd_zpzz_b, uint8_t, H1, DO_SUQADD_B)
+DO_ZPZZ(sve2_suqadd_zpzz_h, uint16_t, H1_2, DO_SUQADD_H)
+DO_ZPZZ(sve2_suqadd_zpzz_s, uint32_t, H1_4, DO_SUQADD_S)
+DO_ZPZZ_D(sve2_suqadd_zpzz_d, uint64_t, do_sve2_suqadd_d)
 
-    return flags;
-}
+#define DO_USQADD_B(N, M) \
+    DO_SAT_BHS((int64_t)(N) + (int8_t)(M), 0, UINT8_MAX)
+#define DO_USQADD_H(N, M) \
+    DO_SAT_BHS((int64_t)(N) + (int16_t)(M), 0, UINT16_MAX)
+#define DO_USQADD_S(N, M) \
+    DO_SAT_BHS((int64_t)(N) + (int32_t)(M), 0, UINT32_MAX)
 
-/* Store zero into every active element of Zd.  We will use this for two
- * and three-operand predicated instructions for which logic dictates a
- * zero result.  In particular, logical shift by element size, which is
- * otherwise undefined on the host.
- *
- * For element sizes smaller than uint64_t, we use tables to expand
- * the N bits of the controlling predicate to a byte mask, and clear
- * those bytes.
- */
-void HELPER(sve_clr_b)(void *vd, void *vg, uint32_t desc)
+static inline uint64_t do_sve2_usqadd_d(uint64_t n, int64_t m)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] &= ~expand_pred_b(pg[H1(i)]);
+    if (m < 0) {
+        uint64_t abs_m = ~((uint64_t)m) + 1;
+
+        return n < abs_m ? 0 : n - abs_m;
     }
+    return UINT64_MAX - n < (uint64_t)m ? UINT64_MAX : n + (uint64_t)m;
+}
+
+DO_ZPZZ(sve2_usqadd_zpzz_b, uint8_t, H1, DO_USQADD_B)
+DO_ZPZZ(sve2_usqadd_zpzz_h, uint16_t, H1_2, DO_USQADD_H)
+DO_ZPZZ(sve2_usqadd_zpzz_s, uint32_t, H1_4, DO_USQADD_S)
+DO_ZPZZ_D(sve2_usqadd_zpzz_d, uint64_t, do_sve2_usqadd_d)
+
+#undef DO_SAT_BHS
+#undef DO_SQADD_B
+#undef DO_SQADD_H
+#undef DO_SQADD_S
+#undef DO_UQADD_B
+#undef DO_UQADD_H
+#undef DO_UQADD_S
+#undef DO_SQSUB_B
+#undef DO_SQSUB_H
+#undef DO_SQSUB_S
+#undef DO_UQSUB_B
+#undef DO_UQSUB_H
+#undef DO_UQSUB_S
+#undef DO_SUQADD_B
+#undef DO_SUQADD_H
+#undef DO_SUQADD_S
+#undef DO_USQADD_B
+#undef DO_USQADD_H
+#undef DO_USQADD_S
+
+/* Because the computation type is at least twice as large as required,
+   these work for both signed and unsigned source types.  */
+static inline uint8_t do_mulh_b(int32_t n, int32_t m)
+{
+    return (n * m) >> 8;
 }
 
-void HELPER(sve_clr_h)(void *vd, void *vg, uint32_t desc)
+static inline uint16_t do_mulh_h(int32_t n, int32_t m)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] &= ~expand_pred_h(pg[H1(i)]);
-    }
+    return (n * m) >> 16;
 }
 
-void HELPER(sve_clr_s)(void *vd, void *vg, uint32_t desc)
+static inline uint32_t do_mulh_s(int64_t n, int64_t m)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] &= ~expand_pred_s(pg[H1(i)]);
-    }
+    return (n * m) >> 32;
 }
 
-void HELPER(sve_clr_d)(void *vd, void *vg, uint32_t desc)
+static inline uint64_t do_smulh_d(uint64_t n, uint64_t m)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-        if (pg[H1(i)] & 1) {
-            d[i] = 0;
-        }
+    uint64_t lo, hi;
+    muls64(&lo, &hi, n, m);
+    return hi;
+}
+
+static inline uint64_t do_umulh_d(uint64_t n, uint64_t m)
+{
+    uint64_t lo, hi;
+    mulu64(&lo, &hi, n, m);
+    return hi;
+}
+
+DO_ZPZZ(sve_mul_zpzz_b, uint8_t, H1, DO_MUL)
+DO_ZPZZ(sve_mul_zpzz_h, uint16_t, H1_2, DO_MUL)
+DO_ZPZZ(sve_mul_zpzz_s, uint32_t, H1_4, DO_MUL)
+DO_ZPZZ_D(sve_mul_zpzz_d, uint64_t, DO_MUL)
+
+DO_ZPZZ(sve_smulh_zpzz_b, int8_t, H1, do_mulh_b)
+DO_ZPZZ(sve_smulh_zpzz_h, int16_t, H1_2, do_mulh_h)
+DO_ZPZZ(sve_smulh_zpzz_s, int32_t, H1_4, do_mulh_s)
+DO_ZPZZ_D(sve_smulh_zpzz_d, uint64_t, do_smulh_d)
+
+DO_ZPZZ(sve_umulh_zpzz_b, uint8_t, H1, do_mulh_b)
+DO_ZPZZ(sve_umulh_zpzz_h, uint16_t, H1_2, do_mulh_h)
+DO_ZPZZ(sve_umulh_zpzz_s, uint32_t, H1_4, do_mulh_s)
+DO_ZPZZ_D(sve_umulh_zpzz_d, uint64_t, do_umulh_d)
+
+#define DO_ZZZ_MULH(NAME, TYPE, H, OP)                                    \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)            \
+{                                                                         \
+    intptr_t i, oprsz = simd_oprsz(desc);                                 \
+                                                                          \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                           \
+        TYPE nn = *(TYPE *)((char *)vn + H(i));                           \
+        TYPE mm = *(TYPE *)((char *)vm + H(i));                           \
+                                                                          \
+        *(TYPE *)((char *)vd + H(i)) = OP(nn, mm);                        \
+    }                                                                     \
+}
+
+#define DO_ZZZ_MULH_D(NAME, OP)                                           \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)            \
+{                                                                         \
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;                             \
+    uint64_t *d = vd, *n = vn, *m = vm;                                   \
+                                                                          \
+    for (i = 0; i < oprsz; i++) {                                         \
+        d[i] = OP(n[i], m[i]);                                            \
+    }                                                                     \
+}
+
+DO_ZZZ_MULH(sve2_smulh_zzz_b, int8_t, H1, do_mulh_b)
+DO_ZZZ_MULH(sve2_smulh_zzz_h, int16_t, H1_2, do_mulh_h)
+DO_ZZZ_MULH(sve2_smulh_zzz_s, int32_t, H1_4, do_mulh_s)
+DO_ZZZ_MULH_D(sve2_smulh_zzz_d, do_smulh_d)
+
+DO_ZZZ_MULH(sve2_umulh_zzz_b, uint8_t, H1, do_mulh_b)
+DO_ZZZ_MULH(sve2_umulh_zzz_h, uint16_t, H1_2, do_mulh_h)
+DO_ZZZ_MULH(sve2_umulh_zzz_s, uint32_t, H1_4, do_mulh_s)
+DO_ZZZ_MULH_D(sve2_umulh_zzz_d, do_umulh_d)
+
+#undef DO_ZZZ_MULH
+#undef DO_ZZZ_MULH_D
+
+static int8_t do_sqrdmlah_b(int8_t n, int8_t m, int8_t a,
+                            bool neg, bool round)
+{
+    int32_t r = (int32_t)n * m;
+
+    if (neg) {
+        r = -r;
+    }
+    r += (int32_t)a * (1 << 7) + (round << 6);
+    r >>= 7;
+
+    if (r != (int8_t)r) {
+        r = (r < 0 ? INT8_MIN : INT8_MAX);
     }
+    return r;
 }
 
-/* Copy Zn into Zd, and store zero into inactive elements.  */
-void HELPER(sve_movz_b)(void *vd, void *vn, void *vg, uint32_t desc)
+static int16_t do_sqrdmlah_h(int16_t n, int16_t m, int16_t a,
+                             bool neg, bool round)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = n[i] & expand_pred_b(pg[H1(i)]);
+    int32_t r = (int32_t)n * m;
+
+    if (neg) {
+        r = -r;
+    }
+    r += (int32_t)a * (1 << 15) + (round << 14);
+    r >>= 15;
+
+    if (r != (int16_t)r) {
+        r = (r < 0 ? INT16_MIN : INT16_MAX);
     }
+    return r;
 }
 
-void HELPER(sve_movz_h)(void *vd, void *vn, void *vg, uint32_t desc)
+static int32_t do_sqrdmlah_s(int32_t n, int32_t m, int32_t a,
+                             bool neg, bool round)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = n[i] & expand_pred_h(pg[H1(i)]);
+    int64_t r = (int64_t)n * m;
+
+    if (neg) {
+        r = -r;
+    }
+    r += (int64_t)a * (1ll << 31) + (round << 30);
+    r >>= 31;
+
+    if (r != (int32_t)r) {
+        r = (r < 0 ? INT32_MIN : INT32_MAX);
     }
+    return r;
 }
 
-void HELPER(sve_movz_s)(void *vd, void *vn, void *vg, uint32_t desc)
+static int64_t do_sat128_d(Int128 r)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = n[i] & expand_pred_s(pg[H1(i)]);
+    int64_t lo = int128_getlo(r);
+    int64_t hi = int128_gethi(r);
+
+    if (hi != (lo >> 63)) {
+        return hi < 0 ? INT64_MIN : INT64_MAX;
     }
+    return lo;
 }
 
-void HELPER(sve_movz_d)(void *vd, void *vn, void *vg, uint32_t desc)
+static int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a,
+                             bool neg, bool round)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
-    for (i = 0; i < opr_sz; i += 1) {
-#ifdef _MSC_VER
-        d[i] = n[i] & ((uint64_t)0 - (uint64_t)(pg[H1(i)] & 1));
-#else
-        d[i] = n[i] & -(uint64_t)(pg[H1(i)] & 1);
-#endif
+    uint64_t lo, hi;
+    Int128 r, t;
+
+    muls64(&lo, &hi, m, n);
+    r = int128_make128(lo, hi);
+    if (neg) {
+        r = int128_neg(r);
+    }
+    if (a) {
+        t = int128_exts64(a);
+        t = int128_lshift(t, 63);
+        r = int128_add(r, t);
     }
+    if (round) {
+        t = int128_exts64(1ll << 62);
+        r = int128_add(r, t);
+    }
+    r = int128_rshift(r, 63);
+
+    return do_sat128_d(r);
+}
+
+#define DO_ZZZ_SQDMULH(NAME, TYPE, H, OP, ROUND)                         \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)           \
+{                                                                        \
+    intptr_t i, oprsz = simd_oprsz(desc);                                \
+                                                                         \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                          \
+        TYPE nn = *(TYPE *)((char *)vn + H(i));                          \
+        TYPE mm = *(TYPE *)((char *)vm + H(i));                          \
+                                                                         \
+        *(TYPE *)((char *)vd + H(i)) = OP(nn, mm, 0, false, ROUND);      \
+    }                                                                    \
+}
+
+#define DO_ZZZ_SQDMULH_D(NAME, ROUND)                                    \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)           \
+{                                                                        \
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;                            \
+    int64_t *d = vd, *n = vn, *m = vm;                                   \
+                                                                         \
+    for (i = 0; i < oprsz; i++) {                                        \
+        d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, ROUND);              \
+    }                                                                    \
+}
+
+DO_ZZZ_SQDMULH(sve2_sqdmulh_b, int8_t, H1, do_sqrdmlah_b, false)
+DO_ZZZ_SQDMULH(sve2_sqdmulh_h, int16_t, H1_2, do_sqrdmlah_h, false)
+DO_ZZZ_SQDMULH(sve2_sqdmulh_s, int32_t, H1_4, do_sqrdmlah_s, false)
+DO_ZZZ_SQDMULH_D(sve2_sqdmulh_d, false)
+
+DO_ZZZ_SQDMULH(sve2_sqrdmulh_b, int8_t, H1, do_sqrdmlah_b, true)
+DO_ZZZ_SQDMULH(sve2_sqrdmulh_h, int16_t, H1_2, do_sqrdmlah_h, true)
+DO_ZZZ_SQDMULH(sve2_sqrdmulh_s, int32_t, H1_4, do_sqrdmlah_s, true)
+DO_ZZZ_SQDMULH_D(sve2_sqrdmulh_d, true)
+
+#define DO_ZZZ_SQRDMLA(NAME, TYPE, H, OP, NEG)                          \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                        \
+    intptr_t i, oprsz = simd_oprsz(desc);                                \
+                                                                         \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                          \
+        TYPE aa = *(TYPE *)((char *)va + H(i));                          \
+        TYPE nn = *(TYPE *)((char *)vn + H(i));                          \
+        TYPE mm = *(TYPE *)((char *)vm + H(i));                          \
+                                                                         \
+        *(TYPE *)((char *)vd + H(i)) = OP(nn, mm, aa, NEG, true);        \
+    }                                                                    \
+}
+
+#define DO_ZZZ_SQRDMLA_D(NAME, NEG)                                      \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                        \
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;                            \
+    int64_t *d = vd, *n = vn, *m = vm, *a = va;                          \
+                                                                         \
+    for (i = 0; i < oprsz; i++) {                                        \
+        d[i] = do_sqrdmlah_d(n[i], m[i], a[i], NEG, true);              \
+    }                                                                    \
+}
+
+DO_ZZZ_SQRDMLA(sve2_sqrdmlah_b, int8_t, H1, do_sqrdmlah_b, false)
+DO_ZZZ_SQRDMLA(sve2_sqrdmlah_h, int16_t, H1_2, do_sqrdmlah_h, false)
+DO_ZZZ_SQRDMLA(sve2_sqrdmlah_s, int32_t, H1_4, do_sqrdmlah_s, false)
+DO_ZZZ_SQRDMLA_D(sve2_sqrdmlah_d, false)
+
+DO_ZZZ_SQRDMLA(sve2_sqrdmlsh_b, int8_t, H1, do_sqrdmlah_b, true)
+DO_ZZZ_SQRDMLA(sve2_sqrdmlsh_h, int16_t, H1_2, do_sqrdmlah_h, true)
+DO_ZZZ_SQRDMLA(sve2_sqrdmlsh_s, int32_t, H1_4, do_sqrdmlah_s, true)
+DO_ZZZ_SQRDMLA_D(sve2_sqrdmlsh_d, true)
+
+#undef DO_ZZZ_SQDMULH
+#undef DO_ZZZ_SQDMULH_D
+#undef DO_ZZZ_SQRDMLA
+#undef DO_ZZZ_SQRDMLA_D
+
+#define DO_ZZX_MUL(NAME, TYPE, H, OP)                                    \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)           \
+{                                                                        \
+    intptr_t i, j, oprsz = simd_oprsz(desc);                             \
+    intptr_t segment = 16 / sizeof(TYPE);                                \
+    intptr_t idx = simd_data(desc);                                      \
+    TYPE *d = vd, *n = vn, *m = vm;                                      \
+                                                                         \
+    for (i = 0; i < oprsz / sizeof(TYPE); i += segment) {                \
+        TYPE mm = m[H(i + idx)];                                         \
+                                                                         \
+        for (j = 0; j < segment; j++) {                                  \
+            d[i + j] = OP(n[i + j], mm);                                 \
+        }                                                                \
+    }                                                                    \
+}
+
+#define DO_ZZX_MUL_D(NAME, OP)                                           \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)           \
+{                                                                        \
+    intptr_t i, j, oprsz = simd_oprsz(desc) / 8;                         \
+    intptr_t idx = simd_data(desc);                                      \
+    uint64_t *d = vd, *n = vn, *m = vm;                                  \
+                                                                         \
+    for (i = 0; i < oprsz; i += 2) {                                     \
+        uint64_t mm = m[i + idx];                                        \
+                                                                         \
+        for (j = 0; j < 2; j++) {                                        \
+            d[i + j] = OP(n[i + j], mm);                                 \
+        }                                                                \
+    }                                                                    \
+}
+
+#define DO_ZZX_SQDMULH(NAME, TYPE, H, OP, ROUND)                         \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)           \
+{                                                                        \
+    intptr_t i, j, oprsz = simd_oprsz(desc);                             \
+    intptr_t segment = 16 / sizeof(TYPE);                                \
+    intptr_t idx = simd_data(desc);                                      \
+    TYPE *d = vd, *n = vn, *m = vm;                                      \
+                                                                         \
+    for (i = 0; i < oprsz / sizeof(TYPE); i += segment) {                \
+        TYPE mm = m[H(i + idx)];                                         \
+                                                                         \
+        for (j = 0; j < segment; j++) {                                  \
+            d[i + j] = OP(n[i + j], mm, 0, false, ROUND);                \
+        }                                                                \
+    }                                                                    \
+}
+
+#define DO_ZZX_SQDMULH_D(NAME, ROUND)                                    \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)           \
+{                                                                        \
+    intptr_t i, j, oprsz = simd_oprsz(desc) / 8;                         \
+    intptr_t idx = simd_data(desc);                                      \
+    int64_t *d = vd, *n = vn, *m = vm;                                   \
+                                                                         \
+    for (i = 0; i < oprsz; i += 2) {                                     \
+        int64_t mm = m[i + idx];                                         \
+                                                                         \
+        for (j = 0; j < 2; j++) {                                        \
+            d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, ROUND);    \
+        }                                                                \
+    }                                                                    \
+}
+
+DO_ZZX_MUL(sve2_mul_idx_h, uint16_t, H2, DO_MUL)
+DO_ZZX_MUL(sve2_mul_idx_s, uint32_t, H4, DO_MUL)
+DO_ZZX_MUL_D(sve2_mul_idx_d, DO_MUL)
+
+DO_ZZX_SQDMULH(sve2_sqdmulh_idx_h, int16_t, H2, do_sqrdmlah_h, false)
+DO_ZZX_SQDMULH(sve2_sqdmulh_idx_s, int32_t, H4, do_sqrdmlah_s, false)
+DO_ZZX_SQDMULH_D(sve2_sqdmulh_idx_d, false)
+
+DO_ZZX_SQDMULH(sve2_sqrdmulh_idx_h, int16_t, H2, do_sqrdmlah_h, true)
+DO_ZZX_SQDMULH(sve2_sqrdmulh_idx_s, int32_t, H4, do_sqrdmlah_s, true)
+DO_ZZX_SQDMULH_D(sve2_sqrdmulh_idx_d, true)
+
+#define DO_ZZX_SQRDMLA(NAME, TYPE, H, OP, NEG)                          \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                        \
+    intptr_t i, j, oprsz = simd_oprsz(desc);                             \
+    intptr_t segment = 16 / sizeof(TYPE);                                \
+    intptr_t idx = simd_data(desc);                                      \
+    TYPE *d = vd, *n = vn, *m = vm, *a = va;                             \
+                                                                         \
+    for (i = 0; i < oprsz / sizeof(TYPE); i += segment) {                \
+        TYPE mm = m[H(i + idx)];                                         \
+                                                                         \
+        for (j = 0; j < segment; j++) {                                  \
+            d[i + j] = OP(n[i + j], mm, a[i + j], NEG, true);            \
+        }                                                                \
+    }                                                                    \
+}
+
+#define DO_ZZX_SQRDMLA_D(NAME, NEG)                                      \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                        \
+    intptr_t i, j, oprsz = simd_oprsz(desc) / 8;                         \
+    intptr_t idx = simd_data(desc);                                      \
+    int64_t *d = vd, *n = vn, *m = vm, *a = va;                          \
+                                                                         \
+    for (i = 0; i < oprsz; i += 2) {                                     \
+        int64_t mm = m[i + idx];                                         \
+                                                                         \
+        for (j = 0; j < 2; j++) {                                        \
+            d[i + j] = do_sqrdmlah_d(n[i + j], mm, a[i + j], NEG, true); \
+        }                                                                \
+    }                                                                    \
+}
+
+DO_ZZX_SQRDMLA(sve2_sqrdmlah_idx_h, int16_t, H2, do_sqrdmlah_h, false)
+DO_ZZX_SQRDMLA(sve2_sqrdmlah_idx_s, int32_t, H4, do_sqrdmlah_s, false)
+DO_ZZX_SQRDMLA_D(sve2_sqrdmlah_idx_d, false)
+
+DO_ZZX_SQRDMLA(sve2_sqrdmlsh_idx_h, int16_t, H2, do_sqrdmlah_h, true)
+DO_ZZX_SQRDMLA(sve2_sqrdmlsh_idx_s, int32_t, H4, do_sqrdmlah_s, true)
+DO_ZZX_SQRDMLA_D(sve2_sqrdmlsh_idx_d, true)
+
+#undef DO_ZZX_MUL
+#undef DO_ZZX_MUL_D
+#undef DO_ZZX_SQDMULH
+#undef DO_ZZX_SQDMULH_D
+#undef DO_ZZX_SQRDMLA
+#undef DO_ZZX_SQRDMLA_D
+
+#define DO_CMLA_FUNC(NAME, TYPE, H, OP)                                \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc) / sizeof(TYPE);               \
+    int rot = simd_data(desc);                                          \
+    int sel_a = rot & 1;                                                \
+    int sel_b = sel_a ^ 1;                                              \
+    bool sub_r = rot == 1 || rot == 2;                                  \
+    bool sub_i = rot >= 2;                                              \
+    TYPE *d = vd, *n = vn, *m = vm, *a = va;                            \
+                                                                        \
+    for (i = 0; i < oprsz; i += 2) {                                    \
+        TYPE elt1_a = n[H(i + sel_a)];                                  \
+        TYPE elt2_a = m[H(i + sel_a)];                                  \
+        TYPE elt2_b = m[H(i + sel_b)];                                  \
+                                                                        \
+        d[H(i)] = OP(elt1_a, elt2_a, a[H(i)], sub_r);                  \
+        d[H(i + 1)] = OP(elt1_a, elt2_b, a[H(i + 1)], sub_i);          \
+    }                                                                   \
 }
 
-/* Three-operand expander, immediate operand, controlled by a predicate.
- */
-#define DO_ZPZI(NAME, TYPE, H, OP)                              \
-void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
-{                                                               \
-    intptr_t i, opr_sz = simd_oprsz(desc);                      \
-    TYPE imm = simd_data(desc);                                 \
-    for (i = 0; i < opr_sz; ) {                                 \
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));         \
-        do {                                                    \
-            if (pg & 1) {                                       \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                 \
-                *(TYPE *)((char *)vd + H(i)) = OP(nn, imm);             \
-            }                                                   \
-            i += sizeof(TYPE), pg >>= sizeof(TYPE);             \
-        } while (i & 15);                                       \
-    }                                                           \
+#define DO_CMLA(N, M, A, S) ((A) + ((N) * (M)) * ((S) ? -1 : 1))
+
+DO_CMLA_FUNC(sve2_cmla_zzzz_b, uint8_t, H1, DO_CMLA)
+DO_CMLA_FUNC(sve2_cmla_zzzz_h, uint16_t, H2, DO_CMLA)
+DO_CMLA_FUNC(sve2_cmla_zzzz_s, uint32_t, H4, DO_CMLA)
+DO_CMLA_FUNC(sve2_cmla_zzzz_d, uint64_t, H8, DO_CMLA)
+
+static int8_t do_sqrdcmlah_b(int8_t n, int8_t m, int8_t a, bool neg)
+{
+    return do_sqrdmlah_b(n, m, a, neg, true);
 }
 
-/* Similarly, specialized for 64-bit operands.  */
-#define DO_ZPZI_D(NAME, TYPE, OP)                               \
-void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
-{                                                               \
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                  \
-    TYPE *d = vd, *n = vn;                                      \
-    TYPE imm = simd_data(desc);                                 \
-    uint8_t *pg = vg;                                           \
-    for (i = 0; i < opr_sz; i += 1) {                           \
-        if (pg[H1(i)] & 1) {                                    \
-            TYPE nn = n[i];                                     \
-            d[i] = OP(nn, imm);                                 \
-        }                                                       \
-    }                                                           \
+static int16_t do_sqrdcmlah_h(int16_t n, int16_t m, int16_t a, bool neg)
+{
+    return do_sqrdmlah_h(n, m, a, neg, true);
 }
 
-#define DO_SHR(N, M)  (N >> M)
-#define DO_SHL(N, M)  (N << M)
+static int32_t do_sqrdcmlah_s(int32_t n, int32_t m, int32_t a, bool neg)
+{
+    return do_sqrdmlah_s(n, m, a, neg, true);
+}
 
-/* Arithmetic shift right for division.  This rounds negative numbers
-   toward zero as per signed division.  Therefore before shifting,
-   when N is negative, add 2**M-1.  */
-#ifdef _MSC_VER
- #define DO_ASRD(N, M) ((N + (N < 0 ? (1 << M) - 1 : 0)) >> M)
-#else
- #define DO_ASRD(N, M) ((N + (N < 0 ? ((__typeof(N))1 << M) - 1 : 0)) >> M)
-#endif
+static int64_t do_sqrdcmlah_d(int64_t n, int64_t m, int64_t a, bool neg)
+{
+    return do_sqrdmlah_d(n, m, a, neg, true);
+}
 
-DO_ZPZI(sve_asr_zpzi_b, int8_t, H1, DO_SHR)
-DO_ZPZI(sve_asr_zpzi_h, int16_t, H1_2, DO_SHR)
-DO_ZPZI(sve_asr_zpzi_s, int32_t, H1_4, DO_SHR)
-DO_ZPZI_D(sve_asr_zpzi_d, int64_t, DO_SHR)
+DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_b, int8_t, H1, do_sqrdcmlah_b)
+DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_h, int16_t, H2, do_sqrdcmlah_h)
+DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_s, int32_t, H4, do_sqrdcmlah_s)
+DO_CMLA_FUNC(sve2_sqrdcmlah_zzzz_d, int64_t, H8, do_sqrdcmlah_d)
 
-DO_ZPZI(sve_lsr_zpzi_b, uint8_t, H1, DO_SHR)
-DO_ZPZI(sve_lsr_zpzi_h, uint16_t, H1_2, DO_SHR)
-DO_ZPZI(sve_lsr_zpzi_s, uint32_t, H1_4, DO_SHR)
-DO_ZPZI_D(sve_lsr_zpzi_d, uint64_t, DO_SHR)
+#define DO_CMLA_IDX_FUNC(NAME, TYPE, H, OP)                            \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                       \
+    intptr_t i, j, oprsz = simd_oprsz(desc);                            \
+    int rot = extract32(desc, SIMD_DATA_SHIFT, 2);                      \
+    int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2) * 2;              \
+    int sel_a = rot & 1;                                                \
+    int sel_b = sel_a ^ 1;                                              \
+    bool sub_r = rot == 1 || rot == 2;                                  \
+    bool sub_i = rot >= 2;                                              \
+    TYPE *d = vd, *n = vn, *m = vm, *a = va;                            \
+                                                                        \
+    for (i = 0; i < oprsz / sizeof(TYPE); i += 16 / sizeof(TYPE)) {     \
+        TYPE elt2_a = m[H(i + idx + sel_a)];                            \
+        TYPE elt2_b = m[H(i + idx + sel_b)];                            \
+                                                                        \
+        for (j = 0; j < 16 / sizeof(TYPE); j += 2) {                    \
+            TYPE elt1_a = n[H(i + j + sel_a)];                          \
+                                                                        \
+            d[H2(i + j)] = OP(elt1_a, elt2_a, a[H(i + j)], sub_r);      \
+            d[H2(i + j + 1)] = OP(elt1_a, elt2_b,                       \
+                                  a[H(i + j + 1)], sub_i);              \
+        }                                                               \
+    }                                                                   \
+}
 
-DO_ZPZI(sve_lsl_zpzi_b, uint8_t, H1, DO_SHL)
-DO_ZPZI(sve_lsl_zpzi_h, uint16_t, H1_2, DO_SHL)
-DO_ZPZI(sve_lsl_zpzi_s, uint32_t, H1_4, DO_SHL)
-DO_ZPZI_D(sve_lsl_zpzi_d, uint64_t, DO_SHL)
+DO_CMLA_IDX_FUNC(sve2_cmla_idx_h, int16_t, H2, DO_CMLA)
+DO_CMLA_IDX_FUNC(sve2_cmla_idx_s, int32_t, H4, DO_CMLA)
 
-DO_ZPZI(sve_asrd_b, int8_t, H1, DO_ASRD)
-DO_ZPZI(sve_asrd_h, int16_t, H1_2, DO_ASRD)
-DO_ZPZI(sve_asrd_s, int32_t, H1_4, DO_ASRD)
-DO_ZPZI_D(sve_asrd_d, int64_t, DO_ASRD)
+DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_h, int16_t, H2, do_sqrdcmlah_h)
+DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_s, int32_t, H4, do_sqrdcmlah_s)
 
-#undef DO_SHR
-#undef DO_SHL
-#undef DO_ASRD
-#undef DO_ZPZI
-#undef DO_ZPZI_D
+#undef DO_CMLA
+#undef DO_CMLA_FUNC
+#undef DO_CMLA_IDX_FUNC
 
-/* Fully general four-operand expander, controlled by a predicate.
- */
-#define DO_ZPZZZ(NAME, TYPE, H, OP)                           \
-void HELPER(NAME)(void *vd, void *va, void *vn, void *vm,     \
-                  void *vg, uint32_t desc)                    \
-{                                                             \
-    intptr_t i, opr_sz = simd_oprsz(desc);                    \
-    for (i = 0; i < opr_sz; ) {                               \
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));       \
-        do {                                                  \
-            if (pg & 1) {                                     \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));               \
-                TYPE mm = *(TYPE *)((char *)vm + H(i));               \
-                TYPE aa = *(TYPE *)((char *)va + H(i));               \
-                *(TYPE *)((char *)vd + H(i)) = OP(aa, nn, mm);        \
-            }                                                 \
-            i += sizeof(TYPE), pg >>= sizeof(TYPE);           \
-        } while (i & 15);                                     \
-    }                                                         \
+static int32_t do_cdot_s(uint32_t n, uint32_t m, int32_t a,
+                         int sel_a, int sel_b, int sub_i)
+{
+    int i;
+
+    for (i = 0; i <= 1; i++) {
+        int32_t elt1_r = (int8_t)(n >> (16 * i));
+        int32_t elt1_i = (int8_t)(n >> (16 * i + 8));
+        int32_t elt2_a = (int8_t)(m >> (16 * i + 8 * sel_a));
+        int32_t elt2_b = (int8_t)(m >> (16 * i + 8 * sel_b));
+
+        a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
+    }
+    return a;
 }
 
-/* Similarly, specialized for 64-bit operands.  */
-#define DO_ZPZZZ_D(NAME, TYPE, OP)                            \
-void HELPER(NAME)(void *vd, void *va, void *vn, void *vm,     \
-                  void *vg, uint32_t desc)                    \
-{                                                             \
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                \
-    TYPE *d = vd, *a = va, *n = vn, *m = vm;                  \
-    uint8_t *pg = vg;                                         \
-    for (i = 0; i < opr_sz; i += 1) {                         \
-        if (pg[H1(i)] & 1) {                                  \
-            TYPE aa = a[i], nn = n[i], mm = m[i];             \
-            d[i] = OP(aa, nn, mm);                            \
-        }                                                     \
-    }                                                         \
-}
-
-#define DO_MLA(A, N, M)  (A + N * M)
-#define DO_MLS(A, N, M)  (A - N * M)
-
-DO_ZPZZZ(sve_mla_b, uint8_t, H1, DO_MLA)
-DO_ZPZZZ(sve_mls_b, uint8_t, H1, DO_MLS)
-
-DO_ZPZZZ(sve_mla_h, uint16_t, H1_2, DO_MLA)
-DO_ZPZZZ(sve_mls_h, uint16_t, H1_2, DO_MLS)
-
-DO_ZPZZZ(sve_mla_s, uint32_t, H1_4, DO_MLA)
-DO_ZPZZZ(sve_mls_s, uint32_t, H1_4, DO_MLS)
-
-DO_ZPZZZ_D(sve_mla_d, uint64_t, DO_MLA)
-DO_ZPZZZ_D(sve_mls_d, uint64_t, DO_MLS)
+static int64_t do_cdot_d(uint64_t n, uint64_t m, int64_t a,
+                         int sel_a, int sel_b, int sub_i)
+{
+    int i;
 
-#undef DO_MLA
-#undef DO_MLS
-#undef DO_ZPZZZ
-#undef DO_ZPZZZ_D
+    for (i = 0; i <= 1; i++) {
+        int64_t elt1_r = (int16_t)(n >> (32 * i));
+        int64_t elt1_i = (int16_t)(n >> (32 * i + 16));
+        int64_t elt2_a = (int16_t)(m >> (32 * i + 16 * sel_a));
+        int64_t elt2_b = (int16_t)(m >> (32 * i + 16 * sel_b));
 
-void HELPER(sve_index_b)(void *vd, uint32_t start,
-                         uint32_t incr, uint32_t desc)
-{
-    intptr_t i, opr_sz = simd_oprsz(desc);
-    uint8_t *d = vd;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[H1(i)] = start + i * incr;
+        a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
     }
+    return a;
 }
 
-void HELPER(sve_index_h)(void *vd, uint32_t start,
-                         uint32_t incr, uint32_t desc)
+void HELPER(sve2_cdot_zzzz_s)(void *vd, void *vn, void *vm,
+                              void *va, uint32_t desc)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 2;
-    uint16_t *d = vd;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[H2(i)] = start + i * incr;
+    intptr_t e, oprsz = simd_oprsz(desc) / 4;
+    int rot = simd_data(desc);
+    int sel_a = rot & 1;
+    int sel_b = sel_a ^ 1;
+    int sub_i = rot == 0 || rot == 3 ? -1 : 1;
+    uint32_t *d = vd, *n = vn, *m = vm, *a = va;
+
+    for (e = 0; e < oprsz; e++) {
+        d[e] = do_cdot_s(n[e], m[e], a[e], sel_a, sel_b, sub_i);
     }
 }
 
-void HELPER(sve_index_s)(void *vd, uint32_t start,
-                         uint32_t incr, uint32_t desc)
+void HELPER(sve2_cdot_zzzz_d)(void *vd, void *vn, void *vm,
+                              void *va, uint32_t desc)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
-    uint32_t *d = vd;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[H4(i)] = start + i * incr;
+    intptr_t e, oprsz = simd_oprsz(desc) / 8;
+    int rot = simd_data(desc);
+    int sel_a = rot & 1;
+    int sel_b = sel_a ^ 1;
+    int sub_i = rot == 0 || rot == 3 ? -1 : 1;
+    uint64_t *d = vd, *n = vn, *m = vm, *a = va;
+
+    for (e = 0; e < oprsz; e++) {
+        d[e] = do_cdot_d(n[e], m[e], a[e], sel_a, sel_b, sub_i);
     }
 }
 
-void HELPER(sve_index_d)(void *vd, uint64_t start,
-                         uint64_t incr, uint32_t desc)
+void HELPER(sve2_cdot_idx_s)(void *vd, void *vn, void *vm,
+                             void *va, uint32_t desc)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = start + i * incr;
+    intptr_t seg, e, oprsz = simd_oprsz(desc) / 4;
+    int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
+    int idx = H4(extract32(desc, SIMD_DATA_SHIFT + 2, 2));
+    int sel_a = rot & 1;
+    int sel_b = sel_a ^ 1;
+    int sub_i = rot == 0 || rot == 3 ? -1 : 1;
+    uint32_t *d = vd, *n = vn, *m = vm, *a = va;
+
+    for (seg = 0; seg < oprsz; seg += 4) {
+        uint32_t seg_m = m[seg + idx];
+
+        for (e = 0; e < 4; e++) {
+            d[seg + e] = do_cdot_s(n[seg + e], seg_m, a[seg + e],
+                                   sel_a, sel_b, sub_i);
+        }
     }
 }
 
-void HELPER(sve_adr_p32)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(sve2_cdot_idx_d)(void *vd, void *vn, void *vm,
+                             void *va, uint32_t desc)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
-    uint32_t sh = simd_data(desc);
-    uint32_t *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = n[i] + (m[i] << sh);
+    intptr_t seg, e, oprsz = simd_oprsz(desc) / 8;
+    int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
+    int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
+    int sel_a = rot & 1;
+    int sel_b = sel_a ^ 1;
+    int sub_i = rot == 0 || rot == 3 ? -1 : 1;
+    uint64_t *d = vd, *n = vn, *m = vm, *a = va;
+
+    for (seg = 0; seg < oprsz; seg += 2) {
+        uint64_t seg_m = m[seg + idx];
+
+        for (e = 0; e < 2; e++) {
+            d[seg + e] = do_cdot_d(n[seg + e], seg_m, a[seg + e],
+                                   sel_a, sel_b, sub_i);
+        }
     }
 }
 
-void HELPER(sve_adr_p64)(void *vd, void *vn, void *vm, uint32_t desc)
-{
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t sh = simd_data(desc);
-    uint64_t *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = n[i] + (m[i] << sh);
-    }
+#define DO_ZZZ_TB(NAME, TYPEW, TYPEN, HW, HN, OP)                       \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    int sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN);     \
+    int sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(TYPEN); \
+    char *d = vd, *n = vn, *m = vm;                                      \
+                                                                        \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                        \
+        TYPEW nn = *(TYPEN *)(n + HN(i + sel1));                        \
+        TYPEW mm = *(TYPEN *)(m + HN(i + sel2));                        \
+                                                                        \
+        *(TYPEW *)(d + HW(i)) = OP(nn, mm);                             \
+    }                                                                   \
 }
 
-void HELPER(sve_adr_s32)(void *vd, void *vn, void *vm, uint32_t desc)
+DO_ZZZ_TB(sve2_saddl_h, int16_t, int8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_TB(sve2_saddl_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_TB(sve2_saddl_d, int64_t, int32_t, H1_8, H1_4, DO_ADD)
+
+DO_ZZZ_TB(sve2_uaddl_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_TB(sve2_uaddl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_TB(sve2_uaddl_d, uint64_t, uint32_t, H1_8, H1_4, DO_ADD)
+
+DO_ZZZ_TB(sve2_ssubl_h, int16_t, int8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_TB(sve2_ssubl_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_TB(sve2_ssubl_d, int64_t, int32_t, H1_8, H1_4, DO_SUB)
+
+DO_ZZZ_TB(sve2_usubl_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_TB(sve2_usubl_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_TB(sve2_usubl_d, uint64_t, uint32_t, H1_8, H1_4, DO_SUB)
+
+DO_ZZZ_TB(sve2_sabdl_h, int16_t, int8_t, H1_2, H1, DO_ABD)
+DO_ZZZ_TB(sve2_sabdl_s, int32_t, int16_t, H1_4, H1_2, DO_ABD)
+DO_ZZZ_TB(sve2_sabdl_d, int64_t, int32_t, H1_8, H1_4, DO_ABD)
+
+DO_ZZZ_TB(sve2_uabdl_h, uint16_t, uint8_t, H1_2, H1, DO_ABD)
+DO_ZZZ_TB(sve2_uabdl_s, uint32_t, uint16_t, H1_4, H1_2, DO_ABD)
+DO_ZZZ_TB(sve2_uabdl_d, uint64_t, uint32_t, H1_8, H1_4, DO_ABD)
+
+DO_ZZZ_TB(sve2_smull_zzz_h, int16_t, int8_t, H1_2, H1, DO_MUL)
+DO_ZZZ_TB(sve2_smull_zzz_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
+DO_ZZZ_TB(sve2_smull_zzz_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)
+
+DO_ZZZ_TB(sve2_umull_zzz_h, uint16_t, uint8_t, H1_2, H1, DO_MUL)
+DO_ZZZ_TB(sve2_umull_zzz_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
+DO_ZZZ_TB(sve2_umull_zzz_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)
+
+static int16_t do_sqdmull_h(int16_t n, int16_t m)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t sh = simd_data(desc);
-    uint64_t *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = n[i] + ((uint64_t)(int32_t)m[i] << sh);
-    }
+    int32_t val = n * m;
+
+    return val == 0x4000 ? 0x7fff : val * 2;
 }
 
-void HELPER(sve_adr_u32)(void *vd, void *vn, void *vm, uint32_t desc)
+static int32_t do_sqdmull_s(int32_t n, int32_t m)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t sh = simd_data(desc);
-    uint64_t *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = n[i] + ((uint64_t)(uint32_t)m[i] << sh);
-    }
+    int64_t val = (int64_t)n * m;
+
+    return val == 0x40000000ll ? 0x7fffffff : val * 2;
 }
 
-void HELPER(sve_fexpa_h)(void *vd, void *vn, uint32_t desc)
+static int64_t do_sqdmull_d(int64_t n, int64_t m)
 {
-    /* These constants are cut-and-paste directly from the ARM pseudocode.  */
-    static const uint16_t coeff[] = {
-        0x0000, 0x0016, 0x002d, 0x0045, 0x005d, 0x0075, 0x008e, 0x00a8,
-        0x00c2, 0x00dc, 0x00f8, 0x0114, 0x0130, 0x014d, 0x016b, 0x0189,
-        0x01a8, 0x01c8, 0x01e8, 0x0209, 0x022b, 0x024e, 0x0271, 0x0295,
-        0x02ba, 0x02e0, 0x0306, 0x032e, 0x0356, 0x037f, 0x03a9, 0x03d4,
-    };
-    intptr_t i, opr_sz = simd_oprsz(desc) / 2;
-    uint16_t *d = vd, *n = vn;
+    int64_t val = n * m;
 
-    for (i = 0; i < opr_sz; i++) {
-        uint16_t nn = n[i];
-        intptr_t idx = extract32(nn, 0, 5);
-        uint16_t exp = extract32(nn, 5, 5);
-        d[i] = coeff[idx] | (exp << 10);
-    }
+    return val == INT64_C(0x4000000000000000) ?
+           INT64_MAX : val * 2;
 }
 
-void HELPER(sve_fexpa_s)(void *vd, void *vn, uint32_t desc)
+DO_ZZZ_TB(sve2_sqdmull_zzz_h, int16_t, int8_t, H1_2, H1, do_sqdmull_h)
+DO_ZZZ_TB(sve2_sqdmull_zzz_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
+DO_ZZZ_TB(sve2_sqdmull_zzz_d, int64_t, int32_t, H1_8, H1_4, do_sqdmull_d)
+
+#undef DO_ZZZ_TB
+
+#define DO_ZZXW(NAME, TYPEW, TYPEN, HW, HN, OP)                         \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, j, oprsz = simd_oprsz(desc);                            \
+    intptr_t sel = (simd_data(desc) & 1) * sizeof(TYPEN);               \
+    intptr_t idx = (simd_data(desc) >> 1) * sizeof(TYPEN);              \
+    char *d = vd, *n = vn, *m = vm;                                      \
+                                                                        \
+    for (i = 0; i < oprsz; i += 16) {                                   \
+        TYPEW mm = *(TYPEN *)(m + HN(i + idx));                         \
+                                                                        \
+        for (j = 0; j < 16; j += sizeof(TYPEW)) {                       \
+            TYPEW nn = *(TYPEN *)(n + HN(i + j + sel));                 \
+                                                                        \
+            *(TYPEW *)(d + HW(i + j)) = OP(nn, mm);                     \
+        }                                                               \
+    }                                                                   \
+}
+
+DO_ZZXW(sve2_smull_idx_s, int32_t, int16_t, H1_4, H1_2, DO_MUL)
+DO_ZZXW(sve2_smull_idx_d, int64_t, int32_t, H1_8, H1_4, DO_MUL)
+DO_ZZXW(sve2_umull_idx_s, uint32_t, uint16_t, H1_4, H1_2, DO_MUL)
+DO_ZZXW(sve2_umull_idx_d, uint64_t, uint32_t, H1_8, H1_4, DO_MUL)
+DO_ZZXW(sve2_sqdmull_idx_s, int32_t, int16_t, H1_4, H1_2, do_sqdmull_s)
+DO_ZZXW(sve2_sqdmull_idx_d, int64_t, int32_t, H1_8, H1_4, do_sqdmull_d)
+
+#undef DO_ZZXW
+
+static int16_t do_smlal_h(int8_t n, int8_t m, int16_t a)
 {
-    /* These constants are cut-and-paste directly from the ARM pseudocode.  */
-    static const uint32_t coeff[] = {
-        0x000000, 0x0164d2, 0x02cd87, 0x043a29,
-        0x05aac3, 0x071f62, 0x08980f, 0x0a14d5,
-        0x0b95c2, 0x0d1adf, 0x0ea43a, 0x1031dc,
-        0x11c3d3, 0x135a2b, 0x14f4f0, 0x16942d,
-        0x1837f0, 0x19e046, 0x1b8d3a, 0x1d3eda,
-        0x1ef532, 0x20b051, 0x227043, 0x243516,
-        0x25fed7, 0x27cd94, 0x29a15b, 0x2b7a3a,
-        0x2d583f, 0x2f3b79, 0x3123f6, 0x3311c4,
-        0x3504f3, 0x36fd92, 0x38fbaf, 0x3aff5b,
-        0x3d08a4, 0x3f179a, 0x412c4d, 0x4346cd,
-        0x45672a, 0x478d75, 0x49b9be, 0x4bec15,
-        0x4e248c, 0x506334, 0x52a81e, 0x54f35b,
-        0x5744fd, 0x599d16, 0x5bfbb8, 0x5e60f5,
-        0x60ccdf, 0x633f89, 0x65b907, 0x68396a,
-        0x6ac0c7, 0x6d4f30, 0x6fe4ba, 0x728177,
-        0x75257d, 0x77d0df, 0x7a83b3, 0x7d3e0c,
-    };
-    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
-    uint32_t *d = vd, *n = vn;
+    uint16_t sum = (uint16_t)a + (uint16_t)(int16_t)(n * m);
 
-    for (i = 0; i < opr_sz; i++) {
-        uint32_t nn = n[i];
-        intptr_t idx = extract32(nn, 0, 6);
-        uint32_t exp = extract32(nn, 6, 8);
-        d[i] = coeff[idx] | (exp << 23);
-    }
+    return (int16_t)sum;
 }
 
-void HELPER(sve_fexpa_d)(void *vd, void *vn, uint32_t desc)
+static int32_t do_smlal_s(int16_t n, int16_t m, int32_t a)
 {
-    /* These constants are cut-and-paste directly from the ARM pseudocode.  */
-    static const uint64_t coeff[] = {
-        0x0000000000000ull, 0x02C9A3E778061ull, 0x059B0D3158574ull,
-        0x0874518759BC8ull, 0x0B5586CF9890Full, 0x0E3EC32D3D1A2ull,
-        0x11301D0125B51ull, 0x1429AAEA92DE0ull, 0x172B83C7D517Bull,
-        0x1A35BEB6FCB75ull, 0x1D4873168B9AAull, 0x2063B88628CD6ull,
-        0x2387A6E756238ull, 0x26B4565E27CDDull, 0x29E9DF51FDEE1ull,
-        0x2D285A6E4030Bull, 0x306FE0A31B715ull, 0x33C08B26416FFull,
-        0x371A7373AA9CBull, 0x3A7DB34E59FF7ull, 0x3DEA64C123422ull,
-        0x4160A21F72E2Aull, 0x44E086061892Dull, 0x486A2B5C13CD0ull,
-        0x4BFDAD5362A27ull, 0x4F9B2769D2CA7ull, 0x5342B569D4F82ull,
-        0x56F4736B527DAull, 0x5AB07DD485429ull, 0x5E76F15AD2148ull,
-        0x6247EB03A5585ull, 0x6623882552225ull, 0x6A09E667F3BCDull,
-        0x6DFB23C651A2Full, 0x71F75E8EC5F74ull, 0x75FEB564267C9ull,
-        0x7A11473EB0187ull, 0x7E2F336CF4E62ull, 0x82589994CCE13ull,
-        0x868D99B4492EDull, 0x8ACE5422AA0DBull, 0x8F1AE99157736ull,
-        0x93737B0CDC5E5ull, 0x97D829FDE4E50ull, 0x9C49182A3F090ull,
-        0xA0C667B5DE565ull, 0xA5503B23E255Dull, 0xA9E6B5579FDBFull,
-        0xAE89F995AD3ADull, 0xB33A2B84F15FBull, 0xB7F76F2FB5E47ull,
-        0xBCC1E904BC1D2ull, 0xC199BDD85529Cull, 0xC67F12E57D14Bull,
-        0xCB720DCEF9069ull, 0xD072D4A07897Cull, 0xD5818DCFBA487ull,
-        0xDA9E603DB3285ull, 0xDFC97337B9B5Full, 0xE502EE78B3FF6ull,
-        0xEA4AFA2A490DAull, 0xEFA1BEE615A27ull, 0xF50765B6E4540ull,
-        0xFA7C1819E90D8ull,
-    };
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
+    uint32_t sum = (uint32_t)a + (uint32_t)(int32_t)(n * m);
 
-    for (i = 0; i < opr_sz; i++) {
-        uint64_t nn = n[i];
-        intptr_t idx = extract32(nn, 0, 6);
-        uint64_t exp = extract32(nn, 6, 11);
-        d[i] = coeff[idx] | (exp << 52);
-    }
+    return (int32_t)sum;
 }
 
-void HELPER(sve_ftssel_h)(void *vd, void *vn, void *vm, uint32_t desc)
+static int64_t do_smlal_d(int32_t n, int32_t m, int64_t a)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 2;
-    uint16_t *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i += 1) {
-        uint16_t nn = n[i];
-        uint16_t mm = m[i];
-        if (mm & 1) {
-            nn = float16_one;
-        }
-        d[i] = nn ^ (mm & 2) << 14;
-    }
+    uint64_t sum = (uint64_t)a + (uint64_t)((int64_t)n * m);
+
+    return (int64_t)sum;
 }
 
-void HELPER(sve_ftssel_s)(void *vd, void *vn, void *vm, uint32_t desc)
+static uint16_t do_umlal_h(uint8_t n, uint8_t m, uint16_t a)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
-    uint32_t *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i += 1) {
-        uint32_t nn = n[i];
-        uint32_t mm = m[i];
-        if (mm & 1) {
-            nn = float32_one;
-        }
-        d[i] = nn ^ (mm & 2) << 30;
-    }
+    return a + n * m;
 }
 
-void HELPER(sve_ftssel_d)(void *vd, void *vn, void *vm, uint32_t desc)
+static uint32_t do_umlal_s(uint16_t n, uint16_t m, uint32_t a)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i];
-        uint64_t mm = m[i];
-        if (mm & 1) {
-            nn = float64_one;
-        }
-        d[i] = nn ^ (mm & 2) << 62;
-    }
+    return a + n * m;
 }
 
-/*
- * Signed saturating addition with scalar operand.
- */
+static uint64_t do_umlal_d(uint32_t n, uint32_t m, uint64_t a)
+{
+    return a + (uint64_t)n * m;
+}
 
-void HELPER(sve_sqaddi_b)(void *d, void *a, int32_t b, uint32_t desc)
+static int16_t do_smlsl_h(int8_t n, int8_t m, int16_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    uint16_t diff = (uint16_t)a - (uint16_t)(int16_t)(n * m);
 
-    for (i = 0; i < oprsz; i += sizeof(int8_t)) {
-        int r = *(int8_t *)((char *)a + i) + b;
-        if (r > INT8_MAX) {
-            r = INT8_MAX;
-        } else if (r < INT8_MIN) {
-            r = INT8_MIN;
-        }
-        *(int8_t *)((char *)d + i) = r;
-    }
+    return (int16_t)diff;
 }
 
-void HELPER(sve_sqaddi_h)(void *d, void *a, int32_t b, uint32_t desc)
+static int32_t do_smlsl_s(int16_t n, int16_t m, int32_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    uint32_t diff = (uint32_t)a - (uint32_t)(int32_t)(n * m);
 
-    for (i = 0; i < oprsz; i += sizeof(int16_t)) {
-        int r = *(int16_t *)((char *)a + i) + b;
-        if (r > INT16_MAX) {
-            r = INT16_MAX;
-        } else if (r < INT16_MIN) {
-            r = INT16_MIN;
-        }
-        *(int16_t *)((char *)d + i) = r;
-    }
+    return (int32_t)diff;
 }
 
-void HELPER(sve_sqaddi_s)(void *d, void *a, int64_t b, uint32_t desc)
+static int64_t do_smlsl_d(int32_t n, int32_t m, int64_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    uint64_t diff = (uint64_t)a - (uint64_t)((int64_t)n * m);
 
-    for (i = 0; i < oprsz; i += sizeof(int32_t)) {
-        int64_t r = *(int32_t *)((char *)a + i) + b;
-        if (r > INT32_MAX) {
-            r = INT32_MAX;
-        } else if (r < INT32_MIN) {
-            r = INT32_MIN;
-        }
-        *(int32_t *)((char *)d + i) = r;
-    }
+    return (int64_t)diff;
 }
 
-void HELPER(sve_sqaddi_d)(void *d, void *a, int64_t b, uint32_t desc)
+static uint16_t do_umlsl_h(uint8_t n, uint8_t m, uint16_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    return a - n * m;
+}
 
-    for (i = 0; i < oprsz; i += sizeof(int64_t)) {
-        int64_t ai = *(int64_t *)((char *)a + i);
-        int64_t r = ai + b;
-        if (((r ^ ai) & ~(ai ^ b)) < 0) {
-            /* Signed overflow.  */
-            r = (r < 0 ? INT64_MAX : INT64_MIN);
-        }
-        *(int64_t *)((char *)d + i) = r;
-    }
+static uint32_t do_umlsl_s(uint16_t n, uint16_t m, uint32_t a)
+{
+    return a - n * m;
 }
 
-/*
- * Unsigned saturating addition with scalar operand.
- */
+static uint64_t do_umlsl_d(uint32_t n, uint32_t m, uint64_t a)
+{
+    return a - (uint64_t)n * m;
+}
 
-void HELPER(sve_uqaddi_b)(void *d, void *a, int32_t b, uint32_t desc)
+static int16_t do_sabal_h(int8_t n, int8_t m, int16_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    uint16_t sum = (uint16_t)a + (uint16_t)(n < m ? m - n : n - m);
 
-    for (i = 0; i < oprsz; i += sizeof(uint8_t)) {
-        int r = *(uint8_t *)((char *)a + i) + b;
-        if (r > UINT8_MAX) {
-            r = UINT8_MAX;
-        } else if (r < 0) {
-            r = 0;
-        }
-        *(uint8_t *)((char *)d + i) = r;
-    }
+    return (int16_t)sum;
 }
 
-void HELPER(sve_uqaddi_h)(void *d, void *a, int32_t b, uint32_t desc)
+static int32_t do_sabal_s(int16_t n, int16_t m, int32_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    uint32_t sum = (uint32_t)a + (uint32_t)(n < m ? m - n : n - m);
 
-    for (i = 0; i < oprsz; i += sizeof(uint16_t)) {
-        int r = *(uint16_t *)((char *)a + i) + b;
-        if (r > UINT16_MAX) {
-            r = UINT16_MAX;
-        } else if (r < 0) {
-            r = 0;
-        }
-        *(uint16_t *)((char *)d + i) = r;
-    }
+    return (int32_t)sum;
 }
 
-void HELPER(sve_uqaddi_s)(void *d, void *a, int64_t b, uint32_t desc)
+static int64_t do_sabal_d(int32_t n, int32_t m, int64_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    uint64_t diff = n < m ? (uint32_t)m - (uint32_t)n :
+                    (uint32_t)n - (uint32_t)m;
 
-    for (i = 0; i < oprsz; i += sizeof(uint32_t)) {
-        int64_t r = *(uint32_t *)((char *)a + i) + b;
-        if (r > UINT32_MAX) {
-            r = UINT32_MAX;
-        } else if (r < 0) {
-            r = 0;
-        }
-        *(uint32_t *)((char *)d + i) = r;
-    }
+    return (int64_t)((uint64_t)a + diff);
 }
 
-void HELPER(sve_uqaddi_d)(void *d, void *a, uint64_t b, uint32_t desc)
+static uint16_t do_uabal_h(uint8_t n, uint8_t m, uint16_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
-
-    for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
-        uint64_t r = *(uint64_t *)((char *)a + i) + b;
-        if (r < b) {
-            r = UINT64_MAX;
-        }
-        *(uint64_t *)((char *)d + i) = r;
-    }
+    return a + (n < m ? m - n : n - m);
 }
 
-void HELPER(sve_uqsubi_d)(void *d, void *a, uint64_t b, uint32_t desc)
+static uint32_t do_uabal_s(uint16_t n, uint16_t m, uint32_t a)
 {
-    intptr_t i, oprsz = simd_oprsz(desc);
+    return a + (n < m ? m - n : n - m);
+}
 
-    for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
-        uint64_t ai = *(uint64_t *)((char *)a + i);
-        *(uint64_t *)((char *)d + i) = (ai < b ? 0 : ai - b);
-    }
+static uint64_t do_uabal_d(uint32_t n, uint32_t m, uint64_t a)
+{
+    return a + (n < m ? (uint64_t)m - n : (uint64_t)n - m);
 }
 
-/* Two operand predicated copy immediate with merge.  All valid immediates
- * can fit within 17 signed bits in the simd_data field.
- */
-void HELPER(sve_cpy_m_b)(void *vd, void *vn, void *vg,
-                         uint64_t mm, uint32_t desc)
+static int16_t do_sqadd_h(int16_t a, int16_t b)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
+    int32_t sum = (int32_t)a + b;
 
-    mm = dup_const(MO_8, mm);
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i];
-        uint64_t pp = expand_pred_b(pg[H1(i)]);
-        d[i] = (mm & pp) | (nn & ~pp);
-    }
+    return MIN(MAX(sum, INT16_MIN), INT16_MAX);
 }
 
-void HELPER(sve_cpy_m_h)(void *vd, void *vn, void *vg,
-                         uint64_t mm, uint32_t desc)
+static int32_t do_sqadd_s(int32_t a, int32_t b)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
+    int64_t sum = (int64_t)a + b;
 
-    mm = dup_const(MO_16, mm);
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i];
-        uint64_t pp = expand_pred_h(pg[H1(i)]);
-        d[i] = (mm & pp) | (nn & ~pp);
-    }
+    return MIN(MAX(sum, INT32_MIN), INT32_MAX);
 }
 
-void HELPER(sve_cpy_m_s)(void *vd, void *vn, void *vg,
-                         uint64_t mm, uint32_t desc)
+static int16_t do_sqsub_h(int16_t a, int16_t b)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
+    int32_t diff = (int32_t)a - b;
 
-    mm = dup_const(MO_32, mm);
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i];
-        uint64_t pp = expand_pred_s(pg[H1(i)]);
-        d[i] = (mm & pp) | (nn & ~pp);
-    }
+    return MIN(MAX(diff, INT16_MIN), INT16_MAX);
 }
 
-void HELPER(sve_cpy_m_d)(void *vd, void *vn, void *vg,
-                         uint64_t mm, uint32_t desc)
+static int32_t do_sqsub_s(int32_t a, int32_t b)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
+    int64_t diff = (int64_t)a - b;
 
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i];
-        d[i] = (pg[H1(i)] & 1 ? mm : nn);
-    }
+    return MIN(MAX(diff, INT32_MIN), INT32_MAX);
 }
 
-void HELPER(sve_cpy_z_b)(void *vd, void *vg, uint64_t val, uint32_t desc)
+static int64_t do_sqadd_d(int64_t a, int64_t b)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
+    int64_t r = (int64_t)((uint64_t)a + (uint64_t)b);
 
-    val = dup_const(MO_8, val);
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = val & expand_pred_b(pg[H1(i)]);
+    if (b > 0 && r < a) {
+        return INT64_MAX;
     }
+    if (b < 0 && r > a) {
+        return INT64_MIN;
+    }
+    return r;
 }
 
-void HELPER(sve_cpy_z_h)(void *vd, void *vg, uint64_t val, uint32_t desc)
+static int64_t do_sqsub_d(int64_t a, int64_t b)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
+    int64_t r = (int64_t)((uint64_t)a - (uint64_t)b);
 
-    val = dup_const(MO_16, val);
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = val & expand_pred_h(pg[H1(i)]);
+    if (b > 0 && r > a) {
+        return INT64_MIN;
     }
+    if (b < 0 && r < a) {
+        return INT64_MAX;
+    }
+    return r;
 }
 
-void HELPER(sve_cpy_z_s)(void *vd, void *vg, uint64_t val, uint32_t desc)
+static int16_t do_sqdmlal_h(int8_t n, int8_t m, int16_t a)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
+    return do_sqadd_h(a, do_sqdmull_h(n, m));
+}
 
-    val = dup_const(MO_32, val);
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = val & expand_pred_s(pg[H1(i)]);
-    }
+static int32_t do_sqdmlal_s(int16_t n, int16_t m, int32_t a)
+{
+    return do_sqadd_s(a, do_sqdmull_s(n, m));
 }
 
-void HELPER(sve_cpy_z_d)(void *vd, void *vg, uint64_t val, uint32_t desc)
+static int64_t do_sqdmlal_d(int32_t n, int32_t m, int64_t a)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd;
-    uint8_t *pg = vg;
+    return do_sqadd_d(a, do_sqdmull_d(n, m));
+}
 
-    for (i = 0; i < opr_sz; i += 1) {
-        d[i] = (pg[H1(i)] & 1 ? val : 0);
-    }
+static int16_t do_sqdmlsl_h(int8_t n, int8_t m, int16_t a)
+{
+    return do_sqsub_h(a, do_sqdmull_h(n, m));
 }
 
-/* Big-endian hosts need to frob the byte indicies.  If the copy
- * happens to be 8-byte aligned, then no frobbing necessary.
- */
-static void swap_memmove(void *vd, void *vs, size_t n)
+static int32_t do_sqdmlsl_s(int16_t n, int16_t m, int32_t a)
 {
-    uintptr_t d = (uintptr_t)vd;
-    uintptr_t s = (uintptr_t)vs;
-    uintptr_t o = (d | s | n) & 7;
-    size_t i;
+    return do_sqsub_s(a, do_sqdmull_s(n, m));
+}
 
-#ifndef HOST_WORDS_BIGENDIAN
-    o = 0;
-#endif
-    switch (o) {
-    case 0:
-        memmove(vd, vs, n);
-        break;
+static int64_t do_sqdmlsl_d(int32_t n, int32_t m, int64_t a)
+{
+    return do_sqsub_d(a, do_sqdmull_d(n, m));
+}
 
-    case 4:
-        if (d < s || d >= s + n) {
-            for (i = 0; i < n; i += 4) {
-                *(uint32_t *)H1_4(d + i) = *(uint32_t *)H1_4(s + i);
-            }
-        } else {
-            for (i = n; i > 0; ) {
-                i -= 4;
-                *(uint32_t *)H1_4(d + i) = *(uint32_t *)H1_4(s + i);
-            }
-        }
-        break;
+#define DO_ZZZWA(NAME, TYPEW, TYPEN, HW, HN, OP)                        \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc)\
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    int sel1 = (simd_data(desc) & 1) * sizeof(TYPEN);                   \
+    int sel2 = ((simd_data(desc) >> 1) & 1) * sizeof(TYPEN);            \
+    char *d = vd, *n = vn, *m = vm, *a = va;                             \
+                                                                        \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                        \
+        TYPEN nn = *(TYPEN *)(n + HN(i + sel1));                        \
+        TYPEN mm = *(TYPEN *)(m + HN(i + sel2));                        \
+        TYPEW aa = *(TYPEW *)(a + HW(i));                               \
+                                                                        \
+        *(TYPEW *)(d + HW(i)) = OP(nn, mm, aa);                         \
+    }                                                                   \
+}
 
-    case 2:
-    case 6:
-        if (d < s || d >= s + n) {
-            for (i = 0; i < n; i += 2) {
-                *(uint16_t *)H1_2(d + i) = *(uint16_t *)H1_2(s + i);
-            }
-        } else {
-            for (i = n; i > 0; ) {
-                i -= 2;
-                *(uint16_t *)H1_2(d + i) = *(uint16_t *)H1_2(s + i);
-            }
-        }
-        break;
+DO_ZZZWA(sve2_smlal_zzzw_h, int16_t, int8_t, H1_2, H1, do_smlal_h)
+DO_ZZZWA(sve2_smlal_zzzw_s, int32_t, int16_t, H1_4, H1_2, do_smlal_s)
+DO_ZZZWA(sve2_smlal_zzzw_d, int64_t, int32_t, H1_8, H1_4, do_smlal_d)
+DO_ZZZWA(sve2_umlal_zzzw_h, uint16_t, uint8_t, H1_2, H1, do_umlal_h)
+DO_ZZZWA(sve2_umlal_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, do_umlal_s)
+DO_ZZZWA(sve2_umlal_zzzw_d, uint64_t, uint32_t, H1_8, H1_4, do_umlal_d)
+DO_ZZZWA(sve2_smlsl_zzzw_h, int16_t, int8_t, H1_2, H1, do_smlsl_h)
+DO_ZZZWA(sve2_smlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2, do_smlsl_s)
+DO_ZZZWA(sve2_smlsl_zzzw_d, int64_t, int32_t, H1_8, H1_4, do_smlsl_d)
+DO_ZZZWA(sve2_umlsl_zzzw_h, uint16_t, uint8_t, H1_2, H1, do_umlsl_h)
+DO_ZZZWA(sve2_umlsl_zzzw_s, uint32_t, uint16_t, H1_4, H1_2, do_umlsl_s)
+DO_ZZZWA(sve2_umlsl_zzzw_d, uint64_t, uint32_t, H1_8, H1_4, do_umlsl_d)
+DO_ZZZWA(sve2_sabal_h, int16_t, int8_t, H1_2, H1, do_sabal_h)
+DO_ZZZWA(sve2_sabal_s, int32_t, int16_t, H1_4, H1_2, do_sabal_s)
+DO_ZZZWA(sve2_sabal_d, int64_t, int32_t, H1_8, H1_4, do_sabal_d)
+DO_ZZZWA(sve2_uabal_h, uint16_t, uint8_t, H1_2, H1, do_uabal_h)
+DO_ZZZWA(sve2_uabal_s, uint32_t, uint16_t, H1_4, H1_2, do_uabal_s)
+DO_ZZZWA(sve2_uabal_d, uint64_t, uint32_t, H1_8, H1_4, do_uabal_d)
+DO_ZZZWA(sve2_sqdmlal_zzzw_h, int16_t, int8_t, H1_2, H1, do_sqdmlal_h)
+DO_ZZZWA(sve2_sqdmlal_zzzw_s, int32_t, int16_t, H1_4, H1_2,
+         do_sqdmlal_s)
+DO_ZZZWA(sve2_sqdmlal_zzzw_d, int64_t, int32_t, H1_8, H1_4,
+         do_sqdmlal_d)
+DO_ZZZWA(sve2_sqdmlsl_zzzw_h, int16_t, int8_t, H1_2, H1, do_sqdmlsl_h)
+DO_ZZZWA(sve2_sqdmlsl_zzzw_s, int32_t, int16_t, H1_4, H1_2,
+         do_sqdmlsl_s)
+DO_ZZZWA(sve2_sqdmlsl_zzzw_d, int64_t, int32_t, H1_8, H1_4,
+         do_sqdmlsl_d)
+
+#undef DO_ZZZWA
+
+#define DO_ZZXWA(NAME, TYPEW, TYPEN, HW, HN, OP)                        \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc)\
+{                                                                       \
+    intptr_t i, j, oprsz = simd_oprsz(desc);                            \
+    intptr_t sel = (simd_data(desc) & 1) * sizeof(TYPEN);               \
+    intptr_t idx = (simd_data(desc) >> 1) * sizeof(TYPEN);              \
+    char *d = vd, *n = vn, *m = vm, *a = va;                             \
+                                                                        \
+    for (i = 0; i < oprsz; i += 16) {                                   \
+        TYPEN mm = *(TYPEN *)(m + HN(i + idx));                         \
+                                                                        \
+        for (j = 0; j < 16; j += sizeof(TYPEW)) {                       \
+            TYPEN nn = *(TYPEN *)(n + HN(i + j + sel));                 \
+            TYPEW aa = *(TYPEW *)(a + HW(i + j));                       \
+                                                                        \
+            *(TYPEW *)(d + HW(i + j)) = OP(nn, mm, aa);                 \
+        }                                                               \
+    }                                                                   \
+}
 
-    default:
-        if (d < s || d >= s + n) {
-            for (i = 0; i < n; i++) {
-                *(uint8_t *)H1(d + i) = *(uint8_t *)H1(s + i);
-            }
-        } else {
-            for (i = n; i > 0; ) {
-                i -= 1;
-                *(uint8_t *)H1(d + i) = *(uint8_t *)H1(s + i);
-            }
-        }
-        break;
-    }
+DO_ZZXWA(sve2_smlal_idx_s, int32_t, int16_t, H1_4, H1_2, do_smlal_s)
+DO_ZZXWA(sve2_smlal_idx_d, int64_t, int32_t, H1_8, H1_4, do_smlal_d)
+DO_ZZXWA(sve2_umlal_idx_s, uint32_t, uint16_t, H1_4, H1_2, do_umlal_s)
+DO_ZZXWA(sve2_umlal_idx_d, uint64_t, uint32_t, H1_8, H1_4, do_umlal_d)
+DO_ZZXWA(sve2_smlsl_idx_s, int32_t, int16_t, H1_4, H1_2, do_smlsl_s)
+DO_ZZXWA(sve2_smlsl_idx_d, int64_t, int32_t, H1_8, H1_4, do_smlsl_d)
+DO_ZZXWA(sve2_umlsl_idx_s, uint32_t, uint16_t, H1_4, H1_2, do_umlsl_s)
+DO_ZZXWA(sve2_umlsl_idx_d, uint64_t, uint32_t, H1_8, H1_4, do_umlsl_d)
+DO_ZZXWA(sve2_sqdmlal_idx_s, int32_t, int16_t, H1_4, H1_2,
+         do_sqdmlal_s)
+DO_ZZXWA(sve2_sqdmlal_idx_d, int64_t, int32_t, H1_8, H1_4,
+         do_sqdmlal_d)
+DO_ZZXWA(sve2_sqdmlsl_idx_s, int32_t, int16_t, H1_4, H1_2,
+         do_sqdmlsl_s)
+DO_ZZXWA(sve2_sqdmlsl_idx_d, int64_t, int32_t, H1_8, H1_4,
+         do_sqdmlsl_d)
+
+#undef DO_ZZXWA
+
+#define DO_ZZZ_WTB(NAME, TYPEW, TYPEN, HW, HN, OP)                      \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    int sel2 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPEN);     \
+    char *d = vd, *n = vn, *m = vm;                                      \
+                                                                        \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                        \
+        TYPEW nn = *(TYPEW *)(n + HW(i));                               \
+        TYPEW mm = *(TYPEN *)(m + HN(i + sel2));                        \
+                                                                        \
+        *(TYPEW *)(d + HW(i)) = OP(nn, mm);                             \
+    }                                                                   \
 }
 
-/* Similarly for memset of 0.  */
-static void swap_memzero(void *vd, size_t n)
-{
-    uintptr_t d = (uintptr_t)vd;
-    uintptr_t o = (d | n) & 7;
-    size_t i;
+DO_ZZZ_WTB(sve2_saddw_h, int16_t, int8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_WTB(sve2_saddw_s, int32_t, int16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_WTB(sve2_saddw_d, int64_t, int32_t, H1_8, H1_4, DO_ADD)
 
-    /* Usually, the first bit of a predicate is set, so N is 0.  */
-    if (likely(n == 0)) {
-        return;
-    }
+DO_ZZZ_WTB(sve2_uaddw_h, uint16_t, uint8_t, H1_2, H1, DO_ADD)
+DO_ZZZ_WTB(sve2_uaddw_s, uint32_t, uint16_t, H1_4, H1_2, DO_ADD)
+DO_ZZZ_WTB(sve2_uaddw_d, uint64_t, uint32_t, H1_8, H1_4, DO_ADD)
 
-#ifndef HOST_WORDS_BIGENDIAN
-    o = 0;
-#endif
-    switch (o) {
-    case 0:
-        memset(vd, 0, n);
-        break;
+DO_ZZZ_WTB(sve2_ssubw_h, int16_t, int8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_WTB(sve2_ssubw_s, int32_t, int16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_WTB(sve2_ssubw_d, int64_t, int32_t, H1_8, H1_4, DO_SUB)
 
-    case 4:
-        for (i = 0; i < n; i += 4) {
-            *(uint32_t *)H1_4(d + i) = 0;
-        }
-        break;
+DO_ZZZ_WTB(sve2_usubw_h, uint16_t, uint8_t, H1_2, H1, DO_SUB)
+DO_ZZZ_WTB(sve2_usubw_s, uint32_t, uint16_t, H1_4, H1_2, DO_SUB)
+DO_ZZZ_WTB(sve2_usubw_d, uint64_t, uint32_t, H1_8, H1_4, DO_SUB)
 
-    case 2:
-    case 6:
-        for (i = 0; i < n; i += 2) {
-            *(uint16_t *)H1_2(d + i) = 0;
-        }
-        break;
+#undef DO_ZZZ_WTB
 
-    default:
-        for (i = 0; i < n; i++) {
-            *(uint8_t *)H1(d + i) = 0;
-        }
-        break;
-    }
+#define DO_ZZI_SHLL(NAME, TYPEW, TYPEN, HW, HN)                         \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                    \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    intptr_t sel = (simd_data(desc) & 1) * sizeof(TYPEN);               \
+    int shift = simd_data(desc) >> 1;                                   \
+    char *d = vd, *n = vn;                                              \
+                                                                        \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                        \
+        TYPEW nn = *(TYPEN *)(n + HN(i + sel));                         \
+                                                                        \
+        *(TYPEW *)(d + HW(i)) = nn << shift;                            \
+    }                                                                   \
 }
 
-void HELPER(sve_ext)(void *vd, void *vn, void *vm, uint32_t desc)
+DO_ZZI_SHLL(sve2_sshll_h, int16_t, int8_t, H1_2, H1)
+DO_ZZI_SHLL(sve2_sshll_s, int32_t, int16_t, H1_4, H1_2)
+DO_ZZI_SHLL(sve2_sshll_d, int64_t, int32_t, H1_8, H1_4)
+
+DO_ZZI_SHLL(sve2_ushll_h, uint16_t, uint8_t, H1_2, H1)
+DO_ZZI_SHLL(sve2_ushll_s, uint32_t, uint16_t, H1_4, H1_2)
+DO_ZZI_SHLL(sve2_ushll_d, uint64_t, uint32_t, H1_8, H1_4)
+
+#undef DO_ZZI_SHLL
+
+static uint64_t do_shrn_lsr(uint64_t x, unsigned shift)
 {
-    intptr_t opr_sz = simd_oprsz(desc);
-    size_t n_ofs = simd_data(desc);
-    size_t n_siz = opr_sz - n_ofs;
+    return shift >= 64 ? 0 : x >> shift;
+}
 
-    if (vd != vm) {
-        swap_memmove(vd, (char *)vn + n_ofs, n_siz);
-        swap_memmove((char *)vd + n_siz, vm, n_ofs);
-    } else if (vd != vn) {
-        swap_memmove((char *)vd + n_siz, vd, n_ofs);
-        swap_memmove(vd, (char *)vn + n_ofs, n_siz);
-    } else {
-        /* vd == vn == vm.  Need temp space.  */
-        ARMVectorReg tmp;
-        swap_memmove(&tmp, vm, n_ofs);
-        swap_memmove(vd, (char *)vd + n_ofs, n_siz);
-        memcpy((char *)vd + n_siz, &tmp, n_ofs);
+static uint64_t do_shrn_urshr(uint64_t x, unsigned shift)
+{
+    if (shift < 64) {
+        return (x >> shift) + ((x >> (shift - 1)) & 1);
+    }
+    if (shift == 64) {
+        return x >> 63;
     }
+    return 0;
 }
 
-#define DO_INSR(NAME, TYPE, H) \
-void HELPER(NAME)(void *vd, void *vn, uint64_t val, uint32_t desc) \
-{                                                                  \
-    intptr_t opr_sz = simd_oprsz(desc);                            \
-    swap_memmove((char *)vd + sizeof(TYPE), vn, opr_sz - sizeof(TYPE));    \
-    *(TYPE *)((char *)vd + H(0)) = val;                                    \
+static int64_t do_shrn_sar(int64_t x, unsigned shift)
+{
+    if (shift >= 64) {
+        return x < 0 ? -1 : 0;
+    }
+    return x >> shift;
 }
 
-DO_INSR(sve_insr_b, uint8_t, H1)
-DO_INSR(sve_insr_h, uint16_t, H1_2)
-DO_INSR(sve_insr_s, uint32_t, H1_4)
-DO_INSR(sve_insr_d, uint64_t, )
-
-#undef DO_INSR
-
-void HELPER(sve_rev_b)(void *vd, void *vn, uint32_t desc)
+static int64_t do_shrn_srshr(int64_t x, unsigned shift)
 {
-    intptr_t i, j, opr_sz = simd_oprsz(desc);
-    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
-        uint64_t f = *(uint64_t *)((char *)vn + i);
-        uint64_t b = *(uint64_t *)((char *)vn + j);
-        *(uint64_t *)((char *)vd + i) = bswap64(b);
-        *(uint64_t *)((char *)vd + j) = bswap64(f);
+    if (shift < 64) {
+        return (x >> shift) + ((x >> (shift - 1)) & 1);
     }
+    return 0;
 }
 
-void HELPER(sve_rev_h)(void *vd, void *vn, uint32_t desc)
+static uint64_t do_shrn_sat_u(int64_t val, uint64_t max)
 {
-    intptr_t i, j, opr_sz = simd_oprsz(desc);
-    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
-        uint64_t f = *(uint64_t *)((char *)vn + i);
-        uint64_t b = *(uint64_t *)((char *)vn + j);
-        *(uint64_t *)((char *)vd + i) = hswap64(b);
-        *(uint64_t *)((char *)vd + j) = hswap64(f);
+    if (val < 0) {
+        return 0;
     }
+    if ((uint64_t)val > max) {
+        return max;
+    }
+    return val;
 }
 
-void HELPER(sve_rev_s)(void *vd, void *vn, uint32_t desc)
+static uint64_t do_shrn_sat_uu(uint64_t val, uint64_t max)
 {
-    intptr_t i, j, opr_sz = simd_oprsz(desc);
-    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
-        uint64_t f = *(uint64_t *)((char *)vn + i);
-        uint64_t b = *(uint64_t *)((char *)vn + j);
-        *(uint64_t *)((char *)vd + i) = rol64(b, 32);
-        *(uint64_t *)((char *)vd + j) = rol64(f, 32);
-    }
+    return val > max ? max : val;
 }
 
-void HELPER(sve_rev_d)(void *vd, void *vn, uint32_t desc)
+static int64_t do_shrn_sat_s(int64_t val, int64_t min, int64_t max)
 {
-    intptr_t i, j, opr_sz = simd_oprsz(desc);
-    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
-        uint64_t f = *(uint64_t *)((char *)vn + i);
-        uint64_t b = *(uint64_t *)((char *)vn + j);
-        *(uint64_t *)((char *)vd + i) = b;
-        *(uint64_t *)((char *)vd + j) = f;
+    if (val < min) {
+        return min;
     }
+    if (val > max) {
+        return max;
+    }
+    return val;
 }
 
-#define DO_TBL(NAME, TYPE, H) \
-void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
-{                                                              \
-    intptr_t i, opr_sz = simd_oprsz(desc);                     \
-    uintptr_t elem = opr_sz / sizeof(TYPE);                    \
-    TYPE *d = vd, *n = vn, *m = vm;                            \
-    ARMVectorReg tmp;                                          \
-    if (unlikely(vd == vn)) {                                  \
-        n = memcpy(&tmp, vn, opr_sz);                          \
-    }                                                          \
-    for (i = 0; i < elem; i++) {                               \
-        TYPE j = m[H(i)];                                      \
-        d[H(i)] = j < elem ? n[H(j)] : 0;                      \
-    }                                                          \
+#define DO_SRA(NAME, TYPE, UTYPE, H, OP)                                \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                   \
+{                                                                      \
+    intptr_t i, oprsz = simd_oprsz(desc);                              \
+    unsigned shift = simd_data(desc);                                  \
+    char *d = vd, *n = vn;                                             \
+                                                                       \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                        \
+        TYPE dd = *(TYPE *)(d + H(i));                                 \
+        TYPE nn = *(TYPE *)(n + H(i));                                 \
+                                                                       \
+        *(TYPE *)(d + H(i)) =                                          \
+            (TYPE)((UTYPE)dd + (UTYPE)(TYPE)OP(nn, shift));            \
+    }                                                                  \
 }
 
-DO_TBL(sve_tbl_b, uint8_t, H1)
-DO_TBL(sve_tbl_h, uint16_t, H2)
-DO_TBL(sve_tbl_s, uint32_t, H4)
-DO_TBL(sve_tbl_d, uint64_t, )
+#define DO_SSRA_H(X, SHIFT) do_shrn_sar((int16_t)(X), SHIFT)
+#define DO_SSRA_S(X, SHIFT) do_shrn_sar((int32_t)(X), SHIFT)
+#define DO_SSRA_D(X, SHIFT) do_shrn_sar((int64_t)(X), SHIFT)
+#define DO_USRA(X, SHIFT) do_shrn_lsr((uint64_t)(X), SHIFT)
+#define DO_SRSRA_H(X, SHIFT) do_shrn_srshr((int16_t)(X), SHIFT)
+#define DO_SRSRA_S(X, SHIFT) do_shrn_srshr((int32_t)(X), SHIFT)
+#define DO_SRSRA_D(X, SHIFT) do_shrn_srshr((int64_t)(X), SHIFT)
+#define DO_URSRA(X, SHIFT) do_shrn_urshr((uint64_t)(X), SHIFT)
+
+DO_SRA(sve2_ssra_b, int8_t, uint8_t, H1, DO_SSRA_H)
+DO_SRA(sve2_ssra_h, int16_t, uint16_t, H1_2, DO_SSRA_H)
+DO_SRA(sve2_ssra_s, int32_t, uint32_t, H1_4, DO_SSRA_S)
+DO_SRA(sve2_ssra_d, int64_t, uint64_t, H1_8, DO_SSRA_D)
+DO_SRA(sve2_usra_b, uint8_t, uint8_t, H1, DO_USRA)
+DO_SRA(sve2_usra_h, uint16_t, uint16_t, H1_2, DO_USRA)
+DO_SRA(sve2_usra_s, uint32_t, uint32_t, H1_4, DO_USRA)
+DO_SRA(sve2_usra_d, uint64_t, uint64_t, H1_8, DO_USRA)
+DO_SRA(sve2_srsra_b, int8_t, uint8_t, H1, DO_SRSRA_H)
+DO_SRA(sve2_srsra_h, int16_t, uint16_t, H1_2, DO_SRSRA_H)
+DO_SRA(sve2_srsra_s, int32_t, uint32_t, H1_4, DO_SRSRA_S)
+DO_SRA(sve2_srsra_d, int64_t, uint64_t, H1_8, DO_SRSRA_D)
+DO_SRA(sve2_ursra_b, uint8_t, uint8_t, H1, DO_URSRA)
+DO_SRA(sve2_ursra_h, uint16_t, uint16_t, H1_2, DO_URSRA)
+DO_SRA(sve2_ursra_s, uint32_t, uint32_t, H1_4, DO_URSRA)
+DO_SRA(sve2_ursra_d, uint64_t, uint64_t, H1_8, DO_URSRA)
+
+#undef DO_SRA
+#undef DO_SSRA_H
+#undef DO_SSRA_S
+#undef DO_SSRA_D
+#undef DO_USRA
+#undef DO_SRSRA_H
+#undef DO_SRSRA_S
+#undef DO_SRSRA_D
+#undef DO_URSRA
+
+#define DO_SRI(NAME, TYPE, H)                                          \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                   \
+{                                                                      \
+    intptr_t i, oprsz = simd_oprsz(desc);                              \
+    unsigned shift = simd_data(desc);                                  \
+    unsigned bits = sizeof(TYPE) * 8;                                  \
+    TYPE mask;                                                         \
+    char *d = vd, *n = vn;                                             \
+                                                                       \
+    if (shift == bits) {                                               \
+        return;                                                        \
+    }                                                                  \
+    mask = (TYPE)((TYPE)-1 >> shift);                                  \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                        \
+        TYPE dd = *(TYPE *)(d + H(i));                                 \
+        TYPE nn = *(TYPE *)(n + H(i));                                 \
+                                                                       \
+        *(TYPE *)(d + H(i)) =                                          \
+            (TYPE)((dd & (TYPE)~mask) | ((nn >> shift) & mask));       \
+    }                                                                  \
+}
 
-#undef TBL
+DO_SRI(sve2_sri_b, uint8_t, H1)
+DO_SRI(sve2_sri_h, uint16_t, H1_2)
+DO_SRI(sve2_sri_s, uint32_t, H1_4)
+DO_SRI(sve2_sri_d, uint64_t, H1_8)
 
-#define DO_UNPK(NAME, TYPED, TYPES, HD, HS) \
-void HELPER(NAME)(void *vd, void *vn, uint32_t desc)           \
-{                                                              \
-    intptr_t i, opr_sz = simd_oprsz(desc);                     \
-    TYPED *d = vd;                                             \
-    TYPES *n = vn;                                             \
-    ARMVectorReg tmp;                                          \
-    if (unlikely((char *)vn - (char *)vd < opr_sz)) {                          \
-        n = memcpy(&tmp, n, opr_sz / 2);                       \
-    }                                                          \
-    for (i = 0; i < opr_sz / sizeof(TYPED); i++) {             \
-        d[HD(i)] = n[HS(i)];                                   \
-    }                                                          \
+#undef DO_SRI
+
+#define DO_SLI(NAME, TYPE, H)                                          \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                   \
+{                                                                      \
+    intptr_t i, oprsz = simd_oprsz(desc);                              \
+    unsigned shift = simd_data(desc);                                  \
+    unsigned bits = sizeof(TYPE) * 8;                                  \
+    TYPE mask = shift == 0 ? 0 : (TYPE)((TYPE)-1 >> (bits - shift));   \
+    char *d = vd, *n = vn;                                             \
+                                                                       \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                        \
+        TYPE dd = *(TYPE *)(d + H(i));                                 \
+        TYPE nn = *(TYPE *)(n + H(i));                                 \
+        TYPE res = (TYPE)((dd & mask) | (TYPE)(nn << shift));          \
+                                                                       \
+        *(TYPE *)(d + H(i)) = res;                                     \
+    }                                                                  \
 }
 
-DO_UNPK(sve_sunpk_h, int16_t, int8_t, H2, H1)
-DO_UNPK(sve_sunpk_s, int32_t, int16_t, H4, H2)
-DO_UNPK(sve_sunpk_d, int64_t, int32_t, , H4)
+DO_SLI(sve2_sli_b, uint8_t, H1)
+DO_SLI(sve2_sli_h, uint16_t, H1_2)
+DO_SLI(sve2_sli_s, uint32_t, H1_4)
+DO_SLI(sve2_sli_d, uint64_t, H1_8)
 
-DO_UNPK(sve_uunpk_h, uint16_t, uint8_t, H2, H1)
-DO_UNPK(sve_uunpk_s, uint32_t, uint16_t, H4, H2)
-DO_UNPK(sve_uunpk_d, uint64_t, uint32_t, , H4)
+#undef DO_SLI
 
-#undef DO_UNPK
+#define DO_SHRNB(NAME, TYPEW, TYPEN, OP)                               \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                   \
+{                                                                      \
+    intptr_t i, oprsz = simd_oprsz(desc);                              \
+    unsigned shift = simd_data(desc);                                  \
+    char *d = vd, *n = vn;                                             \
+                                                                       \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                       \
+        TYPEW nn = *(TYPEW *)(n + i);                                  \
+                                                                       \
+        *(TYPEW *)(d + i) = (TYPEW)(TYPEN)OP(nn, shift);               \
+    }                                                                  \
+}
 
-/* Mask of bits included in the even numbered predicates of width esz.
- * We also use this for expand_bits/compress_bits, and so extend the
- * same pattern out to 16-bit units.
- */
-static const uint64_t even_bit_esz_masks[5] = {
-    0x5555555555555555ull,
-    0x3333333333333333ull,
-    0x0f0f0f0f0f0f0f0full,
-    0x00ff00ff00ff00ffull,
-    0x0000ffff0000ffffull,
-};
+#define DO_SHRNT(NAME, TYPEW, TYPEN, HW, HN, OP)                       \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                   \
+{                                                                      \
+    intptr_t i, oprsz = simd_oprsz(desc);                              \
+    unsigned shift = simd_data(desc);                                  \
+    char *d = vd, *n = vn;                                             \
+                                                                       \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                       \
+        TYPEW nn = *(TYPEW *)(n + HW(i));                              \
+                                                                       \
+        *(TYPEN *)(d + HN(i + sizeof(TYPEN))) = (TYPEN)OP(nn, shift);  \
+    }                                                                  \
+}
 
-/* Zero-extend units of 2**N bits to units of 2**(N+1) bits.
- * For N==0, this corresponds to the operation that in qemu/bitops.h
- * we call half_shuffle64; this algorithm is from Hacker's Delight,
- * section 7-2 Shuffling Bits.
- */
-static uint64_t expand_bits(uint64_t x, int n)
-{
-    int i;
+#define DO_SHRN_U(X, SHIFT) do_shrn_lsr((uint64_t)(X), SHIFT)
+#define DO_RSHRN_U(X, SHIFT) do_shrn_urshr((uint64_t)(X), SHIFT)
+#define DO_SQSHRUN_H(X, SHIFT)                                         \
+    do_shrn_sat_u(do_shrn_sar((int16_t)(X), SHIFT), UINT8_MAX)
+#define DO_SQSHRUN_S(X, SHIFT)                                         \
+    do_shrn_sat_u(do_shrn_sar((int32_t)(X), SHIFT), UINT16_MAX)
+#define DO_SQSHRUN_D(X, SHIFT)                                         \
+    do_shrn_sat_u(do_shrn_sar((int64_t)(X), SHIFT), UINT32_MAX)
+#define DO_SQRSHRUN_H(X, SHIFT)                                        \
+    do_shrn_sat_u(do_shrn_srshr((int16_t)(X), SHIFT), UINT8_MAX)
+#define DO_SQRSHRUN_S(X, SHIFT)                                        \
+    do_shrn_sat_u(do_shrn_srshr((int32_t)(X), SHIFT), UINT16_MAX)
+#define DO_SQRSHRUN_D(X, SHIFT)                                        \
+    do_shrn_sat_u(do_shrn_srshr((int64_t)(X), SHIFT), UINT32_MAX)
+#define DO_SQSHRN_H(X, SHIFT)                                          \
+    do_shrn_sat_s(do_shrn_sar((int16_t)(X), SHIFT), INT8_MIN, INT8_MAX)
+#define DO_SQSHRN_S(X, SHIFT)                                          \
+    do_shrn_sat_s(do_shrn_sar((int32_t)(X), SHIFT), INT16_MIN, INT16_MAX)
+#define DO_SQSHRN_D(X, SHIFT)                                          \
+    do_shrn_sat_s(do_shrn_sar((int64_t)(X), SHIFT), INT32_MIN, INT32_MAX)
+#define DO_SQRSHRN_H(X, SHIFT)                                         \
+    do_shrn_sat_s(do_shrn_srshr((int16_t)(X), SHIFT), INT8_MIN, INT8_MAX)
+#define DO_SQRSHRN_S(X, SHIFT)                                         \
+    do_shrn_sat_s(do_shrn_srshr((int32_t)(X), SHIFT), INT16_MIN,        \
+                  INT16_MAX)
+#define DO_SQRSHRN_D(X, SHIFT)                                         \
+    do_shrn_sat_s(do_shrn_srshr((int64_t)(X), SHIFT), INT32_MIN,        \
+                  INT32_MAX)
+#define DO_UQSHRN_H(X, SHIFT)                                          \
+    do_shrn_sat_uu(do_shrn_lsr((uint16_t)(X), SHIFT), UINT8_MAX)
+#define DO_UQSHRN_S(X, SHIFT)                                          \
+    do_shrn_sat_uu(do_shrn_lsr((uint32_t)(X), SHIFT), UINT16_MAX)
+#define DO_UQSHRN_D(X, SHIFT)                                          \
+    do_shrn_sat_uu(do_shrn_lsr((uint64_t)(X), SHIFT), UINT32_MAX)
+#define DO_UQRSHRN_H(X, SHIFT)                                         \
+    do_shrn_sat_uu(do_shrn_urshr((uint16_t)(X), SHIFT), UINT8_MAX)
+#define DO_UQRSHRN_S(X, SHIFT)                                         \
+    do_shrn_sat_uu(do_shrn_urshr((uint32_t)(X), SHIFT), UINT16_MAX)
+#define DO_UQRSHRN_D(X, SHIFT)                                         \
+    do_shrn_sat_uu(do_shrn_urshr((uint64_t)(X), SHIFT), UINT32_MAX)
+
+DO_SHRNB(sve2_sqshrunb_h, int16_t, uint8_t, DO_SQSHRUN_H)
+DO_SHRNB(sve2_sqshrunb_s, int32_t, uint16_t, DO_SQSHRUN_S)
+DO_SHRNB(sve2_sqshrunb_d, int64_t, uint32_t, DO_SQSHRUN_D)
+DO_SHRNT(sve2_sqshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRUN_H)
+DO_SHRNT(sve2_sqshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRUN_S)
+DO_SHRNT(sve2_sqshrunt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQSHRUN_D)
+DO_SHRNB(sve2_sqrshrunb_h, int16_t, uint8_t, DO_SQRSHRUN_H)
+DO_SHRNB(sve2_sqrshrunb_s, int32_t, uint16_t, DO_SQRSHRUN_S)
+DO_SHRNB(sve2_sqrshrunb_d, int64_t, uint32_t, DO_SQRSHRUN_D)
+DO_SHRNT(sve2_sqrshrunt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRUN_H)
+DO_SHRNT(sve2_sqrshrunt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRUN_S)
+DO_SHRNT(sve2_sqrshrunt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQRSHRUN_D)
+DO_SHRNB(sve2_shrnb_h, uint16_t, uint8_t, DO_SHRN_U)
+DO_SHRNB(sve2_shrnb_s, uint32_t, uint16_t, DO_SHRN_U)
+DO_SHRNB(sve2_shrnb_d, uint64_t, uint32_t, DO_SHRN_U)
+DO_SHRNT(sve2_shrnt_h, uint16_t, uint8_t, H1_2, H1, DO_SHRN_U)
+DO_SHRNT(sve2_shrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_SHRN_U)
+DO_SHRNT(sve2_shrnt_d, uint64_t, uint32_t, H1_8, H1_4, DO_SHRN_U)
+DO_SHRNB(sve2_rshrnb_h, uint16_t, uint8_t, DO_RSHRN_U)
+DO_SHRNB(sve2_rshrnb_s, uint32_t, uint16_t, DO_RSHRN_U)
+DO_SHRNB(sve2_rshrnb_d, uint64_t, uint32_t, DO_RSHRN_U)
+DO_SHRNT(sve2_rshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_RSHRN_U)
+DO_SHRNT(sve2_rshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_RSHRN_U)
+DO_SHRNT(sve2_rshrnt_d, uint64_t, uint32_t, H1_8, H1_4, DO_RSHRN_U)
+DO_SHRNB(sve2_sqshrnb_h, int16_t, uint8_t, DO_SQSHRN_H)
+DO_SHRNB(sve2_sqshrnb_s, int32_t, uint16_t, DO_SQSHRN_S)
+DO_SHRNB(sve2_sqshrnb_d, int64_t, uint32_t, DO_SQSHRN_D)
+DO_SHRNT(sve2_sqshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQSHRN_H)
+DO_SHRNT(sve2_sqshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQSHRN_S)
+DO_SHRNT(sve2_sqshrnt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQSHRN_D)
+DO_SHRNB(sve2_sqrshrnb_h, int16_t, uint8_t, DO_SQRSHRN_H)
+DO_SHRNB(sve2_sqrshrnb_s, int32_t, uint16_t, DO_SQRSHRN_S)
+DO_SHRNB(sve2_sqrshrnb_d, int64_t, uint32_t, DO_SQRSHRN_D)
+DO_SHRNT(sve2_sqrshrnt_h, int16_t, uint8_t, H1_2, H1, DO_SQRSHRN_H)
+DO_SHRNT(sve2_sqrshrnt_s, int32_t, uint16_t, H1_4, H1_2, DO_SQRSHRN_S)
+DO_SHRNT(sve2_sqrshrnt_d, int64_t, uint32_t, H1_8, H1_4, DO_SQRSHRN_D)
+DO_SHRNB(sve2_uqshrnb_h, uint16_t, uint8_t, DO_UQSHRN_H)
+DO_SHRNB(sve2_uqshrnb_s, uint32_t, uint16_t, DO_UQSHRN_S)
+DO_SHRNB(sve2_uqshrnb_d, uint64_t, uint32_t, DO_UQSHRN_D)
+DO_SHRNT(sve2_uqshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQSHRN_H)
+DO_SHRNT(sve2_uqshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQSHRN_S)
+DO_SHRNT(sve2_uqshrnt_d, uint64_t, uint32_t, H1_8, H1_4, DO_UQSHRN_D)
+DO_SHRNB(sve2_uqrshrnb_h, uint16_t, uint8_t, DO_UQRSHRN_H)
+DO_SHRNB(sve2_uqrshrnb_s, uint32_t, uint16_t, DO_UQRSHRN_S)
+DO_SHRNB(sve2_uqrshrnb_d, uint64_t, uint32_t, DO_UQRSHRN_D)
+DO_SHRNT(sve2_uqrshrnt_h, uint16_t, uint8_t, H1_2, H1, DO_UQRSHRN_H)
+DO_SHRNT(sve2_uqrshrnt_s, uint32_t, uint16_t, H1_4, H1_2, DO_UQRSHRN_S)
+DO_SHRNT(sve2_uqrshrnt_d, uint64_t, uint32_t, H1_8, H1_4, DO_UQRSHRN_D)
+
+#undef DO_SHRNB
+#undef DO_SHRNT
+#undef DO_SHRN_U
+#undef DO_RSHRN_U
+#undef DO_SQSHRUN_H
+#undef DO_SQSHRUN_S
+#undef DO_SQSHRUN_D
+#undef DO_SQRSHRUN_H
+#undef DO_SQRSHRUN_S
+#undef DO_SQRSHRUN_D
+#undef DO_SQSHRN_H
+#undef DO_SQSHRN_S
+#undef DO_SQSHRN_D
+#undef DO_SQRSHRN_H
+#undef DO_SQRSHRN_S
+#undef DO_SQRSHRN_D
+#undef DO_UQSHRN_H
+#undef DO_UQSHRN_S
+#undef DO_UQSHRN_D
+#undef DO_UQRSHRN_H
+#undef DO_UQRSHRN_S
+#undef DO_UQRSHRN_D
+
+#define DO_BINOPNB(NAME, TYPEW, TYPEN, SHIFT, ROUND, SUB)               \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    char *d = vd, *n = vn, *m = vm;                                      \
+                                                                        \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                        \
+        TYPEW nn = *(TYPEW *)(n + i);                                   \
+        TYPEW mm = *(TYPEW *)(m + i);                                   \
+        TYPEW val = (SUB) ? nn - mm : nn + mm;                          \
+                                                                        \
+        if (ROUND) {                                                    \
+            val += (TYPEW)1 << ((SHIFT) - 1);                           \
+        }                                                               \
+        *(TYPEW *)(d + i) = (TYPEN)(val >> (SHIFT));                    \
+    }                                                                   \
+}
 
-    x &= 0xffffffffu;
-    for (i = 4; i >= n; i--) {
-        int sh = 1 << i;
-        x = ((x << sh) | x) & even_bit_esz_masks[i];
-    }
-    return x;
+#define DO_BINOPNT(NAME, TYPEW, TYPEN, SHIFT, HW, HN, ROUND, SUB)       \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    char *d = vd, *n = vn, *m = vm;                                      \
+                                                                        \
+    for (i = 0; i < oprsz; i += sizeof(TYPEW)) {                        \
+        TYPEW nn = *(TYPEW *)(n + HW(i));                               \
+        TYPEW mm = *(TYPEW *)(m + HW(i));                               \
+        TYPEW val = (SUB) ? nn - mm : nn + mm;                          \
+                                                                        \
+        if (ROUND) {                                                    \
+            val += (TYPEW)1 << ((SHIFT) - 1);                           \
+        }                                                               \
+        *(TYPEN *)(d + HN(i + sizeof(TYPEN))) =                         \
+            (TYPEN)(val >> (SHIFT));                                    \
+    }                                                                   \
 }
 
-/* Compress units of 2**(N+1) bits to units of 2**N bits.
- * For N==0, this corresponds to the operation that in qemu/bitops.h
- * we call half_unshuffle64; this algorithm is from Hacker's Delight,
- * section 7-2 Shuffling Bits, where it is called an inverse half shuffle.
- */
-static uint64_t compress_bits(uint64_t x, int n)
-{
-    int i;
+DO_BINOPNB(sve2_addhnb_h, uint16_t, uint8_t, 8, false, false)
+DO_BINOPNB(sve2_addhnb_s, uint32_t, uint16_t, 16, false, false)
+DO_BINOPNB(sve2_addhnb_d, uint64_t, uint32_t, 32, false, false)
 
-    for (i = n; i <= 4; i++) {
-        int sh = 1 << i;
-        x &= even_bit_esz_masks[i];
-        x = (x >> sh) | x;
-    }
-    return x & 0xffffffffu;
-}
+DO_BINOPNT(sve2_addhnt_h, uint16_t, uint8_t, 8, H1_2, H1, false, false)
+DO_BINOPNT(sve2_addhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, false, false)
+DO_BINOPNT(sve2_addhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, false, false)
 
-void HELPER(sve_zip_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    int esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
-    intptr_t high = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1);
-    uint64_t *d = vd;
-    intptr_t i;
+DO_BINOPNB(sve2_raddhnb_h, uint16_t, uint8_t, 8, true, false)
+DO_BINOPNB(sve2_raddhnb_s, uint32_t, uint16_t, 16, true, false)
+DO_BINOPNB(sve2_raddhnb_d, uint64_t, uint32_t, 32, true, false)
 
-    if (oprsz <= 8) {
-        uint64_t nn = *(uint64_t *)vn;
-        uint64_t mm = *(uint64_t *)vm;
-        int half = 4 * oprsz;
+DO_BINOPNT(sve2_raddhnt_h, uint16_t, uint8_t, 8, H1_2, H1, true, false)
+DO_BINOPNT(sve2_raddhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, true, false)
+DO_BINOPNT(sve2_raddhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, true, false)
 
-        nn = extract64(nn, high * half, half);
-        mm = extract64(mm, high * half, half);
-        nn = expand_bits(nn, esz);
-        mm = expand_bits(mm, esz);
-        d[0] = nn + (mm << (1 << esz));
-    } else {
-        ARMPredicateReg tmp_n, tmp_m;
+DO_BINOPNB(sve2_subhnb_h, uint16_t, uint8_t, 8, false, true)
+DO_BINOPNB(sve2_subhnb_s, uint32_t, uint16_t, 16, false, true)
+DO_BINOPNB(sve2_subhnb_d, uint64_t, uint32_t, 32, false, true)
 
-        /* We produce output faster than we consume input.
-           Therefore we must be mindful of possible overlap.  */
-        if (((char *)vn - (char *)vd) < (uintptr_t)oprsz) {
-            vn = memcpy(&tmp_n, vn, oprsz);
-        }
-        if (((char *)vm - (char *)vd) < (uintptr_t)oprsz) {
-            vm = memcpy(&tmp_m, vm, oprsz);
-        }
-        if (high) {
-            high = oprsz >> 1;
-        }
+DO_BINOPNT(sve2_subhnt_h, uint16_t, uint8_t, 8, H1_2, H1, false, true)
+DO_BINOPNT(sve2_subhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, false, true)
+DO_BINOPNT(sve2_subhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, false, true)
 
-        if ((high & 3) == 0) {
-            uint32_t *n = vn, *m = vm;
-            high >>= 2;
+DO_BINOPNB(sve2_rsubhnb_h, uint16_t, uint8_t, 8, true, true)
+DO_BINOPNB(sve2_rsubhnb_s, uint32_t, uint16_t, 16, true, true)
+DO_BINOPNB(sve2_rsubhnb_d, uint64_t, uint32_t, 32, true, true)
 
-            for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
-                uint64_t nn = n[H4(high + i)];
-                uint64_t mm = m[H4(high + i)];
+DO_BINOPNT(sve2_rsubhnt_h, uint16_t, uint8_t, 8, H1_2, H1, true, true)
+DO_BINOPNT(sve2_rsubhnt_s, uint32_t, uint16_t, 16, H1_4, H1_2, true, true)
+DO_BINOPNT(sve2_rsubhnt_d, uint64_t, uint32_t, 32, H1_8, H1_4, true, true)
 
-                nn = expand_bits(nn, esz);
-                mm = expand_bits(mm, esz);
-                d[i] = nn + (mm << (1 << esz));
-            }
-        } else {
-            uint8_t *n = vn, *m = vm;
-            uint16_t *d16 = vd;
+#undef DO_BINOPNB
+#undef DO_BINOPNT
 
-            for (i = 0; i < oprsz / 2; i++) {
-                uint16_t nn = n[H1(high + i)];
-                uint16_t mm = m[H1(high + i)];
+#define DO_XTNB(NAME, TYPE, OP)                                        \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                   \
+{                                                                      \
+    intptr_t i, oprsz = simd_oprsz(desc);                              \
+    char *d = vd, *n = vn;                                             \
+                                                                       \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                        \
+        TYPE nn = *(TYPE *)(n + i);                                    \
+        uint64_t val = (uint64_t)OP(nn);                               \
+                                                                       \
+        val &= MAKE_64BIT_MASK(0, sizeof(TYPE) * 4);                   \
+        *(TYPE *)(d + i) = (TYPE)val;                                  \
+    }                                                                  \
+}
 
-                nn = expand_bits(nn, esz);
-                mm = expand_bits(mm, esz);
-                d16[H2(i)] = nn + (mm << (1 << esz));
-            }
-        }
-    }
+#define DO_XTNT(NAME, TYPE, TYPEN, H, OP)                              \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)                   \
+{                                                                      \
+    intptr_t i, oprsz = simd_oprsz(desc);                              \
+    intptr_t odd = H(sizeof(TYPEN));                                   \
+    char *d = vd, *n = vn;                                             \
+                                                                       \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                        \
+        TYPE nn = *(TYPE *)(n + i);                                    \
+                                                                       \
+        *(TYPEN *)(d + i + odd) = (TYPEN)OP(nn);                       \
+    }                                                                  \
 }
 
-void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    int esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
-    int odd = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1) << esz;
-    uint64_t *d = vd, *n = vn, *m = vm;
-    uint64_t l, h;
-    intptr_t i;
+#define DO_SQXTN_H(N)   ((N) < INT8_MIN ? INT8_MIN :                  \
+                         (N) > INT8_MAX ? INT8_MAX : (N))
+#define DO_SQXTN_S(N)   ((N) < INT16_MIN ? INT16_MIN :                \
+                         (N) > INT16_MAX ? INT16_MAX : (N))
+#define DO_SQXTN_D(N)   ((N) < INT32_MIN ? INT32_MIN :                \
+                         (N) > INT32_MAX ? INT32_MAX : (N))
+
+#define DO_UQXTN_H(N)   ((N) > UINT8_MAX ? UINT8_MAX : (N))
+#define DO_UQXTN_S(N)   ((N) > UINT16_MAX ? UINT16_MAX : (N))
+#define DO_UQXTN_D(N)   ((N) > UINT32_MAX ? UINT32_MAX : (N))
+
+#define DO_SQXTUN_H(N)  ((N) < 0 ? 0 : DO_UQXTN_H(N))
+#define DO_SQXTUN_S(N)  ((N) < 0 ? 0 : DO_UQXTN_S(N))
+#define DO_SQXTUN_D(N)  ((N) < 0 ? 0 : DO_UQXTN_D(N))
+
+DO_XTNB(sve2_sqxtnb_h, int16_t, DO_SQXTN_H)
+DO_XTNB(sve2_sqxtnb_s, int32_t, DO_SQXTN_S)
+DO_XTNB(sve2_sqxtnb_d, int64_t, DO_SQXTN_D)
+
+DO_XTNT(sve2_sqxtnt_h, int16_t, int8_t, H1, DO_SQXTN_H)
+DO_XTNT(sve2_sqxtnt_s, int32_t, int16_t, H1_2, DO_SQXTN_S)
+DO_XTNT(sve2_sqxtnt_d, int64_t, int32_t, H1_4, DO_SQXTN_D)
+
+DO_XTNB(sve2_uqxtnb_h, uint16_t, DO_UQXTN_H)
+DO_XTNB(sve2_uqxtnb_s, uint32_t, DO_UQXTN_S)
+DO_XTNB(sve2_uqxtnb_d, uint64_t, DO_UQXTN_D)
+
+DO_XTNT(sve2_uqxtnt_h, uint16_t, uint8_t, H1, DO_UQXTN_H)
+DO_XTNT(sve2_uqxtnt_s, uint32_t, uint16_t, H1_2, DO_UQXTN_S)
+DO_XTNT(sve2_uqxtnt_d, uint64_t, uint32_t, H1_4, DO_UQXTN_D)
+
+DO_XTNB(sve2_sqxtunb_h, int16_t, DO_SQXTUN_H)
+DO_XTNB(sve2_sqxtunb_s, int32_t, DO_SQXTUN_S)
+DO_XTNB(sve2_sqxtunb_d, int64_t, DO_SQXTUN_D)
+
+DO_XTNT(sve2_sqxtunt_h, int16_t, int8_t, H1, DO_SQXTUN_H)
+DO_XTNT(sve2_sqxtunt_s, int32_t, int16_t, H1_2, DO_SQXTUN_S)
+DO_XTNT(sve2_sqxtunt_d, int64_t, int32_t, H1_4, DO_SQXTUN_D)
+
+#undef DO_XTNB
+#undef DO_XTNT
+#undef DO_SQXTN_H
+#undef DO_SQXTN_S
+#undef DO_SQXTN_D
+#undef DO_UQXTN_H
+#undef DO_UQXTN_S
+#undef DO_UQXTN_D
+#undef DO_SQXTUN_H
+#undef DO_SQXTUN_S
+#undef DO_SQXTUN_D
 
-    if (oprsz <= 8) {
-        l = compress_bits(n[0] >> odd, esz);
-        h = compress_bits(m[0] >> odd, esz);
-        d[0] = extract64(l + (h << (4 * oprsz)), 0, 8 * oprsz);
-    } else {
-        ARMPredicateReg tmp_m;
-        intptr_t oprsz_16 = oprsz / 16;
+DO_ZPZZ(sve_sdiv_zpzz_s, int32_t, H1_4, DO_SDIV)
+DO_ZPZZ_D(sve_sdiv_zpzz_d, int64_t, DO_SDIV)
 
-        if (((char *)vm - (char *)vd) < (uintptr_t)oprsz) {
-            m = memcpy(&tmp_m, vm, oprsz);
-        }
+DO_ZPZZ(sve_udiv_zpzz_s, uint32_t, H1_4, DO_UDIV)
+DO_ZPZZ_D(sve_udiv_zpzz_d, uint64_t, DO_UDIV)
 
-        for (i = 0; i < oprsz_16; i++) {
-            l = n[2 * i + 0];
-            h = n[2 * i + 1];
-            l = compress_bits(l >> odd, esz);
-            h = compress_bits(h >> odd, esz);
-            d[i] = l + (h << 32);
-        }
+/* Note that all bits of the shift are significant
+   and not modulo the element size.  */
+#define DO_ASR(N, M)  (N >> MIN(M, sizeof(N) * 8 - 1))
+#define DO_LSR(N, M)  (M < sizeof(N) * 8 ? N >> M : 0)
+#define DO_LSL(N, M)  (M < sizeof(N) * 8 ? N << M : 0)
 
-        /* For VL which is not a power of 2, the results from M do not
-           align nicely with the uint64_t for D.  Put the aligned results
-           from M into TMP_M and then copy it into place afterward.  */
-        if (oprsz & 15) {
-            d[i] = compress_bits(n[2 * i] >> odd, esz);
+DO_ZPZZ(sve_asr_zpzz_b, int8_t, H1, DO_ASR)
+DO_ZPZZ(sve_lsr_zpzz_b, uint8_t, H1_2, DO_LSR)
+DO_ZPZZ(sve_lsl_zpzz_b, uint8_t, H1_4, DO_LSL)
 
-            for (i = 0; i < oprsz_16; i++) {
-                l = m[2 * i + 0];
-                h = m[2 * i + 1];
-                l = compress_bits(l >> odd, esz);
-                h = compress_bits(h >> odd, esz);
-                tmp_m.p[i] = l + (h << 32);
-            }
-            tmp_m.p[i] = compress_bits(m[2 * i] >> odd, esz);
+DO_ZPZZ(sve_asr_zpzz_h, int16_t, H1, DO_ASR)
+DO_ZPZZ(sve_lsr_zpzz_h, uint16_t, H1_2, DO_LSR)
+DO_ZPZZ(sve_lsl_zpzz_h, uint16_t, H1_4, DO_LSL)
 
-            swap_memmove((char *)vd + oprsz / 2, &tmp_m, oprsz / 2);
-        } else {
-            for (i = 0; i < oprsz_16; i++) {
-                l = m[2 * i + 0];
-                h = m[2 * i + 1];
-                l = compress_bits(l >> odd, esz);
-                h = compress_bits(h >> odd, esz);
-                d[oprsz_16 + i] = l + (h << 32);
-            }
-        }
-    }
-}
+DO_ZPZZ(sve_asr_zpzz_s, int32_t, H1, DO_ASR)
+DO_ZPZZ(sve_lsr_zpzz_s, uint32_t, H1_2, DO_LSR)
+DO_ZPZZ(sve_lsl_zpzz_s, uint32_t, H1_4, DO_LSL)
 
-void HELPER(sve_trn_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    uintptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
-    bool odd = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1);
-    uint64_t *d = vd, *n = vn, *m = vm;
-    uint64_t mask;
-    int shr, shl;
-    intptr_t i;
+DO_ZPZZ_D(sve_asr_zpzz_d, int64_t, DO_ASR)
+DO_ZPZZ_D(sve_lsr_zpzz_d, uint64_t, DO_LSR)
+DO_ZPZZ_D(sve_lsl_zpzz_d, uint64_t, DO_LSL)
 
-    shl = 1 << esz;
-    shr = 0;
-    mask = even_bit_esz_masks[esz];
-    if (odd) {
-        mask <<= shl;
-        shr = shl;
-        shl = 0;
-    }
+#undef DO_ZPZZ
+#undef DO_ZPZZ_D
 
-    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
-        uint64_t nn = (n[i] & mask) >> shr;
-        uint64_t mm = (m[i] & mask) << shl;
-        d[i] = nn + mm;
-    }
+/*
+ * Three operand expander, operating on element pairs.
+ * If the slot I is even, use VN {I, I+1}; if the slot I is odd, use
+ * VM {I-1, I}. Load pair inputs before overwriting output.
+ */
+#define DO_ZPZZ_PAIR(NAME, TYPE, H, OP)                                \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{                                                                      \
+    intptr_t i, opr_sz = simd_oprsz(desc);                             \
+    char *d = vd, *n = vn, *m = vm, *g = vg;                            \
+                                                                       \
+    for (i = 0; i < opr_sz; ) {                                        \
+        uint16_t pg = *(uint16_t *)(g + H1_2(i >> 3));                 \
+        do {                                                           \
+            TYPE n0 = *(TYPE *)(n + H(i));                             \
+            TYPE m0 = *(TYPE *)(m + H(i));                             \
+            TYPE n1 = *(TYPE *)(n + H(i + sizeof(TYPE)));              \
+            TYPE m1 = *(TYPE *)(m + H(i + sizeof(TYPE)));              \
+            if (pg & 1) {                                              \
+                *(TYPE *)(d + H(i)) = OP(n0, n1);                      \
+            }                                                          \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);                    \
+            if (pg & 1) {                                              \
+                *(TYPE *)(d + H(i)) = OP(m0, m1);                      \
+            }                                                          \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);                    \
+        } while (i & 15);                                              \
+    }                                                                  \
 }
 
-/* Reverse units of 2**N bits.  */
-static uint64_t reverse_bits_64(uint64_t x, int n)
-{
-    int i, sh;
-
-    x = bswap64(x);
-    for (i = 2, sh = 4; i >= n; i--, sh >>= 1) {
-        uint64_t mask = even_bit_esz_masks[i];
-        x = ((x & mask) << sh) | ((x >> sh) & mask);
-    }
-    return x;
+#define DO_ZPZZ_PAIR_D(NAME, TYPE, OP)                                 \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{                                                                      \
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                         \
+    TYPE *d = vd, *n = vn, *m = vm;                                    \
+    uint8_t *pg = vg;                                                  \
+                                                                       \
+    for (i = 0; i < opr_sz; i += 2) {                                  \
+        TYPE n0 = n[i], n1 = n[i + 1];                                 \
+        TYPE m0 = m[i], m1 = m[i + 1];                                 \
+        if (pg[H1(i)] & 1) {                                           \
+            d[i] = OP(n0, n1);                                         \
+        }                                                              \
+        if (pg[H1(i + 1)] & 1) {                                       \
+            d[i + 1] = OP(m0, m1);                                     \
+        }                                                              \
+    }                                                                  \
 }
 
-static uint8_t reverse_bits_8(uint8_t x, int n)
-{
-    static const uint8_t mask[3] = { 0x55, 0x33, 0x0f };
-    int i, sh;
+DO_ZPZZ_PAIR(sve2_addp_zpzz_b, uint8_t, H1, DO_ADD)
+DO_ZPZZ_PAIR(sve2_addp_zpzz_h, uint16_t, H1_2, DO_ADD)
+DO_ZPZZ_PAIR(sve2_addp_zpzz_s, uint32_t, H1_4, DO_ADD)
+DO_ZPZZ_PAIR_D(sve2_addp_zpzz_d, uint64_t, DO_ADD)
 
-    for (i = 2, sh = 4; i >= n; i--, sh >>= 1) {
-        x = ((x & mask[i]) << sh) | ((x >> sh) & mask[i]);
-    }
-    return x;
-}
+DO_ZPZZ_PAIR(sve2_smaxp_zpzz_b, int8_t, H1, DO_MAX)
+DO_ZPZZ_PAIR(sve2_smaxp_zpzz_h, int16_t, H1_2, DO_MAX)
+DO_ZPZZ_PAIR(sve2_smaxp_zpzz_s, int32_t, H1_4, DO_MAX)
+DO_ZPZZ_PAIR_D(sve2_smaxp_zpzz_d, int64_t, DO_MAX)
 
-void HELPER(sve_rev_p)(void *vd, void *vn, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    int esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
-    intptr_t i, oprsz_2 = oprsz / 2;
+DO_ZPZZ_PAIR(sve2_umaxp_zpzz_b, uint8_t, H1, DO_MAX)
+DO_ZPZZ_PAIR(sve2_umaxp_zpzz_h, uint16_t, H1_2, DO_MAX)
+DO_ZPZZ_PAIR(sve2_umaxp_zpzz_s, uint32_t, H1_4, DO_MAX)
+DO_ZPZZ_PAIR_D(sve2_umaxp_zpzz_d, uint64_t, DO_MAX)
 
-    if (oprsz <= 8) {
-        uint64_t l = *(uint64_t *)vn;
-        l = reverse_bits_64(l << (64 - 8 * oprsz), esz);
-        *(uint64_t *)vd = l;
-    } else if ((oprsz & 15) == 0) {
-        for (i = 0; i < oprsz_2; i += 8) {
-            intptr_t ih = oprsz - 8 - i;
-            uint64_t l = reverse_bits_64(*(uint64_t *)((char *)vn + i), esz);
-            uint64_t h = reverse_bits_64(*(uint64_t *)((char *)vn + ih), esz);
-            *(uint64_t *)((char *)vd + i) = h;
-            *(uint64_t *)((char *)vd + ih) = l;
-        }
-    } else {
-        for (i = 0; i < oprsz_2; i += 1) {
-            intptr_t il = H1(i);
-            intptr_t ih = H1(oprsz - 1 - i);
-            uint8_t l = reverse_bits_8(*(uint8_t *)((char *)vn + il), esz);
-            uint8_t h = reverse_bits_8(*(uint8_t *)((char *)vn + ih), esz);
-            *(uint8_t *)((char *)vd + il) = h;
-            *(uint8_t *)((char *)vd + ih) = l;
-        }
-    }
-}
+DO_ZPZZ_PAIR(sve2_sminp_zpzz_b, int8_t, H1, DO_MIN)
+DO_ZPZZ_PAIR(sve2_sminp_zpzz_h, int16_t, H1_2, DO_MIN)
+DO_ZPZZ_PAIR(sve2_sminp_zpzz_s, int32_t, H1_4, DO_MIN)
+DO_ZPZZ_PAIR_D(sve2_sminp_zpzz_d, int64_t, DO_MIN)
 
-void HELPER(sve_punpk_p)(void *vd, void *vn, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    intptr_t high = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1);
-    uint64_t *d = vd;
-    intptr_t i;
+DO_ZPZZ_PAIR(sve2_uminp_zpzz_b, uint8_t, H1, DO_MIN)
+DO_ZPZZ_PAIR(sve2_uminp_zpzz_h, uint16_t, H1_2, DO_MIN)
+DO_ZPZZ_PAIR(sve2_uminp_zpzz_s, uint32_t, H1_4, DO_MIN)
+DO_ZPZZ_PAIR_D(sve2_uminp_zpzz_d, uint64_t, DO_MIN)
 
-    if (oprsz <= 8) {
-        uint64_t nn = *(uint64_t *)vn;
-        int half = 4 * oprsz;
+#undef DO_ZPZZ_PAIR
+#undef DO_ZPZZ_PAIR_D
 
-        nn = extract64(nn, high * half, half);
-        nn = expand_bits(nn, 0);
-        d[0] = nn;
-    } else {
-        ARMPredicateReg tmp_n;
+#define DO_ZPZZ_PAIR_FP(NAME, TYPE, H, OP)                            \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg,              \
+                  void *status, uint32_t desc)                         \
+{                                                                      \
+    intptr_t i, opr_sz = simd_oprsz(desc);                             \
+    char *d = vd, *n = vn, *m = vm, *g = vg;                            \
+                                                                       \
+    for (i = 0; i < opr_sz; ) {                                        \
+        uint16_t pg = *(uint16_t *)(g + H1_2(i >> 3));                 \
+        do {                                                           \
+            TYPE n0 = *(TYPE *)(n + H(i));                             \
+            TYPE m0 = *(TYPE *)(m + H(i));                             \
+            TYPE n1 = *(TYPE *)(n + H(i + sizeof(TYPE)));              \
+            TYPE m1 = *(TYPE *)(m + H(i + sizeof(TYPE)));              \
+            if (pg & 1) {                                              \
+                *(TYPE *)(d + H(i)) = OP(n0, n1, status);              \
+            }                                                          \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);                    \
+            if (pg & 1) {                                              \
+                *(TYPE *)(d + H(i)) = OP(m0, m1, status);              \
+            }                                                          \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);                    \
+        } while (i & 15);                                              \
+    }                                                                  \
+}
 
-        /* We produce output faster than we consume input.
-           Therefore we must be mindful of possible overlap.  */
-        if (((char *)vn - (char *)vd) < (uintptr_t)oprsz) {
-            vn = memcpy(&tmp_n, vn, oprsz);
-        }
-        if (high) {
-            high = oprsz >> 1;
-        }
+DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_h, float16, H1_2, float16_add)
+DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_s, float32, H1_4, float32_add)
+DO_ZPZZ_PAIR_FP(sve2_faddp_zpzz_d, float64, H1_8, float64_add)
 
-        if ((high & 3) == 0) {
-            uint32_t *n = vn;
-            high >>= 2;
+DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_h, float16, H1_2, float16_maxnum)
+DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_s, float32, H1_4, float32_maxnum)
+DO_ZPZZ_PAIR_FP(sve2_fmaxnmp_zpzz_d, float64, H1_8, float64_maxnum)
 
-            for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
-                uint64_t nn = n[H4(high + i)];
-                d[i] = expand_bits(nn, 0);
-            }
-        } else {
-            uint16_t *d16 = vd;
-            uint8_t *n = vn;
+DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_h, float16, H1_2, float16_minnum)
+DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_s, float32, H1_4, float32_minnum)
+DO_ZPZZ_PAIR_FP(sve2_fminnmp_zpzz_d, float64, H1_8, float64_minnum)
 
-            for (i = 0; i < oprsz / 2; i++) {
-                uint16_t nn = n[H1(high + i)];
-                d16[H2(i)] = expand_bits(nn, 0);
-            }
-        }
-    }
-}
+DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_h, float16, H1_2, float16_max)
+DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_s, float32, H1_4, float32_max)
+DO_ZPZZ_PAIR_FP(sve2_fmaxp_zpzz_d, float64, H1_8, float64_max)
 
-#define DO_ZIP(NAME, TYPE, H) \
-void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)       \
-{                                                                    \
-    intptr_t oprsz = simd_oprsz(desc);                               \
-    intptr_t i, oprsz_2 = oprsz / 2;                                 \
-    ARMVectorReg tmp_n, tmp_m;                                       \
-    /* We produce output faster than we consume input.               \
-       Therefore we must be mindful of possible overlap.  */         \
-    if (unlikely(((char *)vn - (char *)vd) < (uintptr_t)oprsz)) {                    \
-        vn = memcpy(&tmp_n, vn, oprsz_2);                            \
-    }                                                                \
-    if (unlikely(((char *)vm - (char *)vd) < (uintptr_t)oprsz)) {                    \
-        vm = memcpy(&tmp_m, vm, oprsz_2);                            \
-    }                                                                \
-    for (i = 0; i < oprsz_2; i += sizeof(TYPE)) {                    \
-        *(TYPE *)((char *)vd + H(2 * i + 0)) = *(TYPE *)((char *)vn + H(i));         \
-        *(TYPE *)((char *)vd + H(2 * i + sizeof(TYPE))) = *(TYPE *)((char *)vm + H(i)); \
-    }                                                                \
-}
+DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_h, float16, H1_2, float16_min)
+DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_s, float32, H1_4, float32_min)
+DO_ZPZZ_PAIR_FP(sve2_fminp_zpzz_d, float64, H1_8, float64_min)
 
-DO_ZIP(sve_zip_b, uint8_t, H1)
-DO_ZIP(sve_zip_h, uint16_t, H1_2)
-DO_ZIP(sve_zip_s, uint32_t, H1_4)
-DO_ZIP(sve_zip_d, uint64_t, )
+#undef DO_ZPZZ_PAIR_FP
 
-#define DO_UZP(NAME, TYPE, H) \
-void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)         \
-{                                                                      \
-    intptr_t oprsz = simd_oprsz(desc);                                 \
-    intptr_t oprsz_2 = oprsz / 2;                                      \
-    intptr_t odd_ofs = simd_data(desc);                                \
-    intptr_t i;                                                        \
-    ARMVectorReg tmp_m;                                                \
-    if (unlikely(((char *)vm - (char *)vd) < (uintptr_t)oprsz)) {                      \
-        vm = memcpy(&tmp_m, vm, oprsz);                                \
-    }                                                                  \
-    for (i = 0; i < oprsz_2; i += sizeof(TYPE)) {                      \
-        *(TYPE *)((char *)vd + H(i)) = *(TYPE *)((char *)vn + H(2 * i + odd_ofs));     \
-    }                                                                  \
-    for (i = 0; i < oprsz_2; i += sizeof(TYPE)) {                      \
-        *(TYPE *)((char *)vd + H(oprsz_2 + i)) = *(TYPE *)((char *)vm + H(2 * i + odd_ofs)); \
-    }                                                                  \
+/* Three-operand expander, controlled by a predicate, in which the
+ * third operand is "wide".  That is, for D = N op M, the same 64-bit
+ * value of M is used with all of the narrower values of N.
+ */
+#define DO_ZPZW(NAME, TYPE, TYPEW, H, OP)                               \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{                                                                       \
+    intptr_t i, opr_sz = simd_oprsz(desc);                              \
+    for (i = 0; i < opr_sz; ) {                                         \
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(i >> 3));                     \
+        TYPEW mm = *(TYPEW *)((char *)vm + i);                                  \
+        do {                                                            \
+            if (pg & 1) {                                               \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                         \
+                *(TYPE *)((char *)vd + H(i)) = OP(nn, mm);                      \
+            }                                                           \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);                     \
+        } while (i & 7);                                                \
+    }                                                                   \
 }
 
-DO_UZP(sve_uzp_b, uint8_t, H1)
-DO_UZP(sve_uzp_h, uint16_t, H1_2)
-DO_UZP(sve_uzp_s, uint32_t, H1_4)
-DO_UZP(sve_uzp_d, uint64_t, )
+DO_ZPZW(sve_asr_zpzw_b, int8_t, uint64_t, H1, DO_ASR)
+DO_ZPZW(sve_lsr_zpzw_b, uint8_t, uint64_t, H1, DO_LSR)
+DO_ZPZW(sve_lsl_zpzw_b, uint8_t, uint64_t, H1, DO_LSL)
 
-#define DO_TRN(NAME, TYPE, H) \
-void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)         \
-{                                                                      \
-    intptr_t oprsz = simd_oprsz(desc);                                 \
-    intptr_t odd_ofs = simd_data(desc);                                \
-    intptr_t i;                                                        \
-    for (i = 0; i < oprsz; i += 2 * sizeof(TYPE)) {                    \
-        TYPE ae = *(TYPE *)((char *)vn + H(i + odd_ofs));                      \
-        TYPE be = *(TYPE *)((char *)vm + H(i + odd_ofs));                      \
-        *(TYPE *)((char *)vd + H(i + 0)) = ae;                                 \
-        *(TYPE *)((char *)vd + H(i + sizeof(TYPE))) = be;                      \
-    }                                                                  \
-}
+DO_ZPZW(sve_asr_zpzw_h, int16_t, uint64_t, H1_2, DO_ASR)
+DO_ZPZW(sve_lsr_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
+DO_ZPZW(sve_lsl_zpzw_h, uint16_t, uint64_t, H1_2, DO_LSL)
 
-DO_TRN(sve_trn_b, uint8_t, H1)
-DO_TRN(sve_trn_h, uint16_t, H1_2)
-DO_TRN(sve_trn_s, uint32_t, H1_4)
-DO_TRN(sve_trn_d, uint64_t, )
+DO_ZPZW(sve_asr_zpzw_s, int32_t, uint64_t, H1_4, DO_ASR)
+DO_ZPZW(sve_lsr_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
+DO_ZPZW(sve_lsl_zpzw_s, uint32_t, uint64_t, H1_4, DO_LSL)
 
-#undef DO_ZIP
-#undef DO_UZP
-#undef DO_TRN
+#undef DO_ZPZW
 
-void HELPER(sve_compact_s)(void *vd, void *vn, void *vg, uint32_t desc)
-{
-    intptr_t i, j, opr_sz = simd_oprsz(desc) / 4;
-    uint32_t *d = vd, *n = vn;
-    uint8_t *pg = vg;
+/* Fully general two-operand expander, controlled by a predicate.
+ */
+#define DO_ZPZ(NAME, TYPE, H, OP)                               \
+void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
+{                                                               \
+    intptr_t i, opr_sz = simd_oprsz(desc);                      \
+    for (i = 0; i < opr_sz; ) {                                 \
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));         \
+        do {                                                    \
+            if (pg & 1) {                                       \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                 \
+                *(TYPE *)((char *)vd + H(i)) = OP(nn);                  \
+            }                                                   \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);             \
+        } while (i & 15);                                       \
+    }                                                           \
+}
 
-    for (i = j = 0; i < opr_sz; i++) {
-        if (pg[H1(i / 2)] & (i & 1 ? 0x10 : 0x01)) {
+/* Similarly, specialized for 64-bit operands.  */
+#define DO_ZPZ_D(NAME, TYPE, OP)                                \
+void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
+{                                                               \
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                  \
+    TYPE *d = vd, *n = vn;                                      \
+    uint8_t *pg = vg;                                           \
+    for (i = 0; i < opr_sz; i += 1) {                           \
+        if (pg[H1(i)] & 1) {                                    \
+            TYPE nn = n[i];                                     \
+            d[i] = OP(nn);                                      \
+        }                                                       \
+    }                                                           \
+}
+
+#define DO_CLS_B(N)   (clrsb32(N) - 24)
+#define DO_CLS_H(N)   (clrsb32(N) - 16)
+
+DO_ZPZ(sve_cls_b, int8_t, H1, DO_CLS_B)
+DO_ZPZ(sve_cls_h, int16_t, H1_2, DO_CLS_H)
+DO_ZPZ(sve_cls_s, int32_t, H1_4, clrsb32)
+DO_ZPZ_D(sve_cls_d, int64_t, clrsb64)
+
+#define DO_CLZ_B(N)   (clz32(N) - 24)
+#define DO_CLZ_H(N)   (clz32(N) - 16)
+
+DO_ZPZ(sve_clz_b, uint8_t, H1, DO_CLZ_B)
+DO_ZPZ(sve_clz_h, uint16_t, H1_2, DO_CLZ_H)
+DO_ZPZ(sve_clz_s, uint32_t, H1_4, clz32)
+DO_ZPZ_D(sve_clz_d, uint64_t, clz64)
+
+DO_ZPZ(sve_cnt_zpz_b, uint8_t, H1, ctpop8)
+DO_ZPZ(sve_cnt_zpz_h, uint16_t, H1_2, ctpop16)
+DO_ZPZ(sve_cnt_zpz_s, uint32_t, H1_4, ctpop32)
+DO_ZPZ_D(sve_cnt_zpz_d, uint64_t, ctpop64)
+
+#define DO_CNOT(N)    (N == 0)
+
+DO_ZPZ(sve_cnot_b, uint8_t, H1, DO_CNOT)
+DO_ZPZ(sve_cnot_h, uint16_t, H1_2, DO_CNOT)
+DO_ZPZ(sve_cnot_s, uint32_t, H1_4, DO_CNOT)
+DO_ZPZ_D(sve_cnot_d, uint64_t, DO_CNOT)
+
+#ifdef _MSC_VER
+#define DO_FABS16(N)    (N & ((uint16_t)-1 >> 1))
+#define DO_FABS32(N)    (N & ((uint32_t)-1 >> 1))
+#define DO_FABS64(N)    (N & ((uint64_t)-1 >> 1))
+
+DO_ZPZ(sve_fabs_h, uint16_t, H1_2, DO_FABS16)
+DO_ZPZ(sve_fabs_s, uint32_t, H1_4, DO_FABS32)
+DO_ZPZ_D(sve_fabs_d, uint64_t, DO_FABS64)
+#else
+#define DO_FABS(N)    (N & ((__typeof(N))-1 >> 1))
+
+DO_ZPZ(sve_fabs_h, uint16_t, H1_2, DO_FABS)
+DO_ZPZ(sve_fabs_s, uint32_t, H1_4, DO_FABS)
+DO_ZPZ_D(sve_fabs_d, uint64_t, DO_FABS)
+#endif
+
+#ifdef _MSC_VER
+#define DO_FNEG16(N)    (N ^ ~((uint16_t)-1 >> 1))
+#define DO_FNEG32(N)    (N ^ ~((uint32_t)-1 >> 1))
+#define DO_FNEG64(N)    (N ^ ~((uint64_t)-1 >> 1))
+
+DO_ZPZ(sve_fneg_h, uint16_t, H1_2, DO_FNEG16)
+DO_ZPZ(sve_fneg_s, uint32_t, H1_4, DO_FNEG32)
+DO_ZPZ_D(sve_fneg_d, uint64_t, DO_FNEG64)
+#else
+#define DO_FNEG(N)    (N ^ ~((__typeof(N))-1 >> 1))
+
+DO_ZPZ(sve_fneg_h, uint16_t, H1_2, DO_FNEG)
+DO_ZPZ(sve_fneg_s, uint32_t, H1_4, DO_FNEG)
+DO_ZPZ_D(sve_fneg_d, uint64_t, DO_FNEG)
+#endif
+
+#define DO_NOT(N)    (~N)
+
+DO_ZPZ(sve_not_zpz_b, uint8_t, H1, DO_NOT)
+DO_ZPZ(sve_not_zpz_h, uint16_t, H1_2, DO_NOT)
+DO_ZPZ(sve_not_zpz_s, uint32_t, H1_4, DO_NOT)
+DO_ZPZ_D(sve_not_zpz_d, uint64_t, DO_NOT)
+
+#define DO_SXTB(N)    ((int8_t)N)
+#define DO_SXTH(N)    ((int16_t)N)
+#define DO_SXTS(N)    ((int32_t)N)
+#define DO_UXTB(N)    ((uint8_t)N)
+#define DO_UXTH(N)    ((uint16_t)N)
+#define DO_UXTS(N)    ((uint32_t)N)
+
+DO_ZPZ(sve_sxtb_h, uint16_t, H1_2, DO_SXTB)
+DO_ZPZ(sve_sxtb_s, uint32_t, H1_4, DO_SXTB)
+DO_ZPZ(sve_sxth_s, uint32_t, H1_4, DO_SXTH)
+DO_ZPZ_D(sve_sxtb_d, uint64_t, DO_SXTB)
+DO_ZPZ_D(sve_sxth_d, uint64_t, DO_SXTH)
+DO_ZPZ_D(sve_sxtw_d, uint64_t, DO_SXTS)
+
+DO_ZPZ(sve_uxtb_h, uint16_t, H1_2, DO_UXTB)
+DO_ZPZ(sve_uxtb_s, uint32_t, H1_4, DO_UXTB)
+DO_ZPZ(sve_uxth_s, uint32_t, H1_4, DO_UXTH)
+DO_ZPZ_D(sve_uxtb_d, uint64_t, DO_UXTB)
+DO_ZPZ_D(sve_uxth_d, uint64_t, DO_UXTH)
+DO_ZPZ_D(sve_uxtw_d, uint64_t, DO_UXTS)
+
+#ifdef _MSC_VER
+#define DO_ABS(N)    (N < 0 ? (0 - N) : N)
+#else
+#define DO_ABS(N)    (N < 0 ? -N : N)
+#endif
+
+DO_ZPZ(sve_abs_b, int8_t, H1, DO_ABS)
+DO_ZPZ(sve_abs_h, int16_t, H1_2, DO_ABS)
+DO_ZPZ(sve_abs_s, int32_t, H1_4, DO_ABS)
+DO_ZPZ_D(sve_abs_d, int64_t, DO_ABS)
+
+#ifdef _MSC_VER
+#define DO_NEG(N)    (0 - N)
+#else
+#define DO_NEG(N)    (-N)
+#endif
+
+DO_ZPZ(sve_neg_b, uint8_t, H1, DO_NEG)
+DO_ZPZ(sve_neg_h, uint16_t, H1_2, DO_NEG)
+DO_ZPZ(sve_neg_s, uint32_t, H1_4, DO_NEG)
+DO_ZPZ_D(sve_neg_d, uint64_t, DO_NEG)
+
+DO_ZPZ(sve_revb_h, uint16_t, H1_2, bswap16)
+DO_ZPZ(sve_revb_s, uint32_t, H1_4, bswap32)
+DO_ZPZ_D(sve_revb_d, uint64_t, bswap64)
+
+DO_ZPZ(sve_revh_s, uint32_t, H1_4, hswap32)
+DO_ZPZ_D(sve_revh_d, uint64_t, hswap64)
+
+DO_ZPZ_D(sve_revw_d, uint64_t, wswap64)
+
+DO_ZPZ(sve_rbit_b, uint8_t, H1, revbit8)
+DO_ZPZ(sve_rbit_h, uint16_t, H1_2, revbit16)
+DO_ZPZ(sve_rbit_s, uint32_t, H1_4, revbit32)
+DO_ZPZ_D(sve_rbit_d, uint64_t, revbit64)
+
+#define DO_SQABS_B(N) ((N) == INT8_MIN ? INT8_MAX : (N) < 0 ? -(N) : (N))
+#define DO_SQABS_H(N) ((N) == INT16_MIN ? INT16_MAX : (N) < 0 ? -(N) : (N))
+#define DO_SQABS_S(N) ((N) == INT32_MIN ? INT32_MAX : (N) < 0 ? -(N) : (N))
+#define DO_SQABS_D(N) ((N) == INT64_MIN ? INT64_MAX : (N) < 0 ? -(N) : (N))
+
+DO_ZPZ(sve2_sqabs_b, int8_t, H1, DO_SQABS_B)
+DO_ZPZ(sve2_sqabs_h, int16_t, H1_2, DO_SQABS_H)
+DO_ZPZ(sve2_sqabs_s, int32_t, H1_4, DO_SQABS_S)
+DO_ZPZ_D(sve2_sqabs_d, int64_t, DO_SQABS_D)
+
+#define DO_SQNEG_B(N) ((N) == 0x80u ? 0x7fu : (uint8_t)(-(N)))
+#define DO_SQNEG_H(N) ((N) == 0x8000u ? 0x7fffu : (uint16_t)(-(N)))
+#define DO_SQNEG_S(N) \
+    ((N) == 0x80000000u ? 0x7fffffffu : (uint32_t)(-(N)))
+#define DO_SQNEG_D(N)                                                     \
+    ((N) == 0x8000000000000000ull ? 0x7fffffffffffffffull : (uint64_t)(-(N)))
+
+DO_ZPZ(sve2_sqneg_b, uint8_t, H1, DO_SQNEG_B)
+DO_ZPZ(sve2_sqneg_h, uint16_t, H1_2, DO_SQNEG_H)
+DO_ZPZ(sve2_sqneg_s, uint32_t, H1_4, DO_SQNEG_S)
+DO_ZPZ_D(sve2_sqneg_d, uint64_t, DO_SQNEG_D)
+
+static uint32_t do_sve2_urecpe_s(uint32_t n)
+{
+    return HELPER(recpe_u32)(n, NULL);
+}
+
+static uint32_t do_sve2_ursqrte_s(uint32_t n)
+{
+    return HELPER(rsqrte_u32)(n, NULL);
+}
+
+DO_ZPZ(sve2_urecpe_s, uint32_t, H1_4, do_sve2_urecpe_s)
+DO_ZPZ(sve2_ursqrte_s, uint32_t, H1_4, do_sve2_ursqrte_s)
+
+/* Three-operand expander, unpredicated, in which the third operand is "wide".
+ */
+#define DO_ZZW(NAME, TYPE, TYPEW, H, OP)                       \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
+{                                                              \
+    intptr_t i, opr_sz = simd_oprsz(desc);                     \
+    for (i = 0; i < opr_sz; ) {                                \
+        TYPEW mm = *(TYPEW *)((char *)vm + i);                         \
+        do {                                                   \
+            TYPE nn = *(TYPE *)((char *)vn + H(i));                    \
+            *(TYPE *)((char *)vd + H(i)) = OP(nn, mm);                 \
+            i += sizeof(TYPE);                                 \
+        } while (i & 7);                                       \
+    }                                                          \
+}
+
+DO_ZZW(sve_asr_zzw_b, int8_t, uint64_t, H1, DO_ASR)
+DO_ZZW(sve_lsr_zzw_b, uint8_t, uint64_t, H1, DO_LSR)
+DO_ZZW(sve_lsl_zzw_b, uint8_t, uint64_t, H1, DO_LSL)
+
+DO_ZZW(sve_asr_zzw_h, int16_t, uint64_t, H1_2, DO_ASR)
+DO_ZZW(sve_lsr_zzw_h, uint16_t, uint64_t, H1_2, DO_LSR)
+DO_ZZW(sve_lsl_zzw_h, uint16_t, uint64_t, H1_2, DO_LSL)
+
+DO_ZZW(sve_asr_zzw_s, int32_t, uint64_t, H1_4, DO_ASR)
+DO_ZZW(sve_lsr_zzw_s, uint32_t, uint64_t, H1_4, DO_LSR)
+DO_ZZW(sve_lsl_zzw_s, uint32_t, uint64_t, H1_4, DO_LSL)
+
+#undef DO_ZZW
+
+#undef DO_CLS_B
+#undef DO_CLS_H
+#undef DO_CLZ_B
+#undef DO_CLZ_H
+#undef DO_CNOT
+#undef DO_FABS
+#undef DO_FNEG
+#undef DO_ABS
+#undef DO_NEG
+#undef DO_SQABS_B
+#undef DO_SQABS_H
+#undef DO_SQABS_S
+#undef DO_SQABS_D
+#undef DO_SQNEG_B
+#undef DO_SQNEG_H
+#undef DO_SQNEG_S
+#undef DO_SQNEG_D
+#undef DO_ZPZ
+#undef DO_ZPZ_D
+
+/* Two-operand reduction expander, controlled by a predicate.
+ * The difference between TYPERED and TYPERET has to do with
+ * sign-extension.  E.g. for SMAX, TYPERED must be signed,
+ * but TYPERET must be unsigned so that e.g. a 32-bit value
+ * is not sign-extended to the ABI uint64_t return type.
+ */
+/* ??? If we were to vectorize this by hand the reduction ordering
+ * would change.  For integer operands, this is perfectly fine.
+ */
+#define DO_VPZ(NAME, TYPEELT, TYPERED, TYPERET, H, INIT, OP) \
+uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc)   \
+{                                                          \
+    intptr_t i, opr_sz = simd_oprsz(desc);                 \
+    TYPERED ret = INIT;                                    \
+    for (i = 0; i < opr_sz; ) {                            \
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));    \
+        do {                                               \
+            if (pg & 1) {                                  \
+                TYPEELT nn = *(TYPEELT *)((char *)vn + H(i));      \
+                ret = OP(ret, nn);                         \
+            }                                              \
+            i += sizeof(TYPEELT), pg >>= sizeof(TYPEELT);  \
+        } while (i & 15);                                  \
+    }                                                      \
+    return (TYPERET)ret;                                   \
+}
+
+#define DO_VPZ_D(NAME, TYPEE, TYPER, INIT, OP)             \
+uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc)   \
+{                                                          \
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;             \
+    TYPEE *n = vn;                                         \
+    uint8_t *pg = vg;                                      \
+    TYPER ret = INIT;                                      \
+    for (i = 0; i < opr_sz; i += 1) {                      \
+        if (pg[H1(i)] & 1) {                               \
+            TYPEE nn = n[i];                               \
+            ret = OP(ret, nn);                             \
+        }                                                  \
+    }                                                      \
+    return ret;                                            \
+}
+
+DO_VPZ(sve_orv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_ORR)
+DO_VPZ(sve_orv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_ORR)
+DO_VPZ(sve_orv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_ORR)
+DO_VPZ_D(sve_orv_d, uint64_t, uint64_t, 0, DO_ORR)
+
+DO_VPZ(sve_eorv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_EOR)
+DO_VPZ(sve_eorv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_EOR)
+DO_VPZ(sve_eorv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_EOR)
+DO_VPZ_D(sve_eorv_d, uint64_t, uint64_t, 0, DO_EOR)
+
+DO_VPZ(sve_andv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_AND)
+DO_VPZ(sve_andv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_AND)
+DO_VPZ(sve_andv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_AND)
+DO_VPZ_D(sve_andv_d, uint64_t, uint64_t, -1, DO_AND)
+
+DO_VPZ(sve_saddv_b, int8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
+DO_VPZ(sve_saddv_h, int16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
+DO_VPZ(sve_saddv_s, int32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)
+
+DO_VPZ(sve_uaddv_b, uint8_t, uint64_t, uint64_t, H1, 0, DO_ADD)
+DO_VPZ(sve_uaddv_h, uint16_t, uint64_t, uint64_t, H1_2, 0, DO_ADD)
+DO_VPZ(sve_uaddv_s, uint32_t, uint64_t, uint64_t, H1_4, 0, DO_ADD)
+DO_VPZ_D(sve_uaddv_d, uint64_t, uint64_t, 0, DO_ADD)
+
+DO_VPZ(sve_smaxv_b, int8_t, int8_t, uint8_t, H1, INT8_MIN, DO_MAX)
+DO_VPZ(sve_smaxv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MIN, DO_MAX)
+DO_VPZ(sve_smaxv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MIN, DO_MAX)
+DO_VPZ_D(sve_smaxv_d, int64_t, int64_t, INT64_MIN, DO_MAX)
+
+DO_VPZ(sve_umaxv_b, uint8_t, uint8_t, uint8_t, H1, 0, DO_MAX)
+DO_VPZ(sve_umaxv_h, uint16_t, uint16_t, uint16_t, H1_2, 0, DO_MAX)
+DO_VPZ(sve_umaxv_s, uint32_t, uint32_t, uint32_t, H1_4, 0, DO_MAX)
+DO_VPZ_D(sve_umaxv_d, uint64_t, uint64_t, 0, DO_MAX)
+
+DO_VPZ(sve_sminv_b, int8_t, int8_t, uint8_t, H1, INT8_MAX, DO_MIN)
+DO_VPZ(sve_sminv_h, int16_t, int16_t, uint16_t, H1_2, INT16_MAX, DO_MIN)
+DO_VPZ(sve_sminv_s, int32_t, int32_t, uint32_t, H1_4, INT32_MAX, DO_MIN)
+DO_VPZ_D(sve_sminv_d, int64_t, int64_t, INT64_MAX, DO_MIN)
+
+DO_VPZ(sve_uminv_b, uint8_t, uint8_t, uint8_t, H1, -1, DO_MIN)
+DO_VPZ(sve_uminv_h, uint16_t, uint16_t, uint16_t, H1_2, -1, DO_MIN)
+DO_VPZ(sve_uminv_s, uint32_t, uint32_t, uint32_t, H1_4, -1, DO_MIN)
+DO_VPZ_D(sve_uminv_d, uint64_t, uint64_t, -1, DO_MIN)
+
+#undef DO_VPZ
+#undef DO_VPZ_D
+
+/* Two vector operand, one scalar operand, unpredicated.  */
+#define DO_ZZI(NAME, TYPE, OP)                                       \
+void HELPER(NAME)(void *vd, void *vn, uint64_t s64, uint32_t desc)   \
+{                                                                    \
+    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(TYPE);            \
+    TYPE s = s64, *d = vd, *n = vn;                                  \
+    for (i = 0; i < opr_sz; ++i) {                                   \
+        d[i] = OP(n[i], s);                                          \
+    }                                                                \
+}
+
+#define DO_SUBR(X, Y)   (Y - X)
+
+DO_ZZI(sve_subri_b, uint8_t, DO_SUBR)
+DO_ZZI(sve_subri_h, uint16_t, DO_SUBR)
+DO_ZZI(sve_subri_s, uint32_t, DO_SUBR)
+DO_ZZI(sve_subri_d, uint64_t, DO_SUBR)
+
+DO_ZZI(sve_smaxi_b, int8_t, DO_MAX)
+DO_ZZI(sve_smaxi_h, int16_t, DO_MAX)
+DO_ZZI(sve_smaxi_s, int32_t, DO_MAX)
+DO_ZZI(sve_smaxi_d, int64_t, DO_MAX)
+
+DO_ZZI(sve_smini_b, int8_t, DO_MIN)
+DO_ZZI(sve_smini_h, int16_t, DO_MIN)
+DO_ZZI(sve_smini_s, int32_t, DO_MIN)
+DO_ZZI(sve_smini_d, int64_t, DO_MIN)
+
+DO_ZZI(sve_umaxi_b, uint8_t, DO_MAX)
+DO_ZZI(sve_umaxi_h, uint16_t, DO_MAX)
+DO_ZZI(sve_umaxi_s, uint32_t, DO_MAX)
+DO_ZZI(sve_umaxi_d, uint64_t, DO_MAX)
+
+DO_ZZI(sve_umini_b, uint8_t, DO_MIN)
+DO_ZZI(sve_umini_h, uint16_t, DO_MIN)
+DO_ZZI(sve_umini_s, uint32_t, DO_MIN)
+DO_ZZI(sve_umini_d, uint64_t, DO_MIN)
+
+#undef DO_ZZI
+
+#undef DO_AND
+#undef DO_ORR
+#undef DO_EOR
+#undef DO_BIC
+#undef DO_ADD
+#undef DO_SUB
+#undef DO_MAX
+#undef DO_MIN
+#undef DO_ABD
+#undef DO_MUL
+#undef DO_DIV
+#undef DO_ASR
+#undef DO_LSR
+#undef DO_LSL
+#undef DO_SUBR
+
+/* Similar to the ARM LastActiveElement pseudocode function, except the
+   result is multiplied by the element size.  This includes the not found
+   indication; e.g. not found for esz=3 is -8.  */
+static intptr_t last_active_element(uint64_t *g, intptr_t words, intptr_t esz)
+{
+    uint64_t mask = pred_esz_masks[esz];
+    intptr_t i = words;
+
+    do {
+        uint64_t this_g = g[--i] & mask;
+        if (this_g) {
+            return i * 64 + (63 - clz64(this_g));
+        }
+    } while (i > 0);
+    return (intptr_t)-1 << esz;
+}
+
+uint32_t HELPER(sve_pfirst)(void *vd, void *vg, uint32_t words)
+{
+    uint32_t flags = PREDTEST_INIT;
+    uint64_t *d = vd, *g = vg;
+    intptr_t i = 0;
+
+    do {
+        uint64_t this_d = d[i];
+        uint64_t this_g = g[i];
+
+        if (this_g) {
+            if (!(flags & 4)) {
+                /* Set in D the first bit of G.  */
+#ifdef _MSC_VER
+                this_d |= this_g & (0 - this_g);
+#else
+                this_d |= this_g & -this_g;
+#endif
+                d[i] = this_d;
+            }
+            flags = iter_predtest_fwd(this_d, this_g, flags);
+        }
+    } while (++i < words);
+
+    return flags;
+}
+
+uint32_t HELPER(sve_pnext)(void *vd, void *vg, uint32_t pred_desc)
+{
+    intptr_t words = extract32(pred_desc, 0, SIMD_OPRSZ_BITS);
+    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    uint32_t flags = PREDTEST_INIT;
+    uint64_t *d = vd, *g = vg, esz_mask;
+    intptr_t i, next;
+
+    next = last_active_element(vd, words, esz) + (1ULL << esz);
+    esz_mask = pred_esz_masks[esz];
+
+    /* Similar to the pseudocode for pnext, but scaled by ESZ
+       so that we find the correct bit.  */
+    if (next < words * 64) {
+        uint64_t mask = -1;
+
+        if (next & 63) {
+            mask = ~((1ull << (next & 63)) - 1);
+            next &= -64;
+        }
+        do {
+            uint64_t this_g = g[next / 64] & esz_mask & mask;
+            if (this_g != 0) {
+                next = (next & -64) + ctz64(this_g);
+                break;
+            }
+            next += 64;
+            mask = -1;
+        } while (next < words * 64);
+    }
+
+    i = 0;
+    do {
+        uint64_t this_d = 0;
+        if (i == next / 64) {
+            this_d = 1ull << (next & 63);
+        }
+        d[i] = this_d;
+        flags = iter_predtest_fwd(this_d, g[i] & esz_mask, flags);
+    } while (++i < words);
+
+    return flags;
+}
+
+/* Store zero into every active element of Zd.  We will use this for two
+ * and three-operand predicated instructions for which logic dictates a
+ * zero result.  In particular, logical shift by element size, which is
+ * otherwise undefined on the host.
+ *
+ * For element sizes smaller than uint64_t, we use tables to expand
+ * the N bits of the controlling predicate to a byte mask, and clear
+ * those bytes.
+ */
+void HELPER(sve_clr_b)(void *vd, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] &= ~expand_pred_b(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_clr_h)(void *vd, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] &= ~expand_pred_h(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_clr_s)(void *vd, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] &= ~expand_pred_s(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_clr_d)(void *vd, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+        if (pg[H1(i)] & 1) {
+            d[i] = 0;
+        }
+    }
+}
+
+/* Copy Zn into Zd, and store zero into inactive elements.  */
+void HELPER(sve_movz_b)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = n[i] & expand_pred_b(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_movz_h)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = n[i] & expand_pred_h(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_movz_s)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = n[i] & expand_pred_s(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_movz_d)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+    for (i = 0; i < opr_sz; i += 1) {
+#ifdef _MSC_VER
+        d[i] = n[i] & ((uint64_t)0 - (uint64_t)(pg[H1(i)] & 1));
+#else
+        d[i] = n[i] & -(uint64_t)(pg[H1(i)] & 1);
+#endif
+    }
+}
+
+/* Three-operand expander, immediate operand, controlled by a predicate.
+ */
+#define DO_ZPZI(NAME, TYPE, H, OP)                              \
+void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
+{                                                               \
+    intptr_t i, opr_sz = simd_oprsz(desc);                      \
+    TYPE imm = simd_data(desc);                                 \
+    for (i = 0; i < opr_sz; ) {                                 \
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));         \
+        do {                                                    \
+            if (pg & 1) {                                       \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                 \
+                *(TYPE *)((char *)vd + H(i)) = OP(nn, imm);             \
+            }                                                   \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);             \
+        } while (i & 15);                                       \
+    }                                                           \
+}
+
+/* Similarly, specialized for 64-bit operands.  */
+#define DO_ZPZI_D(NAME, TYPE, OP)                               \
+void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)  \
+{                                                               \
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                  \
+    TYPE *d = vd, *n = vn;                                      \
+    TYPE imm = simd_data(desc);                                 \
+    uint8_t *pg = vg;                                           \
+    for (i = 0; i < opr_sz; i += 1) {                           \
+        if (pg[H1(i)] & 1) {                                    \
+            TYPE nn = n[i];                                     \
+            d[i] = OP(nn, imm);                                 \
+        }                                                       \
+    }                                                           \
+}
+
+#define DO_SHR(N, M)  (N >> M)
+#define DO_SHL(N, M)  (N << M)
+
+/* Arithmetic shift right for division.  This rounds negative numbers
+   toward zero as per signed division.  Therefore before shifting,
+   when N is negative, add 2**M-1.  */
+#ifdef _MSC_VER
+ #define DO_ASRD(N, M) ((N + (N < 0 ? (1 << M) - 1 : 0)) >> M)
+#else
+ #define DO_ASRD(N, M) ((N + (N < 0 ? ((__typeof(N))1 << M) - 1 : 0)) >> M)
+#endif
+
+DO_ZPZI(sve_asr_zpzi_b, int8_t, H1, DO_SHR)
+DO_ZPZI(sve_asr_zpzi_h, int16_t, H1_2, DO_SHR)
+DO_ZPZI(sve_asr_zpzi_s, int32_t, H1_4, DO_SHR)
+DO_ZPZI_D(sve_asr_zpzi_d, int64_t, DO_SHR)
+
+DO_ZPZI(sve_lsr_zpzi_b, uint8_t, H1, DO_SHR)
+DO_ZPZI(sve_lsr_zpzi_h, uint16_t, H1_2, DO_SHR)
+DO_ZPZI(sve_lsr_zpzi_s, uint32_t, H1_4, DO_SHR)
+DO_ZPZI_D(sve_lsr_zpzi_d, uint64_t, DO_SHR)
+
+DO_ZPZI(sve_lsl_zpzi_b, uint8_t, H1, DO_SHL)
+DO_ZPZI(sve_lsl_zpzi_h, uint16_t, H1_2, DO_SHL)
+DO_ZPZI(sve_lsl_zpzi_s, uint32_t, H1_4, DO_SHL)
+DO_ZPZI_D(sve_lsl_zpzi_d, uint64_t, DO_SHL)
+
+DO_ZPZI(sve_asrd_b, int8_t, H1, DO_ASRD)
+DO_ZPZI(sve_asrd_h, int16_t, H1_2, DO_ASRD)
+DO_ZPZI(sve_asrd_s, int32_t, H1_4, DO_ASRD)
+DO_ZPZI_D(sve_asrd_d, int64_t, DO_ASRD)
+
+#undef DO_SHR
+#undef DO_SHL
+#undef DO_ASRD
+#undef DO_ZPZI
+#undef DO_ZPZI_D
+
+/* Fully general four-operand expander, controlled by a predicate.
+ */
+#define DO_ZPZZZ(NAME, TYPE, H, OP)                           \
+void HELPER(NAME)(void *vd, void *va, void *vn, void *vm,     \
+                  void *vg, uint32_t desc)                    \
+{                                                             \
+    intptr_t i, opr_sz = simd_oprsz(desc);                    \
+    for (i = 0; i < opr_sz; ) {                               \
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));       \
+        do {                                                  \
+            if (pg & 1) {                                     \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));               \
+                TYPE mm = *(TYPE *)((char *)vm + H(i));               \
+                TYPE aa = *(TYPE *)((char *)va + H(i));               \
+                *(TYPE *)((char *)vd + H(i)) = OP(aa, nn, mm);        \
+            }                                                 \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);           \
+        } while (i & 15);                                     \
+    }                                                         \
+}
+
+/* Similarly, specialized for 64-bit operands.  */
+#define DO_ZPZZZ_D(NAME, TYPE, OP)                            \
+void HELPER(NAME)(void *vd, void *va, void *vn, void *vm,     \
+                  void *vg, uint32_t desc)                    \
+{                                                             \
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;                \
+    TYPE *d = vd, *a = va, *n = vn, *m = vm;                  \
+    uint8_t *pg = vg;                                         \
+    for (i = 0; i < opr_sz; i += 1) {                         \
+        if (pg[H1(i)] & 1) {                                  \
+            TYPE aa = a[i], nn = n[i], mm = m[i];             \
+            d[i] = OP(aa, nn, mm);                            \
+        }                                                     \
+    }                                                         \
+}
+
+#define DO_MLA(A, N, M)  (A + N * M)
+#define DO_MLS(A, N, M)  (A - N * M)
+
+DO_ZPZZZ(sve_mla_b, uint8_t, H1, DO_MLA)
+DO_ZPZZZ(sve_mls_b, uint8_t, H1, DO_MLS)
+
+DO_ZPZZZ(sve_mla_h, uint16_t, H1_2, DO_MLA)
+DO_ZPZZZ(sve_mls_h, uint16_t, H1_2, DO_MLS)
+
+DO_ZPZZZ(sve_mla_s, uint32_t, H1_4, DO_MLA)
+DO_ZPZZZ(sve_mls_s, uint32_t, H1_4, DO_MLS)
+
+DO_ZPZZZ_D(sve_mla_d, uint64_t, DO_MLA)
+DO_ZPZZZ_D(sve_mls_d, uint64_t, DO_MLS)
+
+#undef DO_MLA
+#undef DO_MLS
+#undef DO_ZPZZZ
+#undef DO_ZPZZZ_D
+
+void HELPER(sve_index_b)(void *vd, uint32_t start,
+                         uint32_t incr, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc);
+    uint8_t *d = vd;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[H1(i)] = start + i * incr;
+    }
+}
+
+void HELPER(sve_index_h)(void *vd, uint32_t start,
+                         uint32_t incr, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 2;
+    uint16_t *d = vd;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[H2(i)] = start + i * incr;
+    }
+}
+
+void HELPER(sve_index_s)(void *vd, uint32_t start,
+                         uint32_t incr, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
+    uint32_t *d = vd;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[H4(i)] = start + i * incr;
+    }
+}
+
+void HELPER(sve_index_d)(void *vd, uint64_t start,
+                         uint64_t incr, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = start + i * incr;
+    }
+}
+
+void HELPER(sve_adr_p32)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
+    uint32_t sh = simd_data(desc);
+    uint32_t *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = n[i] + (m[i] << sh);
+    }
+}
+
+void HELPER(sve_adr_p64)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t sh = simd_data(desc);
+    uint64_t *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = n[i] + (m[i] << sh);
+    }
+}
+
+void HELPER(sve_adr_s32)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t sh = simd_data(desc);
+    uint64_t *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = n[i] + ((uint64_t)(int32_t)m[i] << sh);
+    }
+}
+
+void HELPER(sve_adr_u32)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t sh = simd_data(desc);
+    uint64_t *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = n[i] + ((uint64_t)(uint32_t)m[i] << sh);
+    }
+}
+
+void HELPER(sve_fexpa_h)(void *vd, void *vn, uint32_t desc)
+{
+    /* These constants are cut-and-paste directly from the ARM pseudocode.  */
+    static const uint16_t coeff[] = {
+        0x0000, 0x0016, 0x002d, 0x0045, 0x005d, 0x0075, 0x008e, 0x00a8,
+        0x00c2, 0x00dc, 0x00f8, 0x0114, 0x0130, 0x014d, 0x016b, 0x0189,
+        0x01a8, 0x01c8, 0x01e8, 0x0209, 0x022b, 0x024e, 0x0271, 0x0295,
+        0x02ba, 0x02e0, 0x0306, 0x032e, 0x0356, 0x037f, 0x03a9, 0x03d4,
+    };
+    intptr_t i, opr_sz = simd_oprsz(desc) / 2;
+    uint16_t *d = vd, *n = vn;
+
+    for (i = 0; i < opr_sz; i++) {
+        uint16_t nn = n[i];
+        intptr_t idx = extract32(nn, 0, 5);
+        uint16_t exp = extract32(nn, 5, 5);
+        d[i] = coeff[idx] | (exp << 10);
+    }
+}
+
+void HELPER(sve_fexpa_s)(void *vd, void *vn, uint32_t desc)
+{
+    /* These constants are cut-and-paste directly from the ARM pseudocode.  */
+    static const uint32_t coeff[] = {
+        0x000000, 0x0164d2, 0x02cd87, 0x043a29,
+        0x05aac3, 0x071f62, 0x08980f, 0x0a14d5,
+        0x0b95c2, 0x0d1adf, 0x0ea43a, 0x1031dc,
+        0x11c3d3, 0x135a2b, 0x14f4f0, 0x16942d,
+        0x1837f0, 0x19e046, 0x1b8d3a, 0x1d3eda,
+        0x1ef532, 0x20b051, 0x227043, 0x243516,
+        0x25fed7, 0x27cd94, 0x29a15b, 0x2b7a3a,
+        0x2d583f, 0x2f3b79, 0x3123f6, 0x3311c4,
+        0x3504f3, 0x36fd92, 0x38fbaf, 0x3aff5b,
+        0x3d08a4, 0x3f179a, 0x412c4d, 0x4346cd,
+        0x45672a, 0x478d75, 0x49b9be, 0x4bec15,
+        0x4e248c, 0x506334, 0x52a81e, 0x54f35b,
+        0x5744fd, 0x599d16, 0x5bfbb8, 0x5e60f5,
+        0x60ccdf, 0x633f89, 0x65b907, 0x68396a,
+        0x6ac0c7, 0x6d4f30, 0x6fe4ba, 0x728177,
+        0x75257d, 0x77d0df, 0x7a83b3, 0x7d3e0c,
+    };
+    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
+    uint32_t *d = vd, *n = vn;
+
+    for (i = 0; i < opr_sz; i++) {
+        uint32_t nn = n[i];
+        intptr_t idx = extract32(nn, 0, 6);
+        uint32_t exp = extract32(nn, 6, 8);
+        d[i] = coeff[idx] | (exp << 23);
+    }
+}
+
+void HELPER(sve_fexpa_d)(void *vd, void *vn, uint32_t desc)
+{
+    /* These constants are cut-and-paste directly from the ARM pseudocode.  */
+    static const uint64_t coeff[] = {
+        0x0000000000000ull, 0x02C9A3E778061ull, 0x059B0D3158574ull,
+        0x0874518759BC8ull, 0x0B5586CF9890Full, 0x0E3EC32D3D1A2ull,
+        0x11301D0125B51ull, 0x1429AAEA92DE0ull, 0x172B83C7D517Bull,
+        0x1A35BEB6FCB75ull, 0x1D4873168B9AAull, 0x2063B88628CD6ull,
+        0x2387A6E756238ull, 0x26B4565E27CDDull, 0x29E9DF51FDEE1ull,
+        0x2D285A6E4030Bull, 0x306FE0A31B715ull, 0x33C08B26416FFull,
+        0x371A7373AA9CBull, 0x3A7DB34E59FF7ull, 0x3DEA64C123422ull,
+        0x4160A21F72E2Aull, 0x44E086061892Dull, 0x486A2B5C13CD0ull,
+        0x4BFDAD5362A27ull, 0x4F9B2769D2CA7ull, 0x5342B569D4F82ull,
+        0x56F4736B527DAull, 0x5AB07DD485429ull, 0x5E76F15AD2148ull,
+        0x6247EB03A5585ull, 0x6623882552225ull, 0x6A09E667F3BCDull,
+        0x6DFB23C651A2Full, 0x71F75E8EC5F74ull, 0x75FEB564267C9ull,
+        0x7A11473EB0187ull, 0x7E2F336CF4E62ull, 0x82589994CCE13ull,
+        0x868D99B4492EDull, 0x8ACE5422AA0DBull, 0x8F1AE99157736ull,
+        0x93737B0CDC5E5ull, 0x97D829FDE4E50ull, 0x9C49182A3F090ull,
+        0xA0C667B5DE565ull, 0xA5503B23E255Dull, 0xA9E6B5579FDBFull,
+        0xAE89F995AD3ADull, 0xB33A2B84F15FBull, 0xB7F76F2FB5E47ull,
+        0xBCC1E904BC1D2ull, 0xC199BDD85529Cull, 0xC67F12E57D14Bull,
+        0xCB720DCEF9069ull, 0xD072D4A07897Cull, 0xD5818DCFBA487ull,
+        0xDA9E603DB3285ull, 0xDFC97337B9B5Full, 0xE502EE78B3FF6ull,
+        0xEA4AFA2A490DAull, 0xEFA1BEE615A27ull, 0xF50765B6E4540ull,
+        0xFA7C1819E90D8ull,
+    };
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+
+    for (i = 0; i < opr_sz; i++) {
+        uint64_t nn = n[i];
+        intptr_t idx = extract32(nn, 0, 6);
+        uint64_t exp = extract32(nn, 6, 11);
+        d[i] = coeff[idx] | (exp << 52);
+    }
+}
+
+void HELPER(sve_ftssel_h)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 2;
+    uint16_t *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i += 1) {
+        uint16_t nn = n[i];
+        uint16_t mm = m[i];
+        if (mm & 1) {
+            nn = float16_one;
+        }
+        d[i] = nn ^ (mm & 2) << 14;
+    }
+}
+
+void HELPER(sve_ftssel_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 4;
+    uint32_t *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i += 1) {
+        uint32_t nn = n[i];
+        uint32_t mm = m[i];
+        if (mm & 1) {
+            nn = float32_one;
+        }
+        d[i] = nn ^ (mm & 2) << 30;
+    }
+}
+
+void HELPER(sve_ftssel_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i];
+        uint64_t mm = m[i];
+        if (mm & 1) {
+            nn = float64_one;
+        }
+        d[i] = nn ^ (mm & 2) << 62;
+    }
+}
+
+/*
+ * Signed saturating addition with scalar operand.
+ */
+
+void HELPER(sve_sqaddi_b)(void *d, void *a, int32_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(int8_t)) {
+        int r = *(int8_t *)((char *)a + i) + b;
+        if (r > INT8_MAX) {
+            r = INT8_MAX;
+        } else if (r < INT8_MIN) {
+            r = INT8_MIN;
+        }
+        *(int8_t *)((char *)d + i) = r;
+    }
+}
+
+void HELPER(sve_sqaddi_h)(void *d, void *a, int32_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(int16_t)) {
+        int r = *(int16_t *)((char *)a + i) + b;
+        if (r > INT16_MAX) {
+            r = INT16_MAX;
+        } else if (r < INT16_MIN) {
+            r = INT16_MIN;
+        }
+        *(int16_t *)((char *)d + i) = r;
+    }
+}
+
+void HELPER(sve_sqaddi_s)(void *d, void *a, int64_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(int32_t)) {
+        int64_t r = *(int32_t *)((char *)a + i) + b;
+        if (r > INT32_MAX) {
+            r = INT32_MAX;
+        } else if (r < INT32_MIN) {
+            r = INT32_MIN;
+        }
+        *(int32_t *)((char *)d + i) = r;
+    }
+}
+
+void HELPER(sve_sqaddi_d)(void *d, void *a, int64_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(int64_t)) {
+        int64_t ai = *(int64_t *)((char *)a + i);
+        int64_t r = ai + b;
+        if (((r ^ ai) & ~(ai ^ b)) < 0) {
+            /* Signed overflow.  */
+            r = (r < 0 ? INT64_MAX : INT64_MIN);
+        }
+        *(int64_t *)((char *)d + i) = r;
+    }
+}
+
+/*
+ * Unsigned saturating addition with scalar operand.
+ */
+
+void HELPER(sve_uqaddi_b)(void *d, void *a, int32_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(uint8_t)) {
+        int r = *(uint8_t *)((char *)a + i) + b;
+        if (r > UINT8_MAX) {
+            r = UINT8_MAX;
+        } else if (r < 0) {
+            r = 0;
+        }
+        *(uint8_t *)((char *)d + i) = r;
+    }
+}
+
+void HELPER(sve_uqaddi_h)(void *d, void *a, int32_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(uint16_t)) {
+        int r = *(uint16_t *)((char *)a + i) + b;
+        if (r > UINT16_MAX) {
+            r = UINT16_MAX;
+        } else if (r < 0) {
+            r = 0;
+        }
+        *(uint16_t *)((char *)d + i) = r;
+    }
+}
+
+void HELPER(sve_uqaddi_s)(void *d, void *a, int64_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(uint32_t)) {
+        int64_t r = *(uint32_t *)((char *)a + i) + b;
+        if (r > UINT32_MAX) {
+            r = UINT32_MAX;
+        } else if (r < 0) {
+            r = 0;
+        }
+        *(uint32_t *)((char *)d + i) = r;
+    }
+}
+
+void HELPER(sve_uqaddi_d)(void *d, void *a, uint64_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
+        uint64_t r = *(uint64_t *)((char *)a + i) + b;
+        if (r < b) {
+            r = UINT64_MAX;
+        }
+        *(uint64_t *)((char *)d + i) = r;
+    }
+}
+
+void HELPER(sve_uqsubi_d)(void *d, void *a, uint64_t b, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; i += sizeof(uint64_t)) {
+        uint64_t ai = *(uint64_t *)((char *)a + i);
+        *(uint64_t *)((char *)d + i) = (ai < b ? 0 : ai - b);
+    }
+}
+
+/* Two operand predicated copy immediate with merge.  All valid immediates
+ * can fit within 17 signed bits in the simd_data field.
+ */
+void HELPER(sve_cpy_m_b)(void *vd, void *vn, void *vg,
+                         uint64_t mm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+
+    mm = dup_const(MO_8, mm);
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i];
+        uint64_t pp = expand_pred_b(pg[H1(i)]);
+        d[i] = (mm & pp) | (nn & ~pp);
+    }
+}
+
+void HELPER(sve_cpy_m_h)(void *vd, void *vn, void *vg,
+                         uint64_t mm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+
+    mm = dup_const(MO_16, mm);
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i];
+        uint64_t pp = expand_pred_h(pg[H1(i)]);
+        d[i] = (mm & pp) | (nn & ~pp);
+    }
+}
+
+void HELPER(sve_cpy_m_s)(void *vd, void *vn, void *vg,
+                         uint64_t mm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+
+    mm = dup_const(MO_32, mm);
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i];
+        uint64_t pp = expand_pred_s(pg[H1(i)]);
+        d[i] = (mm & pp) | (nn & ~pp);
+    }
+}
+
+void HELPER(sve_cpy_m_d)(void *vd, void *vn, void *vg,
+                         uint64_t mm, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i];
+        d[i] = (pg[H1(i)] & 1 ? mm : nn);
+    }
+}
+
+void HELPER(sve_cpy_z_b)(void *vd, void *vg, uint64_t val, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+
+    val = dup_const(MO_8, val);
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = val & expand_pred_b(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_cpy_z_h)(void *vd, void *vg, uint64_t val, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+
+    val = dup_const(MO_16, val);
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = val & expand_pred_h(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_cpy_z_s)(void *vd, void *vg, uint64_t val, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+
+    val = dup_const(MO_32, val);
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = val & expand_pred_s(pg[H1(i)]);
+    }
+}
+
+void HELPER(sve_cpy_z_d)(void *vd, void *vg, uint64_t val, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd;
+    uint8_t *pg = vg;
+
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = (pg[H1(i)] & 1 ? val : 0);
+    }
+}
+
+/* Big-endian hosts need to frob the byte indicies.  If the copy
+ * happens to be 8-byte aligned, then no frobbing necessary.
+ */
+static void swap_memmove(void *vd, void *vs, size_t n)
+{
+    uintptr_t d = (uintptr_t)vd;
+    uintptr_t s = (uintptr_t)vs;
+    uintptr_t o = (d | s | n) & 7;
+    size_t i;
+
+#ifndef HOST_WORDS_BIGENDIAN
+    o = 0;
+#endif
+    switch (o) {
+    case 0:
+        memmove(vd, vs, n);
+        break;
+
+    case 4:
+        if (d < s || d >= s + n) {
+            for (i = 0; i < n; i += 4) {
+                *(uint32_t *)H1_4(d + i) = *(uint32_t *)H1_4(s + i);
+            }
+        } else {
+            for (i = n; i > 0; ) {
+                i -= 4;
+                *(uint32_t *)H1_4(d + i) = *(uint32_t *)H1_4(s + i);
+            }
+        }
+        break;
+
+    case 2:
+    case 6:
+        if (d < s || d >= s + n) {
+            for (i = 0; i < n; i += 2) {
+                *(uint16_t *)H1_2(d + i) = *(uint16_t *)H1_2(s + i);
+            }
+        } else {
+            for (i = n; i > 0; ) {
+                i -= 2;
+                *(uint16_t *)H1_2(d + i) = *(uint16_t *)H1_2(s + i);
+            }
+        }
+        break;
+
+    default:
+        if (d < s || d >= s + n) {
+            for (i = 0; i < n; i++) {
+                *(uint8_t *)H1(d + i) = *(uint8_t *)H1(s + i);
+            }
+        } else {
+            for (i = n; i > 0; ) {
+                i -= 1;
+                *(uint8_t *)H1(d + i) = *(uint8_t *)H1(s + i);
+            }
+        }
+        break;
+    }
+}
+
+/* Similarly for memset of 0.  */
+static void swap_memzero(void *vd, size_t n)
+{
+    uintptr_t d = (uintptr_t)vd;
+    uintptr_t o = (d | n) & 7;
+    size_t i;
+
+    /* Usually, the first bit of a predicate is set, so N is 0.  */
+    if (likely(n == 0)) {
+        return;
+    }
+
+#ifndef HOST_WORDS_BIGENDIAN
+    o = 0;
+#endif
+    switch (o) {
+    case 0:
+        memset(vd, 0, n);
+        break;
+
+    case 4:
+        for (i = 0; i < n; i += 4) {
+            *(uint32_t *)H1_4(d + i) = 0;
+        }
+        break;
+
+    case 2:
+    case 6:
+        for (i = 0; i < n; i += 2) {
+            *(uint16_t *)H1_2(d + i) = 0;
+        }
+        break;
+
+    default:
+        for (i = 0; i < n; i++) {
+            *(uint8_t *)H1(d + i) = 0;
+        }
+        break;
+    }
+}
+
+void HELPER(sve_ext)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t opr_sz = simd_oprsz(desc);
+    size_t n_ofs = simd_data(desc);
+    size_t n_siz = opr_sz - n_ofs;
+
+    if (vd != vm) {
+        swap_memmove(vd, (char *)vn + n_ofs, n_siz);
+        swap_memmove((char *)vd + n_siz, vm, n_ofs);
+    } else if (vd != vn) {
+        swap_memmove((char *)vd + n_siz, vd, n_ofs);
+        swap_memmove(vd, (char *)vn + n_ofs, n_siz);
+    } else {
+        /* vd == vn == vm.  Need temp space.  */
+        ARMVectorReg tmp;
+        swap_memmove(&tmp, vm, n_ofs);
+        swap_memmove(vd, (char *)vd + n_ofs, n_siz);
+        memcpy((char *)vd + n_siz, &tmp, n_ofs);
+    }
+}
+
+#define DO_INSR(NAME, TYPE, H) \
+void HELPER(NAME)(void *vd, void *vn, uint64_t val, uint32_t desc) \
+{                                                                  \
+    intptr_t opr_sz = simd_oprsz(desc);                            \
+    swap_memmove((char *)vd + sizeof(TYPE), vn, opr_sz - sizeof(TYPE));    \
+    *(TYPE *)((char *)vd + H(0)) = val;                                    \
+}
+
+DO_INSR(sve_insr_b, uint8_t, H1)
+DO_INSR(sve_insr_h, uint16_t, H1_2)
+DO_INSR(sve_insr_s, uint32_t, H1_4)
+DO_INSR(sve_insr_d, uint64_t, )
+
+#undef DO_INSR
+
+void HELPER(sve_rev_b)(void *vd, void *vn, uint32_t desc)
+{
+    intptr_t i, j, opr_sz = simd_oprsz(desc);
+    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
+        uint64_t f = *(uint64_t *)((char *)vn + i);
+        uint64_t b = *(uint64_t *)((char *)vn + j);
+        *(uint64_t *)((char *)vd + i) = bswap64(b);
+        *(uint64_t *)((char *)vd + j) = bswap64(f);
+    }
+}
+
+void HELPER(sve_rev_h)(void *vd, void *vn, uint32_t desc)
+{
+    intptr_t i, j, opr_sz = simd_oprsz(desc);
+    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
+        uint64_t f = *(uint64_t *)((char *)vn + i);
+        uint64_t b = *(uint64_t *)((char *)vn + j);
+        *(uint64_t *)((char *)vd + i) = hswap64(b);
+        *(uint64_t *)((char *)vd + j) = hswap64(f);
+    }
+}
+
+void HELPER(sve_rev_s)(void *vd, void *vn, uint32_t desc)
+{
+    intptr_t i, j, opr_sz = simd_oprsz(desc);
+    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
+        uint64_t f = *(uint64_t *)((char *)vn + i);
+        uint64_t b = *(uint64_t *)((char *)vn + j);
+        *(uint64_t *)((char *)vd + i) = rol64(b, 32);
+        *(uint64_t *)((char *)vd + j) = rol64(f, 32);
+    }
+}
+
+void HELPER(sve_rev_d)(void *vd, void *vn, uint32_t desc)
+{
+    intptr_t i, j, opr_sz = simd_oprsz(desc);
+    for (i = 0, j = opr_sz - 8; i < opr_sz / 2; i += 8, j -= 8) {
+        uint64_t f = *(uint64_t *)((char *)vn + i);
+        uint64_t b = *(uint64_t *)((char *)vn + j);
+        *(uint64_t *)((char *)vd + i) = b;
+        *(uint64_t *)((char *)vd + j) = f;
+    }
+}
+
+typedef void tb_impl_fn(void *, void *, void *, void *, uintptr_t, bool);
+
+static inline void do_tbl1(void *vd, void *vn, void *vm, uint32_t desc,
+                           bool is_tbx, tb_impl_fn *fn)
+{
+    ARMVectorReg scratch;
+    uintptr_t oprsz = simd_oprsz(desc);
+
+    if (unlikely(vd == vn)) {
+        vn = memcpy(&scratch, vn, oprsz);
+    }
+
+    fn(vd, vn, NULL, vm, oprsz, is_tbx);
+}
+
+static inline void do_tbl2(void *vd, void *vn0, void *vn1, void *vm,
+                           uint32_t desc, bool is_tbx, tb_impl_fn *fn)
+{
+    ARMVectorReg scratch;
+    uintptr_t oprsz = simd_oprsz(desc);
+
+    if (unlikely(vd == vn0)) {
+        vn0 = memcpy(&scratch, vn0, oprsz);
+        if (vd == vn1) {
+            vn1 = vn0;
+        }
+    } else if (unlikely(vd == vn1)) {
+        vn1 = memcpy(&scratch, vn1, oprsz);
+    }
+
+    fn(vd, vn0, vn1, vm, oprsz, is_tbx);
+}
+
+#define DO_TB(SUFF, TYPE, H)                                            \
+static inline void do_tb_##SUFF(void *vd, void *vt0, void *vt1,         \
+                                void *vm, uintptr_t oprsz, bool is_tbx) \
+{                                                                       \
+    TYPE *d = vd, *tbl0 = vt0, *tbl1 = vt1, *indexes = vm;              \
+    uintptr_t i, nelem = oprsz / sizeof(TYPE);                          \
+    for (i = 0; i < nelem; i++) {                                       \
+        TYPE index = indexes[H1(i)], val = 0;                           \
+        if (index < nelem) {                                            \
+            val = tbl0[H(index)];                                       \
+        } else {                                                        \
+            index -= nelem;                                             \
+            if (tbl1 && index < nelem) {                                \
+                val = tbl1[H(index)];                                   \
+            } else if (is_tbx) {                                        \
+                continue;                                               \
+            }                                                           \
+        }                                                               \
+        d[H(i)] = val;                                                  \
+    }                                                                   \
+}                                                                       \
+void HELPER(sve_tbl_##SUFF)(void *vd, void *vn, void *vm, uint32_t desc) \
+{                                                                       \
+    do_tbl1(vd, vn, vm, desc, false, do_tb_##SUFF);                     \
+}                                                                       \
+void HELPER(sve2_tbl_##SUFF)(void *vd, void *vn0, void *vn1,            \
+                             void *vm, uint32_t desc)                   \
+{                                                                       \
+    do_tbl2(vd, vn0, vn1, vm, desc, false, do_tb_##SUFF);               \
+}                                                                       \
+void HELPER(sve2_tbx_##SUFF)(void *vd, void *vn, void *vm,              \
+                             uint32_t desc)                             \
+{                                                                       \
+    do_tbl1(vd, vn, vm, desc, true, do_tb_##SUFF);                      \
+}
+
+DO_TB(b, uint8_t, H1)
+DO_TB(h, uint16_t, H2)
+DO_TB(s, uint32_t, H4)
+DO_TB(d, uint64_t, H8)
+
+#undef DO_TB
+
+#define DO_UNPK(NAME, TYPED, TYPES, HD, HS) \
+void HELPER(NAME)(void *vd, void *vn, uint32_t desc)           \
+{                                                              \
+    intptr_t i, opr_sz = simd_oprsz(desc);                     \
+    TYPED *d = vd;                                             \
+    TYPES *n = vn;                                             \
+    ARMVectorReg tmp;                                          \
+    if (unlikely((char *)vn - (char *)vd < opr_sz)) {                          \
+        n = memcpy(&tmp, n, opr_sz / 2);                       \
+    }                                                          \
+    for (i = 0; i < opr_sz / sizeof(TYPED); i++) {             \
+        d[HD(i)] = n[HS(i)];                                   \
+    }                                                          \
+}
+
+DO_UNPK(sve_sunpk_h, int16_t, int8_t, H2, H1)
+DO_UNPK(sve_sunpk_s, int32_t, int16_t, H4, H2)
+DO_UNPK(sve_sunpk_d, int64_t, int32_t, , H4)
+
+DO_UNPK(sve_uunpk_h, uint16_t, uint8_t, H2, H1)
+DO_UNPK(sve_uunpk_s, uint32_t, uint16_t, H4, H2)
+DO_UNPK(sve_uunpk_d, uint64_t, uint32_t, , H4)
+
+#undef DO_UNPK
+
+/* Mask of bits included in the even numbered predicates of width esz.
+ * We also use this for expand_bits/compress_bits, and so extend the
+ * same pattern out to 16-bit units.
+ */
+static const uint64_t even_bit_esz_masks[5] = {
+    0x5555555555555555ull,
+    0x3333333333333333ull,
+    0x0f0f0f0f0f0f0f0full,
+    0x00ff00ff00ff00ffull,
+    0x0000ffff0000ffffull,
+};
+
+/* Zero-extend units of 2**N bits to units of 2**(N+1) bits.
+ * For N==0, this corresponds to the operation that in qemu/bitops.h
+ * we call half_shuffle64; this algorithm is from Hacker's Delight,
+ * section 7-2 Shuffling Bits.
+ */
+static uint64_t expand_bits(uint64_t x, int n)
+{
+    int i;
+
+    x &= 0xffffffffu;
+    for (i = 4; i >= n; i--) {
+        int sh = 1 << i;
+        x = ((x << sh) | x) & even_bit_esz_masks[i];
+    }
+    return x;
+}
+
+/* Compress units of 2**(N+1) bits to units of 2**N bits.
+ * For N==0, this corresponds to the operation that in qemu/bitops.h
+ * we call half_unshuffle64; this algorithm is from Hacker's Delight,
+ * section 7-2 Shuffling Bits, where it is called an inverse half shuffle.
+ */
+static uint64_t compress_bits(uint64_t x, int n)
+{
+    int i;
+
+    for (i = n; i <= 4; i++) {
+        int sh = 1 << i;
+        x &= even_bit_esz_masks[i];
+        x = (x >> sh) | x;
+    }
+    return x & 0xffffffffu;
+}
+
+void HELPER(sve_zip_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    int esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    intptr_t high = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1);
+    uint64_t *d = vd;
+    intptr_t i;
+
+    if (oprsz <= 8) {
+        uint64_t nn = *(uint64_t *)vn;
+        uint64_t mm = *(uint64_t *)vm;
+        int half = 4 * oprsz;
+
+        nn = extract64(nn, high * half, half);
+        mm = extract64(mm, high * half, half);
+        nn = expand_bits(nn, esz);
+        mm = expand_bits(mm, esz);
+        d[0] = nn + (mm << (1 << esz));
+    } else {
+        ARMPredicateReg tmp_n, tmp_m;
+
+        /* We produce output faster than we consume input.
+           Therefore we must be mindful of possible overlap.  */
+        if (((char *)vn - (char *)vd) < (uintptr_t)oprsz) {
+            vn = memcpy(&tmp_n, vn, oprsz);
+        }
+        if (((char *)vm - (char *)vd) < (uintptr_t)oprsz) {
+            vm = memcpy(&tmp_m, vm, oprsz);
+        }
+        if (high) {
+            high = oprsz >> 1;
+        }
+
+        if ((high & 3) == 0) {
+            uint32_t *n = vn, *m = vm;
+            high >>= 2;
+
+            for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
+                uint64_t nn = n[H4(high + i)];
+                uint64_t mm = m[H4(high + i)];
+
+                nn = expand_bits(nn, esz);
+                mm = expand_bits(mm, esz);
+                d[i] = nn + (mm << (1 << esz));
+            }
+        } else {
+            uint8_t *n = vn, *m = vm;
+            uint16_t *d16 = vd;
+
+            for (i = 0; i < oprsz / 2; i++) {
+                uint16_t nn = n[H1(high + i)];
+                uint16_t mm = m[H1(high + i)];
+
+                nn = expand_bits(nn, esz);
+                mm = expand_bits(mm, esz);
+                d16[H2(i)] = nn + (mm << (1 << esz));
+            }
+        }
+    }
+}
+
+void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    int esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    int odd = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1) << esz;
+    uint64_t *d = vd, *n = vn, *m = vm;
+    uint64_t l, h;
+    intptr_t i;
+
+    if (oprsz <= 8) {
+        l = compress_bits(n[0] >> odd, esz);
+        h = compress_bits(m[0] >> odd, esz);
+        d[0] = extract64(l + (h << (4 * oprsz)), 0, 8 * oprsz);
+    } else {
+        ARMPredicateReg tmp_m;
+        intptr_t oprsz_16 = oprsz / 16;
+
+        if (((char *)vm - (char *)vd) < (uintptr_t)oprsz) {
+            m = memcpy(&tmp_m, vm, oprsz);
+        }
+
+        for (i = 0; i < oprsz_16; i++) {
+            l = n[2 * i + 0];
+            h = n[2 * i + 1];
+            l = compress_bits(l >> odd, esz);
+            h = compress_bits(h >> odd, esz);
+            d[i] = l + (h << 32);
+        }
+
+        /* For VL which is not a power of 2, the results from M do not
+           align nicely with the uint64_t for D.  Put the aligned results
+           from M into TMP_M and then copy it into place afterward.  */
+        if (oprsz & 15) {
+            d[i] = compress_bits(n[2 * i] >> odd, esz);
+
+            for (i = 0; i < oprsz_16; i++) {
+                l = m[2 * i + 0];
+                h = m[2 * i + 1];
+                l = compress_bits(l >> odd, esz);
+                h = compress_bits(h >> odd, esz);
+                tmp_m.p[i] = l + (h << 32);
+            }
+            tmp_m.p[i] = compress_bits(m[2 * i] >> odd, esz);
+
+            swap_memmove((char *)vd + oprsz / 2, &tmp_m, oprsz / 2);
+        } else {
+            for (i = 0; i < oprsz_16; i++) {
+                l = m[2 * i + 0];
+                h = m[2 * i + 1];
+                l = compress_bits(l >> odd, esz);
+                h = compress_bits(h >> odd, esz);
+                d[oprsz_16 + i] = l + (h << 32);
+            }
+        }
+    }
+}
+
+void HELPER(sve_trn_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    uintptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    bool odd = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1);
+    uint64_t *d = vd, *n = vn, *m = vm;
+    uint64_t mask;
+    int shr, shl;
+    intptr_t i;
+
+    shl = 1 << esz;
+    shr = 0;
+    mask = even_bit_esz_masks[esz];
+    if (odd) {
+        mask <<= shl;
+        shr = shl;
+        shl = 0;
+    }
+
+    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
+        uint64_t nn = (n[i] & mask) >> shr;
+        uint64_t mm = (m[i] & mask) << shl;
+        d[i] = nn + mm;
+    }
+}
+
+/* Reverse units of 2**N bits.  */
+static uint64_t reverse_bits_64(uint64_t x, int n)
+{
+    int i, sh;
+
+    x = bswap64(x);
+    for (i = 2, sh = 4; i >= n; i--, sh >>= 1) {
+        uint64_t mask = even_bit_esz_masks[i];
+        x = ((x & mask) << sh) | ((x >> sh) & mask);
+    }
+    return x;
+}
+
+static uint8_t reverse_bits_8(uint8_t x, int n)
+{
+    static const uint8_t mask[3] = { 0x55, 0x33, 0x0f };
+    int i, sh;
+
+    for (i = 2, sh = 4; i >= n; i--, sh >>= 1) {
+        x = ((x & mask[i]) << sh) | ((x >> sh) & mask[i]);
+    }
+    return x;
+}
+
+void HELPER(sve_rev_p)(void *vd, void *vn, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    int esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    intptr_t i, oprsz_2 = oprsz / 2;
+
+    if (oprsz <= 8) {
+        uint64_t l = *(uint64_t *)vn;
+        l = reverse_bits_64(l << (64 - 8 * oprsz), esz);
+        *(uint64_t *)vd = l;
+    } else if ((oprsz & 15) == 0) {
+        for (i = 0; i < oprsz_2; i += 8) {
+            intptr_t ih = oprsz - 8 - i;
+            uint64_t l = reverse_bits_64(*(uint64_t *)((char *)vn + i), esz);
+            uint64_t h = reverse_bits_64(*(uint64_t *)((char *)vn + ih), esz);
+            *(uint64_t *)((char *)vd + i) = h;
+            *(uint64_t *)((char *)vd + ih) = l;
+        }
+    } else {
+        for (i = 0; i < oprsz_2; i += 1) {
+            intptr_t il = H1(i);
+            intptr_t ih = H1(oprsz - 1 - i);
+            uint8_t l = reverse_bits_8(*(uint8_t *)((char *)vn + il), esz);
+            uint8_t h = reverse_bits_8(*(uint8_t *)((char *)vn + ih), esz);
+            *(uint8_t *)((char *)vd + il) = h;
+            *(uint8_t *)((char *)vd + ih) = l;
+        }
+    }
+}
+
+void HELPER(sve_punpk_p)(void *vd, void *vn, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    intptr_t high = extract32(pred_desc, SIMD_DATA_SHIFT + 2, 1);
+    uint64_t *d = vd;
+    intptr_t i;
+
+    if (oprsz <= 8) {
+        uint64_t nn = *(uint64_t *)vn;
+        int half = 4 * oprsz;
+
+        nn = extract64(nn, high * half, half);
+        nn = expand_bits(nn, 0);
+        d[0] = nn;
+    } else {
+        ARMPredicateReg tmp_n;
+
+        /* We produce output faster than we consume input.
+           Therefore we must be mindful of possible overlap.  */
+        if (((char *)vn - (char *)vd) < (uintptr_t)oprsz) {
+            vn = memcpy(&tmp_n, vn, oprsz);
+        }
+        if (high) {
+            high = oprsz >> 1;
+        }
+
+        if ((high & 3) == 0) {
+            uint32_t *n = vn;
+            high >>= 2;
+
+            for (i = 0; i < DIV_ROUND_UP(oprsz, 8); i++) {
+                uint64_t nn = n[H4(high + i)];
+                d[i] = expand_bits(nn, 0);
+            }
+        } else {
+            uint16_t *d16 = vd;
+            uint8_t *n = vn;
+
+            for (i = 0; i < oprsz / 2; i++) {
+                uint16_t nn = n[H1(high + i)];
+                d16[H2(i)] = expand_bits(nn, 0);
+            }
+        }
+    }
+}
+
+#define DO_ZIP(NAME, TYPE, H) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)       \
+{                                                                    \
+    intptr_t oprsz = simd_oprsz(desc);                               \
+    intptr_t odd_ofs = simd_data(desc);                              \
+    intptr_t i, oprsz_2 = oprsz / 2;                                 \
+    ARMVectorReg tmp_n, tmp_m;                                       \
+    /* We produce output faster than we consume input.               \
+       Therefore we must be mindful of possible overlap.  */         \
+    if (unlikely(((char *)vn - (char *)vd) < (uintptr_t)oprsz)) {     \
+        vn = memcpy(&tmp_n, vn, oprsz);                              \
+    }                                                                \
+    if (unlikely(((char *)vm - (char *)vd) < (uintptr_t)oprsz)) {     \
+        vm = memcpy(&tmp_m, vm, oprsz);                              \
+    }                                                                \
+    for (i = 0; i < oprsz_2; i += sizeof(TYPE)) {                    \
+        *(TYPE *)((char *)vd + H(2 * i + 0)) =                       \
+            *(TYPE *)((char *)vn + odd_ofs + H(i));                  \
+        *(TYPE *)((char *)vd + H(2 * i + sizeof(TYPE))) =            \
+            *(TYPE *)((char *)vm + odd_ofs + H(i));                  \
+    }                                                                \
+    if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) {                \
+        memset((char *)vd + oprsz - 16, 0, 16);                      \
+    }                                                                \
+}
+
+DO_ZIP(sve_zip_b, uint8_t, H1)
+DO_ZIP(sve_zip_h, uint16_t, H1_2)
+DO_ZIP(sve_zip_s, uint32_t, H1_4)
+DO_ZIP(sve_zip_d, uint64_t, H1_8)
+DO_ZIP(sve2_zip_q, Int128, )
+
+#define DO_UZP(NAME, TYPE, H) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)         \
+{                                                                      \
+    intptr_t oprsz = simd_oprsz(desc);                                 \
+    intptr_t odd_ofs = simd_data(desc);                                \
+    intptr_t i, p;                                                     \
+    ARMVectorReg tmp_m;                                                \
+    if (unlikely(((char *)vm - (char *)vd) < (uintptr_t)oprsz)) {      \
+        vm = memcpy(&tmp_m, vm, oprsz);                                \
+    }                                                                  \
+    i = 0, p = odd_ofs;                                                \
+    do {                                                               \
+        *(TYPE *)((char *)vd + H(i)) = *(TYPE *)((char *)vn + H(p));   \
+        i += sizeof(TYPE), p += 2 * sizeof(TYPE);                      \
+    } while (p < oprsz);                                               \
+    p -= oprsz;                                                        \
+    do {                                                               \
+        *(TYPE *)((char *)vd + H(i)) = *(TYPE *)((char *)vm + H(p));   \
+        i += sizeof(TYPE), p += 2 * sizeof(TYPE);                      \
+    } while (p < oprsz);                                               \
+    tcg_debug_assert(i == oprsz);                                      \
+}
+
+DO_UZP(sve_uzp_b, uint8_t, H1)
+DO_UZP(sve_uzp_h, uint16_t, H1_2)
+DO_UZP(sve_uzp_s, uint32_t, H1_4)
+DO_UZP(sve_uzp_d, uint64_t, H1_8)
+DO_UZP(sve2_uzp_q, Int128, )
+
+#define DO_TRN(NAME, TYPE, H) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)         \
+{                                                                      \
+    intptr_t oprsz = simd_oprsz(desc);                                 \
+    intptr_t odd_ofs = simd_data(desc);                                \
+    intptr_t i;                                                        \
+    for (i = 0; i < oprsz; i += 2 * sizeof(TYPE)) {                    \
+        TYPE ae = *(TYPE *)((char *)vn + H(i + odd_ofs));                      \
+        TYPE be = *(TYPE *)((char *)vm + H(i + odd_ofs));                      \
+        *(TYPE *)((char *)vd + H(i + 0)) = ae;                                 \
+        *(TYPE *)((char *)vd + H(i + sizeof(TYPE))) = be;                      \
+    }                                                                  \
+    if (sizeof(TYPE) == 16 && unlikely(oprsz & 16)) {                  \
+        memset((char *)vd + oprsz - 16, 0, 16);                        \
+    }                                                                  \
+}
+
+DO_TRN(sve_trn_b, uint8_t, H1)
+DO_TRN(sve_trn_h, uint16_t, H1_2)
+DO_TRN(sve_trn_s, uint32_t, H1_4)
+DO_TRN(sve_trn_d, uint64_t, H1_8)
+DO_TRN(sve2_trn_q, Int128, )
+
+#undef DO_ZIP
+#undef DO_UZP
+#undef DO_TRN
+
+void HELPER(sve_compact_s)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+    intptr_t i, j, opr_sz = simd_oprsz(desc) / 4;
+    uint32_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+
+    for (i = j = 0; i < opr_sz; i++) {
+        if (pg[H1(i / 2)] & (i & 1 ? 0x10 : 0x01)) {
             d[H4(j)] = n[H4(i)];
             j++;
         }
     }
-    for (; j < opr_sz; j++) {
-        d[H4(j)] = 0;
+    for (; j < opr_sz; j++) {
+        d[H4(j)] = 0;
+    }
+}
+
+void HELPER(sve_compact_d)(void *vd, void *vn, void *vg, uint32_t desc)
+{
+    intptr_t i, j, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn;
+    uint8_t *pg = vg;
+
+    for (i = j = 0; i < opr_sz; i++) {
+        if (pg[H1(i)] & 1) {
+            d[j] = n[i];
+            j++;
+        }
+    }
+    for (; j < opr_sz; j++) {
+        d[j] = 0;
+    }
+}
+
+/* Similar to the ARM LastActiveElement pseudocode function, except the
+ * result is multiplied by the element size.  This includes the not found
+ * indication; e.g. not found for esz=3 is -8.
+ */
+int32_t HELPER(sve_last_active_element)(void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+
+    return last_active_element(vg, DIV_ROUND_UP(oprsz, 8), esz);
+}
+
+void HELPER(sve_splice)(void *vd, void *vn, void *vm, void *vg, uint32_t desc)
+{
+    intptr_t opr_sz = simd_oprsz(desc) / 8;
+    int esz = simd_data(desc);
+    uint64_t pg, first_g, last_g, len, mask = pred_esz_masks[esz];
+    intptr_t i, first_i, last_i;
+    ARMVectorReg tmp;
+
+    first_i = last_i = 0;
+    first_g = last_g = 0;
+
+    /* Find the extent of the active elements within VG.  */
+    for (i = QEMU_ALIGN_UP(opr_sz, 8) - 8; i >= 0; i -= 8) {
+        pg = *(uint64_t *)((char *)vg + i) & mask;
+        if (pg) {
+            if (last_g == 0) {
+                last_g = pg;
+                last_i = i;
+            }
+            first_g = pg;
+            first_i = i;
+        }
+    }
+
+    len = 0;
+    if (first_g != 0) {
+        first_i = first_i * 8 + ctz64(first_g);
+        last_i = last_i * 8 + 63 - clz64(last_g);
+        len = last_i - first_i + (1ULL << esz);
+        if (vd == vm) {
+            vm = memcpy(&tmp, vm, opr_sz * 8);
+        }
+        swap_memmove(vd, (char *)vn + first_i, len);
+    }
+    swap_memmove((char *)vd + len, vm, opr_sz * 8 - len);
+}
+
+void HELPER(sve_sel_zpzz_b)(void *vd, void *vn, void *vm,
+                            void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm;
+    uint8_t *pg = vg;
+
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i], mm = m[i];
+        uint64_t pp = expand_pred_b(pg[H1(i)]);
+        d[i] = (nn & pp) | (mm & ~pp);
+    }
+}
+
+void HELPER(sve_sel_zpzz_h)(void *vd, void *vn, void *vm,
+                            void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm;
+    uint8_t *pg = vg;
+
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i], mm = m[i];
+        uint64_t pp = expand_pred_h(pg[H1(i)]);
+        d[i] = (nn & pp) | (mm & ~pp);
+    }
+}
+
+void HELPER(sve_sel_zpzz_s)(void *vd, void *vn, void *vm,
+                            void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm;
+    uint8_t *pg = vg;
+
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i], mm = m[i];
+        uint64_t pp = expand_pred_s(pg[H1(i)]);
+        d[i] = (nn & pp) | (mm & ~pp);
+    }
+}
+
+void HELPER(sve_sel_zpzz_d)(void *vd, void *vn, void *vm,
+                            void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm;
+    uint8_t *pg = vg;
+
+    for (i = 0; i < opr_sz; i += 1) {
+        uint64_t nn = n[i], mm = m[i];
+        d[i] = (pg[H1(i)] & 1 ? nn : mm);
+    }
+}
+
+void HELPER(sve_sel_zpzz_q)(void *vd, void *vn, void *vm,
+                            void *vg, uint32_t desc)
+{
+    intptr_t i, opr_sz = simd_oprsz(desc) / 16;
+    Int128 *d = vd, *n = vn, *m = vm;
+    uint16_t *pg = vg;
+
+    for (i = 0; i < opr_sz; i += 1) {
+        d[i] = (pg[H2(i)] & 1 ? n : m)[i];
+    }
+}
+
+/* Two operand comparison controlled by a predicate.
+ * ??? It is very tempting to want to be able to expand this inline
+ * with x86 instructions, e.g.
+ *
+ *    vcmpeqw    zm, zn, %ymm0
+ *    vpmovmskb  %ymm0, %eax
+ *    and        $0x5555, %eax
+ *    and        pg, %eax
+ *
+ * or even aarch64, e.g.
+ *
+ *    // mask = 4000 1000 0400 0100 0040 0010 0004 0001
+ *    cmeq       v0.8h, zn, zm
+ *    and        v0.8h, v0.8h, mask
+ *    addv       h0, v0.8h
+ *    and        v0.8b, pg
+ *
+ * However, coming up with an abstraction that allows vector inputs and
+ * a scalar output, and also handles the byte-ordering of sub-uint64_t
+ * scalar outputs, is tricky.
+ */
+#define DO_CMP_PPZZ(NAME, TYPE, OP, H, MASK)                                 \
+uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{                                                                            \
+    intptr_t opr_sz = simd_oprsz(desc);                                      \
+    uint32_t flags = PREDTEST_INIT;                                          \
+    intptr_t i = opr_sz;                                                     \
+    do {                                                                     \
+        uint64_t out = 0, pg;                                                \
+        do {                                                                 \
+            i -= sizeof(TYPE), out <<= sizeof(TYPE);                         \
+            TYPE nn = *(TYPE *)((char *)vn + H(i));                                  \
+            TYPE mm = *(TYPE *)((char *)vm + H(i));                                  \
+            out |= nn OP mm;                                                 \
+        } while (i & 63);                                                    \
+        pg = *(uint64_t *)((char *)vg + (i >> 3)) & MASK;                            \
+        out &= pg;                                                           \
+        *(uint64_t *)((char *)vd + (i >> 3)) = out;                                  \
+        flags = iter_predtest_bwd(out, pg, flags);                           \
+    } while (i > 0);                                                         \
+    return flags;                                                            \
+}
+
+#define DO_CMP_PPZZ_B(NAME, TYPE, OP) \
+    DO_CMP_PPZZ(NAME, TYPE, OP, H1,   0xffffffffffffffffull)
+#define DO_CMP_PPZZ_H(NAME, TYPE, OP) \
+    DO_CMP_PPZZ(NAME, TYPE, OP, H1_2, 0x5555555555555555ull)
+#define DO_CMP_PPZZ_S(NAME, TYPE, OP) \
+    DO_CMP_PPZZ(NAME, TYPE, OP, H1_4, 0x1111111111111111ull)
+#define DO_CMP_PPZZ_D(NAME, TYPE, OP) \
+    DO_CMP_PPZZ(NAME, TYPE, OP,     , 0x0101010101010101ull)
+
+DO_CMP_PPZZ_B(sve_cmpeq_ppzz_b, uint8_t,  ==)
+DO_CMP_PPZZ_H(sve_cmpeq_ppzz_h, uint16_t, ==)
+DO_CMP_PPZZ_S(sve_cmpeq_ppzz_s, uint32_t, ==)
+DO_CMP_PPZZ_D(sve_cmpeq_ppzz_d, uint64_t, ==)
+
+DO_CMP_PPZZ_B(sve_cmpne_ppzz_b, uint8_t,  !=)
+DO_CMP_PPZZ_H(sve_cmpne_ppzz_h, uint16_t, !=)
+DO_CMP_PPZZ_S(sve_cmpne_ppzz_s, uint32_t, !=)
+DO_CMP_PPZZ_D(sve_cmpne_ppzz_d, uint64_t, !=)
+
+DO_CMP_PPZZ_B(sve_cmpgt_ppzz_b, int8_t,  >)
+DO_CMP_PPZZ_H(sve_cmpgt_ppzz_h, int16_t, >)
+DO_CMP_PPZZ_S(sve_cmpgt_ppzz_s, int32_t, >)
+DO_CMP_PPZZ_D(sve_cmpgt_ppzz_d, int64_t, >)
+
+DO_CMP_PPZZ_B(sve_cmpge_ppzz_b, int8_t,  >=)
+DO_CMP_PPZZ_H(sve_cmpge_ppzz_h, int16_t, >=)
+DO_CMP_PPZZ_S(sve_cmpge_ppzz_s, int32_t, >=)
+DO_CMP_PPZZ_D(sve_cmpge_ppzz_d, int64_t, >=)
+
+DO_CMP_PPZZ_B(sve_cmphi_ppzz_b, uint8_t,  >)
+DO_CMP_PPZZ_H(sve_cmphi_ppzz_h, uint16_t, >)
+DO_CMP_PPZZ_S(sve_cmphi_ppzz_s, uint32_t, >)
+DO_CMP_PPZZ_D(sve_cmphi_ppzz_d, uint64_t, >)
+
+DO_CMP_PPZZ_B(sve_cmphs_ppzz_b, uint8_t,  >=)
+DO_CMP_PPZZ_H(sve_cmphs_ppzz_h, uint16_t, >=)
+DO_CMP_PPZZ_S(sve_cmphs_ppzz_s, uint32_t, >=)
+DO_CMP_PPZZ_D(sve_cmphs_ppzz_d, uint64_t, >=)
+
+#undef DO_CMP_PPZZ_B
+#undef DO_CMP_PPZZ_H
+#undef DO_CMP_PPZZ_S
+#undef DO_CMP_PPZZ_D
+#undef DO_CMP_PPZZ
+
+/* Similar, but the second source is "wide".  */
+#define DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H, MASK)                     \
+uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
+{                                                                            \
+    intptr_t opr_sz = simd_oprsz(desc);                                      \
+    uint32_t flags = PREDTEST_INIT;                                          \
+    intptr_t i = opr_sz;                                                     \
+    do {                                                                     \
+        uint64_t out = 0, pg;                                                \
+        do {                                                                 \
+            TYPEW mm = *(TYPEW *)((char *)vm + i - 8);                               \
+            do {                                                             \
+                i -= sizeof(TYPE), out <<= sizeof(TYPE);                     \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                              \
+                out |= nn OP mm;                                             \
+            } while (i & 7);                                                 \
+        } while (i & 63);                                                    \
+        pg = *(uint64_t *)((char *)vg + (i >> 3)) & MASK;                            \
+        out &= pg;                                                           \
+        *(uint64_t *)((char *)vd + (i >> 3)) = out;                                  \
+        flags = iter_predtest_bwd(out, pg, flags);                           \
+    } while (i > 0);                                                         \
+    return flags;                                                            \
+}
+
+#define DO_CMP_PPZW_B(NAME, TYPE, TYPEW, OP) \
+    DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1,   0xffffffffffffffffull)
+#define DO_CMP_PPZW_H(NAME, TYPE, TYPEW, OP) \
+    DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1_2, 0x5555555555555555ull)
+#define DO_CMP_PPZW_S(NAME, TYPE, TYPEW, OP) \
+    DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1_4, 0x1111111111111111ull)
+
+DO_CMP_PPZW_B(sve_cmpeq_ppzw_b, int8_t,  uint64_t, ==)
+DO_CMP_PPZW_H(sve_cmpeq_ppzw_h, int16_t, uint64_t, ==)
+DO_CMP_PPZW_S(sve_cmpeq_ppzw_s, int32_t, uint64_t, ==)
+
+DO_CMP_PPZW_B(sve_cmpne_ppzw_b, int8_t,  uint64_t, !=)
+DO_CMP_PPZW_H(sve_cmpne_ppzw_h, int16_t, uint64_t, !=)
+DO_CMP_PPZW_S(sve_cmpne_ppzw_s, int32_t, uint64_t, !=)
+
+DO_CMP_PPZW_B(sve_cmpgt_ppzw_b, int8_t,   int64_t, >)
+DO_CMP_PPZW_H(sve_cmpgt_ppzw_h, int16_t,  int64_t, >)
+DO_CMP_PPZW_S(sve_cmpgt_ppzw_s, int32_t,  int64_t, >)
+
+DO_CMP_PPZW_B(sve_cmpge_ppzw_b, int8_t,   int64_t, >=)
+DO_CMP_PPZW_H(sve_cmpge_ppzw_h, int16_t,  int64_t, >=)
+DO_CMP_PPZW_S(sve_cmpge_ppzw_s, int32_t,  int64_t, >=)
+
+DO_CMP_PPZW_B(sve_cmphi_ppzw_b, uint8_t,  uint64_t, >)
+DO_CMP_PPZW_H(sve_cmphi_ppzw_h, uint16_t, uint64_t, >)
+DO_CMP_PPZW_S(sve_cmphi_ppzw_s, uint32_t, uint64_t, >)
+
+DO_CMP_PPZW_B(sve_cmphs_ppzw_b, uint8_t,  uint64_t, >=)
+DO_CMP_PPZW_H(sve_cmphs_ppzw_h, uint16_t, uint64_t, >=)
+DO_CMP_PPZW_S(sve_cmphs_ppzw_s, uint32_t, uint64_t, >=)
+
+DO_CMP_PPZW_B(sve_cmplt_ppzw_b, int8_t,   int64_t, <)
+DO_CMP_PPZW_H(sve_cmplt_ppzw_h, int16_t,  int64_t, <)
+DO_CMP_PPZW_S(sve_cmplt_ppzw_s, int32_t,  int64_t, <)
+
+DO_CMP_PPZW_B(sve_cmple_ppzw_b, int8_t,   int64_t, <=)
+DO_CMP_PPZW_H(sve_cmple_ppzw_h, int16_t,  int64_t, <=)
+DO_CMP_PPZW_S(sve_cmple_ppzw_s, int32_t,  int64_t, <=)
+
+DO_CMP_PPZW_B(sve_cmplo_ppzw_b, uint8_t,  uint64_t, <)
+DO_CMP_PPZW_H(sve_cmplo_ppzw_h, uint16_t, uint64_t, <)
+DO_CMP_PPZW_S(sve_cmplo_ppzw_s, uint32_t, uint64_t, <)
+
+DO_CMP_PPZW_B(sve_cmpls_ppzw_b, uint8_t,  uint64_t, <=)
+DO_CMP_PPZW_H(sve_cmpls_ppzw_h, uint16_t, uint64_t, <=)
+DO_CMP_PPZW_S(sve_cmpls_ppzw_s, uint32_t, uint64_t, <=)
+
+#undef DO_CMP_PPZW_B
+#undef DO_CMP_PPZW_H
+#undef DO_CMP_PPZW_S
+#undef DO_CMP_PPZW
+
+/* Similar, but the second source is immediate.  */
+#define DO_CMP_PPZI(NAME, TYPE, OP, H, MASK)                         \
+uint32_t HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)   \
+{                                                                    \
+    intptr_t opr_sz = simd_oprsz(desc);                              \
+    uint32_t flags = PREDTEST_INIT;                                  \
+    TYPE mm = simd_data(desc);                                       \
+    intptr_t i = opr_sz;                                             \
+    do {                                                             \
+        uint64_t out = 0, pg;                                        \
+        do {                                                         \
+            i -= sizeof(TYPE), out <<= sizeof(TYPE);                 \
+            TYPE nn = *(TYPE *)((char *)vn + H(i));                          \
+            out |= nn OP mm;                                         \
+        } while (i & 63);                                            \
+        pg = *(uint64_t *)((char *)vg + (i >> 3)) & MASK;                    \
+        out &= pg;                                                   \
+        *(uint64_t *)((char *)vd + (i >> 3)) = out;                          \
+        flags = iter_predtest_bwd(out, pg, flags);                   \
+    } while (i > 0);                                                 \
+    return flags;                                                    \
+}
+
+#define DO_CMP_PPZI_B(NAME, TYPE, OP) \
+    DO_CMP_PPZI(NAME, TYPE, OP, H1,   0xffffffffffffffffull)
+#define DO_CMP_PPZI_H(NAME, TYPE, OP) \
+    DO_CMP_PPZI(NAME, TYPE, OP, H1_2, 0x5555555555555555ull)
+#define DO_CMP_PPZI_S(NAME, TYPE, OP) \
+    DO_CMP_PPZI(NAME, TYPE, OP, H1_4, 0x1111111111111111ull)
+#define DO_CMP_PPZI_D(NAME, TYPE, OP) \
+    DO_CMP_PPZI(NAME, TYPE, OP,     , 0x0101010101010101ull)
+
+DO_CMP_PPZI_B(sve_cmpeq_ppzi_b, uint8_t,  ==)
+DO_CMP_PPZI_H(sve_cmpeq_ppzi_h, uint16_t, ==)
+DO_CMP_PPZI_S(sve_cmpeq_ppzi_s, uint32_t, ==)
+DO_CMP_PPZI_D(sve_cmpeq_ppzi_d, uint64_t, ==)
+
+DO_CMP_PPZI_B(sve_cmpne_ppzi_b, uint8_t,  !=)
+DO_CMP_PPZI_H(sve_cmpne_ppzi_h, uint16_t, !=)
+DO_CMP_PPZI_S(sve_cmpne_ppzi_s, uint32_t, !=)
+DO_CMP_PPZI_D(sve_cmpne_ppzi_d, uint64_t, !=)
+
+DO_CMP_PPZI_B(sve_cmpgt_ppzi_b, int8_t,  >)
+DO_CMP_PPZI_H(sve_cmpgt_ppzi_h, int16_t, >)
+DO_CMP_PPZI_S(sve_cmpgt_ppzi_s, int32_t, >)
+DO_CMP_PPZI_D(sve_cmpgt_ppzi_d, int64_t, >)
+
+DO_CMP_PPZI_B(sve_cmpge_ppzi_b, int8_t,  >=)
+DO_CMP_PPZI_H(sve_cmpge_ppzi_h, int16_t, >=)
+DO_CMP_PPZI_S(sve_cmpge_ppzi_s, int32_t, >=)
+DO_CMP_PPZI_D(sve_cmpge_ppzi_d, int64_t, >=)
+
+DO_CMP_PPZI_B(sve_cmphi_ppzi_b, uint8_t,  >)
+DO_CMP_PPZI_H(sve_cmphi_ppzi_h, uint16_t, >)
+DO_CMP_PPZI_S(sve_cmphi_ppzi_s, uint32_t, >)
+DO_CMP_PPZI_D(sve_cmphi_ppzi_d, uint64_t, >)
+
+DO_CMP_PPZI_B(sve_cmphs_ppzi_b, uint8_t,  >=)
+DO_CMP_PPZI_H(sve_cmphs_ppzi_h, uint16_t, >=)
+DO_CMP_PPZI_S(sve_cmphs_ppzi_s, uint32_t, >=)
+DO_CMP_PPZI_D(sve_cmphs_ppzi_d, uint64_t, >=)
+
+DO_CMP_PPZI_B(sve_cmplt_ppzi_b, int8_t,  <)
+DO_CMP_PPZI_H(sve_cmplt_ppzi_h, int16_t, <)
+DO_CMP_PPZI_S(sve_cmplt_ppzi_s, int32_t, <)
+DO_CMP_PPZI_D(sve_cmplt_ppzi_d, int64_t, <)
+
+DO_CMP_PPZI_B(sve_cmple_ppzi_b, int8_t,  <=)
+DO_CMP_PPZI_H(sve_cmple_ppzi_h, int16_t, <=)
+DO_CMP_PPZI_S(sve_cmple_ppzi_s, int32_t, <=)
+DO_CMP_PPZI_D(sve_cmple_ppzi_d, int64_t, <=)
+
+DO_CMP_PPZI_B(sve_cmplo_ppzi_b, uint8_t,  <)
+DO_CMP_PPZI_H(sve_cmplo_ppzi_h, uint16_t, <)
+DO_CMP_PPZI_S(sve_cmplo_ppzi_s, uint32_t, <)
+DO_CMP_PPZI_D(sve_cmplo_ppzi_d, uint64_t, <)
+
+DO_CMP_PPZI_B(sve_cmpls_ppzi_b, uint8_t,  <=)
+DO_CMP_PPZI_H(sve_cmpls_ppzi_h, uint16_t, <=)
+DO_CMP_PPZI_S(sve_cmpls_ppzi_s, uint32_t, <=)
+DO_CMP_PPZI_D(sve_cmpls_ppzi_d, uint64_t, <=)
+
+#undef DO_CMP_PPZI_B
+#undef DO_CMP_PPZI_H
+#undef DO_CMP_PPZI_S
+#undef DO_CMP_PPZI_D
+#undef DO_CMP_PPZI
+
+/* Similar to the ARM LastActive pseudocode function.  */
+static bool last_active_pred(void *vd, void *vg, intptr_t oprsz)
+{
+    intptr_t i;
+
+    for (i = QEMU_ALIGN_UP(oprsz, 8) - 8; i >= 0; i -= 8) {
+        uint64_t pg = *(uint64_t *)((char *)vg + i);
+        if (pg) {
+            return (pow2floor(pg) & *(uint64_t *)((char *)vd + i)) != 0;
+        }
+    }
+    return 0;
+}
+
+/* Compute a mask into RETB that is true for all G, up to and including
+ * (if after) or excluding (if !after) the first G & N.
+ * Return true if BRK found.
+ */
+static bool compute_brk(uint64_t *retb, uint64_t n, uint64_t g,
+                        bool brk, bool after)
+{
+    uint64_t b;
+
+    if (brk) {
+        b = 0;
+    } else if ((g & n) == 0) {
+        /* For all G, no N are set; break not found.  */
+        b = g;
+    } else {
+        /* Break somewhere in N.  Locate it.  */
+        b = g & n;            /* guard true, pred true */
+#ifdef _MSC_VER
+        b = b & (0 - b);      /* first such */
+#else
+        b = b & -b;           /* first such */
+#endif
+        if (after) {
+            b = b | (b - 1);  /* break after same */
+        } else {
+            b = b - 1;        /* break before same */
+        }
+        brk = true;
+    }
+
+    *retb = b;
+    return brk;
+}
+
+/* Compute a zeroing BRK.  */
+static void compute_brk_z(uint64_t *d, uint64_t *n, uint64_t *g,
+                          intptr_t oprsz, bool after)
+{
+    bool brk = false;
+    intptr_t i;
+
+    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
+        uint64_t this_b, this_g = g[i];
+
+        brk = compute_brk(&this_b, n[i], this_g, brk, after);
+        d[i] = this_b & this_g;
+    }
+}
+
+/* Likewise, but also compute flags.  */
+static uint32_t compute_brks_z(uint64_t *d, uint64_t *n, uint64_t *g,
+                               intptr_t oprsz, bool after)
+{
+    uint32_t flags = PREDTEST_INIT;
+    bool brk = false;
+    intptr_t i;
+
+    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
+        uint64_t this_b, this_d, this_g = g[i];
+
+        brk = compute_brk(&this_b, n[i], this_g, brk, after);
+        d[i] = this_d = this_b & this_g;
+        flags = iter_predtest_fwd(this_d, this_g, flags);
+    }
+    return flags;
+}
+
+/* Compute a merging BRK.  */
+static void compute_brk_m(uint64_t *d, uint64_t *n, uint64_t *g,
+                          intptr_t oprsz, bool after)
+{
+    bool brk = false;
+    intptr_t i;
+
+    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
+        uint64_t this_b, this_g = g[i];
+
+        brk = compute_brk(&this_b, n[i], this_g, brk, after);
+        d[i] = (this_b & this_g) | (d[i] & ~this_g);
+    }
+}
+
+/* Likewise, but also compute flags.  */
+static uint32_t compute_brks_m(uint64_t *d, uint64_t *n, uint64_t *g,
+                               intptr_t oprsz, bool after)
+{
+    uint32_t flags = PREDTEST_INIT;
+    bool brk = false;
+    intptr_t i;
+
+    for (i = 0; i < oprsz / 8; ++i) {
+        uint64_t this_b, this_d = d[i], this_g = g[i];
+
+        brk = compute_brk(&this_b, n[i], this_g, brk, after);
+        d[i] = this_d = (this_b & this_g) | (this_d & ~this_g);
+        flags = iter_predtest_fwd(this_d, this_g, flags);
+    }
+    return flags;
+}
+
+static uint32_t do_zero(ARMPredicateReg *d, intptr_t oprsz)
+{
+    /* It is quicker to zero the whole predicate than loop on OPRSZ.
+     * The compiler should turn this into 4 64-bit integer stores.
+     */
+    memset(d, 0, sizeof(ARMPredicateReg));
+    return PREDTEST_INIT;
+}
+
+void HELPER(sve_brkpa)(void *vd, void *vn, void *vm, void *vg,
+                       uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    if (last_active_pred(vn, vg, oprsz)) {
+        compute_brk_z(vd, vm, vg, oprsz, true);
+    } else {
+        do_zero(vd, oprsz);
+    }
+}
+
+uint32_t HELPER(sve_brkpas)(void *vd, void *vn, void *vm, void *vg,
+                            uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    if (last_active_pred(vn, vg, oprsz)) {
+        return compute_brks_z(vd, vm, vg, oprsz, true);
+    } else {
+        return do_zero(vd, oprsz);
+    }
+}
+
+void HELPER(sve_brkpb)(void *vd, void *vn, void *vm, void *vg,
+                       uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    if (last_active_pred(vn, vg, oprsz)) {
+        compute_brk_z(vd, vm, vg, oprsz, false);
+    } else {
+        do_zero(vd, oprsz);
+    }
+}
+
+uint32_t HELPER(sve_brkpbs)(void *vd, void *vn, void *vm, void *vg,
+                            uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    if (last_active_pred(vn, vg, oprsz)) {
+        return compute_brks_z(vd, vm, vg, oprsz, false);
+    } else {
+        return do_zero(vd, oprsz);
+    }
+}
+
+void HELPER(sve_brka_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    compute_brk_z(vd, vn, vg, oprsz, true);
+}
+
+uint32_t HELPER(sve_brkas_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    return compute_brks_z(vd, vn, vg, oprsz, true);
+}
+
+void HELPER(sve_brkb_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    compute_brk_z(vd, vn, vg, oprsz, false);
+}
+
+uint32_t HELPER(sve_brkbs_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    return compute_brks_z(vd, vn, vg, oprsz, false);
+}
+
+void HELPER(sve_brka_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    compute_brk_m(vd, vn, vg, oprsz, true);
+}
+
+uint32_t HELPER(sve_brkas_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    return compute_brks_m(vd, vn, vg, oprsz, true);
+}
+
+void HELPER(sve_brkb_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    compute_brk_m(vd, vn, vg, oprsz, false);
+}
+
+uint32_t HELPER(sve_brkbs_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    return compute_brks_m(vd, vn, vg, oprsz, false);
+}
+
+void HELPER(sve_brkn)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+
+    if (!last_active_pred(vn, vg, oprsz)) {
+        do_zero(vd, oprsz);
+    }
+}
+
+/* As if PredTest(Ones(PL), D, esz).  */
+static uint32_t predtest_ones(ARMPredicateReg *d, intptr_t oprsz,
+                              uint64_t esz_mask)
+{
+    uint32_t flags = PREDTEST_INIT;
+    intptr_t i;
+
+    for (i = 0; i < oprsz / 8; i++) {
+        flags = iter_predtest_fwd(d->p[i], esz_mask, flags);
+    }
+    if (oprsz & 7) {
+        uint64_t mask = ~(0xffffffffffffffffULL << (8 * (oprsz & 7)));
+        flags = iter_predtest_fwd(d->p[i], esz_mask & mask, flags);
     }
+    return flags;
 }
 
-void HELPER(sve_compact_d)(void *vd, void *vn, void *vg, uint32_t desc)
+uint32_t HELPER(sve_brkns)(void *vd, void *vn, void *vg, uint32_t pred_desc)
 {
-    intptr_t i, j, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn;
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+
+    if (last_active_pred(vn, vg, oprsz)) {
+        return predtest_ones(vd, oprsz, -1);
+    } else {
+        return do_zero(vd, oprsz);
+    }
+}
+
+uint64_t HELPER(sve_cntp)(void *vn, void *vg, uint32_t pred_desc)
+{
+    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    uint64_t *n = vn, *g = vg, sum = 0, mask = pred_esz_masks[esz];
+    intptr_t i;
+
+    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
+        uint64_t t = n[i] & g[i] & mask;
+        sum += ctpop64(t);
+    }
+    return sum;
+}
+
+uint32_t HELPER(sve_while)(void *vd, uint32_t count, uint32_t pred_desc)
+{
+    uintptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    uint64_t esz_mask = pred_esz_masks[esz];
+    ARMPredicateReg *d = vd;
+    uint32_t flags;
+    intptr_t i;
+
+    /* Begin with a zero predicate register.  */
+    flags = do_zero(d, oprsz);
+    if (count == 0) {
+        return flags;
+    }
+
+    /* Set all of the requested bits.  */
+    for (i = 0; i < count / 64; ++i) {
+        d->p[i] = esz_mask;
+    }
+    if (count & 63) {
+        d->p[i] = MAKE_64BIT_MASK(0, count & 63) & esz_mask;
+    }
+
+    return predtest_ones(d, oprsz, esz_mask);
+}
+
+/* Recursive reduction on a function;
+ * C.f. the ARM ARM function ReducePredicated.
+ *
+ * While it would be possible to write this without the DATA temporary,
+ * it is much simpler to process the predicate register this way.
+ * The recursion is bounded to depth 7 (128 fp16 elements), so there's
+ * little to gain with a more complex non-recursive form.
+ */
+#define DO_REDUCE(NAME, TYPE, H, FUNC, IDENT)                         \
+static TYPE NAME##_reduce(TYPE *data, float_status *status, uintptr_t n) \
+{                                                                     \
+    if (n == 1) {                                                     \
+        return *data;                                                 \
+    } else {                                                          \
+        uintptr_t half = n / 2;                                       \
+        TYPE lo = NAME##_reduce(data, status, half);                  \
+        TYPE hi = NAME##_reduce(data + half, status, half);           \
+        return TYPE##_##FUNC(lo, hi, status);                         \
+    }                                                                 \
+}                                                                     \
+uint64_t HELPER(NAME)(void *vn, void *vg, void *vs, uint32_t desc)    \
+{                                                                     \
+    uintptr_t i, oprsz = simd_oprsz(desc), maxsz = simd_maxsz(desc);  \
+    TYPE data[sizeof(ARMVectorReg) / sizeof(TYPE)];                   \
+    for (i = 0; i < oprsz; ) {                                        \
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));               \
+        do {                                                          \
+            TYPE nn = *(TYPE *)((char *)vn + H(i));                           \
+            *(TYPE *)((char *)data + i) = (pg & 1 ? nn : IDENT);      \
+            i += sizeof(TYPE), pg >>= sizeof(TYPE);                   \
+        } while (i & 15);                                             \
+    }                                                                 \
+    for (; i < maxsz; i += sizeof(TYPE)) {                            \
+        *(TYPE *)((char *)data + i) = IDENT;                          \
+    }                                                                 \
+    return NAME##_reduce(data, vs, maxsz / sizeof(TYPE));             \
+}
+
+DO_REDUCE(sve_faddv_h, float16, H1_2, add, float16_zero)
+DO_REDUCE(sve_faddv_s, float32, H1_4, add, float32_zero)
+DO_REDUCE(sve_faddv_d, float64,     , add, float64_zero)
+
+/* Identity is floatN_default_nan, without the function call.  */
+DO_REDUCE(sve_fminnmv_h, float16, H1_2, minnum, 0x7E00)
+DO_REDUCE(sve_fminnmv_s, float32, H1_4, minnum, 0x7FC00000)
+DO_REDUCE(sve_fminnmv_d, float64,     , minnum, 0x7FF8000000000000ULL)
+
+DO_REDUCE(sve_fmaxnmv_h, float16, H1_2, maxnum, 0x7E00)
+DO_REDUCE(sve_fmaxnmv_s, float32, H1_4, maxnum, 0x7FC00000)
+DO_REDUCE(sve_fmaxnmv_d, float64,     , maxnum, 0x7FF8000000000000ULL)
+
+DO_REDUCE(sve_fminv_h, float16, H1_2, min, float16_infinity)
+DO_REDUCE(sve_fminv_s, float32, H1_4, min, float32_infinity)
+DO_REDUCE(sve_fminv_d, float64,     , min, float64_infinity)
+
+DO_REDUCE(sve_fmaxv_h, float16, H1_2, max, float16_chs(float16_infinity))
+DO_REDUCE(sve_fmaxv_s, float32, H1_4, max, float32_chs(float32_infinity))
+DO_REDUCE(sve_fmaxv_d, float64,     , max, float64_chs(float64_infinity))
+
+#undef DO_REDUCE
+
+uint64_t HELPER(sve_fadda_h)(uint64_t nn, void *vm, void *vg,
+                             void *status, uint32_t desc)
+{
+    intptr_t i = 0, opr_sz = simd_oprsz(desc);
+    float16 result = nn;
+
+    do {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        do {
+            if (pg & 1) {
+                float16 mm = *(float16 *)((char *)vm + H1_2(i));
+                result = float16_add(result, mm, status);
+            }
+            i += sizeof(float16), pg >>= sizeof(float16);
+        } while (i & 15);
+    } while (i < opr_sz);
+
+    return result;
+}
+
+uint64_t HELPER(sve_fadda_s)(uint64_t nn, void *vm, void *vg,
+                             void *status, uint32_t desc)
+{
+    intptr_t i = 0, opr_sz = simd_oprsz(desc);
+    float32 result = nn;
+
+    do {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        do {
+            if (pg & 1) {
+                float32 mm = *(float32 *)((char *)vm + H1_2(i));
+                result = float32_add(result, mm, status);
+            }
+            i += sizeof(float32), pg >>= sizeof(float32);
+        } while (i & 15);
+    } while (i < opr_sz);
+
+    return result;
+}
+
+uint64_t HELPER(sve_fadda_d)(uint64_t nn, void *vm, void *vg,
+                             void *status, uint32_t desc)
+{
+    intptr_t i = 0, opr_sz = simd_oprsz(desc) / 8;
+    uint64_t *m = vm;
     uint8_t *pg = vg;
 
-    for (i = j = 0; i < opr_sz; i++) {
+    for (i = 0; i < opr_sz; i++) {
         if (pg[H1(i)] & 1) {
-            d[j] = n[i];
-            j++;
+            nn = float64_add(nn, m[i], status);
         }
     }
-    for (; j < opr_sz; j++) {
-        d[j] = 0;
-    }
+
+    return nn;
 }
 
-/* Similar to the ARM LastActiveElement pseudocode function, except the
- * result is multiplied by the element size.  This includes the not found
- * indication; e.g. not found for esz=3 is -8.
+/* Fully general three-operand expander, controlled by a predicate,
+ * With the extra float_status parameter.
  */
-int32_t HELPER(sve_last_active_element)(void *vg, uint32_t pred_desc)
+#define DO_ZPZZ_FP(NAME, TYPE, H, OP)                           \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg,       \
+                  void *status, uint32_t desc)                  \
+{                                                               \
+    intptr_t i = simd_oprsz(desc);                              \
+    uint64_t *g = vg;                                           \
+    do {                                                        \
+        uint64_t pg = g[(i - 1) >> 6];                          \
+        do {                                                    \
+            i -= sizeof(TYPE);                                  \
+            if (likely((pg >> (i & 63)) & 1)) {                 \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                 \
+                TYPE mm = *(TYPE *)((char *)vm + H(i));                 \
+                *(TYPE *)((char *)vd + H(i)) = OP(nn, mm, status);      \
+            }                                                   \
+        } while (i & 63);                                       \
+    } while (i != 0);                                           \
+}
+
+DO_ZPZZ_FP(sve_fadd_h, uint16_t, H1_2, float16_add)
+DO_ZPZZ_FP(sve_fadd_s, uint32_t, H1_4, float32_add)
+DO_ZPZZ_FP(sve_fadd_d, uint64_t,     , float64_add)
+
+DO_ZPZZ_FP(sve_fsub_h, uint16_t, H1_2, float16_sub)
+DO_ZPZZ_FP(sve_fsub_s, uint32_t, H1_4, float32_sub)
+DO_ZPZZ_FP(sve_fsub_d, uint64_t,     , float64_sub)
+
+DO_ZPZZ_FP(sve_fmul_h, uint16_t, H1_2, float16_mul)
+DO_ZPZZ_FP(sve_fmul_s, uint32_t, H1_4, float32_mul)
+DO_ZPZZ_FP(sve_fmul_d, uint64_t,     , float64_mul)
+
+DO_ZPZZ_FP(sve_fdiv_h, uint16_t, H1_2, float16_div)
+DO_ZPZZ_FP(sve_fdiv_s, uint32_t, H1_4, float32_div)
+DO_ZPZZ_FP(sve_fdiv_d, uint64_t,     , float64_div)
+
+DO_ZPZZ_FP(sve_fmin_h, uint16_t, H1_2, float16_min)
+DO_ZPZZ_FP(sve_fmin_s, uint32_t, H1_4, float32_min)
+DO_ZPZZ_FP(sve_fmin_d, uint64_t,     , float64_min)
+
+DO_ZPZZ_FP(sve_fmax_h, uint16_t, H1_2, float16_max)
+DO_ZPZZ_FP(sve_fmax_s, uint32_t, H1_4, float32_max)
+DO_ZPZZ_FP(sve_fmax_d, uint64_t,     , float64_max)
+
+DO_ZPZZ_FP(sve_fminnum_h, uint16_t, H1_2, float16_minnum)
+DO_ZPZZ_FP(sve_fminnum_s, uint32_t, H1_4, float32_minnum)
+DO_ZPZZ_FP(sve_fminnum_d, uint64_t,     , float64_minnum)
+
+DO_ZPZZ_FP(sve_fmaxnum_h, uint16_t, H1_2, float16_maxnum)
+DO_ZPZZ_FP(sve_fmaxnum_s, uint32_t, H1_4, float32_maxnum)
+DO_ZPZZ_FP(sve_fmaxnum_d, uint64_t,     , float64_maxnum)
+
+static inline float16 abd_h(float16 a, float16 b, float_status *s)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
+    return float16_abs(float16_sub(a, b, s));
+}
+
+static inline float32 abd_s(float32 a, float32 b, float_status *s)
+{
+    return float32_abs(float32_sub(a, b, s));
+}
 
-    return last_active_element(vg, DIV_ROUND_UP(oprsz, 8), esz);
+static inline float64 abd_d(float64 a, float64 b, float_status *s)
+{
+    return float64_abs(float64_sub(a, b, s));
 }
 
-void HELPER(sve_splice)(void *vd, void *vn, void *vm, void *vg, uint32_t desc)
+DO_ZPZZ_FP(sve_fabd_h, uint16_t, H1_2, abd_h)
+DO_ZPZZ_FP(sve_fabd_s, uint32_t, H1_4, abd_s)
+DO_ZPZZ_FP(sve_fabd_d, uint64_t,     , abd_d)
+
+static inline float64 scalbn_d(float64 a, int64_t b, float_status *s)
 {
-    intptr_t opr_sz = simd_oprsz(desc) / 8;
-    int esz = simd_data(desc);
-    uint64_t pg, first_g, last_g, len, mask = pred_esz_masks[esz];
-    intptr_t i, first_i, last_i;
-    ARMVectorReg tmp;
+    int b_int = MIN(MAX(b, INT_MIN), INT_MAX);
+    return float64_scalbn(a, b_int, s);
+}
 
-    first_i = last_i = 0;
-    first_g = last_g = 0;
+DO_ZPZZ_FP(sve_fscalbn_h, int16_t, H1_2, float16_scalbn)
+DO_ZPZZ_FP(sve_fscalbn_s, int32_t, H1_4, float32_scalbn)
+DO_ZPZZ_FP(sve_fscalbn_d, int64_t,     , scalbn_d)
 
-    /* Find the extent of the active elements within VG.  */
-    for (i = QEMU_ALIGN_UP(opr_sz, 8) - 8; i >= 0; i -= 8) {
-        pg = *(uint64_t *)((char *)vg + i) & mask;
-        if (pg) {
-            if (last_g == 0) {
-                last_g = pg;
-                last_i = i;
-            }
-            first_g = pg;
-            first_i = i;
-        }
-    }
+DO_ZPZZ_FP(sve_fmulx_h, uint16_t, H1_2, helper_advsimd_mulxh)
+DO_ZPZZ_FP(sve_fmulx_s, uint32_t, H1_4, helper_vfp_mulxs)
+DO_ZPZZ_FP(sve_fmulx_d, uint64_t,     , helper_vfp_mulxd)
 
-    len = 0;
-    if (first_g != 0) {
-        first_i = first_i * 8 + ctz64(first_g);
-        last_i = last_i * 8 + 63 - clz64(last_g);
-        len = last_i - first_i + (1ULL << esz);
-        if (vd == vm) {
-            vm = memcpy(&tmp, vm, opr_sz * 8);
-        }
-        swap_memmove(vd, (char *)vn + first_i, len);
-    }
-    swap_memmove((char *)vd + len, vm, opr_sz * 8 - len);
+#undef DO_ZPZZ_FP
+
+/* Three-operand expander, with one scalar operand, controlled by
+ * a predicate, with the extra float_status parameter.
+ */
+#define DO_ZPZS_FP(NAME, TYPE, H, OP) \
+void HELPER(NAME)(void *vd, void *vn, void *vg, uint64_t scalar,  \
+                  void *status, uint32_t desc)                    \
+{                                                                 \
+    intptr_t i = simd_oprsz(desc);                                \
+    uint64_t *g = vg;                                             \
+    TYPE mm = scalar;                                             \
+    do {                                                          \
+        uint64_t pg = g[(i - 1) >> 6];                            \
+        do {                                                      \
+            i -= sizeof(TYPE);                                    \
+            if (likely((pg >> (i & 63)) & 1)) {                   \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                   \
+                *(TYPE *)((char *)vd + H(i)) = OP(nn, mm, status);        \
+            }                                                     \
+        } while (i & 63);                                         \
+    } while (i != 0);                                             \
 }
 
-void HELPER(sve_sel_zpzz_b)(void *vd, void *vn, void *vm,
-                            void *vg, uint32_t desc)
-{
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn, *m = vm;
-    uint8_t *pg = vg;
+DO_ZPZS_FP(sve_fadds_h, float16, H1_2, float16_add)
+DO_ZPZS_FP(sve_fadds_s, float32, H1_4, float32_add)
+DO_ZPZS_FP(sve_fadds_d, float64,     , float64_add)
 
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i], mm = m[i];
-        uint64_t pp = expand_pred_b(pg[H1(i)]);
-        d[i] = (nn & pp) | (mm & ~pp);
-    }
-}
+DO_ZPZS_FP(sve_fsubs_h, float16, H1_2, float16_sub)
+DO_ZPZS_FP(sve_fsubs_s, float32, H1_4, float32_sub)
+DO_ZPZS_FP(sve_fsubs_d, float64,     , float64_sub)
 
-void HELPER(sve_sel_zpzz_h)(void *vd, void *vn, void *vm,
-                            void *vg, uint32_t desc)
-{
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn, *m = vm;
-    uint8_t *pg = vg;
+DO_ZPZS_FP(sve_fmuls_h, float16, H1_2, float16_mul)
+DO_ZPZS_FP(sve_fmuls_s, float32, H1_4, float32_mul)
+DO_ZPZS_FP(sve_fmuls_d, float64,     , float64_mul)
 
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i], mm = m[i];
-        uint64_t pp = expand_pred_h(pg[H1(i)]);
-        d[i] = (nn & pp) | (mm & ~pp);
-    }
+static inline float16 subr_h(float16 a, float16 b, float_status *s)
+{
+    return float16_sub(b, a, s);
 }
 
-void HELPER(sve_sel_zpzz_s)(void *vd, void *vn, void *vm,
-                            void *vg, uint32_t desc)
+static inline float32 subr_s(float32 a, float32 b, float_status *s)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn, *m = vm;
-    uint8_t *pg = vg;
-
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i], mm = m[i];
-        uint64_t pp = expand_pred_s(pg[H1(i)]);
-        d[i] = (nn & pp) | (mm & ~pp);
-    }
+    return float32_sub(b, a, s);
 }
 
-void HELPER(sve_sel_zpzz_d)(void *vd, void *vn, void *vm,
-                            void *vg, uint32_t desc)
+static inline float64 subr_d(float64 a, float64 b, float_status *s)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *d = vd, *n = vn, *m = vm;
-    uint8_t *pg = vg;
+    return float64_sub(b, a, s);
+}
 
-    for (i = 0; i < opr_sz; i += 1) {
-        uint64_t nn = n[i], mm = m[i];
-        d[i] = (pg[H1(i)] & 1 ? nn : mm);
-    }
+DO_ZPZS_FP(sve_fsubrs_h, float16, H1_2, subr_h)
+DO_ZPZS_FP(sve_fsubrs_s, float32, H1_4, subr_s)
+DO_ZPZS_FP(sve_fsubrs_d, float64,     , subr_d)
+
+DO_ZPZS_FP(sve_fmaxnms_h, float16, H1_2, float16_maxnum)
+DO_ZPZS_FP(sve_fmaxnms_s, float32, H1_4, float32_maxnum)
+DO_ZPZS_FP(sve_fmaxnms_d, float64,     , float64_maxnum)
+
+DO_ZPZS_FP(sve_fminnms_h, float16, H1_2, float16_minnum)
+DO_ZPZS_FP(sve_fminnms_s, float32, H1_4, float32_minnum)
+DO_ZPZS_FP(sve_fminnms_d, float64,     , float64_minnum)
+
+DO_ZPZS_FP(sve_fmaxs_h, float16, H1_2, float16_max)
+DO_ZPZS_FP(sve_fmaxs_s, float32, H1_4, float32_max)
+DO_ZPZS_FP(sve_fmaxs_d, float64,     , float64_max)
+
+DO_ZPZS_FP(sve_fmins_h, float16, H1_2, float16_min)
+DO_ZPZS_FP(sve_fmins_s, float32, H1_4, float32_min)
+DO_ZPZS_FP(sve_fmins_d, float64,     , float64_min)
+
+/* Fully general two-operand expander, controlled by a predicate,
+ * With the extra float_status parameter.
+ */
+#define DO_ZPZ_FP(NAME, TYPE, H, OP)                                  \
+void HELPER(NAME)(void *vd, void *vn, void *vg, void *status, uint32_t desc) \
+{                                                                     \
+    intptr_t i = simd_oprsz(desc);                                    \
+    uint64_t *g = vg;                                                 \
+    do {                                                              \
+        uint64_t pg = g[(i - 1) >> 6];                                \
+        do {                                                          \
+            i -= sizeof(TYPE);                                        \
+            if (likely((pg >> (i & 63)) & 1)) {                       \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                       \
+                *(TYPE *)((char *)vd + H(i)) = OP(nn, status);                \
+            }                                                         \
+        } while (i & 63);                                             \
+    } while (i != 0);                                                 \
 }
 
-/* Two operand comparison controlled by a predicate.
- * ??? It is very tempting to want to be able to expand this inline
- * with x86 instructions, e.g.
- *
- *    vcmpeqw    zm, zn, %ymm0
- *    vpmovmskb  %ymm0, %eax
- *    and        $0x5555, %eax
- *    and        pg, %eax
- *
- * or even aarch64, e.g.
- *
- *    // mask = 4000 1000 0400 0100 0040 0010 0004 0001
- *    cmeq       v0.8h, zn, zm
- *    and        v0.8h, v0.8h, mask
- *    addv       h0, v0.8h
- *    and        v0.8b, pg
- *
- * However, coming up with an abstraction that allows vector inputs and
- * a scalar output, and also handles the byte-ordering of sub-uint64_t
- * scalar outputs, is tricky.
+/* SVE fp16 conversions always use IEEE mode.  Like AdvSIMD, they ignore
+ * FZ16.  When converting from fp16, this affects flushing input denormals;
+ * when converting to fp16, this affects flushing output denormals.
  */
-#define DO_CMP_PPZZ(NAME, TYPE, OP, H, MASK)                                 \
-uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
-{                                                                            \
-    intptr_t opr_sz = simd_oprsz(desc);                                      \
-    uint32_t flags = PREDTEST_INIT;                                          \
-    intptr_t i = opr_sz;                                                     \
-    do {                                                                     \
-        uint64_t out = 0, pg;                                                \
-        do {                                                                 \
-            i -= sizeof(TYPE), out <<= sizeof(TYPE);                         \
-            TYPE nn = *(TYPE *)((char *)vn + H(i));                                  \
-            TYPE mm = *(TYPE *)((char *)vm + H(i));                                  \
-            out |= nn OP mm;                                                 \
-        } while (i & 63);                                                    \
-        pg = *(uint64_t *)((char *)vg + (i >> 3)) & MASK;                            \
-        out &= pg;                                                           \
-        *(uint64_t *)((char *)vd + (i >> 3)) = out;                                  \
-        flags = iter_predtest_bwd(out, pg, flags);                           \
-    } while (i > 0);                                                         \
-    return flags;                                                            \
+static inline float32 sve_f16_to_f32(float16 f, float_status *fpst)
+{
+    flag save = get_flush_inputs_to_zero(fpst);
+    float32 ret;
+
+    set_flush_inputs_to_zero(false, fpst);
+    ret = float16_to_float32(f, true, fpst);
+    set_flush_inputs_to_zero(save, fpst);
+    return ret;
 }
 
-#define DO_CMP_PPZZ_B(NAME, TYPE, OP) \
-    DO_CMP_PPZZ(NAME, TYPE, OP, H1,   0xffffffffffffffffull)
-#define DO_CMP_PPZZ_H(NAME, TYPE, OP) \
-    DO_CMP_PPZZ(NAME, TYPE, OP, H1_2, 0x5555555555555555ull)
-#define DO_CMP_PPZZ_S(NAME, TYPE, OP) \
-    DO_CMP_PPZZ(NAME, TYPE, OP, H1_4, 0x1111111111111111ull)
-#define DO_CMP_PPZZ_D(NAME, TYPE, OP) \
-    DO_CMP_PPZZ(NAME, TYPE, OP,     , 0x0101010101010101ull)
+static inline float64 sve_f16_to_f64(float16 f, float_status *fpst)
+{
+    flag save = get_flush_inputs_to_zero(fpst);
+    float64 ret;
 
-DO_CMP_PPZZ_B(sve_cmpeq_ppzz_b, uint8_t,  ==)
-DO_CMP_PPZZ_H(sve_cmpeq_ppzz_h, uint16_t, ==)
-DO_CMP_PPZZ_S(sve_cmpeq_ppzz_s, uint32_t, ==)
-DO_CMP_PPZZ_D(sve_cmpeq_ppzz_d, uint64_t, ==)
+    set_flush_inputs_to_zero(false, fpst);
+    ret = float16_to_float64(f, true, fpst);
+    set_flush_inputs_to_zero(save, fpst);
+    return ret;
+}
 
-DO_CMP_PPZZ_B(sve_cmpne_ppzz_b, uint8_t,  !=)
-DO_CMP_PPZZ_H(sve_cmpne_ppzz_h, uint16_t, !=)
-DO_CMP_PPZZ_S(sve_cmpne_ppzz_s, uint32_t, !=)
-DO_CMP_PPZZ_D(sve_cmpne_ppzz_d, uint64_t, !=)
+static inline float16 sve_f32_to_f16(float32 f, float_status *fpst)
+{
+    flag save = get_flush_to_zero(fpst);
+    float16 ret;
 
-DO_CMP_PPZZ_B(sve_cmpgt_ppzz_b, int8_t,  >)
-DO_CMP_PPZZ_H(sve_cmpgt_ppzz_h, int16_t, >)
-DO_CMP_PPZZ_S(sve_cmpgt_ppzz_s, int32_t, >)
-DO_CMP_PPZZ_D(sve_cmpgt_ppzz_d, int64_t, >)
+    set_flush_to_zero(false, fpst);
+    ret = float32_to_float16(f, true, fpst);
+    set_flush_to_zero(save, fpst);
+    return ret;
+}
 
-DO_CMP_PPZZ_B(sve_cmpge_ppzz_b, int8_t,  >=)
-DO_CMP_PPZZ_H(sve_cmpge_ppzz_h, int16_t, >=)
-DO_CMP_PPZZ_S(sve_cmpge_ppzz_s, int32_t, >=)
-DO_CMP_PPZZ_D(sve_cmpge_ppzz_d, int64_t, >=)
+static inline float16 sve_f64_to_f16(float64 f, float_status *fpst)
+{
+    flag save = get_flush_to_zero(fpst);
+    float16 ret;
 
-DO_CMP_PPZZ_B(sve_cmphi_ppzz_b, uint8_t,  >)
-DO_CMP_PPZZ_H(sve_cmphi_ppzz_h, uint16_t, >)
-DO_CMP_PPZZ_S(sve_cmphi_ppzz_s, uint32_t, >)
-DO_CMP_PPZZ_D(sve_cmphi_ppzz_d, uint64_t, >)
+    set_flush_to_zero(false, fpst);
+    ret = float64_to_float16(f, true, fpst);
+    set_flush_to_zero(save, fpst);
+    return ret;
+}
 
-DO_CMP_PPZZ_B(sve_cmphs_ppzz_b, uint8_t,  >=)
-DO_CMP_PPZZ_H(sve_cmphs_ppzz_h, uint16_t, >=)
-DO_CMP_PPZZ_S(sve_cmphs_ppzz_s, uint32_t, >=)
-DO_CMP_PPZZ_D(sve_cmphs_ppzz_d, uint64_t, >=)
+static inline int16_t vfp_float16_to_int16_rtz(float16 f, float_status *s)
+{
+    if (float16_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float16_to_int16_round_to_zero(f, s);
+}
 
-#undef DO_CMP_PPZZ_B
-#undef DO_CMP_PPZZ_H
-#undef DO_CMP_PPZZ_S
-#undef DO_CMP_PPZZ_D
-#undef DO_CMP_PPZZ
+static inline int64_t vfp_float16_to_int64_rtz(float16 f, float_status *s)
+{
+    if (float16_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float16_to_int64_round_to_zero(f, s);
+}
 
-/* Similar, but the second source is "wide".  */
-#define DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H, MASK)                     \
-uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
-{                                                                            \
-    intptr_t opr_sz = simd_oprsz(desc);                                      \
-    uint32_t flags = PREDTEST_INIT;                                          \
-    intptr_t i = opr_sz;                                                     \
-    do {                                                                     \
-        uint64_t out = 0, pg;                                                \
-        do {                                                                 \
-            TYPEW mm = *(TYPEW *)((char *)vm + i - 8);                               \
-            do {                                                             \
-                i -= sizeof(TYPE), out <<= sizeof(TYPE);                     \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                              \
-                out |= nn OP mm;                                             \
-            } while (i & 7);                                                 \
-        } while (i & 63);                                                    \
-        pg = *(uint64_t *)((char *)vg + (i >> 3)) & MASK;                            \
-        out &= pg;                                                           \
-        *(uint64_t *)((char *)vd + (i >> 3)) = out;                                  \
-        flags = iter_predtest_bwd(out, pg, flags);                           \
-    } while (i > 0);                                                         \
-    return flags;                                                            \
+static inline int64_t vfp_float32_to_int64_rtz(float32 f, float_status *s)
+{
+    if (float32_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float32_to_int64_round_to_zero(f, s);
 }
 
-#define DO_CMP_PPZW_B(NAME, TYPE, TYPEW, OP) \
-    DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1,   0xffffffffffffffffull)
-#define DO_CMP_PPZW_H(NAME, TYPE, TYPEW, OP) \
-    DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1_2, 0x5555555555555555ull)
-#define DO_CMP_PPZW_S(NAME, TYPE, TYPEW, OP) \
-    DO_CMP_PPZW(NAME, TYPE, TYPEW, OP, H1_4, 0x1111111111111111ull)
+static inline int64_t vfp_float64_to_int64_rtz(float64 f, float_status *s)
+{
+    if (float64_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float64_to_int64_round_to_zero(f, s);
+}
 
-DO_CMP_PPZW_B(sve_cmpeq_ppzw_b, int8_t,  uint64_t, ==)
-DO_CMP_PPZW_H(sve_cmpeq_ppzw_h, int16_t, uint64_t, ==)
-DO_CMP_PPZW_S(sve_cmpeq_ppzw_s, int32_t, uint64_t, ==)
+static inline uint16_t vfp_float16_to_uint16_rtz(float16 f, float_status *s)
+{
+    if (float16_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float16_to_uint16_round_to_zero(f, s);
+}
 
-DO_CMP_PPZW_B(sve_cmpne_ppzw_b, int8_t,  uint64_t, !=)
-DO_CMP_PPZW_H(sve_cmpne_ppzw_h, int16_t, uint64_t, !=)
-DO_CMP_PPZW_S(sve_cmpne_ppzw_s, int32_t, uint64_t, !=)
+static inline uint64_t vfp_float16_to_uint64_rtz(float16 f, float_status *s)
+{
+    if (float16_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float16_to_uint64_round_to_zero(f, s);
+}
 
-DO_CMP_PPZW_B(sve_cmpgt_ppzw_b, int8_t,   int64_t, >)
-DO_CMP_PPZW_H(sve_cmpgt_ppzw_h, int16_t,  int64_t, >)
-DO_CMP_PPZW_S(sve_cmpgt_ppzw_s, int32_t,  int64_t, >)
+static inline uint64_t vfp_float32_to_uint64_rtz(float32 f, float_status *s)
+{
+    if (float32_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float32_to_uint64_round_to_zero(f, s);
+}
 
-DO_CMP_PPZW_B(sve_cmpge_ppzw_b, int8_t,   int64_t, >=)
-DO_CMP_PPZW_H(sve_cmpge_ppzw_h, int16_t,  int64_t, >=)
-DO_CMP_PPZW_S(sve_cmpge_ppzw_s, int32_t,  int64_t, >=)
+static inline uint64_t vfp_float64_to_uint64_rtz(float64 f, float_status *s)
+{
+    if (float64_is_any_nan(f)) {
+        float_raise(float_flag_invalid, s);
+        return 0;
+    }
+    return float64_to_uint64_round_to_zero(f, s);
+}
 
-DO_CMP_PPZW_B(sve_cmphi_ppzw_b, uint8_t,  uint64_t, >)
-DO_CMP_PPZW_H(sve_cmphi_ppzw_h, uint16_t, uint64_t, >)
-DO_CMP_PPZW_S(sve_cmphi_ppzw_s, uint32_t, uint64_t, >)
+DO_ZPZ_FP(sve_fcvt_sh, uint32_t, H1_4, sve_f32_to_f16)
+DO_ZPZ_FP(sve_fcvt_hs, uint32_t, H1_4, sve_f16_to_f32)
+DO_ZPZ_FP(sve_fcvt_dh, uint64_t,     , sve_f64_to_f16)
+DO_ZPZ_FP(sve_fcvt_hd, uint64_t,     , sve_f16_to_f64)
+DO_ZPZ_FP(sve_fcvt_ds, uint64_t,     , float64_to_float32)
+DO_ZPZ_FP(sve_fcvt_sd, uint64_t,     , float32_to_float64)
+DO_ZPZ_FP(sve_bfcvt, uint32_t, H1_4, float32_to_bfloat16)
+
+void HELPER(fmmla_s)(void *vd, void *vn, void *vm, void *va,
+                     void *status, uint32_t desc)
+{
+    intptr_t s, opr_sz = simd_oprsz(desc) / (sizeof(float32) * 4);
+
+    for (s = 0; s < opr_sz; ++s) {
+        float32 *n = (float32 *)((char *)vn + s * sizeof(float32) * 4);
+        float32 *m = (float32 *)((char *)vm + s * sizeof(float32) * 4);
+        float32 *a = (float32 *)((char *)va + s * sizeof(float32) * 4);
+        float32 *d = (float32 *)((char *)vd + s * sizeof(float32) * 4);
+        float32 n00 = n[H4(0)], n01 = n[H4(1)];
+        float32 n10 = n[H4(2)], n11 = n[H4(3)];
+        float32 m00 = m[H4(0)], m01 = m[H4(1)];
+        float32 m10 = m[H4(2)], m11 = m[H4(3)];
+        float32 p0, p1;
+
+        p0 = float32_mul(n00, m00, status);
+        p1 = float32_mul(n01, m01, status);
+        d[H4(0)] = float32_add(a[H4(0)],
+                               float32_add(p0, p1, status), status);
+
+        p0 = float32_mul(n00, m10, status);
+        p1 = float32_mul(n01, m11, status);
+        d[H4(1)] = float32_add(a[H4(1)],
+                               float32_add(p0, p1, status), status);
+
+        p0 = float32_mul(n10, m00, status);
+        p1 = float32_mul(n11, m01, status);
+        d[H4(2)] = float32_add(a[H4(2)],
+                               float32_add(p0, p1, status), status);
+
+        p0 = float32_mul(n10, m10, status);
+        p1 = float32_mul(n11, m11, status);
+        d[H4(3)] = float32_add(a[H4(3)],
+                               float32_add(p0, p1, status), status);
+    }
+}
 
-DO_CMP_PPZW_B(sve_cmphs_ppzw_b, uint8_t,  uint64_t, >=)
-DO_CMP_PPZW_H(sve_cmphs_ppzw_h, uint16_t, uint64_t, >=)
-DO_CMP_PPZW_S(sve_cmphs_ppzw_s, uint32_t, uint64_t, >=)
+void HELPER(fmmla_d)(void *vd, void *vn, void *vm, void *va,
+                     void *status, uint32_t desc)
+{
+    intptr_t s, opr_sz = simd_oprsz(desc) / (sizeof(float64) * 4);
 
-DO_CMP_PPZW_B(sve_cmplt_ppzw_b, int8_t,   int64_t, <)
-DO_CMP_PPZW_H(sve_cmplt_ppzw_h, int16_t,  int64_t, <)
-DO_CMP_PPZW_S(sve_cmplt_ppzw_s, int32_t,  int64_t, <)
+    for (s = 0; s < opr_sz; ++s) {
+        float64 *n = (float64 *)((char *)vn + s * sizeof(float64) * 4);
+        float64 *m = (float64 *)((char *)vm + s * sizeof(float64) * 4);
+        float64 *a = (float64 *)((char *)va + s * sizeof(float64) * 4);
+        float64 *d = (float64 *)((char *)vd + s * sizeof(float64) * 4);
+        float64 n00 = n[0], n01 = n[1], n10 = n[2], n11 = n[3];
+        float64 m00 = m[0], m01 = m[1], m10 = m[2], m11 = m[3];
+        float64 p0, p1;
 
-DO_CMP_PPZW_B(sve_cmple_ppzw_b, int8_t,   int64_t, <=)
-DO_CMP_PPZW_H(sve_cmple_ppzw_h, int16_t,  int64_t, <=)
-DO_CMP_PPZW_S(sve_cmple_ppzw_s, int32_t,  int64_t, <=)
+        p0 = float64_mul(n00, m00, status);
+        p1 = float64_mul(n01, m01, status);
+        d[0] = float64_add(a[0], float64_add(p0, p1, status), status);
 
-DO_CMP_PPZW_B(sve_cmplo_ppzw_b, uint8_t,  uint64_t, <)
-DO_CMP_PPZW_H(sve_cmplo_ppzw_h, uint16_t, uint64_t, <)
-DO_CMP_PPZW_S(sve_cmplo_ppzw_s, uint32_t, uint64_t, <)
+        p0 = float64_mul(n00, m10, status);
+        p1 = float64_mul(n01, m11, status);
+        d[1] = float64_add(a[1], float64_add(p0, p1, status), status);
 
-DO_CMP_PPZW_B(sve_cmpls_ppzw_b, uint8_t,  uint64_t, <=)
-DO_CMP_PPZW_H(sve_cmpls_ppzw_h, uint16_t, uint64_t, <=)
-DO_CMP_PPZW_S(sve_cmpls_ppzw_s, uint32_t, uint64_t, <=)
+        p0 = float64_mul(n10, m00, status);
+        p1 = float64_mul(n11, m01, status);
+        d[2] = float64_add(a[2], float64_add(p0, p1, status), status);
 
-#undef DO_CMP_PPZW_B
-#undef DO_CMP_PPZW_H
-#undef DO_CMP_PPZW_S
-#undef DO_CMP_PPZW
+        p0 = float64_mul(n10, m10, status);
+        p1 = float64_mul(n11, m11, status);
+        d[3] = float64_add(a[3], float64_add(p0, p1, status), status);
+    }
+}
 
-/* Similar, but the second source is immediate.  */
-#define DO_CMP_PPZI(NAME, TYPE, OP, H, MASK)                         \
-uint32_t HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc)   \
-{                                                                    \
-    intptr_t opr_sz = simd_oprsz(desc);                              \
-    uint32_t flags = PREDTEST_INIT;                                  \
-    TYPE mm = simd_data(desc);                                       \
-    intptr_t i = opr_sz;                                             \
-    do {                                                             \
-        uint64_t out = 0, pg;                                        \
-        do {                                                         \
-            i -= sizeof(TYPE), out <<= sizeof(TYPE);                 \
-            TYPE nn = *(TYPE *)((char *)vn + H(i));                          \
-            out |= nn OP mm;                                         \
-        } while (i & 63);                                            \
-        pg = *(uint64_t *)((char *)vg + (i >> 3)) & MASK;                    \
-        out &= pg;                                                   \
-        *(uint64_t *)((char *)vd + (i >> 3)) = out;                          \
-        flags = iter_predtest_bwd(out, pg, flags);                   \
-    } while (i > 0);                                                 \
-    return flags;                                                    \
+#define DO_FCVTNT(NAME, TYPEW, TYPEN, HW, HN, OP)                    \
+void HELPER(NAME)(void *vd, void *vn, void *vg, void *status,         \
+                  uint32_t desc)                                      \
+{                                                                     \
+    intptr_t i = simd_oprsz(desc);                                    \
+    uint64_t *g = vg;                                                 \
+    do {                                                              \
+        uint64_t pg = g[(i - 1) >> 6];                                \
+        do {                                                          \
+            i -= sizeof(TYPEW);                                       \
+            if (likely((pg >> (i & 63)) & 1)) {                       \
+                TYPEW nn = *(TYPEW *)((char *)vn + HW(i));            \
+                *(TYPEN *)((char *)vd + HN(i + sizeof(TYPEN))) =      \
+                    OP(nn, status);                                   \
+            }                                                         \
+        } while (i & 63);                                             \
+    } while (i != 0);                                                 \
 }
 
-#define DO_CMP_PPZI_B(NAME, TYPE, OP) \
-    DO_CMP_PPZI(NAME, TYPE, OP, H1,   0xffffffffffffffffull)
-#define DO_CMP_PPZI_H(NAME, TYPE, OP) \
-    DO_CMP_PPZI(NAME, TYPE, OP, H1_2, 0x5555555555555555ull)
-#define DO_CMP_PPZI_S(NAME, TYPE, OP) \
-    DO_CMP_PPZI(NAME, TYPE, OP, H1_4, 0x1111111111111111ull)
-#define DO_CMP_PPZI_D(NAME, TYPE, OP) \
-    DO_CMP_PPZI(NAME, TYPE, OP,     , 0x0101010101010101ull)
+DO_FCVTNT(sve2_fcvtnt_sh, uint32_t, uint16_t, H1_4, H1_2,
+          sve_f32_to_f16)
+DO_FCVTNT(sve2_fcvtnt_ds, uint64_t, uint32_t, H1_8, H1_4,
+          float64_to_float32)
+DO_FCVTNT(sve_bfcvtnt, uint32_t, uint16_t, H1_4, H1_2,
+          float32_to_bfloat16)
 
-DO_CMP_PPZI_B(sve_cmpeq_ppzi_b, uint8_t,  ==)
-DO_CMP_PPZI_H(sve_cmpeq_ppzi_h, uint16_t, ==)
-DO_CMP_PPZI_S(sve_cmpeq_ppzi_s, uint32_t, ==)
-DO_CMP_PPZI_D(sve_cmpeq_ppzi_d, uint64_t, ==)
+#define DO_FCVTLT(NAME, TYPEW, TYPEN, HW, HN, OP)                    \
+void HELPER(NAME)(void *vd, void *vn, void *vg, void *status,         \
+                  uint32_t desc)                                      \
+{                                                                     \
+    intptr_t i = simd_oprsz(desc);                                    \
+    uint64_t *g = vg;                                                 \
+    do {                                                              \
+        uint64_t pg = g[(i - 1) >> 6];                                \
+        do {                                                          \
+            i -= sizeof(TYPEW);                                       \
+            if (likely((pg >> (i & 63)) & 1)) {                       \
+                TYPEN nn = *(TYPEN *)((char *)vn +                    \
+                                      HN(i + sizeof(TYPEN)));         \
+                *(TYPEW *)((char *)vd + HW(i)) = OP(nn, status);      \
+            }                                                         \
+        } while (i & 63);                                             \
+    } while (i != 0);                                                 \
+}
 
-DO_CMP_PPZI_B(sve_cmpne_ppzi_b, uint8_t,  !=)
-DO_CMP_PPZI_H(sve_cmpne_ppzi_h, uint16_t, !=)
-DO_CMP_PPZI_S(sve_cmpne_ppzi_s, uint32_t, !=)
-DO_CMP_PPZI_D(sve_cmpne_ppzi_d, uint64_t, !=)
+DO_FCVTLT(sve2_fcvtlt_hs, uint32_t, uint16_t, H1_4, H1_2,
+          sve_f16_to_f32)
+DO_FCVTLT(sve2_fcvtlt_sd, uint64_t, uint32_t, H1_8, H1_4,
+          float32_to_float64)
 
-DO_CMP_PPZI_B(sve_cmpgt_ppzi_b, int8_t,  >)
-DO_CMP_PPZI_H(sve_cmpgt_ppzi_h, int16_t, >)
-DO_CMP_PPZI_S(sve_cmpgt_ppzi_s, int32_t, >)
-DO_CMP_PPZI_D(sve_cmpgt_ppzi_d, int64_t, >)
+#undef DO_FCVTLT
+#undef DO_FCVTNT
 
-DO_CMP_PPZI_B(sve_cmpge_ppzi_b, int8_t,  >=)
-DO_CMP_PPZI_H(sve_cmpge_ppzi_h, int16_t, >=)
-DO_CMP_PPZI_S(sve_cmpge_ppzi_s, int32_t, >=)
-DO_CMP_PPZI_D(sve_cmpge_ppzi_d, int64_t, >=)
+DO_ZPZ_FP(sve_fcvtzs_hh, uint16_t, H1_2, vfp_float16_to_int16_rtz)
+DO_ZPZ_FP(sve_fcvtzs_hs, uint32_t, H1_4, helper_vfp_tosizh)
+DO_ZPZ_FP(sve_fcvtzs_ss, uint32_t, H1_4, helper_vfp_tosizs)
+DO_ZPZ_FP(sve_fcvtzs_hd, uint64_t,     , vfp_float16_to_int64_rtz)
+DO_ZPZ_FP(sve_fcvtzs_sd, uint64_t,     , vfp_float32_to_int64_rtz)
+DO_ZPZ_FP(sve_fcvtzs_ds, uint64_t,     , helper_vfp_tosizd)
+DO_ZPZ_FP(sve_fcvtzs_dd, uint64_t,     , vfp_float64_to_int64_rtz)
 
-DO_CMP_PPZI_B(sve_cmphi_ppzi_b, uint8_t,  >)
-DO_CMP_PPZI_H(sve_cmphi_ppzi_h, uint16_t, >)
-DO_CMP_PPZI_S(sve_cmphi_ppzi_s, uint32_t, >)
-DO_CMP_PPZI_D(sve_cmphi_ppzi_d, uint64_t, >)
+DO_ZPZ_FP(sve_fcvtzu_hh, uint16_t, H1_2, vfp_float16_to_uint16_rtz)
+DO_ZPZ_FP(sve_fcvtzu_hs, uint32_t, H1_4, helper_vfp_touizh)
+DO_ZPZ_FP(sve_fcvtzu_ss, uint32_t, H1_4, helper_vfp_touizs)
+DO_ZPZ_FP(sve_fcvtzu_hd, uint64_t,     , vfp_float16_to_uint64_rtz)
+DO_ZPZ_FP(sve_fcvtzu_sd, uint64_t,     , vfp_float32_to_uint64_rtz)
+DO_ZPZ_FP(sve_fcvtzu_ds, uint64_t,     , helper_vfp_touizd)
+DO_ZPZ_FP(sve_fcvtzu_dd, uint64_t,     , vfp_float64_to_uint64_rtz)
 
-DO_CMP_PPZI_B(sve_cmphs_ppzi_b, uint8_t,  >=)
-DO_CMP_PPZI_H(sve_cmphs_ppzi_h, uint16_t, >=)
-DO_CMP_PPZI_S(sve_cmphs_ppzi_s, uint32_t, >=)
-DO_CMP_PPZI_D(sve_cmphs_ppzi_d, uint64_t, >=)
+DO_ZPZ_FP(sve_frint_h, uint16_t, H1_2, helper_advsimd_rinth)
+DO_ZPZ_FP(sve_frint_s, uint32_t, H1_4, helper_rints)
+DO_ZPZ_FP(sve_frint_d, uint64_t,     , helper_rintd)
 
-DO_CMP_PPZI_B(sve_cmplt_ppzi_b, int8_t,  <)
-DO_CMP_PPZI_H(sve_cmplt_ppzi_h, int16_t, <)
-DO_CMP_PPZI_S(sve_cmplt_ppzi_s, int32_t, <)
-DO_CMP_PPZI_D(sve_cmplt_ppzi_d, int64_t, <)
+DO_ZPZ_FP(sve_frintx_h, uint16_t, H1_2, float16_round_to_int)
+DO_ZPZ_FP(sve_frintx_s, uint32_t, H1_4, float32_round_to_int)
+DO_ZPZ_FP(sve_frintx_d, uint64_t,     , float64_round_to_int)
 
-DO_CMP_PPZI_B(sve_cmple_ppzi_b, int8_t,  <=)
-DO_CMP_PPZI_H(sve_cmple_ppzi_h, int16_t, <=)
-DO_CMP_PPZI_S(sve_cmple_ppzi_s, int32_t, <=)
-DO_CMP_PPZI_D(sve_cmple_ppzi_d, int64_t, <=)
+DO_ZPZ_FP(sve_frecpx_h, uint16_t, H1_2, helper_frecpx_f16)
+DO_ZPZ_FP(sve_frecpx_s, uint32_t, H1_4, helper_frecpx_f32)
+DO_ZPZ_FP(sve_frecpx_d, uint64_t,     , helper_frecpx_f64)
 
-DO_CMP_PPZI_B(sve_cmplo_ppzi_b, uint8_t,  <)
-DO_CMP_PPZI_H(sve_cmplo_ppzi_h, uint16_t, <)
-DO_CMP_PPZI_S(sve_cmplo_ppzi_s, uint32_t, <)
-DO_CMP_PPZI_D(sve_cmplo_ppzi_d, uint64_t, <)
+DO_ZPZ_FP(sve_fsqrt_h, uint16_t, H1_2, float16_sqrt)
+DO_ZPZ_FP(sve_fsqrt_s, uint32_t, H1_4, float32_sqrt)
+DO_ZPZ_FP(sve_fsqrt_d, uint64_t,     , float64_sqrt)
 
-DO_CMP_PPZI_B(sve_cmpls_ppzi_b, uint8_t,  <=)
-DO_CMP_PPZI_H(sve_cmpls_ppzi_h, uint16_t, <=)
-DO_CMP_PPZI_S(sve_cmpls_ppzi_s, uint32_t, <=)
-DO_CMP_PPZI_D(sve_cmpls_ppzi_d, uint64_t, <=)
+DO_ZPZ_FP(sve_scvt_hh, uint16_t, H1_2, int16_to_float16)
+DO_ZPZ_FP(sve_scvt_sh, uint32_t, H1_4, int32_to_float16)
+DO_ZPZ_FP(sve_scvt_ss, uint32_t, H1_4, int32_to_float32)
+DO_ZPZ_FP(sve_scvt_sd, uint64_t,     , int32_to_float64)
+DO_ZPZ_FP(sve_scvt_dh, uint64_t,     , int64_to_float16)
+DO_ZPZ_FP(sve_scvt_ds, uint64_t,     , int64_to_float32)
+DO_ZPZ_FP(sve_scvt_dd, uint64_t,     , int64_to_float64)
 
-#undef DO_CMP_PPZI_B
-#undef DO_CMP_PPZI_H
-#undef DO_CMP_PPZI_S
-#undef DO_CMP_PPZI_D
-#undef DO_CMP_PPZI
+DO_ZPZ_FP(sve_ucvt_hh, uint16_t, H1_2, uint16_to_float16)
+DO_ZPZ_FP(sve_ucvt_sh, uint32_t, H1_4, uint32_to_float16)
+DO_ZPZ_FP(sve_ucvt_ss, uint32_t, H1_4, uint32_to_float32)
+DO_ZPZ_FP(sve_ucvt_sd, uint64_t,     , uint32_to_float64)
+DO_ZPZ_FP(sve_ucvt_dh, uint64_t,     , uint64_to_float16)
+DO_ZPZ_FP(sve_ucvt_ds, uint64_t,     , uint64_to_float32)
+DO_ZPZ_FP(sve_ucvt_dd, uint64_t,     , uint64_to_float64)
 
-/* Similar to the ARM LastActive pseudocode function.  */
-static bool last_active_pred(void *vd, void *vg, intptr_t oprsz)
+static int16_t do_float16_logb_as_int(float16 a, float_status *s)
 {
-    intptr_t i;
+    uint32_t frac = (uint32_t)a << (16 + 6);
+    int16_t exp = extract32(a, 10, 5);
 
-    for (i = QEMU_ALIGN_UP(oprsz, 8) - 8; i >= 0; i -= 8) {
-        uint64_t pg = *(uint64_t *)((char *)vg + i);
-        if (pg) {
-            return (pow2floor(pg) & *(uint64_t *)((char *)vd + i)) != 0;
+    if (unlikely(exp == 0)) {
+        if (frac != 0) {
+            if (!get_flush_inputs_to_zero(s)) {
+                return -15 - clz32(frac);
+            }
+            float_raise(float_flag_input_denormal, s);
+        }
+    } else if (unlikely(exp == 0x1f)) {
+        if (frac == 0) {
+            return INT16_MAX;
         }
+    } else {
+        return exp - 15;
     }
-    return 0;
+
+    float_raise(float_flag_invalid, s);
+    return INT16_MIN;
 }
 
-/* Compute a mask into RETB that is true for all G, up to and including
- * (if after) or excluding (if !after) the first G & N.
- * Return true if BRK found.
- */
-static bool compute_brk(uint64_t *retb, uint64_t n, uint64_t g,
-                        bool brk, bool after)
+static int32_t do_float32_logb_as_int(float32 a, float_status *s)
 {
-    uint64_t b;
+    uint32_t frac = a << 9;
+    int32_t exp = extract32(a, 23, 8);
 
-    if (brk) {
-        b = 0;
-    } else if ((g & n) == 0) {
-        /* For all G, no N are set; break not found.  */
-        b = g;
-    } else {
-        /* Break somewhere in N.  Locate it.  */
-        b = g & n;            /* guard true, pred true */
-#ifdef _MSC_VER
-        b = b & (0 - b);      /* first such */
-#else
-        b = b & -b;           /* first such */
-#endif
-        if (after) {
-            b = b | (b - 1);  /* break after same */
-        } else {
-            b = b - 1;        /* break before same */
+    if (unlikely(exp == 0)) {
+        if (frac != 0) {
+            if (!get_flush_inputs_to_zero(s)) {
+                return -127 - clz32(frac);
+            }
+            float_raise(float_flag_input_denormal, s);
         }
-        brk = true;
+    } else if (unlikely(exp == 0xff)) {
+        if (frac == 0) {
+            return INT32_MAX;
+        }
+    } else {
+        return exp - 127;
     }
 
-    *retb = b;
-    return brk;
+    float_raise(float_flag_invalid, s);
+    return INT32_MIN;
 }
 
-/* Compute a zeroing BRK.  */
-static void compute_brk_z(uint64_t *d, uint64_t *n, uint64_t *g,
-                          intptr_t oprsz, bool after)
+static int64_t do_float64_logb_as_int(float64 a, float_status *s)
 {
-    bool brk = false;
-    intptr_t i;
+    uint64_t frac = a << 12;
+    int64_t exp = extract64(a, 52, 11);
 
-    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
-        uint64_t this_b, this_g = g[i];
-
-        brk = compute_brk(&this_b, n[i], this_g, brk, after);
-        d[i] = this_b & this_g;
+    if (unlikely(exp == 0)) {
+        if (frac != 0) {
+            if (!get_flush_inputs_to_zero(s)) {
+                return -1023 - clz64(frac);
+            }
+            float_raise(float_flag_input_denormal, s);
+        }
+    } else if (unlikely(exp == 0x7ff)) {
+        if (frac == 0) {
+            return INT64_MAX;
+        }
+    } else {
+        return exp - 1023;
     }
-}
-
-/* Likewise, but also compute flags.  */
-static uint32_t compute_brks_z(uint64_t *d, uint64_t *n, uint64_t *g,
-                               intptr_t oprsz, bool after)
-{
-    uint32_t flags = PREDTEST_INIT;
-    bool brk = false;
-    intptr_t i;
-
-    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
-        uint64_t this_b, this_d, this_g = g[i];
 
-        brk = compute_brk(&this_b, n[i], this_g, brk, after);
-        d[i] = this_d = this_b & this_g;
-        flags = iter_predtest_fwd(this_d, this_g, flags);
-    }
-    return flags;
+    float_raise(float_flag_invalid, s);
+    return INT64_MIN;
 }
 
-/* Compute a merging BRK.  */
-static void compute_brk_m(uint64_t *d, uint64_t *n, uint64_t *g,
-                          intptr_t oprsz, bool after)
-{
-    bool brk = false;
-    intptr_t i;
+DO_ZPZ_FP(flogb_h, float16, H1_2, do_float16_logb_as_int)
+DO_ZPZ_FP(flogb_s, float32, H1_4, do_float32_logb_as_int)
+DO_ZPZ_FP(flogb_d, float64,     , do_float64_logb_as_int)
 
-    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
-        uint64_t this_b, this_g = g[i];
+#undef DO_ZPZ_FP
 
-        brk = compute_brk(&this_b, n[i], this_g, brk, after);
-        d[i] = (this_b & this_g) | (d[i] & ~this_g);
-    }
-}
+/* 4-operand predicated multiply-add.  This requires 7 operands to pass
+ * "properly", so we need to encode some of the registers into DESC.
+ */
+QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 20 > 32);
 
-/* Likewise, but also compute flags.  */
-static uint32_t compute_brks_m(uint64_t *d, uint64_t *n, uint64_t *g,
-                               intptr_t oprsz, bool after)
+static void do_fmla_zpzzz_h(CPUARMState *env, void *vg, uint32_t desc,
+                            uint16_t neg1, uint16_t neg3)
 {
-    uint32_t flags = PREDTEST_INIT;
-    bool brk = false;
-    intptr_t i;
+    intptr_t i = simd_oprsz(desc);
+    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+    void *vd = &env->vfp.zregs[rd];
+    void *vn = &env->vfp.zregs[rn];
+    void *vm = &env->vfp.zregs[rm];
+    void *va = &env->vfp.zregs[ra];
+    uint64_t *g = vg;
 
-    for (i = 0; i < oprsz / 8; ++i) {
-        uint64_t this_b, this_d = d[i], this_g = g[i];
+    do {
+        uint64_t pg = g[(i - 1) >> 6];
+        do {
+            i -= 2;
+            if (likely((pg >> (i & 63)) & 1)) {
+                float16 e1, e2, e3, r;
 
-        brk = compute_brk(&this_b, n[i], this_g, brk, after);
-        d[i] = this_d = (this_b & this_g) | (this_d & ~this_g);
-        flags = iter_predtest_fwd(this_d, this_g, flags);
-    }
-    return flags;
+                e1 = *(uint16_t *)((char *)vn + H1_2(i)) ^ neg1;
+                e2 = *(uint16_t *)((char *)vm + H1_2(i));
+                e3 = *(uint16_t *)((char *)va + H1_2(i)) ^ neg3;
+                r = float16_muladd(e1, e2, e3, 0, &env->vfp.fp_status_f16);
+                *(uint16_t *)((char *)vd + H1_2(i)) = r;
+            }
+        } while (i & 63);
+    } while (i != 0);
 }
 
-static uint32_t do_zero(ARMPredicateReg *d, intptr_t oprsz)
+void HELPER(sve_fmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    /* It is quicker to zero the whole predicate than loop on OPRSZ.
-     * The compiler should turn this into 4 64-bit integer stores.
-     */
-    memset(d, 0, sizeof(ARMPredicateReg));
-    return PREDTEST_INIT;
+    do_fmla_zpzzz_h(env, vg, desc, 0, 0);
 }
 
-void HELPER(sve_brkpa)(void *vd, void *vn, void *vm, void *vg,
-                       uint32_t pred_desc)
+void HELPER(sve_fmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    if (last_active_pred(vn, vg, oprsz)) {
-        compute_brk_z(vd, vm, vg, oprsz, true);
-    } else {
-        do_zero(vd, oprsz);
-    }
+    do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0);
 }
 
-uint32_t HELPER(sve_brkpas)(void *vd, void *vn, void *vm, void *vg,
-                            uint32_t pred_desc)
+void HELPER(sve_fnmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    if (last_active_pred(vn, vg, oprsz)) {
-        return compute_brks_z(vd, vm, vg, oprsz, true);
-    } else {
-        return do_zero(vd, oprsz);
-    }
+    do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0x8000);
 }
 
-void HELPER(sve_brkpb)(void *vd, void *vn, void *vm, void *vg,
-                       uint32_t pred_desc)
+void HELPER(sve_fnmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    if (last_active_pred(vn, vg, oprsz)) {
-        compute_brk_z(vd, vm, vg, oprsz, false);
-    } else {
-        do_zero(vd, oprsz);
-    }
+    do_fmla_zpzzz_h(env, vg, desc, 0, 0x8000);
 }
 
-uint32_t HELPER(sve_brkpbs)(void *vd, void *vn, void *vm, void *vg,
-                            uint32_t pred_desc)
+static void do_fmla_zpzzz_s(CPUARMState *env, void *vg, uint32_t desc,
+                            uint32_t neg1, uint32_t neg3)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    if (last_active_pred(vn, vg, oprsz)) {
-        return compute_brks_z(vd, vm, vg, oprsz, false);
-    } else {
-        return do_zero(vd, oprsz);
-    }
-}
+    intptr_t i = simd_oprsz(desc);
+    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+    void *vd = &env->vfp.zregs[rd];
+    void *vn = &env->vfp.zregs[rn];
+    void *vm = &env->vfp.zregs[rm];
+    void *va = &env->vfp.zregs[ra];
+    uint64_t *g = vg;
 
-void HELPER(sve_brka_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    compute_brk_z(vd, vn, vg, oprsz, true);
+    do {
+        uint64_t pg = g[(i - 1) >> 6];
+        do {
+            i -= 4;
+            if (likely((pg >> (i & 63)) & 1)) {
+                float32 e1, e2, e3, r;
+
+                e1 = *(uint32_t *)((char *)vn + H1_4(i)) ^ neg1;
+                e2 = *(uint32_t *)((char *)vm + H1_4(i));
+                e3 = *(uint32_t *)((char *)va + H1_4(i)) ^ neg3;
+                r = float32_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
+                *(uint32_t *)((char *)vd + H1_4(i)) = r;
+            }
+        } while (i & 63);
+    } while (i != 0);
 }
 
-uint32_t HELPER(sve_brkas_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+void HELPER(sve_fmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    return compute_brks_z(vd, vn, vg, oprsz, true);
+    do_fmla_zpzzz_s(env, vg, desc, 0, 0);
 }
 
-void HELPER(sve_brkb_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+void HELPER(sve_fmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    compute_brk_z(vd, vn, vg, oprsz, false);
+    do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0);
 }
 
-uint32_t HELPER(sve_brkbs_z)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+void HELPER(sve_fnmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    return compute_brks_z(vd, vn, vg, oprsz, false);
+    do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0x80000000);
 }
 
-void HELPER(sve_brka_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+void HELPER(sve_fnmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    compute_brk_m(vd, vn, vg, oprsz, true);
+    do_fmla_zpzzz_s(env, vg, desc, 0, 0x80000000);
 }
 
-uint32_t HELPER(sve_brkas_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+static void do_fmla_zpzzz_d(CPUARMState *env, void *vg, uint32_t desc,
+                            uint64_t neg1, uint64_t neg3)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    return compute_brks_m(vd, vn, vg, oprsz, true);
+    intptr_t i = simd_oprsz(desc);
+    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+    void *vd = &env->vfp.zregs[rd];
+    void *vn = &env->vfp.zregs[rn];
+    void *vm = &env->vfp.zregs[rm];
+    void *va = &env->vfp.zregs[ra];
+    uint64_t *g = vg;
+
+    do {
+        uint64_t pg = g[(i - 1) >> 6];
+        do {
+            i -= 8;
+            if (likely((pg >> (i & 63)) & 1)) {
+                float64 e1, e2, e3, r;
+
+                e1 = *(uint64_t *)((char *)vn + i) ^ neg1;
+                e2 = *(uint64_t *)((char *)vm + i);
+                e3 = *(uint64_t *)((char *)va + i) ^ neg3;
+                r = float64_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
+                *(uint64_t *)((char *)vd + i) = r;
+            }
+        } while (i & 63);
+    } while (i != 0);
 }
 
-void HELPER(sve_brkb_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+void HELPER(sve_fmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    compute_brk_m(vd, vn, vg, oprsz, false);
+    do_fmla_zpzzz_d(env, vg, desc, 0, 0);
 }
 
-uint32_t HELPER(sve_brkbs_m)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+void HELPER(sve_fmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    return compute_brks_m(vd, vn, vg, oprsz, false);
+    do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, 0);
 }
 
-void HELPER(sve_brkn)(void *vd, void *vn, void *vg, uint32_t pred_desc)
+void HELPER(sve_fnmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-
-    if (!last_active_pred(vn, vg, oprsz)) {
-        do_zero(vd, oprsz);
-    }
+    do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, INT64_MIN);
 }
 
-/* As if PredTest(Ones(PL), D, esz).  */
-static uint32_t predtest_ones(ARMPredicateReg *d, intptr_t oprsz,
-                              uint64_t esz_mask)
+void HELPER(sve_fnmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    uint32_t flags = PREDTEST_INIT;
-    intptr_t i;
+    do_fmla_zpzzz_d(env, vg, desc, 0, INT64_MIN);
+}
 
-    for (i = 0; i < oprsz / 8; i++) {
-        flags = iter_predtest_fwd(d->p[i], esz_mask, flags);
-    }
-    if (oprsz & 7) {
-        uint64_t mask = ~(0xffffffffffffffffULL << (8 * (oprsz & 7)));
-        flags = iter_predtest_fwd(d->p[i], esz_mask & mask, flags);
-    }
-    return flags;
+/* Two operand floating-point comparison controlled by a predicate.
+ * Unlike the integer version, we are not allowed to optimistically
+ * compare operands, since the comparison may have side effects wrt
+ * the FPSR.
+ */
+#define DO_FPCMP_PPZZ(NAME, TYPE, H, OP)                                \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg,               \
+                  void *status, uint32_t desc)                          \
+{                                                                       \
+    intptr_t i = simd_oprsz(desc), j = (i - 1) >> 6;                    \
+    uint64_t *d = vd, *g = vg;                                          \
+    do {                                                                \
+        uint64_t out = 0, pg = g[j];                                    \
+        do {                                                            \
+            i -= sizeof(TYPE), out <<= sizeof(TYPE);                    \
+            if (likely((pg >> (i & 63)) & 1)) {                         \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));                         \
+                TYPE mm = *(TYPE *)((char *)vm + H(i));                         \
+                out |= OP(TYPE, nn, mm, status);                        \
+            }                                                           \
+        } while (i & 63);                                               \
+        d[j--] = out;                                                   \
+    } while (i > 0);                                                    \
 }
 
-uint32_t HELPER(sve_brkns)(void *vd, void *vn, void *vg, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
+#define DO_FPCMP_PPZZ_H(NAME, OP) \
+    DO_FPCMP_PPZZ(NAME##_h, float16, H1_2, OP)
+#define DO_FPCMP_PPZZ_S(NAME, OP) \
+    DO_FPCMP_PPZZ(NAME##_s, float32, H1_4, OP)
+#define DO_FPCMP_PPZZ_D(NAME, OP) \
+    DO_FPCMP_PPZZ(NAME##_d, float64,     , OP)
 
-    if (last_active_pred(vn, vg, oprsz)) {
-        return predtest_ones(vd, oprsz, -1);
-    } else {
-        return do_zero(vd, oprsz);
-    }
-}
+#define DO_FPCMP_PPZZ_ALL(NAME, OP) \
+    DO_FPCMP_PPZZ_H(NAME, OP)   \
+    DO_FPCMP_PPZZ_S(NAME, OP)   \
+    DO_FPCMP_PPZZ_D(NAME, OP)
 
-uint64_t HELPER(sve_cntp)(void *vn, void *vg, uint32_t pred_desc)
-{
-    intptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
-    uint64_t *n = vn, *g = vg, sum = 0, mask = pred_esz_masks[esz];
-    intptr_t i;
+#define DO_FCMGE(TYPE, X, Y, ST)  TYPE##_compare(Y, X, ST) <= 0
+#define DO_FCMGT(TYPE, X, Y, ST)  TYPE##_compare(Y, X, ST) < 0
+#define DO_FCMLE(TYPE, X, Y, ST)  TYPE##_compare(X, Y, ST) <= 0
+#define DO_FCMLT(TYPE, X, Y, ST)  TYPE##_compare(X, Y, ST) < 0
+#define DO_FCMEQ(TYPE, X, Y, ST)  TYPE##_compare_quiet(X, Y, ST) == 0
+#define DO_FCMNE(TYPE, X, Y, ST)  TYPE##_compare_quiet(X, Y, ST) != 0
+#define DO_FCMUO(TYPE, X, Y, ST)  \
+    TYPE##_compare_quiet(X, Y, ST) == float_relation_unordered
+#define DO_FACGE(TYPE, X, Y, ST)  \
+    TYPE##_compare(TYPE##_abs(Y), TYPE##_abs(X), ST) <= 0
+#define DO_FACGT(TYPE, X, Y, ST)  \
+    TYPE##_compare(TYPE##_abs(Y), TYPE##_abs(X), ST) < 0
 
-    for (i = 0; i < DIV_ROUND_UP(oprsz, 8); ++i) {
-        uint64_t t = n[i] & g[i] & mask;
-        sum += ctpop64(t);
-    }
-    return sum;
+DO_FPCMP_PPZZ_ALL(sve_fcmge, DO_FCMGE)
+DO_FPCMP_PPZZ_ALL(sve_fcmgt, DO_FCMGT)
+DO_FPCMP_PPZZ_ALL(sve_fcmeq, DO_FCMEQ)
+DO_FPCMP_PPZZ_ALL(sve_fcmne, DO_FCMNE)
+DO_FPCMP_PPZZ_ALL(sve_fcmuo, DO_FCMUO)
+DO_FPCMP_PPZZ_ALL(sve_facge, DO_FACGE)
+DO_FPCMP_PPZZ_ALL(sve_facgt, DO_FACGT)
+
+#undef DO_FPCMP_PPZZ_ALL
+#undef DO_FPCMP_PPZZ_D
+#undef DO_FPCMP_PPZZ_S
+#undef DO_FPCMP_PPZZ_H
+#undef DO_FPCMP_PPZZ
+
+/* One operand floating-point comparison against zero, controlled
+ * by a predicate.
+ */
+#define DO_FPCMP_PPZ0(NAME, TYPE, H, OP)                   \
+void HELPER(NAME)(void *vd, void *vn, void *vg,            \
+                  void *status, uint32_t desc)             \
+{                                                          \
+    intptr_t i = simd_oprsz(desc), j = (i - 1) >> 6;       \
+    uint64_t *d = vd, *g = vg;                             \
+    do {                                                   \
+        uint64_t out = 0, pg = g[j];                       \
+        do {                                               \
+            i -= sizeof(TYPE), out <<= sizeof(TYPE);       \
+            if ((pg >> (i & 63)) & 1) {                    \
+                TYPE nn = *(TYPE *)((char *)vn + H(i));            \
+                out |= OP(TYPE, nn, 0, status);            \
+            }                                              \
+        } while (i & 63);                                  \
+        d[j--] = out;                                      \
+    } while (i > 0);                                       \
 }
 
-uint32_t HELPER(sve_while)(void *vd, uint32_t count, uint32_t pred_desc)
-{
-    uintptr_t oprsz = extract32(pred_desc, 0, SIMD_OPRSZ_BITS) + 2;
-    intptr_t esz = extract32(pred_desc, SIMD_DATA_SHIFT, 2);
-    uint64_t esz_mask = pred_esz_masks[esz];
-    ARMPredicateReg *d = vd;
-    uint32_t flags;
-    intptr_t i;
+#define DO_FPCMP_PPZ0_H(NAME, OP) \
+    DO_FPCMP_PPZ0(NAME##_h, float16, H1_2, OP)
+#define DO_FPCMP_PPZ0_S(NAME, OP) \
+    DO_FPCMP_PPZ0(NAME##_s, float32, H1_4, OP)
+#define DO_FPCMP_PPZ0_D(NAME, OP) \
+    DO_FPCMP_PPZ0(NAME##_d, float64,     , OP)
 
-    /* Begin with a zero predicate register.  */
-    flags = do_zero(d, oprsz);
-    if (count == 0) {
-        return flags;
-    }
+#define DO_FPCMP_PPZ0_ALL(NAME, OP) \
+    DO_FPCMP_PPZ0_H(NAME, OP)   \
+    DO_FPCMP_PPZ0_S(NAME, OP)   \
+    DO_FPCMP_PPZ0_D(NAME, OP)
 
-    /* Set all of the requested bits.  */
-    for (i = 0; i < count / 64; ++i) {
-        d->p[i] = esz_mask;
-    }
-    if (count & 63) {
-        d->p[i] = MAKE_64BIT_MASK(0, count & 63) & esz_mask;
+DO_FPCMP_PPZ0_ALL(sve_fcmge0, DO_FCMGE)
+DO_FPCMP_PPZ0_ALL(sve_fcmgt0, DO_FCMGT)
+DO_FPCMP_PPZ0_ALL(sve_fcmle0, DO_FCMLE)
+DO_FPCMP_PPZ0_ALL(sve_fcmlt0, DO_FCMLT)
+DO_FPCMP_PPZ0_ALL(sve_fcmeq0, DO_FCMEQ)
+DO_FPCMP_PPZ0_ALL(sve_fcmne0, DO_FCMNE)
+
+/* FP Trig Multiply-Add. */
+
+void HELPER(sve_ftmad_h)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
+{
+    static const float16 coeff[16] = {
+        0x3c00, 0xb155, 0x2030, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+        0x3c00, 0xb800, 0x293a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    };
+    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float16);
+    intptr_t x = simd_data(desc);
+    float16 *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i++) {
+        float16 mm = m[i];
+        intptr_t xx = x;
+        if (float16_is_neg(mm)) {
+            mm = float16_abs(mm);
+            xx += 8;
+        }
+        d[i] = float16_muladd(n[i], mm, coeff[xx], 0, vs);
     }
+}
 
-    return predtest_ones(d, oprsz, esz_mask);
+void HELPER(sve_ftmad_s)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
+{
+    static const float32 coeff[16] = {
+        0x3f800000, 0xbe2aaaab, 0x3c088886, 0xb95008b9,
+        0x36369d6d, 0x00000000, 0x00000000, 0x00000000,
+        0x3f800000, 0xbf000000, 0x3d2aaaa6, 0xbab60705,
+        0x37cd37cc, 0x00000000, 0x00000000, 0x00000000,
+    };
+    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float32);
+    intptr_t x = simd_data(desc);
+    float32 *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i++) {
+        float32 mm = m[i];
+        intptr_t xx = x;
+        if (float32_is_neg(mm)) {
+            mm = float32_abs(mm);
+            xx += 8;
+        }
+        d[i] = float32_muladd(n[i], mm, coeff[xx], 0, vs);
+    }
 }
 
-/* Recursive reduction on a function;
- * C.f. the ARM ARM function ReducePredicated.
- *
- * While it would be possible to write this without the DATA temporary,
- * it is much simpler to process the predicate register this way.
- * The recursion is bounded to depth 7 (128 fp16 elements), so there's
- * little to gain with a more complex non-recursive form.
- */
-#define DO_REDUCE(NAME, TYPE, H, FUNC, IDENT)                         \
-static TYPE NAME##_reduce(TYPE *data, float_status *status, uintptr_t n) \
-{                                                                     \
-    if (n == 1) {                                                     \
-        return *data;                                                 \
-    } else {                                                          \
-        uintptr_t half = n / 2;                                       \
-        TYPE lo = NAME##_reduce(data, status, half);                  \
-        TYPE hi = NAME##_reduce(data + half, status, half);           \
-        return TYPE##_##FUNC(lo, hi, status);                         \
-    }                                                                 \
-}                                                                     \
-uint64_t HELPER(NAME)(void *vn, void *vg, void *vs, uint32_t desc)    \
-{                                                                     \
-    uintptr_t i, oprsz = simd_oprsz(desc), maxsz = simd_maxsz(desc);  \
-    TYPE data[sizeof(ARMVectorReg) / sizeof(TYPE)];                   \
-    for (i = 0; i < oprsz; ) {                                        \
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));               \
-        do {                                                          \
-            TYPE nn = *(TYPE *)((char *)vn + H(i));                           \
-            *(TYPE *)((char *)data + i) = (pg & 1 ? nn : IDENT);      \
-            i += sizeof(TYPE), pg >>= sizeof(TYPE);                   \
-        } while (i & 15);                                             \
-    }                                                                 \
-    for (; i < maxsz; i += sizeof(TYPE)) {                            \
-        *(TYPE *)((char *)data + i) = IDENT;                          \
-    }                                                                 \
-    return NAME##_reduce(data, vs, maxsz / sizeof(TYPE));             \
+void HELPER(sve_ftmad_d)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
+{
+    static const float64 coeff[16] = {
+        0x3ff0000000000000ull, 0xbfc5555555555543ull,
+        0x3f8111111110f30cull, 0xbf2a01a019b92fc6ull,
+        0x3ec71de351f3d22bull, 0xbe5ae5e2b60f7b91ull,
+        0x3de5d8408868552full, 0x0000000000000000ull,
+        0x3ff0000000000000ull, 0xbfe0000000000000ull,
+        0x3fa5555555555536ull, 0xbf56c16c16c13a0bull,
+        0x3efa01a019b1e8d8ull, 0xbe927e4f7282f468ull,
+        0x3e21ee96d2641b13ull, 0xbda8f76380fbb401ull,
+    };
+    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float64);
+    intptr_t x = simd_data(desc);
+    float64 *d = vd, *n = vn, *m = vm;
+    for (i = 0; i < opr_sz; i++) {
+        float64 mm = m[i];
+        intptr_t xx = x;
+        if (float64_is_neg(mm)) {
+            mm = float64_abs(mm);
+            xx += 8;
+        }
+        d[i] = float64_muladd(n[i], mm, coeff[xx], 0, vs);
+    }
 }
 
-DO_REDUCE(sve_faddv_h, float16, H1_2, add, float16_zero)
-DO_REDUCE(sve_faddv_s, float32, H1_4, add, float32_zero)
-DO_REDUCE(sve_faddv_d, float64,     , add, float64_zero)
+/*
+ * FP Complex Add
+ */
 
-/* Identity is floatN_default_nan, without the function call.  */
-DO_REDUCE(sve_fminnmv_h, float16, H1_2, minnum, 0x7E00)
-DO_REDUCE(sve_fminnmv_s, float32, H1_4, minnum, 0x7FC00000)
-DO_REDUCE(sve_fminnmv_d, float64,     , minnum, 0x7FF8000000000000ULL)
+void HELPER(sve_fcadd_h)(void *vd, void *vn, void *vm, void *vg,
+                         void *vs, uint32_t desc)
+{
+    intptr_t j, i = simd_oprsz(desc);
+    uint64_t *g = vg;
+    float16 neg_imag = float16_set_sign(0, simd_data(desc));
+    float16 neg_real = float16_chs(neg_imag);
 
-DO_REDUCE(sve_fmaxnmv_h, float16, H1_2, maxnum, 0x7E00)
-DO_REDUCE(sve_fmaxnmv_s, float32, H1_4, maxnum, 0x7FC00000)
-DO_REDUCE(sve_fmaxnmv_d, float64,     , maxnum, 0x7FF8000000000000ULL)
+    do {
+        uint64_t pg = g[(i - 1) >> 6];
+        do {
+            float16 e0, e1, e2, e3;
 
-DO_REDUCE(sve_fminv_h, float16, H1_2, min, float16_infinity)
-DO_REDUCE(sve_fminv_s, float32, H1_4, min, float32_infinity)
-DO_REDUCE(sve_fminv_d, float64,     , min, float64_infinity)
+            /* I holds the real index; J holds the imag index.  */
+            j = i - sizeof(float16);
+            i -= 2 * sizeof(float16);
 
-DO_REDUCE(sve_fmaxv_h, float16, H1_2, max, float16_chs(float16_infinity))
-DO_REDUCE(sve_fmaxv_s, float32, H1_4, max, float32_chs(float32_infinity))
-DO_REDUCE(sve_fmaxv_d, float64,     , max, float64_chs(float64_infinity))
+            e0 = *(float16 *)((char *)vn + H1_2(i));
+            e1 = *(float16 *)((char *)vm + H1_2(j)) ^ neg_real;
+            e2 = *(float16 *)((char *)vn + H1_2(j));
+            e3 = *(float16 *)((char *)vm + H1_2(i)) ^ neg_imag;
 
-#undef DO_REDUCE
+            if (likely((pg >> (i & 63)) & 1)) {
+                *(float16 *)((char *)vd + H1_2(i)) = float16_add(e0, e1, vs);
+            }
+            if (likely((pg >> (j & 63)) & 1)) {
+                *(float16 *)((char *)vd + H1_2(j)) = float16_add(e2, e3, vs);
+            }
+        } while (i & 63);
+    } while (i != 0);
+}
 
-uint64_t HELPER(sve_fadda_h)(uint64_t nn, void *vm, void *vg,
-                             void *status, uint32_t desc)
+void HELPER(sve_fcadd_s)(void *vd, void *vn, void *vm, void *vg,
+                         void *vs, uint32_t desc)
 {
-    intptr_t i = 0, opr_sz = simd_oprsz(desc);
-    float16 result = nn;
+    intptr_t j, i = simd_oprsz(desc);
+    uint64_t *g = vg;
+    float32 neg_imag = float32_set_sign(0, simd_data(desc));
+    float32 neg_real = float32_chs(neg_imag);
 
     do {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        uint64_t pg = g[(i - 1) >> 6];
         do {
-            if (pg & 1) {
-                float16 mm = *(float16 *)((char *)vm + H1_2(i));
-                result = float16_add(result, mm, status);
-            }
-            i += sizeof(float16), pg >>= sizeof(float16);
-        } while (i & 15);
-    } while (i < opr_sz);
+            float32 e0, e1, e2, e3;
 
-    return result;
+            /* I holds the real index; J holds the imag index.  */
+            j = i - sizeof(float32);
+            i -= 2 * sizeof(float32);
+
+            e0 = *(float32 *)((char *)vn + H1_2(i));
+            e1 = *(float32 *)((char *)vm + H1_2(j)) ^ neg_real;
+            e2 = *(float32 *)((char *)vn + H1_2(j));
+            e3 = *(float32 *)((char *)vm + H1_2(i)) ^ neg_imag;
+
+            if (likely((pg >> (i & 63)) & 1)) {
+                *(float32 *)((char *)vd + H1_2(i)) = float32_add(e0, e1, vs);
+            }
+            if (likely((pg >> (j & 63)) & 1)) {
+                *(float32 *)((char *)vd + H1_2(j)) = float32_add(e2, e3, vs);
+            }
+        } while (i & 63);
+    } while (i != 0);
 }
 
-uint64_t HELPER(sve_fadda_s)(uint64_t nn, void *vm, void *vg,
-                             void *status, uint32_t desc)
+void HELPER(sve_fcadd_d)(void *vd, void *vn, void *vm, void *vg,
+                         void *vs, uint32_t desc)
 {
-    intptr_t i = 0, opr_sz = simd_oprsz(desc);
-    float32 result = nn;
+    intptr_t j, i = simd_oprsz(desc);
+    uint64_t *g = vg;
+    float64 neg_imag = float64_set_sign(0, simd_data(desc));
+    float64 neg_real = float64_chs(neg_imag);
 
     do {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        uint64_t pg = g[(i - 1) >> 6];
         do {
-            if (pg & 1) {
-                float32 mm = *(float32 *)((char *)vm + H1_2(i));
-                result = float32_add(result, mm, status);
-            }
-            i += sizeof(float32), pg >>= sizeof(float32);
-        } while (i & 15);
-    } while (i < opr_sz);
-
-    return result;
-}
+            float64 e0, e1, e2, e3;
 
-uint64_t HELPER(sve_fadda_d)(uint64_t nn, void *vm, void *vg,
-                             void *status, uint32_t desc)
-{
-    intptr_t i = 0, opr_sz = simd_oprsz(desc) / 8;
-    uint64_t *m = vm;
-    uint8_t *pg = vg;
+            /* I holds the real index; J holds the imag index.  */
+            j = i - sizeof(float64);
+            i -= 2 * sizeof(float64);
 
-    for (i = 0; i < opr_sz; i++) {
-        if (pg[H1(i)] & 1) {
-            nn = float64_add(nn, m[i], status);
-        }
-    }
+            e0 = *(float64 *)((char *)vn + H1_2(i));
+            e1 = *(float64 *)((char *)vm + H1_2(j)) ^ neg_real;
+            e2 = *(float64 *)((char *)vn + H1_2(j));
+            e3 = *(float64 *)((char *)vm + H1_2(i)) ^ neg_imag;
 
-    return nn;
+            if (likely((pg >> (i & 63)) & 1)) {
+                *(float64 *)((char *)vd + H1_2(i)) = float64_add(e0, e1, vs);
+            }
+            if (likely((pg >> (j & 63)) & 1)) {
+                *(float64 *)((char *)vd + H1_2(j)) = float64_add(e2, e3, vs);
+            }
+        } while (i & 63);
+    } while (i != 0);
 }
 
-/* Fully general three-operand expander, controlled by a predicate,
- * With the extra float_status parameter.
+/*
+ * FP Complex Multiply
  */
-#define DO_ZPZZ_FP(NAME, TYPE, H, OP)                           \
-void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg,       \
-                  void *status, uint32_t desc)                  \
-{                                                               \
-    intptr_t i = simd_oprsz(desc);                              \
-    uint64_t *g = vg;                                           \
-    do {                                                        \
-        uint64_t pg = g[(i - 1) >> 6];                          \
-        do {                                                    \
-            i -= sizeof(TYPE);                                  \
-            if (likely((pg >> (i & 63)) & 1)) {                 \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                 \
-                TYPE mm = *(TYPE *)((char *)vm + H(i));                 \
-                *(TYPE *)((char *)vd + H(i)) = OP(nn, mm, status);      \
-            }                                                   \
-        } while (i & 63);                                       \
-    } while (i != 0);                                           \
-}
-
-DO_ZPZZ_FP(sve_fadd_h, uint16_t, H1_2, float16_add)
-DO_ZPZZ_FP(sve_fadd_s, uint32_t, H1_4, float32_add)
-DO_ZPZZ_FP(sve_fadd_d, uint64_t,     , float64_add)
-
-DO_ZPZZ_FP(sve_fsub_h, uint16_t, H1_2, float16_sub)
-DO_ZPZZ_FP(sve_fsub_s, uint32_t, H1_4, float32_sub)
-DO_ZPZZ_FP(sve_fsub_d, uint64_t,     , float64_sub)
-
-DO_ZPZZ_FP(sve_fmul_h, uint16_t, H1_2, float16_mul)
-DO_ZPZZ_FP(sve_fmul_s, uint32_t, H1_4, float32_mul)
-DO_ZPZZ_FP(sve_fmul_d, uint64_t,     , float64_mul)
 
-DO_ZPZZ_FP(sve_fdiv_h, uint16_t, H1_2, float16_div)
-DO_ZPZZ_FP(sve_fdiv_s, uint32_t, H1_4, float32_div)
-DO_ZPZZ_FP(sve_fdiv_d, uint64_t,     , float64_div)
+QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 22 > 32);
 
-DO_ZPZZ_FP(sve_fmin_h, uint16_t, H1_2, float16_min)
-DO_ZPZZ_FP(sve_fmin_s, uint32_t, H1_4, float32_min)
-DO_ZPZZ_FP(sve_fmin_d, uint64_t,     , float64_min)
+void HELPER(sve_fcmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+{
+    intptr_t j, i = simd_oprsz(desc);
+    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+    unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
+    bool flip = rot & 1;
+    float16 neg_imag, neg_real;
+    void *vd = &env->vfp.zregs[rd];
+    void *vn = &env->vfp.zregs[rn];
+    void *vm = &env->vfp.zregs[rm];
+    void *va = &env->vfp.zregs[ra];
+    uint64_t *g = vg;
 
-DO_ZPZZ_FP(sve_fmax_h, uint16_t, H1_2, float16_max)
-DO_ZPZZ_FP(sve_fmax_s, uint32_t, H1_4, float32_max)
-DO_ZPZZ_FP(sve_fmax_d, uint64_t,     , float64_max)
+    neg_imag = float16_set_sign(0, (rot & 2) != 0);
+    neg_real = float16_set_sign(0, rot == 1 || rot == 2);
 
-DO_ZPZZ_FP(sve_fminnum_h, uint16_t, H1_2, float16_minnum)
-DO_ZPZZ_FP(sve_fminnum_s, uint32_t, H1_4, float32_minnum)
-DO_ZPZZ_FP(sve_fminnum_d, uint64_t,     , float64_minnum)
+    do {
+        uint64_t pg = g[(i - 1) >> 6];
+        do {
+            float16 e1, e2, e3, e4, nr, ni, mr, mi, d;
 
-DO_ZPZZ_FP(sve_fmaxnum_h, uint16_t, H1_2, float16_maxnum)
-DO_ZPZZ_FP(sve_fmaxnum_s, uint32_t, H1_4, float32_maxnum)
-DO_ZPZZ_FP(sve_fmaxnum_d, uint64_t,     , float64_maxnum)
+            /* I holds the real index; J holds the imag index.  */
+            j = i - sizeof(float16);
+            i -= 2 * sizeof(float16);
 
-static inline float16 abd_h(float16 a, float16 b, float_status *s)
-{
-    return float16_abs(float16_sub(a, b, s));
-}
+            nr = *(float16 *)((char *)vn + H1_2(i));
+            ni = *(float16 *)((char *)vn + H1_2(j));
+            mr = *(float16 *)((char *)vm + H1_2(i));
+            mi = *(float16 *)((char *)vm + H1_2(j));
 
-static inline float32 abd_s(float32 a, float32 b, float_status *s)
-{
-    return float32_abs(float32_sub(a, b, s));
+            e2 = (flip ? ni : nr);
+            e1 = (flip ? mi : mr) ^ neg_real;
+            e4 = e2;
+            e3 = (flip ? mr : mi) ^ neg_imag;
+
+            if (likely((pg >> (i & 63)) & 1)) {
+                d = *(float16 *)((char *)va + H1_2(i));
+                d = float16_muladd(e2, e1, d, 0, &env->vfp.fp_status_f16);
+                *(float16 *)((char *)vd + H1_2(i)) = d;
+            }
+            if (likely((pg >> (j & 63)) & 1)) {
+                d = *(float16 *)((char *)va + H1_2(j));
+                d = float16_muladd(e4, e3, d, 0, &env->vfp.fp_status_f16);
+                *(float16 *)((char *)vd + H1_2(j)) = d;
+            }
+        } while (i & 63);
+    } while (i != 0);
 }
 
-static inline float64 abd_d(float64 a, float64 b, float_status *s)
+void HELPER(sve_fcmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
 {
-    return float64_abs(float64_sub(a, b, s));
-}
+    intptr_t j, i = simd_oprsz(desc);
+    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+    unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
+    bool flip = rot & 1;
+    float32 neg_imag, neg_real;
+    void *vd = &env->vfp.zregs[rd];
+    void *vn = &env->vfp.zregs[rn];
+    void *vm = &env->vfp.zregs[rm];
+    void *va = &env->vfp.zregs[ra];
+    uint64_t *g = vg;
 
-DO_ZPZZ_FP(sve_fabd_h, uint16_t, H1_2, abd_h)
-DO_ZPZZ_FP(sve_fabd_s, uint32_t, H1_4, abd_s)
-DO_ZPZZ_FP(sve_fabd_d, uint64_t,     , abd_d)
+    neg_imag = float32_set_sign(0, (rot & 2) != 0);
+    neg_real = float32_set_sign(0, rot == 1 || rot == 2);
 
-static inline float64 scalbn_d(float64 a, int64_t b, float_status *s)
-{
-    int b_int = MIN(MAX(b, INT_MIN), INT_MAX);
-    return float64_scalbn(a, b_int, s);
-}
+    do {
+        uint64_t pg = g[(i - 1) >> 6];
+        do {
+            float32 e1, e2, e3, e4, nr, ni, mr, mi, d;
 
-DO_ZPZZ_FP(sve_fscalbn_h, int16_t, H1_2, float16_scalbn)
-DO_ZPZZ_FP(sve_fscalbn_s, int32_t, H1_4, float32_scalbn)
-DO_ZPZZ_FP(sve_fscalbn_d, int64_t,     , scalbn_d)
+            /* I holds the real index; J holds the imag index.  */
+            j = i - sizeof(float32);
+            i -= 2 * sizeof(float32);
 
-DO_ZPZZ_FP(sve_fmulx_h, uint16_t, H1_2, helper_advsimd_mulxh)
-DO_ZPZZ_FP(sve_fmulx_s, uint32_t, H1_4, helper_vfp_mulxs)
-DO_ZPZZ_FP(sve_fmulx_d, uint64_t,     , helper_vfp_mulxd)
+            nr = *(float32 *)((char *)vn + H1_2(i));
+            ni = *(float32 *)((char *)vn + H1_2(j));
+            mr = *(float32 *)((char *)vm + H1_2(i));
+            mi = *(float32 *)((char *)vm + H1_2(j));
 
-#undef DO_ZPZZ_FP
+            e2 = (flip ? ni : nr);
+            e1 = (flip ? mi : mr) ^ neg_real;
+            e4 = e2;
+            e3 = (flip ? mr : mi) ^ neg_imag;
 
-/* Three-operand expander, with one scalar operand, controlled by
- * a predicate, with the extra float_status parameter.
- */
-#define DO_ZPZS_FP(NAME, TYPE, H, OP) \
-void HELPER(NAME)(void *vd, void *vn, void *vg, uint64_t scalar,  \
-                  void *status, uint32_t desc)                    \
-{                                                                 \
-    intptr_t i = simd_oprsz(desc);                                \
-    uint64_t *g = vg;                                             \
-    TYPE mm = scalar;                                             \
-    do {                                                          \
-        uint64_t pg = g[(i - 1) >> 6];                            \
-        do {                                                      \
-            i -= sizeof(TYPE);                                    \
-            if (likely((pg >> (i & 63)) & 1)) {                   \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                   \
-                *(TYPE *)((char *)vd + H(i)) = OP(nn, mm, status);        \
-            }                                                     \
-        } while (i & 63);                                         \
-    } while (i != 0);                                             \
+            if (likely((pg >> (i & 63)) & 1)) {
+                d = *(float32 *)((char *)va + H1_2(i));
+                d = float32_muladd(e2, e1, d, 0, &env->vfp.fp_status);
+                *(float32 *)((char *)vd + H1_2(i)) = d;
+            }
+            if (likely((pg >> (j & 63)) & 1)) {
+                d = *(float32 *)((char *)va + H1_2(j));
+                d = float32_muladd(e4, e3, d, 0, &env->vfp.fp_status);
+                *(float32 *)((char *)vd + H1_2(j)) = d;
+            }
+        } while (i & 63);
+    } while (i != 0);
 }
 
-DO_ZPZS_FP(sve_fadds_h, float16, H1_2, float16_add)
-DO_ZPZS_FP(sve_fadds_s, float32, H1_4, float32_add)
-DO_ZPZS_FP(sve_fadds_d, float64,     , float64_add)
-
-DO_ZPZS_FP(sve_fsubs_h, float16, H1_2, float16_sub)
-DO_ZPZS_FP(sve_fsubs_s, float32, H1_4, float32_sub)
-DO_ZPZS_FP(sve_fsubs_d, float64,     , float64_sub)
+void HELPER(sve_fcmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+{
+    intptr_t j, i = simd_oprsz(desc);
+    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
+    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
+    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
+    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
+    unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
+    bool flip = rot & 1;
+    float64 neg_imag, neg_real;
+    void *vd = &env->vfp.zregs[rd];
+    void *vn = &env->vfp.zregs[rn];
+    void *vm = &env->vfp.zregs[rm];
+    void *va = &env->vfp.zregs[ra];
+    uint64_t *g = vg;
 
-DO_ZPZS_FP(sve_fmuls_h, float16, H1_2, float16_mul)
-DO_ZPZS_FP(sve_fmuls_s, float32, H1_4, float32_mul)
-DO_ZPZS_FP(sve_fmuls_d, float64,     , float64_mul)
+    neg_imag = float64_set_sign(0, (rot & 2) != 0);
+    neg_real = float64_set_sign(0, rot == 1 || rot == 2);
 
-static inline float16 subr_h(float16 a, float16 b, float_status *s)
-{
-    return float16_sub(b, a, s);
-}
+    do {
+        uint64_t pg = g[(i - 1) >> 6];
+        do {
+            float64 e1, e2, e3, e4, nr, ni, mr, mi, d;
 
-static inline float32 subr_s(float32 a, float32 b, float_status *s)
-{
-    return float32_sub(b, a, s);
-}
+            /* I holds the real index; J holds the imag index.  */
+            j = i - sizeof(float64);
+            i -= 2 * sizeof(float64);
 
-static inline float64 subr_d(float64 a, float64 b, float_status *s)
-{
-    return float64_sub(b, a, s);
-}
+            nr = *(float64 *)((char *)vn + H1_2(i));
+            ni = *(float64 *)((char *)vn + H1_2(j));
+            mr = *(float64 *)((char *)vm + H1_2(i));
+            mi = *(float64 *)((char *)vm + H1_2(j));
 
-DO_ZPZS_FP(sve_fsubrs_h, float16, H1_2, subr_h)
-DO_ZPZS_FP(sve_fsubrs_s, float32, H1_4, subr_s)
-DO_ZPZS_FP(sve_fsubrs_d, float64,     , subr_d)
+            e2 = (flip ? ni : nr);
+            e1 = (flip ? mi : mr) ^ neg_real;
+            e4 = e2;
+            e3 = (flip ? mr : mi) ^ neg_imag;
 
-DO_ZPZS_FP(sve_fmaxnms_h, float16, H1_2, float16_maxnum)
-DO_ZPZS_FP(sve_fmaxnms_s, float32, H1_4, float32_maxnum)
-DO_ZPZS_FP(sve_fmaxnms_d, float64,     , float64_maxnum)
+            if (likely((pg >> (i & 63)) & 1)) {
+                d = *(float64 *)((char *)va + H1_2(i));
+                d = float64_muladd(e2, e1, d, 0, &env->vfp.fp_status);
+                *(float64 *)((char *)vd + H1_2(i)) = d;
+            }
+            if (likely((pg >> (j & 63)) & 1)) {
+                d = *(float64 *)((char *)va + H1_2(j));
+                d = float64_muladd(e4, e3, d, 0, &env->vfp.fp_status);
+                *(float64 *)((char *)vd + H1_2(j)) = d;
+            }
+        } while (i & 63);
+    } while (i != 0);
+}
 
-DO_ZPZS_FP(sve_fminnms_h, float16, H1_2, float16_minnum)
-DO_ZPZS_FP(sve_fminnms_s, float32, H1_4, float32_minnum)
-DO_ZPZS_FP(sve_fminnms_d, float64,     , float64_minnum)
+/*
+ * Load contiguous data, protected by a governing predicate.
+ */
 
-DO_ZPZS_FP(sve_fmaxs_h, float16, H1_2, float16_max)
-DO_ZPZS_FP(sve_fmaxs_s, float32, H1_4, float32_max)
-DO_ZPZS_FP(sve_fmaxs_d, float64,     , float64_max)
+/*
+ * Load elements into @vd, controlled by @vg, from @host + @mem_ofs.
+ * Memory is valid through @host + @mem_max.  The register element
+ * indicies are inferred from @mem_ofs, as modified by the types for
+ * which the helper is built.  Return the @mem_ofs of the first element
+ * not loaded (which is @mem_max if they are all loaded).
+ *
+ * For softmmu, we have fully validated the guest page.  For user-only,
+ * we cannot fully validate without taking the mmap lock, but since we
+ * know the access is within one host page, if any access is valid they
+ * all must be valid.  However, when @vg is all false, it may be that
+ * no access is valid.
+ */
+typedef intptr_t sve_ld1_host_fn(void *vd, void *vg, void *host,
+                                 intptr_t mem_ofs, intptr_t mem_max);
 
-DO_ZPZS_FP(sve_fmins_h, float16, H1_2, float16_min)
-DO_ZPZS_FP(sve_fmins_s, float32, H1_4, float32_min)
-DO_ZPZS_FP(sve_fmins_d, float64,     , float64_min)
+/*
+ * Load one element into @vd + @reg_off from (@env, @vaddr, @ra).
+ * The controlling predicate is known to be true.
+ */
+typedef void sve_ld1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off,
+                            target_ulong vaddr, TCGMemOpIdx oi, uintptr_t ra);
+typedef sve_ld1_tlb_fn sve_st1_tlb_fn;
 
-/* Fully general two-operand expander, controlled by a predicate,
- * With the extra float_status parameter.
+/*
+ * Generate the above primitives.
  */
-#define DO_ZPZ_FP(NAME, TYPE, H, OP)                                  \
-void HELPER(NAME)(void *vd, void *vn, void *vg, void *status, uint32_t desc) \
-{                                                                     \
-    intptr_t i = simd_oprsz(desc);                                    \
-    uint64_t *g = vg;                                                 \
-    do {                                                              \
-        uint64_t pg = g[(i - 1) >> 6];                                \
-        do {                                                          \
-            i -= sizeof(TYPE);                                        \
-            if (likely((pg >> (i & 63)) & 1)) {                       \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                       \
-                *(TYPE *)((char *)vd + H(i)) = OP(nn, status);                \
-            }                                                         \
-        } while (i & 63);                                             \
-    } while (i != 0);                                                 \
+
+#define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \
+static intptr_t sve_##NAME##_host(void *vd, void *vg, void *host,           \
+                                  intptr_t mem_off, const intptr_t mem_max) \
+{                                                                           \
+    intptr_t reg_off = mem_off * (sizeof(TYPEE) / sizeof(TYPEM));           \
+    uint64_t *pg = vg;                                                      \
+    while (mem_off + sizeof(TYPEM) <= mem_max) {                            \
+        TYPEM val = 0;                                                      \
+        if (likely((pg[reg_off >> 6] >> (reg_off & 63)) & 1)) {             \
+            val = HOST((char *)host + mem_off);                                     \
+        }                                                                   \
+        *(TYPEE *)((char *)vd + H(reg_off)) = val;                                  \
+        mem_off += sizeof(TYPEM), reg_off += sizeof(TYPEE);                 \
+    }                                                                       \
+    return mem_off;                                                         \
 }
 
-/* SVE fp16 conversions always use IEEE mode.  Like AdvSIMD, they ignore
- * FZ16.  When converting from fp16, this affects flushing input denormals;
- * when converting to fp16, this affects flushing output denormals.
- */
-static inline float32 sve_f16_to_f32(float16 f, float_status *fpst)
-{
-    flag save = get_flush_inputs_to_zero(fpst);
-    float32 ret;
-
-    set_flush_inputs_to_zero(false, fpst);
-    ret = float16_to_float32(f, true, fpst);
-    set_flush_inputs_to_zero(save, fpst);
-    return ret;
+#define DO_LD_TLB(NAME, H, TYPEE, TYPEM, HOST, MOEND, TLB) \
+static void sve_##NAME##_tlb(CPUARMState *env, void *vd, intptr_t reg_off,  \
+                             target_ulong addr, TCGMemOpIdx oi, uintptr_t ra)  \
+{                                                                           \
+    TYPEM val = TLB(env, addr, oi, ra);                                     \
+    *(TYPEE *)((char *)vd + H(reg_off)) = val;                                      \
 }
 
-static inline float64 sve_f16_to_f64(float16 f, float_status *fpst)
-{
-    flag save = get_flush_inputs_to_zero(fpst);
-    float64 ret;
+#define DO_LD_PRIM_1(NAME, H, TE, TM)                   \
+    DO_LD_HOST(NAME, H, TE, TM, ldub_p)                 \
+    DO_LD_TLB(NAME, H, TE, TM, ldub_p, 0, helper_ret_ldub_mmu)
 
-    set_flush_inputs_to_zero(false, fpst);
-    ret = float16_to_float64(f, true, fpst);
-    set_flush_inputs_to_zero(save, fpst);
-    return ret;
-}
+DO_LD_PRIM_1(ld1bb,  H1,   uint8_t,  uint8_t)
+DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t)
+DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t,  int8_t)
+DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t)
+DO_LD_PRIM_1(ld1bss, H1_4, uint32_t,  int8_t)
+DO_LD_PRIM_1(ld1bdu,     , uint64_t, uint8_t)
+DO_LD_PRIM_1(ld1bds,     , uint64_t,  int8_t)
 
-static inline float16 sve_f32_to_f16(float32 f, float_status *fpst)
-{
-    flag save = get_flush_to_zero(fpst);
-    float16 ret;
+#define DO_LD_PRIM_2(NAME, end, MOEND, H, TE, TM, PH, PT)  \
+    DO_LD_HOST(NAME##_##end, H, TE, TM, PH##_##end##_p)    \
+    DO_LD_TLB(NAME##_##end, H, TE, TM, PH##_##end##_p,     \
+              MOEND, helper_##end##_##PT##_mmu)
 
-    set_flush_to_zero(false, fpst);
-    ret = float32_to_float16(f, true, fpst);
-    set_flush_to_zero(save, fpst);
-    return ret;
-}
+DO_LD_PRIM_2(ld1hh,  le, MO_LE, H1_2, uint16_t, uint16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hsu, le, MO_LE, H1_4, uint32_t, uint16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hss, le, MO_LE, H1_4, uint32_t,  int16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hdu, le, MO_LE,     , uint64_t, uint16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hds, le, MO_LE,     , uint64_t,  int16_t, lduw, lduw)
 
-static inline float16 sve_f64_to_f16(float64 f, float_status *fpst)
-{
-    flag save = get_flush_to_zero(fpst);
-    float16 ret;
+DO_LD_PRIM_2(ld1ss,  le, MO_LE, H1_4, uint32_t, uint32_t, ldl, ldul)
+DO_LD_PRIM_2(ld1sdu, le, MO_LE,     , uint64_t, uint32_t, ldl, ldul)
+DO_LD_PRIM_2(ld1sds, le, MO_LE,     , uint64_t,  int32_t, ldl, ldul)
 
-    set_flush_to_zero(false, fpst);
-    ret = float64_to_float16(f, true, fpst);
-    set_flush_to_zero(save, fpst);
-    return ret;
-}
+DO_LD_PRIM_2(ld1dd,  le, MO_LE,     , uint64_t, uint64_t, ldq, ldq)
 
-static inline int16_t vfp_float16_to_int16_rtz(float16 f, float_status *s)
-{
-    if (float16_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
-    }
-    return float16_to_int16_round_to_zero(f, s);
-}
+DO_LD_PRIM_2(ld1hh,  be, MO_BE, H1_2, uint16_t, uint16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hsu, be, MO_BE, H1_4, uint32_t, uint16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hss, be, MO_BE, H1_4, uint32_t,  int16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hdu, be, MO_BE,     , uint64_t, uint16_t, lduw, lduw)
+DO_LD_PRIM_2(ld1hds, be, MO_BE,     , uint64_t,  int16_t, lduw, lduw)
 
-static inline int64_t vfp_float16_to_int64_rtz(float16 f, float_status *s)
-{
-    if (float16_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
-    }
-    return float16_to_int64_round_to_zero(f, s);
-}
+DO_LD_PRIM_2(ld1ss,  be, MO_BE, H1_4, uint32_t, uint32_t, ldl, ldul)
+DO_LD_PRIM_2(ld1sdu, be, MO_BE,     , uint64_t, uint32_t, ldl, ldul)
+DO_LD_PRIM_2(ld1sds, be, MO_BE,     , uint64_t,  int32_t, ldl, ldul)
 
-static inline int64_t vfp_float32_to_int64_rtz(float32 f, float_status *s)
-{
-    if (float32_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
-    }
-    return float32_to_int64_round_to_zero(f, s);
-}
+DO_LD_PRIM_2(ld1dd,  be, MO_BE,     , uint64_t, uint64_t, ldq, ldq)
 
-static inline int64_t vfp_float64_to_int64_rtz(float64 f, float_status *s)
-{
-    if (float64_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
-    }
-    return float64_to_int64_round_to_zero(f, s);
-}
+#undef DO_LD_TLB
+#undef DO_LD_HOST
+#undef DO_LD_PRIM_1
+#undef DO_LD_PRIM_2
 
-static inline uint16_t vfp_float16_to_uint16_rtz(float16 f, float_status *s)
+/*
+ * Skip through a sequence of inactive elements in the guarding predicate @vg,
+ * beginning at @reg_off bounded by @reg_max.  Return the offset of the active
+ * element >= @reg_off, or @reg_max if there were no active elements at all.
+ */
+static intptr_t find_next_active(uint64_t *vg, intptr_t reg_off,
+                                 intptr_t reg_max, int esz)
 {
-    if (float16_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
-    }
-    return float16_to_uint16_round_to_zero(f, s);
-}
+    uint64_t pg_mask = pred_esz_masks[esz];
+    uint64_t pg = (vg[reg_off >> 6] & pg_mask) >> (reg_off & 63);
 
-static inline uint64_t vfp_float16_to_uint64_rtz(float16 f, float_status *s)
-{
-    if (float16_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
+    /* In normal usage, the first element is active.  */
+    if (likely(pg & 1)) {
+        return reg_off;
     }
-    return float16_to_uint64_round_to_zero(f, s);
-}
 
-static inline uint64_t vfp_float32_to_uint64_rtz(float32 f, float_status *s)
-{
-    if (float32_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
+    if (pg == 0) {
+        reg_off &= -64;
+        do {
+            reg_off += 64;
+            if (unlikely(reg_off >= reg_max)) {
+                /* The entire predicate was false.  */
+                return reg_max;
+            }
+            pg = vg[reg_off >> 6] & pg_mask;
+        } while (pg == 0);
     }
-    return float32_to_uint64_round_to_zero(f, s);
+    reg_off += ctz64(pg);
+
+    /* We should never see an out of range predicate bit set.  */
+    tcg_debug_assert(reg_off < reg_max);
+    return reg_off;
 }
 
-static inline uint64_t vfp_float64_to_uint64_rtz(float64 f, float_status *s)
+/*
+ * Return the maximum offset <= @mem_max which is still within the page
+ * referenced by @base + @mem_off.
+ */
+static intptr_t max_for_page(struct uc_struct *uc, target_ulong base, intptr_t mem_off,
+                             intptr_t mem_max)
 {
-    if (float64_is_any_nan(f)) {
-        float_raise(float_flag_invalid, s);
-        return 0;
-    }
-    return float64_to_uint64_round_to_zero(f, s);
+    target_ulong addr = base + mem_off;
+    intptr_t split = -(intptr_t)(addr | TARGET_PAGE_MASK);
+    return MIN(split, mem_max - mem_off) + mem_off;
 }
 
-DO_ZPZ_FP(sve_fcvt_sh, uint32_t, H1_4, sve_f32_to_f16)
-DO_ZPZ_FP(sve_fcvt_hs, uint32_t, H1_4, sve_f16_to_f32)
-DO_ZPZ_FP(sve_fcvt_dh, uint64_t,     , sve_f64_to_f16)
-DO_ZPZ_FP(sve_fcvt_hd, uint64_t,     , sve_f16_to_f64)
-DO_ZPZ_FP(sve_fcvt_ds, uint64_t,     , float64_to_float32)
-DO_ZPZ_FP(sve_fcvt_sd, uint64_t,     , float32_to_float64)
-
-DO_ZPZ_FP(sve_fcvtzs_hh, uint16_t, H1_2, vfp_float16_to_int16_rtz)
-DO_ZPZ_FP(sve_fcvtzs_hs, uint32_t, H1_4, helper_vfp_tosizh)
-DO_ZPZ_FP(sve_fcvtzs_ss, uint32_t, H1_4, helper_vfp_tosizs)
-DO_ZPZ_FP(sve_fcvtzs_hd, uint64_t,     , vfp_float16_to_int64_rtz)
-DO_ZPZ_FP(sve_fcvtzs_sd, uint64_t,     , vfp_float32_to_int64_rtz)
-DO_ZPZ_FP(sve_fcvtzs_ds, uint64_t,     , helper_vfp_tosizd)
-DO_ZPZ_FP(sve_fcvtzs_dd, uint64_t,     , vfp_float64_to_int64_rtz)
-
-DO_ZPZ_FP(sve_fcvtzu_hh, uint16_t, H1_2, vfp_float16_to_uint16_rtz)
-DO_ZPZ_FP(sve_fcvtzu_hs, uint32_t, H1_4, helper_vfp_touizh)
-DO_ZPZ_FP(sve_fcvtzu_ss, uint32_t, H1_4, helper_vfp_touizs)
-DO_ZPZ_FP(sve_fcvtzu_hd, uint64_t,     , vfp_float16_to_uint64_rtz)
-DO_ZPZ_FP(sve_fcvtzu_sd, uint64_t,     , vfp_float32_to_uint64_rtz)
-DO_ZPZ_FP(sve_fcvtzu_ds, uint64_t,     , helper_vfp_touizd)
-DO_ZPZ_FP(sve_fcvtzu_dd, uint64_t,     , vfp_float64_to_uint64_rtz)
-
-DO_ZPZ_FP(sve_frint_h, uint16_t, H1_2, helper_advsimd_rinth)
-DO_ZPZ_FP(sve_frint_s, uint32_t, H1_4, helper_rints)
-DO_ZPZ_FP(sve_frint_d, uint64_t,     , helper_rintd)
-
-DO_ZPZ_FP(sve_frintx_h, uint16_t, H1_2, float16_round_to_int)
-DO_ZPZ_FP(sve_frintx_s, uint32_t, H1_4, float32_round_to_int)
-DO_ZPZ_FP(sve_frintx_d, uint64_t,     , float64_round_to_int)
-
-DO_ZPZ_FP(sve_frecpx_h, uint16_t, H1_2, helper_frecpx_f16)
-DO_ZPZ_FP(sve_frecpx_s, uint32_t, H1_4, helper_frecpx_f32)
-DO_ZPZ_FP(sve_frecpx_d, uint64_t,     , helper_frecpx_f64)
-
-DO_ZPZ_FP(sve_fsqrt_h, uint16_t, H1_2, float16_sqrt)
-DO_ZPZ_FP(sve_fsqrt_s, uint32_t, H1_4, float32_sqrt)
-DO_ZPZ_FP(sve_fsqrt_d, uint64_t,     , float64_sqrt)
-
-DO_ZPZ_FP(sve_scvt_hh, uint16_t, H1_2, int16_to_float16)
-DO_ZPZ_FP(sve_scvt_sh, uint32_t, H1_4, int32_to_float16)
-DO_ZPZ_FP(sve_scvt_ss, uint32_t, H1_4, int32_to_float32)
-DO_ZPZ_FP(sve_scvt_sd, uint64_t,     , int32_to_float64)
-DO_ZPZ_FP(sve_scvt_dh, uint64_t,     , int64_to_float16)
-DO_ZPZ_FP(sve_scvt_ds, uint64_t,     , int64_to_float32)
-DO_ZPZ_FP(sve_scvt_dd, uint64_t,     , int64_to_float64)
-
-DO_ZPZ_FP(sve_ucvt_hh, uint16_t, H1_2, uint16_to_float16)
-DO_ZPZ_FP(sve_ucvt_sh, uint32_t, H1_4, uint32_to_float16)
-DO_ZPZ_FP(sve_ucvt_ss, uint32_t, H1_4, uint32_to_float32)
-DO_ZPZ_FP(sve_ucvt_sd, uint64_t,     , uint32_to_float64)
-DO_ZPZ_FP(sve_ucvt_dh, uint64_t,     , uint64_to_float16)
-DO_ZPZ_FP(sve_ucvt_ds, uint64_t,     , uint64_to_float32)
-DO_ZPZ_FP(sve_ucvt_dd, uint64_t,     , uint64_to_float64)
-
-#undef DO_ZPZ_FP
+/* These are normally defined only for CONFIG_USER_ONLY in <exec/cpu_ldst.h> */
+static inline void set_helper_retaddr(uintptr_t ra) { }
+static inline void clear_helper_retaddr(void) { }
 
-/* 4-operand predicated multiply-add.  This requires 7 operands to pass
- * "properly", so we need to encode some of the registers into DESC.
+/*
+ * The result of tlb_vaddr_to_host for user-only is just g2h(x),
+ * which is always non-null.  Elide the useless test.
  */
-QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 20 > 32);
+static inline bool test_host_page(void *host)
+{
+    return likely(host != NULL);
+}
 
-static void do_fmla_zpzzz_h(CPUARMState *env, void *vg, uint32_t desc,
-                            uint16_t neg1, uint16_t neg3)
+static uint32_t sve_mte_desc(CPUARMState *env, TCGMemOpIdx oi,
+                             bool is_write, int size)
 {
-    intptr_t i = simd_oprsz(desc);
-    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
-    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
-    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
-    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
-    void *vd = &env->vfp.zregs[rd];
-    void *vn = &env->vfp.zregs[rn];
-    void *vm = &env->vfp.zregs[rm];
-    void *va = &env->vfp.zregs[ra];
-    uint64_t *g = vg;
+    uint32_t desc = 0;
 
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            i -= 2;
-            if (likely((pg >> (i & 63)) & 1)) {
-                float16 e1, e2, e3, r;
+    FIELD_DP32(desc, MTEDESC, MIDX, get_mmuidx(oi), desc);
+    FIELD_DP32(desc, MTEDESC, TBI,
+               FIELD_EX32(env->hflags, TBFLAG_A64, TBID), desc);
+    FIELD_DP32(desc, MTEDESC, TCMA,
+               FIELD_EX32(env->hflags, TBFLAG_A64, TCMA), desc);
+    FIELD_DP32(desc, MTEDESC, WRITE, is_write, desc);
+    FIELD_DP32(desc, MTEDESC, SIZEM1, size - 1, desc);
+    return desc;
+}
 
-                e1 = *(uint16_t *)((char *)vn + H1_2(i)) ^ neg1;
-                e2 = *(uint16_t *)((char *)vm + H1_2(i));
-                e3 = *(uint16_t *)((char *)va + H1_2(i)) ^ neg3;
-                r = float16_muladd(e1, e2, e3, 0, &env->vfp.fp_status_f16);
-                *(uint16_t *)((char *)vd + H1_2(i)) = r;
-            }
-        } while (i & 63);
-    } while (i != 0);
+static target_ulong sve_mte_check_addr(CPUARMState *env, TCGMemOpIdx oi,
+                                       target_ulong addr, bool is_write,
+                                       int size, uintptr_t ra)
+{
+    return mte_check(env, sve_mte_desc(env, oi, is_write, size), addr, ra);
 }
 
-void HELPER(sve_fmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+static bool sve_mte_probe_addr(CPUARMState *env, TCGMemOpIdx oi,
+                               target_ulong addr, bool is_write, int size)
 {
-    do_fmla_zpzzz_h(env, vg, desc, 0, 0);
+    return mte_probe(env, sve_mte_desc(env, oi, is_write, size), addr);
 }
 
-void HELPER(sve_fmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+static target_ulong sve_mte_clean_addr(CPUARMState *env, TCGMemOpIdx oi,
+                                       target_ulong addr, bool is_write,
+                                       int size)
 {
-    do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0);
+    uint32_t desc = sve_mte_desc(env, oi, is_write, size);
+    int bit55 = extract64(addr, 55, 1);
+
+    if (!tbi_check(desc, bit55)) {
+        return addr;
+    }
+    return addr & ~MAKE_64BIT_MASK(56, 8);
 }
 
-void HELPER(sve_fnmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+static void sve_probe_write_addr(CPUARMState *env, target_ulong addr,
+                                 int size, TCGMemOpIdx oi, uintptr_t ra)
 {
-    do_fmla_zpzzz_h(env, vg, desc, 0x8000, 0x8000);
+    struct uc_struct *uc = env->uc;
+    int mmu_idx = get_mmuidx(oi);
+
+    while (size > 0) {
+        target_ulong page_left = -(addr | TARGET_PAGE_MASK);
+        int probe_size = MIN(size, (int)page_left);
+        target_ulong paddr;
+        MemoryRegion *mr;
+
+        if (!tlb_vaddr_to_paddr(env, addr, MMU_DATA_STORE, mmu_idx, &paddr)) {
+            uc->invalid_addr = addr;
+            uc->invalid_error = UC_ERR_MMU_WRITE;
+            cpu_exit(uc->cpu);
+            cpu_loop_exit_restore(env_cpu(env), ra);
+        }
+
+        mr = uc->memory_mapping(uc, paddr);
+        if (mr == NULL) {
+            uc->invalid_addr = paddr;
+            uc->invalid_error = UC_ERR_WRITE_UNMAPPED;
+            cpu_exit(uc->cpu);
+            cpu_loop_exit_restore(env_cpu(env), ra);
+        }
+        if (!(mr->perms & UC_PROT_WRITE)) {
+            uc->invalid_addr = paddr;
+            uc->invalid_error = UC_ERR_WRITE_PROT;
+            cpu_exit(uc->cpu);
+            cpu_loop_exit_restore(env_cpu(env), ra);
+        }
+
+        probe_access(env, addr, probe_size, MMU_DATA_STORE, mmu_idx, ra);
+        addr += probe_size;
+        size -= probe_size;
+    }
 }
 
-void HELPER(sve_fnmls_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+static bool sve_can_read_addr(CPUARMState *env, target_ulong addr,
+                              int size, TCGMemOpIdx oi)
 {
-    do_fmla_zpzzz_h(env, vg, desc, 0, 0x8000);
+    struct uc_struct *uc = env->uc;
+    int mmu_idx = get_mmuidx(oi);
+
+    while (size > 0) {
+        target_ulong page_left = -(addr | TARGET_PAGE_MASK);
+        int probe_size = MIN(size, (int)page_left);
+        target_ulong paddr;
+        MemoryRegion *mr;
+
+        if (!tlb_vaddr_to_paddr(env, addr, MMU_DATA_LOAD, mmu_idx, &paddr)) {
+            return false;
+        }
+
+        mr = uc->memory_mapping(uc, paddr);
+        if (mr == NULL || !mr->ram || !(mr->perms & UC_PROT_READ)) {
+            return false;
+        }
+
+        addr += probe_size;
+        size -= probe_size;
+    }
+    return true;
 }
 
-static void do_fmla_zpzzz_s(CPUARMState *env, void *vg, uint32_t desc,
-                            uint32_t neg1, uint32_t neg3)
+static void sve_probe_stN_r_pages(CPUARMState *env, void *vg,
+                                  target_ulong addr, uint32_t desc,
+                                  const int esize, const int msize, int n,
+                                  bool mte, uintptr_t ra)
 {
-    intptr_t i = simd_oprsz(desc);
-    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
-    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
-    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
-    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
-    void *vd = &env->vfp.zregs[rd];
-    void *vn = &env->vfp.zregs[rn];
-    void *vm = &env->vfp.zregs[rm];
-    void *va = &env->vfp.zregs[ra];
-    uint64_t *g = vg;
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    intptr_t i, oprsz = simd_oprsz(desc);
 
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            i -= 4;
-            if (likely((pg >> (i & 63)) & 1)) {
-                float32 e1, e2, e3, r;
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
 
-                e1 = *(uint32_t *)((char *)vn + H1_4(i)) ^ neg1;
-                e2 = *(uint32_t *)((char *)vm + H1_4(i));
-                e3 = *(uint32_t *)((char *)va + H1_4(i)) ^ neg3;
-                r = float32_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
-                *(uint32_t *)((char *)vd + H1_4(i)) = r;
+        do {
+            if (pg & 1) {
+                target_ulong clean_addr = addr;
+                int j;
+
+                if (mte) {
+                    clean_addr = sve_mte_clean_addr(env, oi, addr, true,
+                                                    n * msize);
+                }
+                for (j = 0; j < n; j++) {
+                    sve_probe_write_addr(env, clean_addr + j * msize,
+                                         msize, oi, ra);
+                }
             }
-        } while (i & 63);
-    } while (i != 0);
+            i += esize, pg >>= esize;
+            addr += n * msize;
+        } while (i & 15);
+    }
 }
 
-void HELPER(sve_fmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+static void sve_check_stN_r_mte(CPUARMState *env, void *vg,
+                                target_ulong addr, uint32_t desc,
+                                const int esize, const int msize, int n,
+                                uintptr_t ra)
 {
-    do_fmla_zpzzz_s(env, vg, desc, 0, 0);
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+
+        do {
+            if (pg & 1) {
+                sve_mte_check_addr(env, oi, addr, true, n * msize, ra);
+            }
+            i += esize, pg >>= esize;
+            addr += n * msize;
+        } while (i & 15);
+    }
 }
 
-void HELPER(sve_fmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+/*
+ * Common helper for all contiguous one-register predicated loads.
+ */
+static void sve_ld1_r(CPUARMState *env, void *vg, const target_ulong addr,
+                      uint32_t desc, const uintptr_t retaddr,
+                      const int esz, const int msz,
+                      sve_ld1_host_fn *host_fn,
+                      sve_ld1_tlb_fn *tlb_fn)
 {
-    do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0);
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int mmu_idx = get_mmuidx(oi);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    void *vd = &env->vfp.zregs[rd];
+    const int diffsz = esz - msz;
+    const intptr_t reg_max = simd_oprsz(desc);
+    const intptr_t mem_max = reg_max >> diffsz;
+    ARMVectorReg scratch;
+    void *host;
+    intptr_t split, reg_off, mem_off;
+
+    /* Find the first active element.  */
+    reg_off = find_next_active(vg, 0, reg_max, esz);
+    if (unlikely(reg_off == reg_max)) {
+        /* The entire predicate was false; no load occurs.  */
+        memset(vd, 0, reg_max);
+        return;
+    }
+    mem_off = reg_off >> diffsz;
+    set_helper_retaddr(retaddr);
+
+    /*
+     * If the (remaining) load is entirely within a single page, then:
+     * For softmmu, and the tlb hits, then no faults will occur;
+     * For user-only, either the first load will fault or none will.
+     * We can thus perform the load directly to the destination and
+     * Vd will be unmodified on any exception path.
+     */
+    split = max_for_page(env->uc, addr, mem_off, mem_max);
+    if (likely(split == mem_max)) {
+        host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
+        if (test_host_page(host)) {
+            mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, mem_max);
+            tcg_debug_assert(mem_off == mem_max);
+            clear_helper_retaddr();
+            /* After having taken any fault, zero leading inactive elements. */
+            swap_memzero(vd, reg_off);
+            return;
+        }
+    }
+
+    /*
+     * Perform the predicated read into a temporary, thus ensuring
+     * if the load of the last element faults, Vd is not modified.
+     */
+    memset(&scratch, 0, reg_max);
+    goto start;
+    while (1) {
+        reg_off = find_next_active(vg, reg_off, reg_max, esz);
+        if (reg_off >= reg_max) {
+            break;
+        }
+        mem_off = reg_off >> diffsz;
+        split = max_for_page(env->uc, addr, mem_off, mem_max);
+
+    start:
+        if (split - mem_off >= (1ULL << msz)) {
+            /* At least one whole element on this page.  */
+            host = tlb_vaddr_to_host(env, addr + mem_off,
+                                     MMU_DATA_LOAD, mmu_idx);
+            if (host) {
+                mem_off = host_fn(&scratch, vg, (char *)host - mem_off,
+                                  mem_off, split);
+                reg_off = mem_off << diffsz;
+                continue;
+            }
+        }
+
+        /*
+         * Perform one normal read.  This may fault, longjmping out to the
+         * main loop in order to raise an exception.  It may succeed, and
+         * as a side-effect load the TLB entry for the next round.  Finally,
+         * in the extremely unlikely case we're performing this operation
+         * on I/O memory, it may succeed but not bring in the TLB entry.
+         * But even then we have still made forward progress.
+         */
+        tlb_fn(env, &scratch, reg_off, addr + mem_off, oi, retaddr);
+        reg_off += 1ULL << esz;
+    }
+
+    clear_helper_retaddr();
+    memcpy(vd, &scratch, reg_max);
 }
 
-void HELPER(sve_fnmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+static void sve_ld1_r_mte(CPUARMState *env, void *vg, target_ulong addr,
+                          uint32_t desc, const uintptr_t retaddr,
+                          const int esz, const int msz,
+                          sve_ld1_tlb_fn *tlb_fn)
 {
-    do_fmla_zpzzz_s(env, vg, desc, 0x80000000, 0x80000000);
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    void *vd = &env->vfp.zregs[rd];
+    const int diffsz = esz - msz;
+    const intptr_t reg_max = simd_oprsz(desc);
+    const int msize = 1 << msz;
+    ARMVectorReg scratch = { 0 };
+    intptr_t reg_off = 0;
+
+    set_helper_retaddr(retaddr);
+    while (true) {
+        target_ulong clean_addr;
+        intptr_t mem_off;
+
+        reg_off = find_next_active(vg, reg_off, reg_max, esz);
+        if (reg_off >= reg_max) {
+            break;
+        }
+
+        mem_off = reg_off >> diffsz;
+        clean_addr = sve_mte_check_addr(env, oi, addr + mem_off,
+                                        false, msize, retaddr);
+        tlb_fn(env, &scratch, reg_off, clean_addr, oi, retaddr);
+        reg_off += 1 << esz;
+    }
+    clear_helper_retaddr();
+
+    memcpy(vd, &scratch, reg_max);
+}
+
+#define DO_LD1_1(NAME, ESZ) \
+void HELPER(sve_##NAME##_r)(CPUARMState *env, void *vg,        \
+                            target_ulong addr, uint32_t desc)  \
+{                                                              \
+    sve_ld1_r(env, vg, addr, desc, GETPC(), ESZ, 0,            \
+              sve_##NAME##_host, sve_##NAME##_tlb);            \
+}                                                              \
+void HELPER(sve_##NAME##_r_mte)(CPUARMState *env, void *vg,    \
+                                target_ulong addr, uint32_t desc) \
+{                                                              \
+    sve_ld1_r_mte(env, vg, addr, desc, GETPC(), ESZ, 0,        \
+                  sve_##NAME##_tlb);                           \
+}
+
+#define DO_LD1_2(NAME, ESZ, MSZ) \
+void HELPER(sve_##NAME##_le_r)(CPUARMState *env, void *vg,        \
+                               target_ulong addr, uint32_t desc)  \
+{                                                                 \
+    sve_ld1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,             \
+              sve_##NAME##_le_host, sve_##NAME##_le_tlb);         \
+}                                                                 \
+void HELPER(sve_##NAME##_be_r)(CPUARMState *env, void *vg,        \
+                               target_ulong addr, uint32_t desc)  \
+{                                                                 \
+    sve_ld1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,             \
+              sve_##NAME##_be_host, sve_##NAME##_be_tlb);         \
+}                                                                 \
+void HELPER(sve_##NAME##_le_r_mte)(CPUARMState *env, void *vg,    \
+                                   target_ulong addr, uint32_t desc) \
+{                                                                 \
+    sve_ld1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ,         \
+                  sve_##NAME##_le_tlb);                           \
+}                                                                 \
+void HELPER(sve_##NAME##_be_r_mte)(CPUARMState *env, void *vg,    \
+                                   target_ulong addr, uint32_t desc) \
+{                                                                 \
+    sve_ld1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ,         \
+                  sve_##NAME##_be_tlb);                           \
 }
 
-void HELPER(sve_fnmls_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
-{
-    do_fmla_zpzzz_s(env, vg, desc, 0, 0x80000000);
-}
+DO_LD1_1(ld1bb,  0)
+DO_LD1_1(ld1bhu, 1)
+DO_LD1_1(ld1bhs, 1)
+DO_LD1_1(ld1bsu, 2)
+DO_LD1_1(ld1bss, 2)
+DO_LD1_1(ld1bdu, 3)
+DO_LD1_1(ld1bds, 3)
+
+DO_LD1_2(ld1hh,  1, 1)
+DO_LD1_2(ld1hsu, 2, 1)
+DO_LD1_2(ld1hss, 2, 1)
+DO_LD1_2(ld1hdu, 3, 1)
+DO_LD1_2(ld1hds, 3, 1)
+
+DO_LD1_2(ld1ss,  2, 2)
+DO_LD1_2(ld1sdu, 3, 2)
+DO_LD1_2(ld1sds, 3, 2)
+
+DO_LD1_2(ld1dd,  3, 3)
 
-static void do_fmla_zpzzz_d(CPUARMState *env, void *vg, uint32_t desc,
-                            uint64_t neg1, uint64_t neg3)
+#undef DO_LD1_1
+#undef DO_LD1_2
+
+/*
+ * Common helpers for all contiguous 2,3,4-register predicated loads.
+ */
+static void sve_ld2_r(CPUARMState *env, void *vg, target_ulong addr,
+                      uint32_t desc, int size, uintptr_t ra,
+                      sve_ld1_tlb_fn *tlb_fn)
 {
-    intptr_t i = simd_oprsz(desc);
-    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
-    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
-    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
-    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
-    void *vd = &env->vfp.zregs[rd];
-    void *vn = &env->vfp.zregs[rn];
-    void *vm = &env->vfp.zregs[rm];
-    void *va = &env->vfp.zregs[ra];
-    uint64_t *g = vg;
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    ARMVectorReg scratch[2] = { 0 };
 
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
         do {
-            i -= 8;
-            if (likely((pg >> (i & 63)) & 1)) {
-                float64 e1, e2, e3, r;
-
-                e1 = *(uint64_t *)((char *)vn + i) ^ neg1;
-                e2 = *(uint64_t *)((char *)vm + i);
-                e3 = *(uint64_t *)((char *)va + i) ^ neg3;
-                r = float64_muladd(e1, e2, e3, 0, &env->vfp.fp_status);
-                *(uint64_t *)((char *)vd + i) = r;
+            if (pg & 1) {
+                tlb_fn(env, &scratch[0], i, addr, oi, ra);
+                tlb_fn(env, &scratch[1], i, addr + size, oi, ra);
             }
-        } while (i & 63);
-    } while (i != 0);
-}
-
-void HELPER(sve_fmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
-{
-    do_fmla_zpzzz_d(env, vg, desc, 0, 0);
-}
+            i += size, pg >>= size;
+            addr += 2 * size;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
 
-void HELPER(sve_fmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
-{
-    do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, 0);
+    /* Wait until all exceptions have been raised to write back.  */
+    memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
+    memcpy(&env->vfp.zregs[(rd + 1) & 31], &scratch[1], oprsz);
 }
 
-void HELPER(sve_fnmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+static void sve_ld3_r(CPUARMState *env, void *vg, target_ulong addr,
+                      uint32_t desc, int size, uintptr_t ra,
+                      sve_ld1_tlb_fn *tlb_fn)
 {
-    do_fmla_zpzzz_d(env, vg, desc, INT64_MIN, INT64_MIN);
-}
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    ARMVectorReg scratch[3] = { 0 };
 
-void HELPER(sve_fnmls_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
-{
-    do_fmla_zpzzz_d(env, vg, desc, 0, INT64_MIN);
-}
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        do {
+            if (pg & 1) {
+                tlb_fn(env, &scratch[0], i, addr, oi, ra);
+                tlb_fn(env, &scratch[1], i, addr + size, oi, ra);
+                tlb_fn(env, &scratch[2], i, addr + 2 * size, oi, ra);
+            }
+            i += size, pg >>= size;
+            addr += 3 * size;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
 
-/* Two operand floating-point comparison controlled by a predicate.
- * Unlike the integer version, we are not allowed to optimistically
- * compare operands, since the comparison may have side effects wrt
- * the FPSR.
- */
-#define DO_FPCMP_PPZZ(NAME, TYPE, H, OP)                                \
-void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg,               \
-                  void *status, uint32_t desc)                          \
-{                                                                       \
-    intptr_t i = simd_oprsz(desc), j = (i - 1) >> 6;                    \
-    uint64_t *d = vd, *g = vg;                                          \
-    do {                                                                \
-        uint64_t out = 0, pg = g[j];                                    \
-        do {                                                            \
-            i -= sizeof(TYPE), out <<= sizeof(TYPE);                    \
-            if (likely((pg >> (i & 63)) & 1)) {                         \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));                         \
-                TYPE mm = *(TYPE *)((char *)vm + H(i));                         \
-                out |= OP(TYPE, nn, mm, status);                        \
-            }                                                           \
-        } while (i & 63);                                               \
-        d[j--] = out;                                                   \
-    } while (i > 0);                                                    \
+    /* Wait until all exceptions have been raised to write back.  */
+    memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
+    memcpy(&env->vfp.zregs[(rd + 1) & 31], &scratch[1], oprsz);
+    memcpy(&env->vfp.zregs[(rd + 2) & 31], &scratch[2], oprsz);
 }
 
-#define DO_FPCMP_PPZZ_H(NAME, OP) \
-    DO_FPCMP_PPZZ(NAME##_h, float16, H1_2, OP)
-#define DO_FPCMP_PPZZ_S(NAME, OP) \
-    DO_FPCMP_PPZZ(NAME##_s, float32, H1_4, OP)
-#define DO_FPCMP_PPZZ_D(NAME, OP) \
-    DO_FPCMP_PPZZ(NAME##_d, float64,     , OP)
-
-#define DO_FPCMP_PPZZ_ALL(NAME, OP) \
-    DO_FPCMP_PPZZ_H(NAME, OP)   \
-    DO_FPCMP_PPZZ_S(NAME, OP)   \
-    DO_FPCMP_PPZZ_D(NAME, OP)
-
-#define DO_FCMGE(TYPE, X, Y, ST)  TYPE##_compare(Y, X, ST) <= 0
-#define DO_FCMGT(TYPE, X, Y, ST)  TYPE##_compare(Y, X, ST) < 0
-#define DO_FCMLE(TYPE, X, Y, ST)  TYPE##_compare(X, Y, ST) <= 0
-#define DO_FCMLT(TYPE, X, Y, ST)  TYPE##_compare(X, Y, ST) < 0
-#define DO_FCMEQ(TYPE, X, Y, ST)  TYPE##_compare_quiet(X, Y, ST) == 0
-#define DO_FCMNE(TYPE, X, Y, ST)  TYPE##_compare_quiet(X, Y, ST) != 0
-#define DO_FCMUO(TYPE, X, Y, ST)  \
-    TYPE##_compare_quiet(X, Y, ST) == float_relation_unordered
-#define DO_FACGE(TYPE, X, Y, ST)  \
-    TYPE##_compare(TYPE##_abs(Y), TYPE##_abs(X), ST) <= 0
-#define DO_FACGT(TYPE, X, Y, ST)  \
-    TYPE##_compare(TYPE##_abs(Y), TYPE##_abs(X), ST) < 0
-
-DO_FPCMP_PPZZ_ALL(sve_fcmge, DO_FCMGE)
-DO_FPCMP_PPZZ_ALL(sve_fcmgt, DO_FCMGT)
-DO_FPCMP_PPZZ_ALL(sve_fcmeq, DO_FCMEQ)
-DO_FPCMP_PPZZ_ALL(sve_fcmne, DO_FCMNE)
-DO_FPCMP_PPZZ_ALL(sve_fcmuo, DO_FCMUO)
-DO_FPCMP_PPZZ_ALL(sve_facge, DO_FACGE)
-DO_FPCMP_PPZZ_ALL(sve_facgt, DO_FACGT)
+static void sve_ld4_r(CPUARMState *env, void *vg, target_ulong addr,
+                      uint32_t desc, int size, uintptr_t ra,
+                      sve_ld1_tlb_fn *tlb_fn)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    ARMVectorReg scratch[4] = { 0 };
 
-#undef DO_FPCMP_PPZZ_ALL
-#undef DO_FPCMP_PPZZ_D
-#undef DO_FPCMP_PPZZ_S
-#undef DO_FPCMP_PPZZ_H
-#undef DO_FPCMP_PPZZ
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        do {
+            if (pg & 1) {
+                tlb_fn(env, &scratch[0], i, addr, oi, ra);
+                tlb_fn(env, &scratch[1], i, addr + size, oi, ra);
+                tlb_fn(env, &scratch[2], i, addr + 2 * size, oi, ra);
+                tlb_fn(env, &scratch[3], i, addr + 3 * size, oi, ra);
+            }
+            i += size, pg >>= size;
+            addr += 4 * size;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
 
-/* One operand floating-point comparison against zero, controlled
- * by a predicate.
- */
-#define DO_FPCMP_PPZ0(NAME, TYPE, H, OP)                   \
-void HELPER(NAME)(void *vd, void *vn, void *vg,            \
-                  void *status, uint32_t desc)             \
-{                                                          \
-    intptr_t i = simd_oprsz(desc), j = (i - 1) >> 6;       \
-    uint64_t *d = vd, *g = vg;                             \
-    do {                                                   \
-        uint64_t out = 0, pg = g[j];                       \
-        do {                                               \
-            i -= sizeof(TYPE), out <<= sizeof(TYPE);       \
-            if ((pg >> (i & 63)) & 1) {                    \
-                TYPE nn = *(TYPE *)((char *)vn + H(i));            \
-                out |= OP(TYPE, nn, 0, status);            \
-            }                                              \
-        } while (i & 63);                                  \
-        d[j--] = out;                                      \
-    } while (i > 0);                                       \
+    /* Wait until all exceptions have been raised to write back.  */
+    memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
+    memcpy(&env->vfp.zregs[(rd + 1) & 31], &scratch[1], oprsz);
+    memcpy(&env->vfp.zregs[(rd + 2) & 31], &scratch[2], oprsz);
+    memcpy(&env->vfp.zregs[(rd + 3) & 31], &scratch[3], oprsz);
 }
 
-#define DO_FPCMP_PPZ0_H(NAME, OP) \
-    DO_FPCMP_PPZ0(NAME##_h, float16, H1_2, OP)
-#define DO_FPCMP_PPZ0_S(NAME, OP) \
-    DO_FPCMP_PPZ0(NAME##_s, float32, H1_4, OP)
-#define DO_FPCMP_PPZ0_D(NAME, OP) \
-    DO_FPCMP_PPZ0(NAME##_d, float64,     , OP)
+static void sve_ldN_r_mte(CPUARMState *env, void *vg, target_ulong addr,
+                          uint32_t desc, int size, int n, uintptr_t ra,
+                          sve_ld1_tlb_fn *tlb_fn)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    ARMVectorReg scratch[4] = { 0 };
+    int j;
 
-#define DO_FPCMP_PPZ0_ALL(NAME, OP) \
-    DO_FPCMP_PPZ0_H(NAME, OP)   \
-    DO_FPCMP_PPZ0_S(NAME, OP)   \
-    DO_FPCMP_PPZ0_D(NAME, OP)
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
 
-DO_FPCMP_PPZ0_ALL(sve_fcmge0, DO_FCMGE)
-DO_FPCMP_PPZ0_ALL(sve_fcmgt0, DO_FCMGT)
-DO_FPCMP_PPZ0_ALL(sve_fcmle0, DO_FCMLE)
-DO_FPCMP_PPZ0_ALL(sve_fcmlt0, DO_FCMLT)
-DO_FPCMP_PPZ0_ALL(sve_fcmeq0, DO_FCMEQ)
-DO_FPCMP_PPZ0_ALL(sve_fcmne0, DO_FCMNE)
+        do {
+            if (pg & 1) {
+                target_ulong clean_addr =
+                    sve_mte_check_addr(env, oi, addr, false, n * size, ra);
 
-/* FP Trig Multiply-Add. */
+                for (j = 0; j < n; j++) {
+                    tlb_fn(env, &scratch[j], i, clean_addr + j * size,
+                           oi, ra);
+                }
+            }
+            i += size, pg >>= size;
+            addr += n * size;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
 
-void HELPER(sve_ftmad_h)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
-{
-    static const float16 coeff[16] = {
-        0x3c00, 0xb155, 0x2030, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-        0x3c00, 0xb800, 0x293a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    };
-    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float16);
-    intptr_t x = simd_data(desc);
-    float16 *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i++) {
-        float16 mm = m[i];
-        intptr_t xx = x;
-        if (float16_is_neg(mm)) {
-            mm = float16_abs(mm);
-            xx += 8;
-        }
-        d[i] = float16_muladd(n[i], mm, coeff[xx], 0, vs);
+    for (j = 0; j < n; j++) {
+        memcpy(&env->vfp.zregs[(rd + j) & 31], &scratch[j], oprsz);
     }
 }
 
-void HELPER(sve_ftmad_s)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
-{
-    static const float32 coeff[16] = {
-        0x3f800000, 0xbe2aaaab, 0x3c088886, 0xb95008b9,
-        0x36369d6d, 0x00000000, 0x00000000, 0x00000000,
-        0x3f800000, 0xbf000000, 0x3d2aaaa6, 0xbab60705,
-        0x37cd37cc, 0x00000000, 0x00000000, 0x00000000,
-    };
-    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float32);
-    intptr_t x = simd_data(desc);
-    float32 *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i++) {
-        float32 mm = m[i];
-        intptr_t xx = x;
-        if (float32_is_neg(mm)) {
-            mm = float32_abs(mm);
-            xx += 8;
-        }
-        d[i] = float32_muladd(n[i], mm, coeff[xx], 0, vs);
-    }
+#define DO_LDN_1(N) \
+void QEMU_FLATTEN HELPER(sve_ld##N##bb_r) \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)  \
+{                                                                   \
+    sve_ld##N##_r(env, vg, addr, desc, 1, GETPC(), sve_ld1bb_tlb);  \
+}                                                                   \
+void QEMU_FLATTEN HELPER(sve_ld##N##bb_r_mte)                       \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)  \
+{                                                                   \
+    sve_ldN_r_mte(env, vg, addr, desc, 1, N, GETPC(),               \
+                  sve_ld1bb_tlb);                                   \
 }
 
-void HELPER(sve_ftmad_d)(void *vd, void *vn, void *vm, void *vs, uint32_t desc)
-{
-    static const float64 coeff[16] = {
-        0x3ff0000000000000ull, 0xbfc5555555555543ull,
-        0x3f8111111110f30cull, 0xbf2a01a019b92fc6ull,
-        0x3ec71de351f3d22bull, 0xbe5ae5e2b60f7b91ull,
-        0x3de5d8408868552full, 0x0000000000000000ull,
-        0x3ff0000000000000ull, 0xbfe0000000000000ull,
-        0x3fa5555555555536ull, 0xbf56c16c16c13a0bull,
-        0x3efa01a019b1e8d8ull, 0xbe927e4f7282f468ull,
-        0x3e21ee96d2641b13ull, 0xbda8f76380fbb401ull,
-    };
-    intptr_t i, opr_sz = simd_oprsz(desc) / sizeof(float64);
-    intptr_t x = simd_data(desc);
-    float64 *d = vd, *n = vn, *m = vm;
-    for (i = 0; i < opr_sz; i++) {
-        float64 mm = m[i];
-        intptr_t xx = x;
-        if (float64_is_neg(mm)) {
-            mm = float64_abs(mm);
-            xx += 8;
-        }
-        d[i] = float64_muladd(n[i], mm, coeff[xx], 0, vs);
-    }
+#define DO_LDN_2(N, SUFF, SIZE)                                       \
+void QEMU_FLATTEN HELPER(sve_ld##N##SUFF##_le_r)                      \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_ld##N##_r(env, vg, addr, desc, SIZE, GETPC(),                 \
+                  sve_ld1##SUFF##_le_tlb);                            \
+}                                                                     \
+void QEMU_FLATTEN HELPER(sve_ld##N##SUFF##_be_r)                      \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_ld##N##_r(env, vg, addr, desc, SIZE, GETPC(),                 \
+                  sve_ld1##SUFF##_be_tlb);                            \
+}                                                                     \
+void QEMU_FLATTEN HELPER(sve_ld##N##SUFF##_le_r_mte)                  \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_ldN_r_mte(env, vg, addr, desc, SIZE, N, GETPC(),              \
+                  sve_ld1##SUFF##_le_tlb);                            \
+}                                                                     \
+void QEMU_FLATTEN HELPER(sve_ld##N##SUFF##_be_r_mte)                  \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_ldN_r_mte(env, vg, addr, desc, SIZE, N, GETPC(),              \
+                  sve_ld1##SUFF##_be_tlb);                            \
 }
 
-/*
- * FP Complex Add
- */
+DO_LDN_1(2)
+DO_LDN_1(3)
+DO_LDN_1(4)
 
-void HELPER(sve_fcadd_h)(void *vd, void *vn, void *vm, void *vg,
-                         void *vs, uint32_t desc)
-{
-    intptr_t j, i = simd_oprsz(desc);
-    uint64_t *g = vg;
-    float16 neg_imag = float16_set_sign(0, simd_data(desc));
-    float16 neg_real = float16_chs(neg_imag);
+DO_LDN_2(2, hh, 2)
+DO_LDN_2(3, hh, 2)
+DO_LDN_2(4, hh, 2)
 
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            float16 e0, e1, e2, e3;
+DO_LDN_2(2, ss, 4)
+DO_LDN_2(3, ss, 4)
+DO_LDN_2(4, ss, 4)
 
-            /* I holds the real index; J holds the imag index.  */
-            j = i - sizeof(float16);
-            i -= 2 * sizeof(float16);
+DO_LDN_2(2, dd, 8)
+DO_LDN_2(3, dd, 8)
+DO_LDN_2(4, dd, 8)
 
-            e0 = *(float16 *)((char *)vn + H1_2(i));
-            e1 = *(float16 *)((char *)vm + H1_2(j)) ^ neg_real;
-            e2 = *(float16 *)((char *)vn + H1_2(j));
-            e3 = *(float16 *)((char *)vm + H1_2(i)) ^ neg_imag;
+#undef DO_LDN_1
+#undef DO_LDN_2
 
-            if (likely((pg >> (i & 63)) & 1)) {
-                *(float16 *)((char *)vd + H1_2(i)) = float16_add(e0, e1, vs);
-            }
-            if (likely((pg >> (j & 63)) & 1)) {
-                *(float16 *)((char *)vd + H1_2(j)) = float16_add(e2, e3, vs);
-            }
-        } while (i & 63);
-    } while (i != 0);
-}
+/*
+ * Load contiguous data, first-fault and no-fault.
+ *
+ * For user-only, one could argue that we should hold the mmap_lock during
+ * the operation so that there is no race between page_check_range and the
+ * load operation.  However, unmapping pages out from under a running thread
+ * is extraordinarily unlikely.  This theoretical race condition also affects
+ * linux-user/ in its get_user/put_user macros.
+ *
+ * TODO: Construct some helpers, written in assembly, that interact with
+ * handle_cpu_signal to produce memory ops which can properly report errors
+ * without racing.
+ */
 
-void HELPER(sve_fcadd_s)(void *vd, void *vn, void *vm, void *vg,
-                         void *vs, uint32_t desc)
+/* Fault on byte I.  All bits in FFR from I are cleared.  The vector
+ * result from I is CONSTRAINED UNPREDICTABLE; we choose the MERGE
+ * option, which leaves subsequent data unchanged.
+ */
+static void record_fault(CPUARMState *env, uintptr_t i, uintptr_t oprsz)
 {
-    intptr_t j, i = simd_oprsz(desc);
-    uint64_t *g = vg;
-    float32 neg_imag = float32_set_sign(0, simd_data(desc));
-    float32 neg_real = float32_chs(neg_imag);
-
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            float32 e0, e1, e2, e3;
+    uint64_t *ffr = env->vfp.pregs[FFR_PRED_NUM].p;
 
-            /* I holds the real index; J holds the imag index.  */
-            j = i - sizeof(float32);
-            i -= 2 * sizeof(float32);
+    if (i & 63) {
+        ffr[i / 64] &= MAKE_64BIT_MASK(0, i & 63);
+        i = ROUND_UP(i, 64);
+    }
+    for (; i < oprsz; i += 64) {
+        ffr[i / 64] = 0;
+    }
+}
 
-            e0 = *(float32 *)((char *)vn + H1_2(i));
-            e1 = *(float32 *)((char *)vm + H1_2(j)) ^ neg_real;
-            e2 = *(float32 *)((char *)vn + H1_2(j));
-            e3 = *(float32 *)((char *)vm + H1_2(i)) ^ neg_imag;
+static bool sve_pred_test(void *vg, intptr_t reg_off)
+{
+    uint64_t *pg = vg;
 
-            if (likely((pg >> (i & 63)) & 1)) {
-                *(float32 *)((char *)vd + H1_2(i)) = float32_add(e0, e1, vs);
-            }
-            if (likely((pg >> (j & 63)) & 1)) {
-                *(float32 *)((char *)vd + H1_2(j)) = float32_add(e2, e3, vs);
-            }
-        } while (i & 63);
-    } while (i != 0);
+    return (pg[reg_off >> 6] >> (reg_off & 63)) & 1;
 }
 
-void HELPER(sve_fcadd_d)(void *vd, void *vn, void *vm, void *vg,
-                         void *vs, uint32_t desc)
+/*
+ * Common helper for all contiguous first-fault loads.
+ */
+static void sve_ldff1_r(CPUARMState *env, void *vg, const target_ulong addr,
+                        uint32_t desc, const uintptr_t retaddr,
+                        const int esz, const int msz,
+                        sve_ld1_host_fn *host_fn,
+                        sve_ld1_tlb_fn *tlb_fn)
 {
-    intptr_t j, i = simd_oprsz(desc);
-    uint64_t *g = vg;
-    float64 neg_imag = float64_set_sign(0, simd_data(desc));
-    float64 neg_real = float64_chs(neg_imag);
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int mmu_idx = get_mmuidx(oi);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    void *vd = &env->vfp.zregs[rd];
+    const int diffsz = esz - msz;
+    const intptr_t reg_max = simd_oprsz(desc);
+    const intptr_t mem_max = reg_max >> diffsz;
+    intptr_t split, reg_off, mem_off;
+    bool is_split;
+    void *host;
 
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            float64 e0, e1, e2, e3;
+    /* Skip to the first active element.  */
+    reg_off = find_next_active(vg, 0, reg_max, esz);
+    if (unlikely(reg_off == reg_max)) {
+        /* The entire predicate was false; no load occurs.  */
+        memset(vd, 0, reg_max);
+        return;
+    }
+    mem_off = reg_off >> diffsz;
+    set_helper_retaddr(retaddr);
 
-            /* I holds the real index; J holds the imag index.  */
-            j = i - sizeof(float64);
-            i -= 2 * sizeof(float64);
+    /*
+     * If the (remaining) load is entirely within a single page, then:
+     * For softmmu, and the tlb hits, then no faults will occur;
+     * For user-only, either the first load will fault or none will.
+     * We can thus perform the load directly to the destination and
+     * Vd will be unmodified on any exception path.
+     */
+    split = max_for_page(env->uc, addr, mem_off, mem_max);
+    if (likely(split == mem_max)) {
+        host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
+        if (test_host_page(host)) {
+            mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, mem_max);
+            tcg_debug_assert(mem_off == mem_max);
+            clear_helper_retaddr();
+            /* After any fault, zero any leading inactive elements.  */
+            swap_memzero(vd, reg_off);
+            return;
+        }
+    }
 
-            e0 = *(float64 *)((char *)vn + H1_2(i));
-            e1 = *(float64 *)((char *)vm + H1_2(j)) ^ neg_real;
-            e2 = *(float64 *)((char *)vn + H1_2(j));
-            e3 = *(float64 *)((char *)vm + H1_2(i)) ^ neg_imag;
+    /*
+     * Perform one normal read, which will fault or not.
+     * But it is likely to bring the page into the tlb.
+     */
+    is_split = max_for_page(env->uc, addr, mem_off,
+                            mem_off + (1ULL << msz)) <
+               mem_off + (1ULL << msz);
+    tlb_fn(env, vd, reg_off, addr + mem_off, oi, retaddr);
 
-            if (likely((pg >> (i & 63)) & 1)) {
-                *(float64 *)((char *)vd + H1_2(i)) = float64_add(e0, e1, vs);
-            }
-            if (likely((pg >> (j & 63)) & 1)) {
-                *(float64 *)((char *)vd + H1_2(j)) = float64_add(e2, e3, vs);
-            }
-        } while (i & 63);
-    } while (i != 0);
+    /* After any fault, zero any leading predicated false elts.  */
+    swap_memzero(vd, reg_off);
+    mem_off += 1ULL << msz;
+    reg_off += 1ULL << esz;
+    if (is_split) {
+        swap_memzero((char *)vd + reg_off, reg_max - reg_off);
+        clear_helper_retaddr();
+        record_fault(env, reg_off, reg_max);
+        return;
+    }
+
+    /* Try again to read the balance of the page.  */
+    split = max_for_page(env->uc, addr, mem_off - 1, mem_max);
+    if (split >= (1ULL << msz)) {
+        host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
+        if (host) {
+            mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, split);
+            reg_off = mem_off << diffsz;
+        }
+    }
+
+    clear_helper_retaddr();
+    record_fault(env, reg_off, reg_max);
 }
 
 /*
- * FP Complex Multiply
+ * Common helper for all contiguous no-fault loads.
  */
-
-QEMU_BUILD_BUG_ON(SIMD_DATA_SHIFT + 22 > 32);
-
-void HELPER(sve_fcmla_zpzzz_h)(CPUARMState *env, void *vg, uint32_t desc)
+static void sve_ldnf1_r(CPUARMState *env, void *vg, const target_ulong addr,
+                        uint32_t desc, const uintptr_t retaddr,
+                        const int esz, const int msz,
+                        sve_ld1_host_fn *host_fn,
+                        sve_ld1_tlb_fn *tlb_fn)
 {
-    intptr_t j, i = simd_oprsz(desc);
-    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
-    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
-    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
-    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
-    unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
-    bool flip = rot & 1;
-    float16 neg_imag, neg_real;
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
     void *vd = &env->vfp.zregs[rd];
-    void *vn = &env->vfp.zregs[rn];
-    void *vm = &env->vfp.zregs[rm];
-    void *va = &env->vfp.zregs[ra];
-    uint64_t *g = vg;
-
-    neg_imag = float16_set_sign(0, (rot & 2) != 0);
-    neg_real = float16_set_sign(0, rot == 1 || rot == 2);
-
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            float16 e1, e2, e3, e4, nr, ni, mr, mi, d;
+    const int diffsz = esz - msz;
+    const intptr_t reg_max = simd_oprsz(desc);
+    const intptr_t mem_max = reg_max >> diffsz;
+    const int mmu_idx = get_mmuidx(oi);
+    const intptr_t msize = 1ULL << msz;
+    intptr_t split, reg_off, mem_off;
+    void *host;
 
-            /* I holds the real index; J holds the imag index.  */
-            j = i - sizeof(float16);
-            i -= 2 * sizeof(float16);
+    /* There will be no fault, so we may modify in advance.  */
+    memset(vd, 0, reg_max);
 
-            nr = *(float16 *)((char *)vn + H1_2(i));
-            ni = *(float16 *)((char *)vn + H1_2(j));
-            mr = *(float16 *)((char *)vm + H1_2(i));
-            mi = *(float16 *)((char *)vm + H1_2(j));
+    /* Skip to the first active element.  */
+    reg_off = find_next_active(vg, 0, reg_max, esz);
+    if (unlikely(reg_off == reg_max)) {
+        /* The entire predicate was false; no load occurs.  */
+        return;
+    }
+    mem_off = reg_off >> diffsz;
 
-            e2 = (flip ? ni : nr);
-            e1 = (flip ? mi : mr) ^ neg_real;
-            e4 = e2;
-            e3 = (flip ? mr : mi) ^ neg_imag;
+    /*
+     * If the address is not in the TLB, we have no way to bring the
+     * entry into the TLB without also risking a fault.  Note that
+     * the corollary is that we never load from an address not in RAM.
+     *
+     * This last is out of spec, in a weird corner case.
+     * Per the MemNF/MemSingleNF pseudocode, a NF load from Device memory
+     * must not actually hit the bus -- it returns UNKNOWN data instead.
+     * But if you map non-RAM with Normal memory attributes and do a NF
+     * load then it should access the bus.  (Nobody ought actually do this
+     * in the real world, obviously.)
+     *
+     * Then there are the annoying special cases with watchpoints...
+     * TODO: Add a form of non-faulting loads using cc->tlb_fill(probe=true).
+     */
+    host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
+    split = max_for_page(env->uc, addr, mem_off, mem_max);
+    if (host && split >= msize) {
+        mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, split);
+        reg_off = mem_off << diffsz;
+    } else if (split > 0 && split < msize && mem_off + msize <= mem_max &&
+               sve_can_read_addr(env, addr + mem_off, msize, oi)) {
+        tlb_fn(env, vd, reg_off, addr + mem_off, oi, retaddr);
+        reg_off += 1ULL << esz;
+    }
 
-            if (likely((pg >> (i & 63)) & 1)) {
-                d = *(float16 *)((char *)va + H1_2(i));
-                d = float16_muladd(e2, e1, d, 0, &env->vfp.fp_status_f16);
-                *(float16 *)((char *)vd + H1_2(i)) = d;
-            }
-            if (likely((pg >> (j & 63)) & 1)) {
-                d = *(float16 *)((char *)va + H1_2(j));
-                d = float16_muladd(e4, e3, d, 0, &env->vfp.fp_status_f16);
-                *(float16 *)((char *)vd + H1_2(j)) = d;
-            }
-        } while (i & 63);
-    } while (i != 0);
+    record_fault(env, reg_off, reg_max);
 }
 
-void HELPER(sve_fcmla_zpzzz_s)(CPUARMState *env, void *vg, uint32_t desc)
+static void sve_ldff1_r_mte(CPUARMState *env, void *vg, target_ulong addr,
+                            uint32_t desc, uintptr_t retaddr,
+                            int esz, int msz, sve_ld1_host_fn *host_fn,
+                            sve_ld1_tlb_fn *tlb_fn)
 {
-    intptr_t j, i = simd_oprsz(desc);
-    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
-    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
-    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
-    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
-    unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
-    bool flip = rot & 1;
-    float32 neg_imag, neg_real;
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int mmu_idx = get_mmuidx(oi);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
     void *vd = &env->vfp.zregs[rd];
-    void *vn = &env->vfp.zregs[rn];
-    void *vm = &env->vfp.zregs[rm];
-    void *va = &env->vfp.zregs[ra];
-    uint64_t *g = vg;
-
-    neg_imag = float32_set_sign(0, (rot & 2) != 0);
-    neg_real = float32_set_sign(0, rot == 1 || rot == 2);
+    const int diffsz = esz - msz;
+    const intptr_t reg_max = simd_oprsz(desc);
+    const intptr_t mem_max = reg_max >> diffsz;
+    const int msize = 1 << msz;
+    const int rstep = 1 << esz;
+    intptr_t reg_off, mem_off;
 
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            float32 e1, e2, e3, e4, nr, ni, mr, mi, d;
+    reg_off = find_next_active(vg, 0, reg_max, esz);
+    if (unlikely(reg_off == reg_max)) {
+        memset(vd, 0, reg_max);
+        return;
+    }
 
-            /* I holds the real index; J holds the imag index.  */
-            j = i - sizeof(float32);
-            i -= 2 * sizeof(float32);
+    set_helper_retaddr(retaddr);
+    mem_off = reg_off >> diffsz;
+    tlb_fn(env, vd, reg_off,
+           sve_mte_check_addr(env, oi, addr + mem_off,
+                              false, msize, retaddr),
+           oi, retaddr);
 
-            nr = *(float32 *)((char *)vn + H1_2(i));
-            ni = *(float32 *)((char *)vn + H1_2(j));
-            mr = *(float32 *)((char *)vm + H1_2(i));
-            mi = *(float32 *)((char *)vm + H1_2(j));
+    swap_memzero(vd, reg_off);
+    reg_off += rstep;
 
-            e2 = (flip ? ni : nr);
-            e1 = (flip ? mi : mr) ^ neg_real;
-            e4 = e2;
-            e3 = (flip ? mr : mi) ^ neg_imag;
+    while (reg_off < reg_max) {
+        bool active = sve_pred_test(vg, reg_off);
 
-            if (likely((pg >> (i & 63)) & 1)) {
-                d = *(float32 *)((char *)va + H1_2(i));
-                d = float32_muladd(e2, e1, d, 0, &env->vfp.fp_status);
-                *(float32 *)((char *)vd + H1_2(i)) = d;
+        mem_off = reg_off >> diffsz;
+        if (active) {
+            target_ulong tagged_addr = addr + mem_off;
+            target_ulong clean_addr =
+                sve_mte_clean_addr(env, oi, tagged_addr, false, msize);
+            void *host;
+
+            if (mem_off >= mem_max ||
+                max_for_page(env->uc, clean_addr, 0, msize) < msize) {
+                break;
             }
-            if (likely((pg >> (j & 63)) & 1)) {
-                d = *(float32 *)((char *)va + H1_2(j));
-                d = float32_muladd(e4, e3, d, 0, &env->vfp.fp_status);
-                *(float32 *)((char *)vd + H1_2(j)) = d;
+
+            host = tlb_vaddr_to_host(env, clean_addr, MMU_DATA_LOAD,
+                                     mmu_idx);
+            if (!host ||
+                !sve_mte_probe_addr(env, oi, tagged_addr, false, msize)) {
+                break;
             }
-        } while (i & 63);
-    } while (i != 0);
+            host_fn(vd, vg, (char *)host - mem_off, mem_off,
+                    mem_off + msize);
+        } else {
+            host_fn(vd, vg, NULL, mem_off, mem_off + msize);
+        }
+        reg_off += rstep;
+    }
+
+    clear_helper_retaddr();
+    record_fault(env, reg_off, reg_max);
 }
 
-void HELPER(sve_fcmla_zpzzz_d)(CPUARMState *env, void *vg, uint32_t desc)
+static void sve_ldnf1_r_mte(CPUARMState *env, void *vg, target_ulong addr,
+                            uint32_t desc, int esz, int msz,
+                            sve_ld1_host_fn *host_fn)
 {
-    intptr_t j, i = simd_oprsz(desc);
-    unsigned rd = extract32(desc, SIMD_DATA_SHIFT, 5);
-    unsigned rn = extract32(desc, SIMD_DATA_SHIFT + 5, 5);
-    unsigned rm = extract32(desc, SIMD_DATA_SHIFT + 10, 5);
-    unsigned ra = extract32(desc, SIMD_DATA_SHIFT + 15, 5);
-    unsigned rot = extract32(desc, SIMD_DATA_SHIFT + 20, 2);
-    bool flip = rot & 1;
-    float64 neg_imag, neg_real;
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int mmu_idx = get_mmuidx(oi);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
     void *vd = &env->vfp.zregs[rd];
-    void *vn = &env->vfp.zregs[rn];
-    void *vm = &env->vfp.zregs[rm];
-    void *va = &env->vfp.zregs[ra];
-    uint64_t *g = vg;
+    const int diffsz = esz - msz;
+    const intptr_t reg_max = simd_oprsz(desc);
+    const intptr_t mem_max = reg_max >> diffsz;
+    const int msize = 1 << msz;
+    intptr_t reg_off, mem_off;
 
-    neg_imag = float64_set_sign(0, (rot & 2) != 0);
-    neg_real = float64_set_sign(0, rot == 1 || rot == 2);
+    memset(vd, 0, reg_max);
 
-    do {
-        uint64_t pg = g[(i - 1) >> 6];
-        do {
-            float64 e1, e2, e3, e4, nr, ni, mr, mi, d;
+    reg_off = find_next_active(vg, 0, reg_max, esz);
+    while (reg_off < reg_max) {
+        target_ulong tagged_addr;
+        target_ulong clean_addr;
+        void *host;
 
-            /* I holds the real index; J holds the imag index.  */
-            j = i - sizeof(float64);
-            i -= 2 * sizeof(float64);
+        mem_off = reg_off >> diffsz;
+        tagged_addr = addr + mem_off;
+        clean_addr = sve_mte_clean_addr(env, oi, tagged_addr, false, msize);
+        if (mem_off >= mem_max ||
+            max_for_page(env->uc, clean_addr, 0, msize) < msize) {
+            break;
+        }
 
-            nr = *(float64 *)((char *)vn + H1_2(i));
-            ni = *(float64 *)((char *)vn + H1_2(j));
-            mr = *(float64 *)((char *)vm + H1_2(i));
-            mi = *(float64 *)((char *)vm + H1_2(j));
+        host = tlb_vaddr_to_host(env, clean_addr, MMU_DATA_LOAD, mmu_idx);
+        if (!host ||
+            !sve_mte_probe_addr(env, oi, tagged_addr, false, msize)) {
+            break;
+        }
+
+        host_fn(vd, vg, (char *)host - mem_off, mem_off, mem_off + msize);
+        reg_off += 1 << esz;
+        if (reg_off < reg_max) {
+            reg_off = find_next_active(vg, reg_off, reg_max, esz);
+        }
+    }
+
+    record_fault(env, reg_off, reg_max);
+}
+
+#define DO_LDFF1_LDNF1_1(PART, ESZ) \
+void HELPER(sve_ldff1##PART##_r)(CPUARMState *env, void *vg,            \
+                                 target_ulong addr, uint32_t desc)      \
+{                                                                       \
+    sve_ldff1_r(env, vg, addr, desc, GETPC(), ESZ, 0,                   \
+                sve_ld1##PART##_host, sve_ld1##PART##_tlb);             \
+}                                                                       \
+void HELPER(sve_ldnf1##PART##_r)(CPUARMState *env, void *vg,            \
+                                 target_ulong addr, uint32_t desc)      \
+{                                                                       \
+    sve_ldnf1_r(env, vg, addr, desc, GETPC(), ESZ, 0,                   \
+                sve_ld1##PART##_host, sve_ld1##PART##_tlb);             \
+}                                                                       \
+void HELPER(sve_ldff1##PART##_r_mte)(CPUARMState *env, void *vg,        \
+                                     target_ulong addr, uint32_t desc)  \
+{                                                                       \
+    sve_ldff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, 0,               \
+                    sve_ld1##PART##_host, sve_ld1##PART##_tlb);         \
+}                                                                       \
+void HELPER(sve_ldnf1##PART##_r_mte)(CPUARMState *env, void *vg,        \
+                                     target_ulong addr, uint32_t desc)  \
+{                                                                       \
+    sve_ldnf1_r_mte(env, vg, addr, desc, ESZ, 0, sve_ld1##PART##_host); \
+}
+
+#define DO_LDFF1_LDNF1_2(PART, ESZ, MSZ) \
+void HELPER(sve_ldff1##PART##_le_r)(CPUARMState *env, void *vg,         \
+                                    target_ulong addr, uint32_t desc)   \
+{                                                                       \
+    sve_ldff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,                 \
+                sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb);       \
+}                                                                       \
+void HELPER(sve_ldnf1##PART##_le_r)(CPUARMState *env, void *vg,         \
+                                    target_ulong addr, uint32_t desc)   \
+{                                                                       \
+    sve_ldnf1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,                 \
+                sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb);       \
+}                                                                       \
+void HELPER(sve_ldff1##PART##_be_r)(CPUARMState *env, void *vg,         \
+                                    target_ulong addr, uint32_t desc)   \
+{                                                                       \
+    sve_ldff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,                 \
+                sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb);       \
+}                                                                       \
+void HELPER(sve_ldnf1##PART##_be_r)(CPUARMState *env, void *vg,         \
+                                    target_ulong addr, uint32_t desc)   \
+{                                                                       \
+    sve_ldnf1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,                 \
+                sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb);       \
+}                                                                       \
+void HELPER(sve_ldff1##PART##_le_r_mte)(CPUARMState *env, void *vg,     \
+                                        target_ulong addr, uint32_t desc) \
+{                                                                       \
+    sve_ldff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ,             \
+                    sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb);   \
+}                                                                       \
+void HELPER(sve_ldnf1##PART##_le_r_mte)(CPUARMState *env, void *vg,     \
+                                        target_ulong addr, uint32_t desc) \
+{                                                                       \
+    sve_ldnf1_r_mte(env, vg, addr, desc, ESZ, MSZ,                      \
+                    sve_ld1##PART##_le_host);                           \
+}                                                                       \
+void HELPER(sve_ldff1##PART##_be_r_mte)(CPUARMState *env, void *vg,     \
+                                        target_ulong addr, uint32_t desc) \
+{                                                                       \
+    sve_ldff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ,             \
+                    sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb);   \
+}                                                                       \
+void HELPER(sve_ldnf1##PART##_be_r_mte)(CPUARMState *env, void *vg,     \
+                                        target_ulong addr, uint32_t desc) \
+{                                                                       \
+    sve_ldnf1_r_mte(env, vg, addr, desc, ESZ, MSZ,                      \
+                    sve_ld1##PART##_be_host);                           \
+}
 
-            e2 = (flip ? ni : nr);
-            e1 = (flip ? mi : mr) ^ neg_real;
-            e4 = e2;
-            e3 = (flip ? mr : mi) ^ neg_imag;
+DO_LDFF1_LDNF1_1(bb,  0)
+DO_LDFF1_LDNF1_1(bhu, 1)
+DO_LDFF1_LDNF1_1(bhs, 1)
+DO_LDFF1_LDNF1_1(bsu, 2)
+DO_LDFF1_LDNF1_1(bss, 2)
+DO_LDFF1_LDNF1_1(bdu, 3)
+DO_LDFF1_LDNF1_1(bds, 3)
 
-            if (likely((pg >> (i & 63)) & 1)) {
-                d = *(float64 *)((char *)va + H1_2(i));
-                d = float64_muladd(e2, e1, d, 0, &env->vfp.fp_status);
-                *(float64 *)((char *)vd + H1_2(i)) = d;
-            }
-            if (likely((pg >> (j & 63)) & 1)) {
-                d = *(float64 *)((char *)va + H1_2(j));
-                d = float64_muladd(e4, e3, d, 0, &env->vfp.fp_status);
-                *(float64 *)((char *)vd + H1_2(j)) = d;
-            }
-        } while (i & 63);
-    } while (i != 0);
-}
+DO_LDFF1_LDNF1_2(hh,  1, 1)
+DO_LDFF1_LDNF1_2(hsu, 2, 1)
+DO_LDFF1_LDNF1_2(hss, 2, 1)
+DO_LDFF1_LDNF1_2(hdu, 3, 1)
+DO_LDFF1_LDNF1_2(hds, 3, 1)
 
-/*
- * Load contiguous data, protected by a governing predicate.
- */
+DO_LDFF1_LDNF1_2(ss,  2, 2)
+DO_LDFF1_LDNF1_2(sdu, 3, 2)
+DO_LDFF1_LDNF1_2(sds, 3, 2)
 
-/*
- * Load elements into @vd, controlled by @vg, from @host + @mem_ofs.
- * Memory is valid through @host + @mem_max.  The register element
- * indicies are inferred from @mem_ofs, as modified by the types for
- * which the helper is built.  Return the @mem_ofs of the first element
- * not loaded (which is @mem_max if they are all loaded).
- *
- * For softmmu, we have fully validated the guest page.  For user-only,
- * we cannot fully validate without taking the mmap lock, but since we
- * know the access is within one host page, if any access is valid they
- * all must be valid.  However, when @vg is all false, it may be that
- * no access is valid.
- */
-typedef intptr_t sve_ld1_host_fn(void *vd, void *vg, void *host,
-                                 intptr_t mem_ofs, intptr_t mem_max);
+DO_LDFF1_LDNF1_2(dd,  3, 3)
 
-/*
- * Load one element into @vd + @reg_off from (@env, @vaddr, @ra).
- * The controlling predicate is known to be true.
- */
-typedef void sve_ld1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off,
-                            target_ulong vaddr, TCGMemOpIdx oi, uintptr_t ra);
-typedef sve_ld1_tlb_fn sve_st1_tlb_fn;
+#undef DO_LDFF1_LDNF1_1
+#undef DO_LDFF1_LDNF1_2
 
 /*
- * Generate the above primitives.
+ * Store contiguous data, protected by a governing predicate.
  */
 
-#define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \
-static intptr_t sve_##NAME##_host(void *vd, void *vg, void *host,           \
-                                  intptr_t mem_off, const intptr_t mem_max) \
-{                                                                           \
-    intptr_t reg_off = mem_off * (sizeof(TYPEE) / sizeof(TYPEM));           \
-    uint64_t *pg = vg;                                                      \
-    while (mem_off + sizeof(TYPEM) <= mem_max) {                            \
-        TYPEM val = 0;                                                      \
-        if (likely((pg[reg_off >> 6] >> (reg_off & 63)) & 1)) {             \
-            val = HOST((char *)host + mem_off);                                     \
-        }                                                                   \
-        *(TYPEE *)((char *)vd + H(reg_off)) = val;                                  \
-        mem_off += sizeof(TYPEM), reg_off += sizeof(TYPEE);                 \
-    }                                                                       \
-    return mem_off;                                                         \
-}
-
-#define DO_LD_TLB(NAME, H, TYPEE, TYPEM, HOST, MOEND, TLB) \
+#define DO_ST_TLB(NAME, H, TYPEM, HOST, MOEND, TLB) \
 static void sve_##NAME##_tlb(CPUARMState *env, void *vd, intptr_t reg_off,  \
-                             target_ulong addr, TCGMemOpIdx oi, uintptr_t ra)  \
+                             target_ulong addr, TCGMemOpIdx oi, uintptr_t ra) \
 {                                                                           \
-    TYPEM val = TLB(env, addr, oi, ra);                                     \
-    *(TYPEE *)((char *)vd + H(reg_off)) = val;                                      \
+    TLB(env, addr, *(TYPEM *)((char *)vd + H(reg_off)), oi, ra);                    \
 }
 
-#define DO_LD_PRIM_1(NAME, H, TE, TM)                   \
-    DO_LD_HOST(NAME, H, TE, TM, ldub_p)                 \
-    DO_LD_TLB(NAME, H, TE, TM, ldub_p, 0, helper_ret_ldub_mmu)
-
-DO_LD_PRIM_1(ld1bb,  H1,   uint8_t,  uint8_t)
-DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t)
-DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t,  int8_t)
-DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t)
-DO_LD_PRIM_1(ld1bss, H1_4, uint32_t,  int8_t)
-DO_LD_PRIM_1(ld1bdu,     , uint64_t, uint8_t)
-DO_LD_PRIM_1(ld1bds,     , uint64_t,  int8_t)
-
-#define DO_LD_PRIM_2(NAME, end, MOEND, H, TE, TM, PH, PT)  \
-    DO_LD_HOST(NAME##_##end, H, TE, TM, PH##_##end##_p)    \
-    DO_LD_TLB(NAME##_##end, H, TE, TM, PH##_##end##_p,     \
-              MOEND, helper_##end##_##PT##_mmu)
+DO_ST_TLB(st1bb,   H1,  uint8_t, stb_p, 0, helper_ret_stb_mmu)
+DO_ST_TLB(st1bh, H1_2, uint16_t, stb_p, 0, helper_ret_stb_mmu)
+DO_ST_TLB(st1bs, H1_4, uint32_t, stb_p, 0, helper_ret_stb_mmu)
+DO_ST_TLB(st1bd,     , uint64_t, stb_p, 0, helper_ret_stb_mmu)
 
-DO_LD_PRIM_2(ld1hh,  le, MO_LE, H1_2, uint16_t, uint16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hsu, le, MO_LE, H1_4, uint32_t, uint16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hss, le, MO_LE, H1_4, uint32_t,  int16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hdu, le, MO_LE,     , uint64_t, uint16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hds, le, MO_LE,     , uint64_t,  int16_t, lduw, lduw)
+DO_ST_TLB(st1hh_le, H1_2, uint16_t, stw_le_p, MO_LE, helper_le_stw_mmu)
+DO_ST_TLB(st1hs_le, H1_4, uint32_t, stw_le_p, MO_LE, helper_le_stw_mmu)
+DO_ST_TLB(st1hd_le,     , uint64_t, stw_le_p, MO_LE, helper_le_stw_mmu)
 
-DO_LD_PRIM_2(ld1ss,  le, MO_LE, H1_4, uint32_t, uint32_t, ldl, ldul)
-DO_LD_PRIM_2(ld1sdu, le, MO_LE,     , uint64_t, uint32_t, ldl, ldul)
-DO_LD_PRIM_2(ld1sds, le, MO_LE,     , uint64_t,  int32_t, ldl, ldul)
+DO_ST_TLB(st1ss_le, H1_4, uint32_t, stl_le_p, MO_LE, helper_le_stl_mmu)
+DO_ST_TLB(st1sd_le,     , uint64_t, stl_le_p, MO_LE, helper_le_stl_mmu)
 
-DO_LD_PRIM_2(ld1dd,  le, MO_LE,     , uint64_t, uint64_t, ldq, ldq)
+DO_ST_TLB(st1dd_le,     , uint64_t, stq_le_p, MO_LE, helper_le_stq_mmu)
 
-DO_LD_PRIM_2(ld1hh,  be, MO_BE, H1_2, uint16_t, uint16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hsu, be, MO_BE, H1_4, uint32_t, uint16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hss, be, MO_BE, H1_4, uint32_t,  int16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hdu, be, MO_BE,     , uint64_t, uint16_t, lduw, lduw)
-DO_LD_PRIM_2(ld1hds, be, MO_BE,     , uint64_t,  int16_t, lduw, lduw)
+DO_ST_TLB(st1hh_be, H1_2, uint16_t, stw_be_p, MO_BE, helper_be_stw_mmu)
+DO_ST_TLB(st1hs_be, H1_4, uint32_t, stw_be_p, MO_BE, helper_be_stw_mmu)
+DO_ST_TLB(st1hd_be,     , uint64_t, stw_be_p, MO_BE, helper_be_stw_mmu)
 
-DO_LD_PRIM_2(ld1ss,  be, MO_BE, H1_4, uint32_t, uint32_t, ldl, ldul)
-DO_LD_PRIM_2(ld1sdu, be, MO_BE,     , uint64_t, uint32_t, ldl, ldul)
-DO_LD_PRIM_2(ld1sds, be, MO_BE,     , uint64_t,  int32_t, ldl, ldul)
+DO_ST_TLB(st1ss_be, H1_4, uint32_t, stl_be_p, MO_BE, helper_be_stl_mmu)
+DO_ST_TLB(st1sd_be,     , uint64_t, stl_be_p, MO_BE, helper_be_stl_mmu)
 
-DO_LD_PRIM_2(ld1dd,  be, MO_BE,     , uint64_t, uint64_t, ldq, ldq)
+DO_ST_TLB(st1dd_be,     , uint64_t, stq_be_p, MO_BE, helper_be_stq_mmu)
 
-#undef DO_LD_TLB
-#undef DO_LD_HOST
-#undef DO_LD_PRIM_1
-#undef DO_LD_PRIM_2
+#undef DO_ST_TLB
 
 /*
- * Skip through a sequence of inactive elements in the guarding predicate @vg,
- * beginning at @reg_off bounded by @reg_max.  Return the offset of the active
- * element >= @reg_off, or @reg_max if there were no active elements at all.
+ * Common helpers for all contiguous 1,2,3,4-register predicated stores.
  */
-static intptr_t find_next_active(uint64_t *vg, intptr_t reg_off,
-                                 intptr_t reg_max, int esz)
+static void sve_st1_r(CPUARMState *env, void *vg, target_ulong addr,
+                      uint32_t desc, const uintptr_t ra,
+                      const int esize, const int msize,
+                      sve_st1_tlb_fn *tlb_fn)
 {
-    uint64_t pg_mask = pred_esz_masks[esz];
-    uint64_t pg = (vg[reg_off >> 6] & pg_mask) >> (reg_off & 63);
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    void *vd = &env->vfp.zregs[rd];
 
-    /* In normal usage, the first element is active.  */
-    if (likely(pg & 1)) {
-        return reg_off;
-    }
+    sve_probe_stN_r_pages(env, vg, addr, desc, esize, msize, 1, false, ra);
 
-    if (pg == 0) {
-        reg_off &= -64;
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
         do {
-            reg_off += 64;
-            if (unlikely(reg_off >= reg_max)) {
-                /* The entire predicate was false.  */
-                return reg_max;
+            if (pg & 1) {
+                tlb_fn(env, vd, i, addr, oi, ra);
             }
-            pg = vg[reg_off >> 6] & pg_mask;
-        } while (pg == 0);
+            i += esize, pg >>= esize;
+            addr += msize;
+        } while (i & 15);
     }
-    reg_off += ctz64(pg);
+    clear_helper_retaddr();
+}
 
-    /* We should never see an out of range predicate bit set.  */
-    tcg_debug_assert(reg_off < reg_max);
-    return reg_off;
+static void sve_st2_r(CPUARMState *env, void *vg, target_ulong addr,
+                      uint32_t desc, const uintptr_t ra,
+                      const int esize, const int msize,
+                      sve_st1_tlb_fn *tlb_fn)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    void *d1 = &env->vfp.zregs[rd];
+    void *d2 = &env->vfp.zregs[(rd + 1) & 31];
+
+    sve_probe_stN_r_pages(env, vg, addr, desc, esize, msize, 2, false, ra);
+
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        do {
+            if (pg & 1) {
+                tlb_fn(env, d1, i, addr, oi, ra);
+                tlb_fn(env, d2, i, addr + msize, oi, ra);
+            }
+            i += esize, pg >>= esize;
+            addr += 2 * msize;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
 }
 
-/*
- * Return the maximum offset <= @mem_max which is still within the page
- * referenced by @base + @mem_off.
- */
-static intptr_t max_for_page(struct uc_struct *uc, target_ulong base, intptr_t mem_off,
-                             intptr_t mem_max)
+static void sve_st3_r(CPUARMState *env, void *vg, target_ulong addr,
+                      uint32_t desc, const uintptr_t ra,
+                      const int esize, const int msize,
+                      sve_st1_tlb_fn *tlb_fn)
 {
-    target_ulong addr = base + mem_off;
-    intptr_t split = -(intptr_t)(addr | TARGET_PAGE_MASK);
-    return MIN(split, mem_max - mem_off) + mem_off;
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    void *d1 = &env->vfp.zregs[rd];
+    void *d2 = &env->vfp.zregs[(rd + 1) & 31];
+    void *d3 = &env->vfp.zregs[(rd + 2) & 31];
+
+    sve_probe_stN_r_pages(env, vg, addr, desc, esize, msize, 3, false, ra);
+
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        do {
+            if (pg & 1) {
+                tlb_fn(env, d1, i, addr, oi, ra);
+                tlb_fn(env, d2, i, addr + msize, oi, ra);
+                tlb_fn(env, d3, i, addr + 2 * msize, oi, ra);
+            }
+            i += esize, pg >>= esize;
+            addr += 3 * msize;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
 }
 
-/* These are normally defined only for CONFIG_USER_ONLY in <exec/cpu_ldst.h> */
-static inline void set_helper_retaddr(uintptr_t ra) { }
-static inline void clear_helper_retaddr(void) { }
+static void sve_st4_r(CPUARMState *env, void *vg, target_ulong addr,
+                      uint32_t desc, const uintptr_t ra,
+                      const int esize, const int msize,
+                      sve_st1_tlb_fn *tlb_fn)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    intptr_t i, oprsz = simd_oprsz(desc);
+    void *d1 = &env->vfp.zregs[rd];
+    void *d2 = &env->vfp.zregs[(rd + 1) & 31];
+    void *d3 = &env->vfp.zregs[(rd + 2) & 31];
+    void *d4 = &env->vfp.zregs[(rd + 3) & 31];
+
+    sve_probe_stN_r_pages(env, vg, addr, desc, esize, msize, 4, false, ra);
 
-/*
- * The result of tlb_vaddr_to_host for user-only is just g2h(x),
- * which is always non-null.  Elide the useless test.
- */
-static inline bool test_host_page(void *host)
-{
-    return likely(host != NULL);
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+        do {
+            if (pg & 1) {
+                tlb_fn(env, d1, i, addr, oi, ra);
+                tlb_fn(env, d2, i, addr + msize, oi, ra);
+                tlb_fn(env, d3, i, addr + 2 * msize, oi, ra);
+                tlb_fn(env, d4, i, addr + 3 * msize, oi, ra);
+            }
+            i += esize, pg >>= esize;
+            addr += 4 * msize;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
 }
 
-/*
- * Common helper for all contiguous one-register predicated loads.
- */
-static void sve_ld1_r(CPUARMState *env, void *vg, const target_ulong addr,
-                      uint32_t desc, const uintptr_t retaddr,
-                      const int esz, const int msz,
-                      sve_ld1_host_fn *host_fn,
-                      sve_ld1_tlb_fn *tlb_fn)
+static void sve_stN_r_mte(CPUARMState *env, void *vg, target_ulong addr,
+                          uint32_t desc, const uintptr_t ra,
+                          const int esize, const int msize, int n,
+                          sve_st1_tlb_fn *tlb_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const int mmu_idx = get_mmuidx(oi);
     const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    void *vd = &env->vfp.zregs[rd];
-    const int diffsz = esz - msz;
-    const intptr_t reg_max = simd_oprsz(desc);
-    const intptr_t mem_max = reg_max >> diffsz;
-    ARMVectorReg scratch;
-    void *host;
-    intptr_t split, reg_off, mem_off;
+    intptr_t i, oprsz = simd_oprsz(desc);
+    void *vd[4];
+    int j;
 
-    /* Find the first active element.  */
-    reg_off = find_next_active(vg, 0, reg_max, esz);
-    if (unlikely(reg_off == reg_max)) {
-        /* The entire predicate was false; no load occurs.  */
-        memset(vd, 0, reg_max);
-        return;
+    for (j = 0; j < n; j++) {
+        vd[j] = &env->vfp.zregs[(rd + j) & 31];
     }
-    mem_off = reg_off >> diffsz;
-    set_helper_retaddr(retaddr);
 
-    /*
-     * If the (remaining) load is entirely within a single page, then:
-     * For softmmu, and the tlb hits, then no faults will occur;
-     * For user-only, either the first load will fault or none will.
-     * We can thus perform the load directly to the destination and
-     * Vd will be unmodified on any exception path.
-     */
-    split = max_for_page(env->uc, addr, mem_off, mem_max);
-    if (likely(split == mem_max)) {
-        host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
-        if (test_host_page(host)) {
-            mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, mem_max);
-            tcg_debug_assert(mem_off == mem_max);
-            clear_helper_retaddr();
-            /* After having taken any fault, zero leading inactive elements. */
-            swap_memzero(vd, reg_off);
-            return;
-        }
-    }
+    sve_probe_stN_r_pages(env, vg, addr, desc, esize, msize, n, true, ra);
+    sve_check_stN_r_mte(env, vg, addr, desc, esize, msize, n, ra);
 
-    /*
-     * Perform the predicated read into a temporary, thus ensuring
-     * if the load of the last element faults, Vd is not modified.
-     */
-    memset(&scratch, 0, reg_max);
-    goto start;
-    while (1) {
-        reg_off = find_next_active(vg, reg_off, reg_max, esz);
-        if (reg_off >= reg_max) {
-            break;
-        }
-        mem_off = reg_off >> diffsz;
-        split = max_for_page(env->uc, addr, mem_off, mem_max);
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
 
-    start:
-        if (split - mem_off >= (1ULL << msz)) {
-            /* At least one whole element on this page.  */
-            host = tlb_vaddr_to_host(env, addr + mem_off,
-                                     MMU_DATA_LOAD, mmu_idx);
-            if (host) {
-                mem_off = host_fn(&scratch, vg, (char *)host - mem_off,
-                                  mem_off, split);
-                reg_off = mem_off << diffsz;
-                continue;
-            }
-        }
+        do {
+            if (pg & 1) {
+                target_ulong clean_addr =
+                    sve_mte_clean_addr(env, oi, addr, true, n * msize);
 
-        /*
-         * Perform one normal read.  This may fault, longjmping out to the
-         * main loop in order to raise an exception.  It may succeed, and
-         * as a side-effect load the TLB entry for the next round.  Finally,
-         * in the extremely unlikely case we're performing this operation
-         * on I/O memory, it may succeed but not bring in the TLB entry.
-         * But even then we have still made forward progress.
-         */
-        tlb_fn(env, &scratch, reg_off, addr + mem_off, oi, retaddr);
-        reg_off += 1ULL << esz;
+                for (j = 0; j < n; j++) {
+                    tlb_fn(env, vd[j], i, clean_addr + j * msize, oi, ra);
+                }
+            }
+            i += esize, pg >>= esize;
+            addr += n * msize;
+        } while (i & 15);
     }
-
     clear_helper_retaddr();
-    memcpy(vd, &scratch, reg_max);
 }
 
-#define DO_LD1_1(NAME, ESZ) \
-void HELPER(sve_##NAME##_r)(CPUARMState *env, void *vg,        \
-                            target_ulong addr, uint32_t desc)  \
-{                                                              \
-    sve_ld1_r(env, vg, addr, desc, GETPC(), ESZ, 0,            \
-              sve_##NAME##_host, sve_##NAME##_tlb);            \
+#define DO_STN_1(N, NAME, ESIZE) \
+void QEMU_FLATTEN HELPER(sve_st##N##NAME##_r) \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)  \
+{                                                                   \
+    sve_st##N##_r(env, vg, addr, desc, GETPC(), ESIZE, 1,           \
+                  sve_st1##NAME##_tlb);                             \
+}                                                                   \
+void QEMU_FLATTEN HELPER(sve_st##N##NAME##_r_mte)                   \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)  \
+{                                                                   \
+    sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESIZE, 1, N,        \
+                  sve_st1##NAME##_tlb);                             \
 }
 
-#define DO_LD1_2(NAME, ESZ, MSZ) \
-void HELPER(sve_##NAME##_le_r)(CPUARMState *env, void *vg,        \
-                               target_ulong addr, uint32_t desc)  \
-{                                                                 \
-    sve_ld1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,             \
-              sve_##NAME##_le_host, sve_##NAME##_le_tlb);         \
-}                                                                 \
-void HELPER(sve_##NAME##_be_r)(CPUARMState *env, void *vg,        \
-                               target_ulong addr, uint32_t desc)  \
-{                                                                 \
-    sve_ld1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,             \
-              sve_##NAME##_be_host, sve_##NAME##_be_tlb);         \
+#define DO_STN_2(N, NAME, ESIZE, MSIZE) \
+void QEMU_FLATTEN HELPER(sve_st##N##NAME##_le_r) \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_st##N##_r(env, vg, addr, desc, GETPC(), ESIZE, MSIZE,         \
+                  sve_st1##NAME##_le_tlb);                            \
+}                                                                     \
+void QEMU_FLATTEN HELPER(sve_st##N##NAME##_be_r)                      \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_st##N##_r(env, vg, addr, desc, GETPC(), ESIZE, MSIZE,         \
+                  sve_st1##NAME##_be_tlb);                            \
+}                                                                     \
+void QEMU_FLATTEN HELPER(sve_st##N##NAME##_le_r_mte)                  \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESIZE, MSIZE, N,      \
+                  sve_st1##NAME##_le_tlb);                            \
+}                                                                     \
+void QEMU_FLATTEN HELPER(sve_st##N##NAME##_be_r_mte)                  \
+    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
+{                                                                     \
+    sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESIZE, MSIZE, N,      \
+                  sve_st1##NAME##_be_tlb);                            \
 }
 
-DO_LD1_1(ld1bb,  0)
-DO_LD1_1(ld1bhu, 1)
-DO_LD1_1(ld1bhs, 1)
-DO_LD1_1(ld1bsu, 2)
-DO_LD1_1(ld1bss, 2)
-DO_LD1_1(ld1bdu, 3)
-DO_LD1_1(ld1bds, 3)
+DO_STN_1(1, bb, 1)
+DO_STN_1(1, bh, 2)
+DO_STN_1(1, bs, 4)
+DO_STN_1(1, bd, 8)
+DO_STN_1(2, bb, 1)
+DO_STN_1(3, bb, 1)
+DO_STN_1(4, bb, 1)
 
-DO_LD1_2(ld1hh,  1, 1)
-DO_LD1_2(ld1hsu, 2, 1)
-DO_LD1_2(ld1hss, 2, 1)
-DO_LD1_2(ld1hdu, 3, 1)
-DO_LD1_2(ld1hds, 3, 1)
+DO_STN_2(1, hh, 2, 2)
+DO_STN_2(1, hs, 4, 2)
+DO_STN_2(1, hd, 8, 2)
+DO_STN_2(2, hh, 2, 2)
+DO_STN_2(3, hh, 2, 2)
+DO_STN_2(4, hh, 2, 2)
 
-DO_LD1_2(ld1ss,  2, 2)
-DO_LD1_2(ld1sdu, 3, 2)
-DO_LD1_2(ld1sds, 3, 2)
+DO_STN_2(1, ss, 4, 4)
+DO_STN_2(1, sd, 8, 4)
+DO_STN_2(2, ss, 4, 4)
+DO_STN_2(3, ss, 4, 4)
+DO_STN_2(4, ss, 4, 4)
 
-DO_LD1_2(ld1dd,  3, 3)
+DO_STN_2(1, dd, 8, 8)
+DO_STN_2(2, dd, 8, 8)
+DO_STN_2(3, dd, 8, 8)
+DO_STN_2(4, dd, 8, 8)
 
-#undef DO_LD1_1
-#undef DO_LD1_2
+#undef DO_STN_1
+#undef DO_STN_2
 
 /*
- * Common helpers for all contiguous 2,3,4-register predicated loads.
+ * Loads with a vector index.
  */
-static void sve_ld2_r(CPUARMState *env, void *vg, target_ulong addr,
-                      uint32_t desc, int size, uintptr_t ra,
-                      sve_ld1_tlb_fn *tlb_fn)
+
+/*
+ * Load the element at @reg + @reg_ofs, sign or zero-extend as needed.
+ */
+typedef target_ulong zreg_off_fn(void *reg, intptr_t reg_ofs);
+
+static target_ulong off_zsu_s(void *reg, intptr_t reg_ofs)
+{
+    return *(uint32_t *)((char *)reg + H1_4(reg_ofs));
+}
+
+static target_ulong off_zss_s(void *reg, intptr_t reg_ofs)
+{
+    return *(int32_t *)((char *)reg + H1_4(reg_ofs));
+}
+
+static target_ulong off_zsu_d(void *reg, intptr_t reg_ofs)
+{
+    return (uint32_t)*(uint64_t *)((char *)reg + reg_ofs);
+}
+
+static target_ulong off_zss_d(void *reg, intptr_t reg_ofs)
+{
+    return (int32_t)*(uint64_t *)((char *)reg + reg_ofs);
+}
+
+static target_ulong off_zd_d(void *reg, intptr_t reg_ofs)
+{
+    return *(uint64_t *)((char *)reg + reg_ofs);
+}
+
+static void sve_ld1_zs(CPUARMState *env, void *vd, void *vg, void *vm,
+                       target_ulong base, uint32_t desc, uintptr_t ra,
+                       zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
     intptr_t i, oprsz = simd_oprsz(desc);
-    ARMVectorReg scratch[2] = { 0 };
+    ARMVectorReg scratch = { 0 };
 
     set_helper_retaddr(ra);
     for (i = 0; i < oprsz; ) {
         uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
         do {
-            if (pg & 1) {
-                tlb_fn(env, &scratch[0], i, addr, oi, ra);
-                tlb_fn(env, &scratch[1], i, addr + size, oi, ra);
+            if (likely(pg & 1)) {
+                target_ulong off = off_fn(vm, i);
+                tlb_fn(env, &scratch, i, base + (off << scale), oi, ra);
             }
-            i += size, pg >>= size;
-            addr += 2 * size;
+            i += 4, pg >>= 4;
         } while (i & 15);
     }
     clear_helper_retaddr();
 
     /* Wait until all exceptions have been raised to write back.  */
-    memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
-    memcpy(&env->vfp.zregs[(rd + 1) & 31], &scratch[1], oprsz);
+    memcpy(vd, &scratch, oprsz);
 }
 
-static void sve_ld3_r(CPUARMState *env, void *vg, target_ulong addr,
-                      uint32_t desc, int size, uintptr_t ra,
-                      sve_ld1_tlb_fn *tlb_fn)
+static void sve_ld1_zd(CPUARMState *env, void *vd, void *vg, void *vm,
+                       target_ulong base, uint32_t desc, uintptr_t ra,
+                       zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    intptr_t i, oprsz = simd_oprsz(desc);
-    ARMVectorReg scratch[3] = { 0 };
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    ARMVectorReg scratch = { 0 };
 
     set_helper_retaddr(ra);
-    for (i = 0; i < oprsz; ) {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
-        do {
-            if (pg & 1) {
-                tlb_fn(env, &scratch[0], i, addr, oi, ra);
-                tlb_fn(env, &scratch[1], i, addr + size, oi, ra);
-                tlb_fn(env, &scratch[2], i, addr + 2 * size, oi, ra);
-            }
-            i += size, pg >>= size;
-            addr += 3 * size;
-        } while (i & 15);
+    for (i = 0; i < oprsz; i++) {
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(i));
+        if (likely(pg & 1)) {
+            target_ulong off = off_fn(vm, i * 8);
+            tlb_fn(env, &scratch, i * 8, base + (off << scale), oi, ra);
+        }
     }
     clear_helper_retaddr();
 
     /* Wait until all exceptions have been raised to write back.  */
-    memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
-    memcpy(&env->vfp.zregs[(rd + 1) & 31], &scratch[1], oprsz);
-    memcpy(&env->vfp.zregs[(rd + 2) & 31], &scratch[2], oprsz);
+    memcpy(vd, &scratch, oprsz * 8);
 }
 
-static void sve_ld4_r(CPUARMState *env, void *vg, target_ulong addr,
-                      uint32_t desc, int size, uintptr_t ra,
-                      sve_ld1_tlb_fn *tlb_fn)
+static void sve_ld1_zs_mte(CPUARMState *env, void *vd, void *vg, void *vm,
+                           target_ulong base, uint32_t desc, uintptr_t ra,
+                           zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
     intptr_t i, oprsz = simd_oprsz(desc);
-    ARMVectorReg scratch[4] = { 0 };
+    ARMVectorReg scratch = { 0 };
 
     set_helper_retaddr(ra);
     for (i = 0; i < oprsz; ) {
         uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+
         do {
-            if (pg & 1) {
-                tlb_fn(env, &scratch[0], i, addr, oi, ra);
-                tlb_fn(env, &scratch[1], i, addr + size, oi, ra);
-                tlb_fn(env, &scratch[2], i, addr + 2 * size, oi, ra);
-                tlb_fn(env, &scratch[3], i, addr + 3 * size, oi, ra);
+            if (likely(pg & 1)) {
+                target_ulong tagged_addr = base + (off_fn(vm, i) << scale);
+                target_ulong clean_addr =
+                    sve_mte_check_addr(env, oi, tagged_addr,
+                                       false, msize, ra);
+
+                tlb_fn(env, &scratch, i, clean_addr, oi, ra);
             }
-            i += size, pg >>= size;
-            addr += 4 * size;
+            i += 4, pg >>= 4;
         } while (i & 15);
     }
     clear_helper_retaddr();
 
-    /* Wait until all exceptions have been raised to write back.  */
-    memcpy(&env->vfp.zregs[rd], &scratch[0], oprsz);
-    memcpy(&env->vfp.zregs[(rd + 1) & 31], &scratch[1], oprsz);
-    memcpy(&env->vfp.zregs[(rd + 2) & 31], &scratch[2], oprsz);
-    memcpy(&env->vfp.zregs[(rd + 3) & 31], &scratch[3], oprsz);
-}
-
-#define DO_LDN_1(N) \
-void QEMU_FLATTEN HELPER(sve_ld##N##bb_r) \
-    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)  \
-{                                                                   \
-    sve_ld##N##_r(env, vg, addr, desc, 1, GETPC(), sve_ld1bb_tlb);  \
-}
-
-#define DO_LDN_2(N, SUFF, SIZE)                                       \
-void QEMU_FLATTEN HELPER(sve_ld##N##SUFF##_le_r)                      \
-    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
-{                                                                     \
-    sve_ld##N##_r(env, vg, addr, desc, SIZE, GETPC(),                 \
-                  sve_ld1##SUFF##_le_tlb);                            \
-}                                                                     \
-void QEMU_FLATTEN HELPER(sve_ld##N##SUFF##_be_r)                      \
-    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
-{                                                                     \
-    sve_ld##N##_r(env, vg, addr, desc, SIZE, GETPC(),                 \
-                  sve_ld1##SUFF##_be_tlb);                            \
-}
-
-DO_LDN_1(2)
-DO_LDN_1(3)
-DO_LDN_1(4)
-
-DO_LDN_2(2, hh, 2)
-DO_LDN_2(3, hh, 2)
-DO_LDN_2(4, hh, 2)
-
-DO_LDN_2(2, ss, 4)
-DO_LDN_2(3, ss, 4)
-DO_LDN_2(4, ss, 4)
-
-DO_LDN_2(2, dd, 8)
-DO_LDN_2(3, dd, 8)
-DO_LDN_2(4, dd, 8)
-
-#undef DO_LDN_1
-#undef DO_LDN_2
-
-/*
- * Load contiguous data, first-fault and no-fault.
- *
- * For user-only, one could argue that we should hold the mmap_lock during
- * the operation so that there is no race between page_check_range and the
- * load operation.  However, unmapping pages out from under a running thread
- * is extraordinarily unlikely.  This theoretical race condition also affects
- * linux-user/ in its get_user/put_user macros.
- *
- * TODO: Construct some helpers, written in assembly, that interact with
- * handle_cpu_signal to produce memory ops which can properly report errors
- * without racing.
- */
-
-/* Fault on byte I.  All bits in FFR from I are cleared.  The vector
- * result from I is CONSTRAINED UNPREDICTABLE; we choose the MERGE
- * option, which leaves subsequent data unchanged.
- */
-static void record_fault(CPUARMState *env, uintptr_t i, uintptr_t oprsz)
-{
-    uint64_t *ffr = env->vfp.pregs[FFR_PRED_NUM].p;
-
-    if (i & 63) {
-        ffr[i / 64] &= MAKE_64BIT_MASK(0, i & 63);
-        i = ROUND_UP(i, 64);
-    }
-    for (; i < oprsz; i += 64) {
-        ffr[i / 64] = 0;
-    }
+    memcpy(vd, &scratch, oprsz);
 }
 
-/*
- * Common helper for all contiguous first-fault loads.
- */
-static void sve_ldff1_r(CPUARMState *env, void *vg, const target_ulong addr,
-                        uint32_t desc, const uintptr_t retaddr,
-                        const int esz, const int msz,
-                        sve_ld1_host_fn *host_fn,
-                        sve_ld1_tlb_fn *tlb_fn)
+static void sve_ld1_zd_mte(CPUARMState *env, void *vd, void *vg, void *vm,
+                           target_ulong base, uint32_t desc, uintptr_t ra,
+                           zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const int mmu_idx = get_mmuidx(oi);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    void *vd = &env->vfp.zregs[rd];
-    const int diffsz = esz - msz;
-    const intptr_t reg_max = simd_oprsz(desc);
-    const intptr_t mem_max = reg_max >> diffsz;
-    intptr_t split, reg_off, mem_off;
-    void *host;
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    ARMVectorReg scratch = { 0 };
 
-    /* Skip to the first active element.  */
-    reg_off = find_next_active(vg, 0, reg_max, esz);
-    if (unlikely(reg_off == reg_max)) {
-        /* The entire predicate was false; no load occurs.  */
-        memset(vd, 0, reg_max);
-        return;
-    }
-    mem_off = reg_off >> diffsz;
-    set_helper_retaddr(retaddr);
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; i++) {
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(i));
 
-    /*
-     * If the (remaining) load is entirely within a single page, then:
-     * For softmmu, and the tlb hits, then no faults will occur;
-     * For user-only, either the first load will fault or none will.
-     * We can thus perform the load directly to the destination and
-     * Vd will be unmodified on any exception path.
-     */
-    split = max_for_page(env->uc, addr, mem_off, mem_max);
-    if (likely(split == mem_max)) {
-        host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
-        if (test_host_page(host)) {
-            mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, mem_max);
-            tcg_debug_assert(mem_off == mem_max);
-            clear_helper_retaddr();
-            /* After any fault, zero any leading inactive elements.  */
-            swap_memzero(vd, reg_off);
-            return;
+        if (likely(pg & 1)) {
+            intptr_t reg_off = i * 8;
+            target_ulong tagged_addr =
+                base + (off_fn(vm, reg_off) << scale);
+            target_ulong clean_addr =
+                sve_mte_check_addr(env, oi, tagged_addr,
+                                   false, msize, ra);
+
+            tlb_fn(env, &scratch, reg_off, clean_addr, oi, ra);
         }
     }
+    clear_helper_retaddr();
 
-    /*
-     * Perform one normal read, which will fault or not.
-     * But it is likely to bring the page into the tlb.
-     */
-    tlb_fn(env, vd, reg_off, addr + mem_off, oi, retaddr);
-
-    /* After any fault, zero any leading predicated false elts.  */
-    swap_memzero(vd, reg_off);
-    mem_off += 1ULL << msz;
-    reg_off += 1ULL << esz;
+    memcpy(vd, &scratch, oprsz * 8);
+}
 
-    /* Try again to read the balance of the page.  */
-    split = max_for_page(env->uc, addr, mem_off - 1, mem_max);
-    if (split >= (1ULL << msz)) {
-        host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
-        if (host) {
-            mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, split);
-            reg_off = mem_off << diffsz;
-        }
-    }
+#define DO_LD1_ZPZ_S(MEM, OFS) \
+void QEMU_FLATTEN HELPER(sve_ld##MEM##_##OFS) \
+    (CPUARMState *env, void *vd, void *vg, void *vm,         \
+     target_ulong base, uint32_t desc)                       \
+{                                                            \
+    sve_ld1_zs(env, vd, vg, vm, base, desc, GETPC(),         \
+               off_##OFS##_s, sve_ld1##MEM##_tlb);            \
+}                                                            \
+void QEMU_FLATTEN HELPER(sve_ld##MEM##_##OFS##_mte)          \
+    (CPUARMState *env, void *vd, void *vg, void *vm,         \
+     target_ulong base, uint32_t desc)                       \
+{                                                            \
+    sve_ld1_zs_mte(env, vd, vg, vm, base, desc, GETPC(),     \
+                   off_##OFS##_s, sve_ld1##MEM##_tlb);       \
+}
 
-    clear_helper_retaddr();
-    record_fault(env, reg_off, reg_max);
+#define DO_LD1_ZPZ_D(MEM, OFS) \
+void QEMU_FLATTEN HELPER(sve_ld##MEM##_##OFS) \
+    (CPUARMState *env, void *vd, void *vg, void *vm,         \
+     target_ulong base, uint32_t desc)                       \
+{                                                            \
+    sve_ld1_zd(env, vd, vg, vm, base, desc, GETPC(),         \
+               off_##OFS##_d, sve_ld1##MEM##_tlb);           \
+}                                                            \
+void QEMU_FLATTEN HELPER(sve_ld##MEM##_##OFS##_mte)          \
+    (CPUARMState *env, void *vd, void *vg, void *vm,         \
+     target_ulong base, uint32_t desc)                       \
+{                                                            \
+    sve_ld1_zd_mte(env, vd, vg, vm, base, desc, GETPC(),     \
+                   off_##OFS##_d, sve_ld1##MEM##_tlb);       \
 }
 
-/*
- * Common helper for all contiguous no-fault loads.
- */
-static void sve_ldnf1_r(CPUARMState *env, void *vg, const target_ulong addr,
-                        uint32_t desc, const int esz, const int msz,
-                        sve_ld1_host_fn *host_fn)
-{
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    void *vd = &env->vfp.zregs[rd];
-    const int diffsz = esz - msz;
-    const intptr_t reg_max = simd_oprsz(desc);
-    const intptr_t mem_max = reg_max >> diffsz;
-    const int mmu_idx = cpu_mmu_index(env, false);
-    intptr_t split, reg_off, mem_off;
-    void *host;
+DO_LD1_ZPZ_S(bsu, zsu)
+DO_LD1_ZPZ_S(bsu, zss)
+DO_LD1_ZPZ_D(bdu, zsu)
+DO_LD1_ZPZ_D(bdu, zss)
+DO_LD1_ZPZ_D(bdu, zd)
+
+DO_LD1_ZPZ_S(bss, zsu)
+DO_LD1_ZPZ_S(bss, zss)
+DO_LD1_ZPZ_D(bds, zsu)
+DO_LD1_ZPZ_D(bds, zss)
+DO_LD1_ZPZ_D(bds, zd)
+
+DO_LD1_ZPZ_S(hsu_le, zsu)
+DO_LD1_ZPZ_S(hsu_le, zss)
+DO_LD1_ZPZ_D(hdu_le, zsu)
+DO_LD1_ZPZ_D(hdu_le, zss)
+DO_LD1_ZPZ_D(hdu_le, zd)
 
-    /* There will be no fault, so we may modify in advance.  */
-    memset(vd, 0, reg_max);
+DO_LD1_ZPZ_S(hsu_be, zsu)
+DO_LD1_ZPZ_S(hsu_be, zss)
+DO_LD1_ZPZ_D(hdu_be, zsu)
+DO_LD1_ZPZ_D(hdu_be, zss)
+DO_LD1_ZPZ_D(hdu_be, zd)
 
-    /* Skip to the first active element.  */
-    reg_off = find_next_active(vg, 0, reg_max, esz);
-    if (unlikely(reg_off == reg_max)) {
-        /* The entire predicate was false; no load occurs.  */
-        return;
-    }
-    mem_off = reg_off >> diffsz;
+DO_LD1_ZPZ_S(hss_le, zsu)
+DO_LD1_ZPZ_S(hss_le, zss)
+DO_LD1_ZPZ_D(hds_le, zsu)
+DO_LD1_ZPZ_D(hds_le, zss)
+DO_LD1_ZPZ_D(hds_le, zd)
 
-    /*
-     * If the address is not in the TLB, we have no way to bring the
-     * entry into the TLB without also risking a fault.  Note that
-     * the corollary is that we never load from an address not in RAM.
-     *
-     * This last is out of spec, in a weird corner case.
-     * Per the MemNF/MemSingleNF pseudocode, a NF load from Device memory
-     * must not actually hit the bus -- it returns UNKNOWN data instead.
-     * But if you map non-RAM with Normal memory attributes and do a NF
-     * load then it should access the bus.  (Nobody ought actually do this
-     * in the real world, obviously.)
-     *
-     * Then there are the annoying special cases with watchpoints...
-     * TODO: Add a form of non-faulting loads using cc->tlb_fill(probe=true).
-     */
-    host = tlb_vaddr_to_host(env, addr + mem_off, MMU_DATA_LOAD, mmu_idx);
-    split = max_for_page(env->uc, addr, mem_off, mem_max);
-    if (host && split >= (1ULL << msz)) {
-        mem_off = host_fn(vd, vg, (char *)host - mem_off, mem_off, split);
-        reg_off = mem_off << diffsz;
-    }
+DO_LD1_ZPZ_S(hss_be, zsu)
+DO_LD1_ZPZ_S(hss_be, zss)
+DO_LD1_ZPZ_D(hds_be, zsu)
+DO_LD1_ZPZ_D(hds_be, zss)
+DO_LD1_ZPZ_D(hds_be, zd)
 
-    record_fault(env, reg_off, reg_max);
-}
+DO_LD1_ZPZ_S(ss_le, zsu)
+DO_LD1_ZPZ_S(ss_le, zss)
+DO_LD1_ZPZ_D(sdu_le, zsu)
+DO_LD1_ZPZ_D(sdu_le, zss)
+DO_LD1_ZPZ_D(sdu_le, zd)
 
-#define DO_LDFF1_LDNF1_1(PART, ESZ) \
-void HELPER(sve_ldff1##PART##_r)(CPUARMState *env, void *vg,            \
-                                 target_ulong addr, uint32_t desc)      \
-{                                                                       \
-    sve_ldff1_r(env, vg, addr, desc, GETPC(), ESZ, 0,                   \
-                sve_ld1##PART##_host, sve_ld1##PART##_tlb);             \
-}                                                                       \
-void HELPER(sve_ldnf1##PART##_r)(CPUARMState *env, void *vg,            \
-                                 target_ulong addr, uint32_t desc)      \
-{                                                                       \
-    sve_ldnf1_r(env, vg, addr, desc, ESZ, 0, sve_ld1##PART##_host);     \
-}
+DO_LD1_ZPZ_S(ss_be, zsu)
+DO_LD1_ZPZ_S(ss_be, zss)
+DO_LD1_ZPZ_D(sdu_be, zsu)
+DO_LD1_ZPZ_D(sdu_be, zss)
+DO_LD1_ZPZ_D(sdu_be, zd)
 
-#define DO_LDFF1_LDNF1_2(PART, ESZ, MSZ) \
-void HELPER(sve_ldff1##PART##_le_r)(CPUARMState *env, void *vg,         \
-                                    target_ulong addr, uint32_t desc)   \
-{                                                                       \
-    sve_ldff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,                 \
-                sve_ld1##PART##_le_host, sve_ld1##PART##_le_tlb);       \
-}                                                                       \
-void HELPER(sve_ldnf1##PART##_le_r)(CPUARMState *env, void *vg,         \
-                                    target_ulong addr, uint32_t desc)   \
-{                                                                       \
-    sve_ldnf1_r(env, vg, addr, desc, ESZ, MSZ, sve_ld1##PART##_le_host); \
-}                                                                       \
-void HELPER(sve_ldff1##PART##_be_r)(CPUARMState *env, void *vg,         \
-                                    target_ulong addr, uint32_t desc)   \
-{                                                                       \
-    sve_ldff1_r(env, vg, addr, desc, GETPC(), ESZ, MSZ,                 \
-                sve_ld1##PART##_be_host, sve_ld1##PART##_be_tlb);       \
-}                                                                       \
-void HELPER(sve_ldnf1##PART##_be_r)(CPUARMState *env, void *vg,         \
-                                    target_ulong addr, uint32_t desc)   \
-{                                                                       \
-    sve_ldnf1_r(env, vg, addr, desc, ESZ, MSZ, sve_ld1##PART##_be_host); \
-}
+DO_LD1_ZPZ_D(sds_le, zsu)
+DO_LD1_ZPZ_D(sds_le, zss)
+DO_LD1_ZPZ_D(sds_le, zd)
 
-DO_LDFF1_LDNF1_1(bb,  0)
-DO_LDFF1_LDNF1_1(bhu, 1)
-DO_LDFF1_LDNF1_1(bhs, 1)
-DO_LDFF1_LDNF1_1(bsu, 2)
-DO_LDFF1_LDNF1_1(bss, 2)
-DO_LDFF1_LDNF1_1(bdu, 3)
-DO_LDFF1_LDNF1_1(bds, 3)
+DO_LD1_ZPZ_D(sds_be, zsu)
+DO_LD1_ZPZ_D(sds_be, zss)
+DO_LD1_ZPZ_D(sds_be, zd)
 
-DO_LDFF1_LDNF1_2(hh,  1, 1)
-DO_LDFF1_LDNF1_2(hsu, 2, 1)
-DO_LDFF1_LDNF1_2(hss, 2, 1)
-DO_LDFF1_LDNF1_2(hdu, 3, 1)
-DO_LDFF1_LDNF1_2(hds, 3, 1)
+DO_LD1_ZPZ_D(dd_le, zsu)
+DO_LD1_ZPZ_D(dd_le, zss)
+DO_LD1_ZPZ_D(dd_le, zd)
 
-DO_LDFF1_LDNF1_2(ss,  2, 2)
-DO_LDFF1_LDNF1_2(sdu, 3, 2)
-DO_LDFF1_LDNF1_2(sds, 3, 2)
+DO_LD1_ZPZ_D(dd_be, zsu)
+DO_LD1_ZPZ_D(dd_be, zss)
+DO_LD1_ZPZ_D(dd_be, zd)
 
-DO_LDFF1_LDNF1_2(dd,  3, 3)
+#undef DO_LD1_ZPZ_S
+#undef DO_LD1_ZPZ_D
 
-#undef DO_LDFF1_LDNF1_1
-#undef DO_LDFF1_LDNF1_2
+/* First fault loads with a vector index.  */
 
-/*
- * Store contiguous data, protected by a governing predicate.
+/* Load one element into VD+REG_OFF from (ENV,VADDR) without faulting.
+ * The controlling predicate is known to be true.  Return true if the
+ * load was successful.
  */
+typedef bool sve_ld1_nf_fn(CPUARMState *env, void *vd, intptr_t reg_off,
+                           target_ulong vaddr, int mmu_idx);
 
-#define DO_ST_TLB(NAME, H, TYPEM, HOST, MOEND, TLB) \
-static void sve_##NAME##_tlb(CPUARMState *env, void *vd, intptr_t reg_off,  \
-                             target_ulong addr, TCGMemOpIdx oi, uintptr_t ra) \
+#ifdef _MSC_VER
+#define DO_LD_NF(NAME, H, TYPEE, TYPEM, HOST) \
+static bool sve_ld##NAME##_nf(CPUARMState *env, void *vd, intptr_t reg_off, \
+                              target_ulong addr, int mmu_idx)               \
 {                                                                           \
-    TLB(env, addr, *(TYPEM *)((char *)vd + H(reg_off)), oi, ra);                    \
+    struct uc_struct *uc = env->uc;                                         \
+    target_ulong next_page = 0ULL - (addr | TARGET_PAGE_MASK);                    \
+    if (likely(next_page - addr >= sizeof(TYPEM))) {                        \
+        void *host = tlb_vaddr_to_host(env, addr, MMU_DATA_LOAD, mmu_idx);  \
+        if (likely(host)) {                                                 \
+            TYPEM val = HOST(host);                                         \
+            *(TYPEE *)((char *)vd + H(reg_off)) = val;                              \
+            return true;                                                    \
+        }                                                                   \
+    }                                                                       \
+    return false;                                                           \
 }
+#else
+#define DO_LD_NF(NAME, H, TYPEE, TYPEM, HOST) \
+static bool sve_ld##NAME##_nf(CPUARMState *env, void *vd, intptr_t reg_off, \
+                              target_ulong addr, int mmu_idx)               \
+{                                                                           \
+    struct uc_struct *uc = env->uc;                                         \
+    target_ulong next_page = -(addr | TARGET_PAGE_MASK);                    \
+    if (likely(next_page - addr >= sizeof(TYPEM))) {                        \
+        void *host = tlb_vaddr_to_host(env, addr, MMU_DATA_LOAD, mmu_idx);  \
+        if (likely(host)) {                                                 \
+            TYPEM val = HOST(host);                                         \
+            *(TYPEE *)((char *)vd + H(reg_off)) = val;                              \
+            return true;                                                    \
+        }                                                                   \
+    }                                                                       \
+    return false;                                                           \
+}
+#endif
 
-DO_ST_TLB(st1bb,   H1,  uint8_t, stb_p, 0, helper_ret_stb_mmu)
-DO_ST_TLB(st1bh, H1_2, uint16_t, stb_p, 0, helper_ret_stb_mmu)
-DO_ST_TLB(st1bs, H1_4, uint32_t, stb_p, 0, helper_ret_stb_mmu)
-DO_ST_TLB(st1bd,     , uint64_t, stb_p, 0, helper_ret_stb_mmu)
-
-DO_ST_TLB(st1hh_le, H1_2, uint16_t, stw_le_p, MO_LE, helper_le_stw_mmu)
-DO_ST_TLB(st1hs_le, H1_4, uint32_t, stw_le_p, MO_LE, helper_le_stw_mmu)
-DO_ST_TLB(st1hd_le,     , uint64_t, stw_le_p, MO_LE, helper_le_stw_mmu)
-
-DO_ST_TLB(st1ss_le, H1_4, uint32_t, stl_le_p, MO_LE, helper_le_stl_mmu)
-DO_ST_TLB(st1sd_le,     , uint64_t, stl_le_p, MO_LE, helper_le_stl_mmu)
-
-DO_ST_TLB(st1dd_le,     , uint64_t, stq_le_p, MO_LE, helper_le_stq_mmu)
-
-DO_ST_TLB(st1hh_be, H1_2, uint16_t, stw_be_p, MO_BE, helper_be_stw_mmu)
-DO_ST_TLB(st1hs_be, H1_4, uint32_t, stw_be_p, MO_BE, helper_be_stw_mmu)
-DO_ST_TLB(st1hd_be,     , uint64_t, stw_be_p, MO_BE, helper_be_stw_mmu)
+DO_LD_NF(bsu, H1_4, uint32_t, uint8_t, ldub_p)
+DO_LD_NF(bss, H1_4, uint32_t,  int8_t, ldsb_p)
+DO_LD_NF(bdu,     , uint64_t, uint8_t, ldub_p)
+DO_LD_NF(bds,     , uint64_t,  int8_t, ldsb_p)
 
-DO_ST_TLB(st1ss_be, H1_4, uint32_t, stl_be_p, MO_BE, helper_be_stl_mmu)
-DO_ST_TLB(st1sd_be,     , uint64_t, stl_be_p, MO_BE, helper_be_stl_mmu)
+DO_LD_NF(hsu_le, H1_4, uint32_t, uint16_t, lduw_le_p)
+DO_LD_NF(hss_le, H1_4, uint32_t,  int16_t, ldsw_le_p)
+DO_LD_NF(hsu_be, H1_4, uint32_t, uint16_t, lduw_be_p)
+DO_LD_NF(hss_be, H1_4, uint32_t,  int16_t, ldsw_be_p)
+DO_LD_NF(hdu_le,     , uint64_t, uint16_t, lduw_le_p)
+DO_LD_NF(hds_le,     , uint64_t,  int16_t, ldsw_le_p)
+DO_LD_NF(hdu_be,     , uint64_t, uint16_t, lduw_be_p)
+DO_LD_NF(hds_be,     , uint64_t,  int16_t, ldsw_be_p)
 
-DO_ST_TLB(st1dd_be,     , uint64_t, stq_be_p, MO_BE, helper_be_stq_mmu)
+DO_LD_NF(ss_le,  H1_4, uint32_t, uint32_t, ldl_le_p)
+DO_LD_NF(ss_be,  H1_4, uint32_t, uint32_t, ldl_be_p)
+DO_LD_NF(sdu_le,     , uint64_t, uint32_t, ldl_le_p)
+DO_LD_NF(sds_le,     , uint64_t,  int32_t, ldl_le_p)
+DO_LD_NF(sdu_be,     , uint64_t, uint32_t, ldl_be_p)
+DO_LD_NF(sds_be,     , uint64_t,  int32_t, ldl_be_p)
 
-#undef DO_ST_TLB
+DO_LD_NF(dd_le,      , uint64_t, uint64_t, ldq_le_p)
+DO_LD_NF(dd_be,      , uint64_t, uint64_t, ldq_be_p)
 
 /*
- * Common helpers for all contiguous 1,2,3,4-register predicated stores.
+ * Common helper for all gather first-faulting loads.
  */
-static void sve_st1_r(CPUARMState *env, void *vg, target_ulong addr,
-                      uint32_t desc, const uintptr_t ra,
-                      const int esize, const int msize,
-                      sve_st1_tlb_fn *tlb_fn)
+static inline void sve_ldff1_zs(CPUARMState *env, void *vd, void *vg, void *vm,
+                                target_ulong base, uint32_t desc, uintptr_t ra,
+                                zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn,
+                                sve_ld1_nf_fn *nonfault_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    intptr_t i, oprsz = simd_oprsz(desc);
-    void *vd = &env->vfp.zregs[rd];
+    const int mmu_idx = get_mmuidx(oi);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    intptr_t reg_off, reg_max = simd_oprsz(desc);
+    target_ulong addr;
 
-    set_helper_retaddr(ra);
-    for (i = 0; i < oprsz; ) {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
-        do {
-            if (pg & 1) {
-                tlb_fn(env, vd, i, addr, oi, ra);
+    /* Skip to the first true predicate.  */
+    reg_off = find_next_active(vg, 0, reg_max, MO_32);
+    if (likely(reg_off < reg_max)) {
+        /* Perform one normal read, which will fault or not.  */
+        set_helper_retaddr(ra);
+        addr = off_fn(vm, reg_off);
+        addr = base + (addr << scale);
+        tlb_fn(env, vd, reg_off, addr, oi, ra);
+
+        /* The rest of the reads will be non-faulting.  */
+        clear_helper_retaddr();
+    }
+
+    /* After any fault, zero the leading predicated false elements.  */
+    swap_memzero(vd, reg_off);
+
+    while (likely((reg_off += 4) < reg_max)) {
+        uint64_t pg = *(uint64_t *)((char *)vg + (reg_off >> 6) * 8);
+        if (likely((pg >> (reg_off & 63)) & 1)) {
+            addr = off_fn(vm, reg_off);
+            addr = base + (addr << scale);
+            if (!nonfault_fn(env, vd, reg_off, addr, mmu_idx)) {
+                record_fault(env, reg_off, reg_max);
+                break;
             }
-            i += esize, pg >>= esize;
-            addr += msize;
-        } while (i & 15);
+        } else {
+            *(uint32_t *)((char *)vd + H1_4(reg_off)) = 0;
+        }
     }
-    clear_helper_retaddr();
 }
 
-static void sve_st2_r(CPUARMState *env, void *vg, target_ulong addr,
-                      uint32_t desc, const uintptr_t ra,
-                      const int esize, const int msize,
-                      sve_st1_tlb_fn *tlb_fn)
+static inline void sve_ldff1_zd(CPUARMState *env, void *vd, void *vg, void *vm,
+                                target_ulong base, uint32_t desc, uintptr_t ra,
+                                zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn,
+                                sve_ld1_nf_fn *nonfault_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    intptr_t i, oprsz = simd_oprsz(desc);
-    void *d1 = &env->vfp.zregs[rd];
-    void *d2 = &env->vfp.zregs[(rd + 1) & 31];
+    const int mmu_idx = get_mmuidx(oi);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    intptr_t reg_off, reg_max = simd_oprsz(desc);
+    target_ulong addr;
 
-    set_helper_retaddr(ra);
-    for (i = 0; i < oprsz; ) {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
-        do {
-            if (pg & 1) {
-                tlb_fn(env, d1, i, addr, oi, ra);
-                tlb_fn(env, d2, i, addr + msize, oi, ra);
+    /* Skip to the first true predicate.  */
+    reg_off = find_next_active(vg, 0, reg_max, MO_64);
+    if (likely(reg_off < reg_max)) {
+        /* Perform one normal read, which will fault or not.  */
+        set_helper_retaddr(ra);
+        addr = off_fn(vm, reg_off);
+        addr = base + (addr << scale);
+        tlb_fn(env, vd, reg_off, addr, oi, ra);
+
+        /* The rest of the reads will be non-faulting.  */
+        clear_helper_retaddr();
+    }
+
+    /* After any fault, zero the leading predicated false elements.  */
+    swap_memzero(vd, reg_off);
+
+    while (likely((reg_off += 8) < reg_max)) {
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(reg_off >> 3));
+        if (likely(pg & 1)) {
+            addr = off_fn(vm, reg_off);
+            addr = base + (addr << scale);
+            if (!nonfault_fn(env, vd, reg_off, addr, mmu_idx)) {
+                record_fault(env, reg_off, reg_max);
+                break;
             }
-            i += esize, pg >>= esize;
-            addr += 2 * msize;
-        } while (i & 15);
+        } else {
+            *(uint64_t *)((char *)vd + reg_off) = 0;
+        }
     }
-    clear_helper_retaddr();
 }
 
-static void sve_st3_r(CPUARMState *env, void *vg, target_ulong addr,
-                      uint32_t desc, const uintptr_t ra,
-                      const int esize, const int msize,
-                      sve_st1_tlb_fn *tlb_fn)
+static inline void sve_ldff1_zs_mte(CPUARMState *env, void *vd, void *vg,
+                                    void *vm, target_ulong base,
+                                    uint32_t desc, uintptr_t ra,
+                                    zreg_off_fn *off_fn,
+                                    sve_ld1_tlb_fn *tlb_fn,
+                                    sve_ld1_nf_fn *nonfault_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    intptr_t i, oprsz = simd_oprsz(desc);
-    void *d1 = &env->vfp.zregs[rd];
-    void *d2 = &env->vfp.zregs[(rd + 1) & 31];
-    void *d3 = &env->vfp.zregs[(rd + 2) & 31];
+    const int mmu_idx = get_mmuidx(oi);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
+    intptr_t reg_off, reg_max = simd_oprsz(desc);
+    target_ulong tagged_addr, clean_addr;
 
-    set_helper_retaddr(ra);
-    for (i = 0; i < oprsz; ) {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
-        do {
-            if (pg & 1) {
-                tlb_fn(env, d1, i, addr, oi, ra);
-                tlb_fn(env, d2, i, addr + msize, oi, ra);
-                tlb_fn(env, d3, i, addr + 2 * msize, oi, ra);
+    reg_off = find_next_active(vg, 0, reg_max, MO_32);
+    if (likely(reg_off < reg_max)) {
+        set_helper_retaddr(ra);
+        tagged_addr = base + (off_fn(vm, reg_off) << scale);
+        clean_addr = sve_mte_check_addr(env, oi, tagged_addr,
+                                        false, msize, ra);
+        tlb_fn(env, vd, reg_off, clean_addr, oi, ra);
+        clear_helper_retaddr();
+    }
+
+    swap_memzero(vd, reg_off);
+
+    while (likely((reg_off += 4) < reg_max)) {
+        uint64_t pg = *(uint64_t *)((char *)vg + (reg_off >> 6) * 8);
+
+        if (likely((pg >> (reg_off & 63)) & 1)) {
+            void *host;
+
+            tagged_addr = base + (off_fn(vm, reg_off) << scale);
+            clean_addr = sve_mte_clean_addr(env, oi, tagged_addr,
+                                            false, msize);
+            if (max_for_page(env->uc, clean_addr, 0, msize) < msize) {
+                record_fault(env, reg_off, reg_max);
+                break;
             }
-            i += esize, pg >>= esize;
-            addr += 3 * msize;
-        } while (i & 15);
+            host = tlb_vaddr_to_host(env, clean_addr, MMU_DATA_LOAD, mmu_idx);
+            if (!host ||
+                !sve_mte_probe_addr(env, oi, tagged_addr, false, msize) ||
+                !nonfault_fn(env, vd, reg_off, clean_addr, mmu_idx)) {
+                record_fault(env, reg_off, reg_max);
+                break;
+            }
+        } else {
+            *(uint32_t *)((char *)vd + H1_4(reg_off)) = 0;
+        }
     }
-    clear_helper_retaddr();
 }
 
-static void sve_st4_r(CPUARMState *env, void *vg, target_ulong addr,
-                      uint32_t desc, const uintptr_t ra,
-                      const int esize, const int msize,
-                      sve_st1_tlb_fn *tlb_fn)
+static inline void sve_ldff1_zd_mte(CPUARMState *env, void *vd, void *vg,
+                                    void *vm, target_ulong base,
+                                    uint32_t desc, uintptr_t ra,
+                                    zreg_off_fn *off_fn,
+                                    sve_ld1_tlb_fn *tlb_fn,
+                                    sve_ld1_nf_fn *nonfault_fn)
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const unsigned rd = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 5);
-    intptr_t i, oprsz = simd_oprsz(desc);
-    void *d1 = &env->vfp.zregs[rd];
-    void *d2 = &env->vfp.zregs[(rd + 1) & 31];
-    void *d3 = &env->vfp.zregs[(rd + 2) & 31];
-    void *d4 = &env->vfp.zregs[(rd + 3) & 31];
+    const int mmu_idx = get_mmuidx(oi);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
+    intptr_t reg_off, reg_max = simd_oprsz(desc);
+    target_ulong tagged_addr, clean_addr;
 
-    set_helper_retaddr(ra);
-    for (i = 0; i < oprsz; ) {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
-        do {
-            if (pg & 1) {
-                tlb_fn(env, d1, i, addr, oi, ra);
-                tlb_fn(env, d2, i, addr + msize, oi, ra);
-                tlb_fn(env, d3, i, addr + 2 * msize, oi, ra);
-                tlb_fn(env, d4, i, addr + 3 * msize, oi, ra);
+    reg_off = find_next_active(vg, 0, reg_max, MO_64);
+    if (likely(reg_off < reg_max)) {
+        set_helper_retaddr(ra);
+        tagged_addr = base + (off_fn(vm, reg_off) << scale);
+        clean_addr = sve_mte_check_addr(env, oi, tagged_addr,
+                                        false, msize, ra);
+        tlb_fn(env, vd, reg_off, clean_addr, oi, ra);
+        clear_helper_retaddr();
+    }
+
+    swap_memzero(vd, reg_off);
+
+    while (likely((reg_off += 8) < reg_max)) {
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(reg_off >> 3));
+
+        if (likely(pg & 1)) {
+            void *host;
+
+            tagged_addr = base + (off_fn(vm, reg_off) << scale);
+            clean_addr = sve_mte_clean_addr(env, oi, tagged_addr,
+                                            false, msize);
+            if (max_for_page(env->uc, clean_addr, 0, msize) < msize) {
+                record_fault(env, reg_off, reg_max);
+                break;
             }
-            i += esize, pg >>= esize;
-            addr += 4 * msize;
-        } while (i & 15);
+            host = tlb_vaddr_to_host(env, clean_addr, MMU_DATA_LOAD, mmu_idx);
+            if (!host ||
+                !sve_mte_probe_addr(env, oi, tagged_addr, false, msize) ||
+                !nonfault_fn(env, vd, reg_off, clean_addr, mmu_idx)) {
+                record_fault(env, reg_off, reg_max);
+                break;
+            }
+        } else {
+            *(uint64_t *)((char *)vd + reg_off) = 0;
+        }
     }
-    clear_helper_retaddr();
 }
 
-#define DO_STN_1(N, NAME, ESIZE) \
-void QEMU_FLATTEN HELPER(sve_st##N##NAME##_r) \
-    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)  \
-{                                                                   \
-    sve_st##N##_r(env, vg, addr, desc, GETPC(), ESIZE, 1,           \
-                  sve_st1##NAME##_tlb);                             \
+#define DO_LDFF1_ZPZ_S(MEM, OFS) \
+void HELPER(sve_ldff##MEM##_##OFS)                                      \
+    (CPUARMState *env, void *vd, void *vg, void *vm,                    \
+     target_ulong base, uint32_t desc)                                  \
+{                                                                       \
+    sve_ldff1_zs(env, vd, vg, vm, base, desc, GETPC(),                  \
+                  off_##OFS##_s, sve_ld1##MEM##_tlb, sve_ld##MEM##_nf);  \
+}                                                                       \
+void HELPER(sve_ldff##MEM##_##OFS##_mte)                                \
+    (CPUARMState *env, void *vd, void *vg, void *vm,                    \
+     target_ulong base, uint32_t desc)                                  \
+{                                                                       \
+    sve_ldff1_zs_mte(env, vd, vg, vm, base, desc, GETPC(),              \
+                     off_##OFS##_s, sve_ld1##MEM##_tlb,                 \
+                     sve_ld##MEM##_nf);                                 \
 }
 
-#define DO_STN_2(N, NAME, ESIZE, MSIZE) \
-void QEMU_FLATTEN HELPER(sve_st##N##NAME##_le_r) \
-    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
-{                                                                     \
-    sve_st##N##_r(env, vg, addr, desc, GETPC(), ESIZE, MSIZE,         \
-                  sve_st1##NAME##_le_tlb);                            \
-}                                                                     \
-void QEMU_FLATTEN HELPER(sve_st##N##NAME##_be_r)                      \
-    (CPUARMState *env, void *vg, target_ulong addr, uint32_t desc)    \
-{                                                                     \
-    sve_st##N##_r(env, vg, addr, desc, GETPC(), ESIZE, MSIZE,         \
-                  sve_st1##NAME##_be_tlb);                            \
+#define DO_LDFF1_ZPZ_D(MEM, OFS) \
+void HELPER(sve_ldff##MEM##_##OFS)                                      \
+    (CPUARMState *env, void *vd, void *vg, void *vm,                    \
+     target_ulong base, uint32_t desc)                                  \
+{                                                                       \
+    sve_ldff1_zd(env, vd, vg, vm, base, desc, GETPC(),                  \
+                  off_##OFS##_d, sve_ld1##MEM##_tlb, sve_ld##MEM##_nf);  \
+}                                                                       \
+void HELPER(sve_ldff##MEM##_##OFS##_mte)                                \
+    (CPUARMState *env, void *vd, void *vg, void *vm,                    \
+     target_ulong base, uint32_t desc)                                  \
+{                                                                       \
+    sve_ldff1_zd_mte(env, vd, vg, vm, base, desc, GETPC(),              \
+                     off_##OFS##_d, sve_ld1##MEM##_tlb,                 \
+                     sve_ld##MEM##_nf);                                 \
 }
 
-DO_STN_1(1, bb, 1)
-DO_STN_1(1, bh, 2)
-DO_STN_1(1, bs, 4)
-DO_STN_1(1, bd, 8)
-DO_STN_1(2, bb, 1)
-DO_STN_1(3, bb, 1)
-DO_STN_1(4, bb, 1)
+DO_LDFF1_ZPZ_S(bsu, zsu)
+DO_LDFF1_ZPZ_S(bsu, zss)
+DO_LDFF1_ZPZ_D(bdu, zsu)
+DO_LDFF1_ZPZ_D(bdu, zss)
+DO_LDFF1_ZPZ_D(bdu, zd)
 
-DO_STN_2(1, hh, 2, 2)
-DO_STN_2(1, hs, 4, 2)
-DO_STN_2(1, hd, 8, 2)
-DO_STN_2(2, hh, 2, 2)
-DO_STN_2(3, hh, 2, 2)
-DO_STN_2(4, hh, 2, 2)
+DO_LDFF1_ZPZ_S(bss, zsu)
+DO_LDFF1_ZPZ_S(bss, zss)
+DO_LDFF1_ZPZ_D(bds, zsu)
+DO_LDFF1_ZPZ_D(bds, zss)
+DO_LDFF1_ZPZ_D(bds, zd)
 
-DO_STN_2(1, ss, 4, 4)
-DO_STN_2(1, sd, 8, 4)
-DO_STN_2(2, ss, 4, 4)
-DO_STN_2(3, ss, 4, 4)
-DO_STN_2(4, ss, 4, 4)
+DO_LDFF1_ZPZ_S(hsu_le, zsu)
+DO_LDFF1_ZPZ_S(hsu_le, zss)
+DO_LDFF1_ZPZ_D(hdu_le, zsu)
+DO_LDFF1_ZPZ_D(hdu_le, zss)
+DO_LDFF1_ZPZ_D(hdu_le, zd)
 
-DO_STN_2(1, dd, 8, 8)
-DO_STN_2(2, dd, 8, 8)
-DO_STN_2(3, dd, 8, 8)
-DO_STN_2(4, dd, 8, 8)
+DO_LDFF1_ZPZ_S(hsu_be, zsu)
+DO_LDFF1_ZPZ_S(hsu_be, zss)
+DO_LDFF1_ZPZ_D(hdu_be, zsu)
+DO_LDFF1_ZPZ_D(hdu_be, zss)
+DO_LDFF1_ZPZ_D(hdu_be, zd)
+
+DO_LDFF1_ZPZ_S(hss_le, zsu)
+DO_LDFF1_ZPZ_S(hss_le, zss)
+DO_LDFF1_ZPZ_D(hds_le, zsu)
+DO_LDFF1_ZPZ_D(hds_le, zss)
+DO_LDFF1_ZPZ_D(hds_le, zd)
+
+DO_LDFF1_ZPZ_S(hss_be, zsu)
+DO_LDFF1_ZPZ_S(hss_be, zss)
+DO_LDFF1_ZPZ_D(hds_be, zsu)
+DO_LDFF1_ZPZ_D(hds_be, zss)
+DO_LDFF1_ZPZ_D(hds_be, zd)
+
+DO_LDFF1_ZPZ_S(ss_le,  zsu)
+DO_LDFF1_ZPZ_S(ss_le,  zss)
+DO_LDFF1_ZPZ_D(sdu_le, zsu)
+DO_LDFF1_ZPZ_D(sdu_le, zss)
+DO_LDFF1_ZPZ_D(sdu_le, zd)
+
+DO_LDFF1_ZPZ_S(ss_be,  zsu)
+DO_LDFF1_ZPZ_S(ss_be,  zss)
+DO_LDFF1_ZPZ_D(sdu_be, zsu)
+DO_LDFF1_ZPZ_D(sdu_be, zss)
+DO_LDFF1_ZPZ_D(sdu_be, zd)
+
+DO_LDFF1_ZPZ_D(sds_le, zsu)
+DO_LDFF1_ZPZ_D(sds_le, zss)
+DO_LDFF1_ZPZ_D(sds_le, zd)
 
-#undef DO_STN_1
-#undef DO_STN_2
+DO_LDFF1_ZPZ_D(sds_be, zsu)
+DO_LDFF1_ZPZ_D(sds_be, zss)
+DO_LDFF1_ZPZ_D(sds_be, zd)
 
-/*
- * Loads with a vector index.
- */
+DO_LDFF1_ZPZ_D(dd_le, zsu)
+DO_LDFF1_ZPZ_D(dd_le, zss)
+DO_LDFF1_ZPZ_D(dd_le, zd)
 
-/*
- * Load the element at @reg + @reg_ofs, sign or zero-extend as needed.
- */
-typedef target_ulong zreg_off_fn(void *reg, intptr_t reg_ofs);
+DO_LDFF1_ZPZ_D(dd_be, zsu)
+DO_LDFF1_ZPZ_D(dd_be, zss)
+DO_LDFF1_ZPZ_D(dd_be, zd)
 
-static target_ulong off_zsu_s(void *reg, intptr_t reg_ofs)
+void HELPER(sve2_eor3)(void *vd, void *vn, void *vm, void *vk,
+                       uint32_t desc)
 {
-    return *(uint32_t *)((char *)reg + H1_4(reg_ofs));
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+    for (i = 0; i < oprsz; i++) {
+        d[i] = n[i] ^ m[i] ^ k[i];
+    }
 }
 
-static target_ulong off_zss_s(void *reg, intptr_t reg_ofs)
+void HELPER(sve2_bcax)(void *vd, void *vn, void *vm, void *vk,
+                       uint32_t desc)
 {
-    return *(int32_t *)((char *)reg + H1_4(reg_ofs));
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+    for (i = 0; i < oprsz; i++) {
+        d[i] = n[i] ^ (m[i] & ~k[i]);
+    }
 }
 
-static target_ulong off_zsu_d(void *reg, intptr_t reg_ofs)
+void HELPER(sve2_bsl1n)(void *vd, void *vn, void *vm, void *vk,
+                        uint32_t desc)
 {
-    return (uint32_t)*(uint64_t *)((char *)reg + reg_ofs);
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+    for (i = 0; i < oprsz; i++) {
+        d[i] = (~n[i] & k[i]) | (m[i] & ~k[i]);
+    }
 }
 
-static target_ulong off_zss_d(void *reg, intptr_t reg_ofs)
+void HELPER(sve2_bsl2n)(void *vd, void *vn, void *vm, void *vk,
+                        uint32_t desc)
 {
-    return (int32_t)*(uint64_t *)((char *)reg + reg_ofs);
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+    for (i = 0; i < oprsz; i++) {
+        d[i] = (n[i] & k[i]) | (~m[i] & ~k[i]);
+    }
 }
 
-static target_ulong off_zd_d(void *reg, intptr_t reg_ofs)
+void HELPER(sve2_nbsl)(void *vd, void *vn, void *vm, void *vk,
+                       uint32_t desc)
 {
-    return *(uint64_t *)((char *)reg + reg_ofs);
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    uint64_t *d = vd, *n = vn, *m = vm, *k = vk;
+
+    for (i = 0; i < oprsz; i++) {
+        d[i] = ~((n[i] & k[i]) | (m[i] & ~k[i]));
+    }
 }
 
-static void sve_ld1_zs(CPUARMState *env, void *vd, void *vg, void *vm,
-                       target_ulong base, uint32_t desc, uintptr_t ra,
-                       zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
+void HELPER(sve2_xar_b)(void *vd, void *vn, void *vm, uint32_t desc)
 {
-    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
-    intptr_t i, oprsz = simd_oprsz(desc);
-    ARMVectorReg scratch = { 0 };
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    int shr = simd_data(desc);
+    int shl = 8 - shr;
+    uint64_t mask = dup_const(MO_8, 0xff >> shr);
+    uint64_t *d = vd, *n = vn, *m = vm;
 
-    set_helper_retaddr(ra);
-    for (i = 0; i < oprsz; ) {
-        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
-        do {
-            if (likely(pg & 1)) {
-                target_ulong off = off_fn(vm, i);
-                tlb_fn(env, &scratch, i, base + (off << scale), oi, ra);
-            }
-            i += 4, pg >>= 4;
-        } while (i & 15);
+    for (i = 0; i < oprsz; i++) {
+        uint64_t t = n[i] ^ m[i];
+        d[i] = ((t >> shr) & mask) | ((t << shl) & ~mask);
     }
-    clear_helper_retaddr();
-
-    /* Wait until all exceptions have been raised to write back.  */
-    memcpy(vd, &scratch, oprsz);
 }
 
-static void sve_ld1_zd(CPUARMState *env, void *vd, void *vg, void *vm,
-                       target_ulong base, uint32_t desc, uintptr_t ra,
-                       zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
+void HELPER(sve2_xar_h)(void *vd, void *vn, void *vm, uint32_t desc)
 {
-    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
     intptr_t i, oprsz = simd_oprsz(desc) / 8;
-    ARMVectorReg scratch = { 0 };
+    int shr = simd_data(desc);
+    int shl = 16 - shr;
+    uint64_t mask = dup_const(MO_16, 0xffff >> shr);
+    uint64_t *d = vd, *n = vn, *m = vm;
 
-    set_helper_retaddr(ra);
     for (i = 0; i < oprsz; i++) {
-        uint8_t pg = *(uint8_t *)((char *)vg + H1(i));
-        if (likely(pg & 1)) {
-            target_ulong off = off_fn(vm, i * 8);
-            tlb_fn(env, &scratch, i * 8, base + (off << scale), oi, ra);
-        }
+        uint64_t t = n[i] ^ m[i];
+        d[i] = ((t >> shr) & mask) | ((t << shl) & ~mask);
     }
-    clear_helper_retaddr();
-
-    /* Wait until all exceptions have been raised to write back.  */
-    memcpy(vd, &scratch, oprsz * 8);
 }
 
-#define DO_LD1_ZPZ_S(MEM, OFS) \
-void QEMU_FLATTEN HELPER(sve_ld##MEM##_##OFS) \
-    (CPUARMState *env, void *vd, void *vg, void *vm,         \
-     target_ulong base, uint32_t desc)                       \
-{                                                            \
-    sve_ld1_zs(env, vd, vg, vm, base, desc, GETPC(),         \
-              off_##OFS##_s, sve_ld1##MEM##_tlb);            \
-}
+void HELPER(sve2_xar_s)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc) / 4;
+    int shr = simd_data(desc);
+    uint32_t *d = vd, *n = vn, *m = vm;
 
-#define DO_LD1_ZPZ_D(MEM, OFS) \
-void QEMU_FLATTEN HELPER(sve_ld##MEM##_##OFS) \
-    (CPUARMState *env, void *vd, void *vg, void *vm,         \
-     target_ulong base, uint32_t desc)                       \
-{                                                            \
-    sve_ld1_zd(env, vd, vg, vm, base, desc, GETPC(),         \
-               off_##OFS##_d, sve_ld1##MEM##_tlb);           \
+    for (i = 0; i < oprsz; i++) {
+        d[i] = ror32(n[i] ^ m[i], shr);
+    }
 }
 
-DO_LD1_ZPZ_S(bsu, zsu)
-DO_LD1_ZPZ_S(bsu, zss)
-DO_LD1_ZPZ_D(bdu, zsu)
-DO_LD1_ZPZ_D(bdu, zss)
-DO_LD1_ZPZ_D(bdu, zd)
+void HELPER(sve2_xar_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+    int shr = simd_data(desc);
+    uint64_t *d = vd, *n = vn, *m = vm;
 
-DO_LD1_ZPZ_S(bss, zsu)
-DO_LD1_ZPZ_S(bss, zss)
-DO_LD1_ZPZ_D(bds, zsu)
-DO_LD1_ZPZ_D(bds, zss)
-DO_LD1_ZPZ_D(bds, zd)
+    for (i = 0; i < oprsz; i++) {
+        d[i] = ror64(n[i] ^ m[i], shr);
+    }
+}
 
-DO_LD1_ZPZ_S(hsu_le, zsu)
-DO_LD1_ZPZ_S(hsu_le, zss)
-DO_LD1_ZPZ_D(hdu_le, zsu)
-DO_LD1_ZPZ_D(hdu_le, zss)
-DO_LD1_ZPZ_D(hdu_le, zd)
+#define DO_ZZZ_NTB(NAME, TYPE, H, OP)                                   \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    intptr_t sel1 = extract32(desc, SIMD_DATA_SHIFT, 1) * sizeof(TYPE); \
+    intptr_t sel2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1)             \
+                   * sizeof(TYPE);                                      \
+                                                                        \
+    for (i = 0; i < oprsz; i += 2 * sizeof(TYPE)) {                     \
+        TYPE nn = *(TYPE *)((char *)vn + H(i + sel1));                  \
+        TYPE mm = *(TYPE *)((char *)vm + H(i + sel2));                  \
+                                                                        \
+        *(TYPE *)((char *)vd + H(i + sel1)) = OP(nn, mm);               \
+    }                                                                   \
+}
 
-DO_LD1_ZPZ_S(hsu_be, zsu)
-DO_LD1_ZPZ_S(hsu_be, zss)
-DO_LD1_ZPZ_D(hdu_be, zsu)
-DO_LD1_ZPZ_D(hdu_be, zss)
-DO_LD1_ZPZ_D(hdu_be, zd)
+#define DO_EORIL(N, M) ((N) ^ (M))
 
-DO_LD1_ZPZ_S(hss_le, zsu)
-DO_LD1_ZPZ_S(hss_le, zss)
-DO_LD1_ZPZ_D(hds_le, zsu)
-DO_LD1_ZPZ_D(hds_le, zss)
-DO_LD1_ZPZ_D(hds_le, zd)
+DO_ZZZ_NTB(sve2_eoril_b, uint8_t, H1, DO_EORIL)
+DO_ZZZ_NTB(sve2_eoril_h, uint16_t, H1_2, DO_EORIL)
+DO_ZZZ_NTB(sve2_eoril_s, uint32_t, H1_4, DO_EORIL)
+DO_ZZZ_NTB(sve2_eoril_d, uint64_t, H1_8, DO_EORIL)
 
-DO_LD1_ZPZ_S(hss_be, zsu)
-DO_LD1_ZPZ_S(hss_be, zss)
-DO_LD1_ZPZ_D(hds_be, zsu)
-DO_LD1_ZPZ_D(hds_be, zss)
-DO_LD1_ZPZ_D(hds_be, zd)
+#undef DO_EORIL
+#undef DO_ZZZ_NTB
 
-DO_LD1_ZPZ_S(ss_le, zsu)
-DO_LD1_ZPZ_S(ss_le, zss)
-DO_LD1_ZPZ_D(sdu_le, zsu)
-DO_LD1_ZPZ_D(sdu_le, zss)
-DO_LD1_ZPZ_D(sdu_le, zd)
+#define DO_BITPERM(NAME, TYPE, OP)                                      \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+                                                                        \
+    for (i = 0; i < oprsz; i += sizeof(TYPE)) {                         \
+        TYPE nn = *(TYPE *)((char *)vn + i);                            \
+        TYPE mm = *(TYPE *)((char *)vm + i);                            \
+                                                                        \
+        *(TYPE *)((char *)vd + i) = OP(nn, mm, sizeof(TYPE) * 8);        \
+    }                                                                   \
+}
 
-DO_LD1_ZPZ_S(ss_be, zsu)
-DO_LD1_ZPZ_S(ss_be, zss)
-DO_LD1_ZPZ_D(sdu_be, zsu)
-DO_LD1_ZPZ_D(sdu_be, zss)
-DO_LD1_ZPZ_D(sdu_be, zd)
+static uint64_t bitextract(uint64_t data, uint64_t mask, int n)
+{
+    uint64_t res = 0;
+    int db, rb = 0;
 
-DO_LD1_ZPZ_D(sds_le, zsu)
-DO_LD1_ZPZ_D(sds_le, zss)
-DO_LD1_ZPZ_D(sds_le, zd)
+    for (db = 0; db < n; ++db) {
+        if ((mask >> db) & 1) {
+            res |= ((data >> db) & 1) << rb;
+            ++rb;
+        }
+    }
+    return res;
+}
 
-DO_LD1_ZPZ_D(sds_be, zsu)
-DO_LD1_ZPZ_D(sds_be, zss)
-DO_LD1_ZPZ_D(sds_be, zd)
+DO_BITPERM(sve2_bext_b, uint8_t, bitextract)
+DO_BITPERM(sve2_bext_h, uint16_t, bitextract)
+DO_BITPERM(sve2_bext_s, uint32_t, bitextract)
+DO_BITPERM(sve2_bext_d, uint64_t, bitextract)
 
-DO_LD1_ZPZ_D(dd_le, zsu)
-DO_LD1_ZPZ_D(dd_le, zss)
-DO_LD1_ZPZ_D(dd_le, zd)
+static uint64_t bitdeposit(uint64_t data, uint64_t mask, int n)
+{
+    uint64_t res = 0;
+    int rb, db = 0;
 
-DO_LD1_ZPZ_D(dd_be, zsu)
-DO_LD1_ZPZ_D(dd_be, zss)
-DO_LD1_ZPZ_D(dd_be, zd)
+    for (rb = 0; rb < n; ++rb) {
+        if ((mask >> rb) & 1) {
+            res |= ((data >> db) & 1) << rb;
+            ++db;
+        }
+    }
+    return res;
+}
 
-#undef DO_LD1_ZPZ_S
-#undef DO_LD1_ZPZ_D
+DO_BITPERM(sve2_bdep_b, uint8_t, bitdeposit)
+DO_BITPERM(sve2_bdep_h, uint16_t, bitdeposit)
+DO_BITPERM(sve2_bdep_s, uint32_t, bitdeposit)
+DO_BITPERM(sve2_bdep_d, uint64_t, bitdeposit)
 
-/* First fault loads with a vector index.  */
+static uint64_t bitgroup(uint64_t data, uint64_t mask, int n)
+{
+    uint64_t resm = 0, resu = 0;
+    int db, rbm = 0, rbu = 0;
 
-/* Load one element into VD+REG_OFF from (ENV,VADDR) without faulting.
- * The controlling predicate is known to be true.  Return true if the
- * load was successful.
- */
-typedef bool sve_ld1_nf_fn(CPUARMState *env, void *vd, intptr_t reg_off,
-                           target_ulong vaddr, int mmu_idx);
+    for (db = 0; db < n; ++db) {
+        uint64_t val = (data >> db) & 1;
 
-#ifdef _MSC_VER
-#define DO_LD_NF(NAME, H, TYPEE, TYPEM, HOST) \
-static bool sve_ld##NAME##_nf(CPUARMState *env, void *vd, intptr_t reg_off, \
-                              target_ulong addr, int mmu_idx)               \
-{                                                                           \
-    struct uc_struct *uc = env->uc;                                         \
-    target_ulong next_page = 0ULL - (addr | TARGET_PAGE_MASK);                    \
-    if (likely(next_page - addr >= sizeof(TYPEM))) {                        \
-        void *host = tlb_vaddr_to_host(env, addr, MMU_DATA_LOAD, mmu_idx);  \
-        if (likely(host)) {                                                 \
-            TYPEM val = HOST(host);                                         \
-            *(TYPEE *)((char *)vd + H(reg_off)) = val;                              \
-            return true;                                                    \
-        }                                                                   \
-    }                                                                       \
-    return false;                                                           \
-}
-#else
-#define DO_LD_NF(NAME, H, TYPEE, TYPEM, HOST) \
-static bool sve_ld##NAME##_nf(CPUARMState *env, void *vd, intptr_t reg_off, \
-                              target_ulong addr, int mmu_idx)               \
-{                                                                           \
-    struct uc_struct *uc = env->uc;                                         \
-    target_ulong next_page = -(addr | TARGET_PAGE_MASK);                    \
-    if (likely(next_page - addr >= sizeof(TYPEM))) {                        \
-        void *host = tlb_vaddr_to_host(env, addr, MMU_DATA_LOAD, mmu_idx);  \
-        if (likely(host)) {                                                 \
-            TYPEM val = HOST(host);                                         \
-            *(TYPEE *)((char *)vd + H(reg_off)) = val;                              \
-            return true;                                                    \
-        }                                                                   \
-    }                                                                       \
-    return false;                                                           \
+        if ((mask >> db) & 1) {
+            resm |= val << rbm++;
+        } else {
+            resu |= val << rbu++;
+        }
+    }
+    return resm | (resu << rbm);
+}
+
+DO_BITPERM(sve2_bgrp_b, uint8_t, bitgroup)
+DO_BITPERM(sve2_bgrp_h, uint16_t, bitgroup)
+DO_BITPERM(sve2_bgrp_s, uint32_t, bitgroup)
+DO_BITPERM(sve2_bgrp_d, uint64_t, bitgroup)
+
+#undef DO_BITPERM
+
+#define DO_CADD_ADD_B(N, M) ((int8_t)((uint8_t)(N) + (uint8_t)(M)))
+#define DO_CADD_ADD_H(N, M) ((int16_t)((uint16_t)(N) + (uint16_t)(M)))
+#define DO_CADD_ADD_S(N, M) ((int32_t)((uint32_t)(N) + (uint32_t)(M)))
+#define DO_CADD_ADD_D(N, M) ((int64_t)((uint64_t)(N) + (uint64_t)(M)))
+
+#define DO_CADD_SUB_B(N, M) ((int8_t)((uint8_t)(N) - (uint8_t)(M)))
+#define DO_CADD_SUB_H(N, M) ((int16_t)((uint16_t)(N) - (uint16_t)(M)))
+#define DO_CADD_SUB_S(N, M) ((int32_t)((uint32_t)(N) - (uint32_t)(M)))
+#define DO_CADD_SUB_D(N, M) ((int64_t)((uint64_t)(N) - (uint64_t)(M)))
+
+#define DO_CADD_SAT(VAL, MIN, MAX) \
+    ((VAL) > (MAX) ? (MAX) : (VAL) < (MIN) ? (MIN) : (VAL))
+
+#define DO_CADD_SQADD_B(N, M) \
+    ((int8_t)DO_CADD_SAT((int64_t)(N) + (M), INT8_MIN, INT8_MAX))
+#define DO_CADD_SQADD_H(N, M) \
+    ((int16_t)DO_CADD_SAT((int64_t)(N) + (M), INT16_MIN, INT16_MAX))
+#define DO_CADD_SQADD_S(N, M) \
+    ((int32_t)DO_CADD_SAT((int64_t)(N) + (M), INT32_MIN, INT32_MAX))
+#define DO_CADD_SQADD_D(N, M) do_sve2_sqadd_d(N, M)
+
+#define DO_CADD_SQSUB_B(N, M) \
+    ((int8_t)DO_CADD_SAT((int64_t)(N) - (M), INT8_MIN, INT8_MAX))
+#define DO_CADD_SQSUB_H(N, M) \
+    ((int16_t)DO_CADD_SAT((int64_t)(N) - (M), INT16_MIN, INT16_MAX))
+#define DO_CADD_SQSUB_S(N, M) \
+    ((int32_t)DO_CADD_SAT((int64_t)(N) - (M), INT32_MIN, INT32_MAX))
+#define DO_CADD_SQSUB_D(N, M) do_sve2_sqsub_d(N, M)
+
+#define DO_CADD(NAME, TYPE, H, ADD_OP, SUB_OP)                          \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    int sub_r = simd_data(desc);                                        \
+                                                                        \
+    for (i = 0; i < oprsz; i += 2 * sizeof(TYPE)) {                     \
+        TYPE acc_r = *(TYPE *)((char *)vn + H(i));                      \
+        TYPE acc_i = *(TYPE *)((char *)vn + H(i + sizeof(TYPE)));       \
+        TYPE el2_r = *(TYPE *)((char *)vm + H(i));                      \
+        TYPE el2_i = *(TYPE *)((char *)vm + H(i + sizeof(TYPE)));       \
+                                                                        \
+        if (sub_r) {                                                    \
+            acc_r = ADD_OP(acc_r, el2_i);                               \
+            acc_i = SUB_OP(acc_i, el2_r);                               \
+        } else {                                                        \
+            acc_r = SUB_OP(acc_r, el2_i);                               \
+            acc_i = ADD_OP(acc_i, el2_r);                               \
+        }                                                               \
+        *(TYPE *)((char *)vd + H(i)) = acc_r;                           \
+        *(TYPE *)((char *)vd + H(i + sizeof(TYPE))) = acc_i;            \
+    }                                                                   \
 }
-#endif
 
-DO_LD_NF(bsu, H1_4, uint32_t, uint8_t, ldub_p)
-DO_LD_NF(bss, H1_4, uint32_t,  int8_t, ldsb_p)
-DO_LD_NF(bdu,     , uint64_t, uint8_t, ldub_p)
-DO_LD_NF(bds,     , uint64_t,  int8_t, ldsb_p)
+DO_CADD(sve2_cadd_b, int8_t, H1, DO_CADD_ADD_B, DO_CADD_SUB_B)
+DO_CADD(sve2_cadd_h, int16_t, H1_2, DO_CADD_ADD_H, DO_CADD_SUB_H)
+DO_CADD(sve2_cadd_s, int32_t, H1_4, DO_CADD_ADD_S, DO_CADD_SUB_S)
+DO_CADD(sve2_cadd_d, int64_t, H1_8, DO_CADD_ADD_D, DO_CADD_SUB_D)
+
+DO_CADD(sve2_sqcadd_b, int8_t, H1, DO_CADD_SQADD_B, DO_CADD_SQSUB_B)
+DO_CADD(sve2_sqcadd_h, int16_t, H1_2, DO_CADD_SQADD_H, DO_CADD_SQSUB_H)
+DO_CADD(sve2_sqcadd_s, int32_t, H1_4, DO_CADD_SQADD_S, DO_CADD_SQSUB_S)
+DO_CADD(sve2_sqcadd_d, int64_t, H1_8, DO_CADD_SQADD_D, DO_CADD_SQSUB_D)
+
+#undef DO_CADD
+#undef DO_CADD_ADD_B
+#undef DO_CADD_ADD_H
+#undef DO_CADD_ADD_S
+#undef DO_CADD_ADD_D
+#undef DO_CADD_SUB_B
+#undef DO_CADD_SUB_H
+#undef DO_CADD_SUB_S
+#undef DO_CADD_SUB_D
+#undef DO_CADD_SAT
+#undef DO_CADD_SQADD_B
+#undef DO_CADD_SQADD_H
+#undef DO_CADD_SQADD_S
+#undef DO_CADD_SQADD_D
+#undef DO_CADD_SQSUB_B
+#undef DO_CADD_SQSUB_H
+#undef DO_CADD_SQSUB_S
+#undef DO_CADD_SQSUB_D
+
+static inline bool do_match2(uint64_t n, uint64_t m0, uint64_t m1, int esz)
+{
+    int bits = 8 << esz;
+    uint64_t ones = dup_const(esz, 1);
+    uint64_t signs = ones << (bits - 1);
+    uint64_t cmp0, cmp1;
+
+    cmp1 = dup_const(esz, n);
+    cmp0 = cmp1 ^ m0;
+    cmp1 = cmp1 ^ m1;
+    cmp0 = (cmp0 - ones) & ~cmp0;
+    cmp1 = (cmp1 - ones) & ~cmp1;
+    return (cmp0 | cmp1) & signs;
+}
+
+static inline uint32_t do_match(void *vd, void *vn, void *vm, void *vg,
+                                uint32_t desc, int esz, bool nmatch)
+{
+    uint16_t esz_mask = pred_esz_masks[esz];
+    intptr_t oprsz = simd_oprsz(desc);
+    uint32_t flags = PREDTEST_INIT;
+    intptr_t i, j, k;
 
-DO_LD_NF(hsu_le, H1_4, uint32_t, uint16_t, lduw_le_p)
-DO_LD_NF(hss_le, H1_4, uint32_t,  int16_t, ldsw_le_p)
-DO_LD_NF(hsu_be, H1_4, uint32_t, uint16_t, lduw_be_p)
-DO_LD_NF(hss_be, H1_4, uint32_t,  int16_t, ldsw_be_p)
-DO_LD_NF(hdu_le,     , uint64_t, uint16_t, lduw_le_p)
-DO_LD_NF(hds_le,     , uint64_t,  int16_t, ldsw_le_p)
-DO_LD_NF(hdu_be,     , uint64_t, uint16_t, lduw_be_p)
-DO_LD_NF(hds_be,     , uint64_t,  int16_t, ldsw_be_p)
+    for (i = 0; i < oprsz; i += 16) {
+        uint64_t m0 = *(uint64_t *)((char *)vm + i);
+        uint64_t m1 = *(uint64_t *)((char *)vm + i + 8);
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3)) & esz_mask;
+        uint16_t out = 0;
 
-DO_LD_NF(ss_le,  H1_4, uint32_t, uint32_t, ldl_le_p)
-DO_LD_NF(ss_be,  H1_4, uint32_t, uint32_t, ldl_be_p)
-DO_LD_NF(sdu_le,     , uint64_t, uint32_t, ldl_le_p)
-DO_LD_NF(sds_le,     , uint64_t,  int32_t, ldl_le_p)
-DO_LD_NF(sdu_be,     , uint64_t, uint32_t, ldl_be_p)
-DO_LD_NF(sds_be,     , uint64_t,  int32_t, ldl_be_p)
+        for (j = 0; j < 16; j += 8) {
+            uint64_t n = *(uint64_t *)((char *)vn + i + j);
+
+            for (k = 0; k < 8; k += 1 << esz) {
+                if (pg & (1 << (j + k))) {
+                    bool o = do_match2(n >> (k * 8), m0, m1, esz);
+
+                    out |= (o ^ nmatch) << (j + k);
+                }
+            }
+        }
+        *(uint16_t *)((char *)vd + H1_2(i >> 3)) = out;
+        flags = iter_predtest_fwd(out, pg, flags);
+    }
+    return flags;
+}
 
-DO_LD_NF(dd_le,      , uint64_t, uint64_t, ldq_le_p)
-DO_LD_NF(dd_be,      , uint64_t, uint64_t, ldq_be_p)
+#define DO_PPZZ_MATCH(NAME, ESZ, INV)                                    \
+uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg,            \
+                      uint32_t desc)                                     \
+{                                                                        \
+    return do_match(vd, vn, vm, vg, desc, ESZ, INV);                     \
+}
 
-/*
- * Common helper for all gather first-faulting loads.
- */
-static inline void sve_ldff1_zs(CPUARMState *env, void *vd, void *vg, void *vm,
-                                target_ulong base, uint32_t desc, uintptr_t ra,
-                                zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn,
-                                sve_ld1_nf_fn *nonfault_fn)
-{
-    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const int mmu_idx = get_mmuidx(oi);
-    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
-    intptr_t reg_off, reg_max = simd_oprsz(desc);
-    target_ulong addr;
+DO_PPZZ_MATCH(sve2_match_ppzz_b, MO_8, false)
+DO_PPZZ_MATCH(sve2_match_ppzz_h, MO_16, false)
+DO_PPZZ_MATCH(sve2_nmatch_ppzz_b, MO_8, true)
+DO_PPZZ_MATCH(sve2_nmatch_ppzz_h, MO_16, true)
 
-    /* Skip to the first true predicate.  */
-    reg_off = find_next_active(vg, 0, reg_max, MO_32);
-    if (likely(reg_off < reg_max)) {
-        /* Perform one normal read, which will fault or not.  */
-        set_helper_retaddr(ra);
-        addr = off_fn(vm, reg_off);
-        addr = base + (addr << scale);
-        tlb_fn(env, vd, reg_off, addr, oi, ra);
+#undef DO_PPZZ_MATCH
 
-        /* The rest of the reads will be non-faulting.  */
-        clear_helper_retaddr();
+void HELPER(sve2_histcnt_s)(void *vd, void *vn, void *vm, void *vg,
+                            uint32_t desc)
+{
+    ARMVectorReg scratch;
+    intptr_t i, j;
+    intptr_t oprsz = simd_oprsz(desc);
+    uint32_t *d = vd, *n = vn, *m = vm;
+    uint8_t *pg = vg;
+
+    if (d == n) {
+        n = memcpy(&scratch, n, oprsz);
+        if (d == m) {
+            m = n;
+        }
+    } else if (d == m) {
+        m = memcpy(&scratch, m, oprsz);
     }
 
-    /* After any fault, zero the leading predicated false elements.  */
-    swap_memzero(vd, reg_off);
+    for (i = 0; i < oprsz; i += 4) {
+        uint64_t count = 0;
+        uint8_t pred = pg[H1(i >> 3)] >> (i & 7);
 
-    while (likely((reg_off += 4) < reg_max)) {
-        uint64_t pg = *(uint64_t *)((char *)vg + (reg_off >> 6) * 8);
-        if (likely((pg >> (reg_off & 63)) & 1)) {
-            addr = off_fn(vm, reg_off);
-            addr = base + (addr << scale);
-            if (!nonfault_fn(env, vd, reg_off, addr, mmu_idx)) {
-                record_fault(env, reg_off, reg_max);
-                break;
+        if (pred & 1) {
+            uint32_t nn = n[H4(i >> 2)];
+
+            for (j = 0; j <= i; j += 4) {
+                pred = pg[H1(j >> 3)] >> (j & 7);
+                if ((pred & 1) && nn == m[H4(j >> 2)]) {
+                    count++;
+                }
             }
-        } else {
-            *(uint32_t *)((char *)vd + H1_4(reg_off)) = 0;
         }
+        d[H4(i >> 2)] = count;
     }
 }
 
-static inline void sve_ldff1_zd(CPUARMState *env, void *vd, void *vg, void *vm,
-                                target_ulong base, uint32_t desc, uintptr_t ra,
-                                zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn,
-                                sve_ld1_nf_fn *nonfault_fn)
+void HELPER(sve2_histcnt_d)(void *vd, void *vn, void *vm, void *vg,
+                            uint32_t desc)
 {
-    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
-    const int mmu_idx = get_mmuidx(oi);
-    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
-    intptr_t reg_off, reg_max = simd_oprsz(desc);
-    target_ulong addr;
-
-    /* Skip to the first true predicate.  */
-    reg_off = find_next_active(vg, 0, reg_max, MO_64);
-    if (likely(reg_off < reg_max)) {
-        /* Perform one normal read, which will fault or not.  */
-        set_helper_retaddr(ra);
-        addr = off_fn(vm, reg_off);
-        addr = base + (addr << scale);
-        tlb_fn(env, vd, reg_off, addr, oi, ra);
+    ARMVectorReg scratch;
+    intptr_t i, j;
+    intptr_t oprsz = simd_oprsz(desc);
+    uint64_t *d = vd, *n = vn, *m = vm;
+    uint8_t *pg = vg;
 
-        /* The rest of the reads will be non-faulting.  */
-        clear_helper_retaddr();
+    if (d == n) {
+        n = memcpy(&scratch, n, oprsz);
+        if (d == m) {
+            m = n;
+        }
+    } else if (d == m) {
+        m = memcpy(&scratch, m, oprsz);
     }
 
-    /* After any fault, zero the leading predicated false elements.  */
-    swap_memzero(vd, reg_off);
+    for (i = 0; i < oprsz / 8; i++) {
+        uint64_t count = 0;
 
-    while (likely((reg_off += 8) < reg_max)) {
-        uint8_t pg = *(uint8_t *)((char *)vg + H1(reg_off >> 3));
-        if (likely(pg & 1)) {
-            addr = off_fn(vm, reg_off);
-            addr = base + (addr << scale);
-            if (!nonfault_fn(env, vd, reg_off, addr, mmu_idx)) {
-                record_fault(env, reg_off, reg_max);
-                break;
+        if (pg[H1(i)] & 1) {
+            uint64_t nn = n[i];
+
+            for (j = 0; j <= i; j++) {
+                if ((pg[H1(j)] & 1) && nn == m[j]) {
+                    count++;
+                }
             }
-        } else {
-            *(uint64_t *)((char *)vd + reg_off) = 0;
         }
+        d[i] = count;
     }
 }
 
-#define DO_LDFF1_ZPZ_S(MEM, OFS) \
-void HELPER(sve_ldff##MEM##_##OFS)                                      \
-    (CPUARMState *env, void *vd, void *vg, void *vm,                    \
-     target_ulong base, uint32_t desc)                                  \
-{                                                                       \
-    sve_ldff1_zs(env, vd, vg, vm, base, desc, GETPC(),                  \
-                 off_##OFS##_s, sve_ld1##MEM##_tlb, sve_ld##MEM##_nf);  \
-}
+static inline uint64_t do_histseg_cnt(uint8_t n, uint64_t m0, uint64_t m1)
+{
+    const uint64_t mask = dup_const(MO_8, 0x7f);
+    uint64_t cmp0, cmp1;
 
-#define DO_LDFF1_ZPZ_D(MEM, OFS) \
-void HELPER(sve_ldff##MEM##_##OFS)                                      \
-    (CPUARMState *env, void *vd, void *vg, void *vm,                    \
-     target_ulong base, uint32_t desc)                                  \
-{                                                                       \
-    sve_ldff1_zd(env, vd, vg, vm, base, desc, GETPC(),                  \
-                 off_##OFS##_d, sve_ld1##MEM##_tlb, sve_ld##MEM##_nf);  \
+    cmp1 = dup_const(MO_8, n);
+    cmp0 = cmp1 ^ m0;
+    cmp1 = cmp1 ^ m1;
+    cmp0 = ~(((cmp0 & mask) + mask) | cmp0 | mask);
+    cmp1 = ~(((cmp1 & mask) + mask) | cmp1 | mask);
+
+    return ctpop64(cmp0 | (cmp1 >> 1));
 }
 
-DO_LDFF1_ZPZ_S(bsu, zsu)
-DO_LDFF1_ZPZ_S(bsu, zss)
-DO_LDFF1_ZPZ_D(bdu, zsu)
-DO_LDFF1_ZPZ_D(bdu, zss)
-DO_LDFF1_ZPZ_D(bdu, zd)
+void HELPER(sve2_histseg)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t i, j;
+    intptr_t oprsz = simd_oprsz(desc);
 
-DO_LDFF1_ZPZ_S(bss, zsu)
-DO_LDFF1_ZPZ_S(bss, zss)
-DO_LDFF1_ZPZ_D(bds, zsu)
-DO_LDFF1_ZPZ_D(bds, zss)
-DO_LDFF1_ZPZ_D(bds, zd)
+    for (i = 0; i < oprsz; i += 16) {
+        uint64_t n0 = *(uint64_t *)((char *)vn + i);
+        uint64_t m0 = *(uint64_t *)((char *)vm + i);
+        uint64_t n1 = *(uint64_t *)((char *)vn + i + 8);
+        uint64_t m1 = *(uint64_t *)((char *)vm + i + 8);
+        uint64_t out0 = 0;
+        uint64_t out1 = 0;
 
-DO_LDFF1_ZPZ_S(hsu_le, zsu)
-DO_LDFF1_ZPZ_S(hsu_le, zss)
-DO_LDFF1_ZPZ_D(hdu_le, zsu)
-DO_LDFF1_ZPZ_D(hdu_le, zss)
-DO_LDFF1_ZPZ_D(hdu_le, zd)
+        for (j = 0; j < 64; j += 8) {
+            uint64_t cnt0 = do_histseg_cnt(n0 >> j, m0, m1);
+            uint64_t cnt1 = do_histseg_cnt(n1 >> j, m0, m1);
 
-DO_LDFF1_ZPZ_S(hsu_be, zsu)
-DO_LDFF1_ZPZ_S(hsu_be, zss)
-DO_LDFF1_ZPZ_D(hdu_be, zsu)
-DO_LDFF1_ZPZ_D(hdu_be, zss)
-DO_LDFF1_ZPZ_D(hdu_be, zd)
+            out0 |= cnt0 << j;
+            out1 |= cnt1 << j;
+        }
+        *(uint64_t *)((char *)vd + i) = out0;
+        *(uint64_t *)((char *)vd + i + 8) = out1;
+    }
+}
 
-DO_LDFF1_ZPZ_S(hss_le, zsu)
-DO_LDFF1_ZPZ_S(hss_le, zss)
-DO_LDFF1_ZPZ_D(hds_le, zsu)
-DO_LDFF1_ZPZ_D(hds_le, zss)
-DO_LDFF1_ZPZ_D(hds_le, zd)
+void HELPER(sve2_adcl_s)(void *vd, void *vn, void *vm, void *va,
+                         uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+    int sel = H4(extract32(desc, SIMD_DATA_SHIFT, 1));
+    uint32_t inv = -extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+    uint32_t *a = va, *n = vn;
+    uint64_t *d = vd, *m = vm;
 
-DO_LDFF1_ZPZ_S(hss_be, zsu)
-DO_LDFF1_ZPZ_S(hss_be, zss)
-DO_LDFF1_ZPZ_D(hds_be, zsu)
-DO_LDFF1_ZPZ_D(hds_be, zss)
-DO_LDFF1_ZPZ_D(hds_be, zd)
+    for (i = 0; i < oprsz / 8; ++i) {
+        uint32_t e1 = a[2 * i + H4(0)];
+        uint32_t e2 = n[2 * i + sel] ^ inv;
+        uint64_t c = extract64(m[i], 32, 1);
 
-DO_LDFF1_ZPZ_S(ss_le,  zsu)
-DO_LDFF1_ZPZ_S(ss_le,  zss)
-DO_LDFF1_ZPZ_D(sdu_le, zsu)
-DO_LDFF1_ZPZ_D(sdu_le, zss)
-DO_LDFF1_ZPZ_D(sdu_le, zd)
+        d[i] = c + e1 + e2;
+    }
+}
 
-DO_LDFF1_ZPZ_S(ss_be,  zsu)
-DO_LDFF1_ZPZ_S(ss_be,  zss)
-DO_LDFF1_ZPZ_D(sdu_be, zsu)
-DO_LDFF1_ZPZ_D(sdu_be, zss)
-DO_LDFF1_ZPZ_D(sdu_be, zd)
+void HELPER(sve2_adcl_d)(void *vd, void *vn, void *vm, void *va,
+                         uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc);
+    int sel = extract32(desc, SIMD_DATA_SHIFT, 1);
+    uint64_t inv = -(uint64_t)extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+    uint64_t *d = vd, *a = va, *n = vn, *m = vm;
+
+    for (i = 0; i < oprsz / 8; i += 2) {
+        Int128 e1 = int128_make64(a[i]);
+        Int128 e2 = int128_make64(n[i + sel] ^ inv);
+        Int128 c = int128_make64(m[i + 1] & 1);
+        Int128 r = int128_add(int128_add(e1, e2), c);
+
+        d[i] = int128_getlo(r);
+        d[i + 1] = int128_gethi(r);
+    }
+}
 
-DO_LDFF1_ZPZ_D(sds_le, zsu)
-DO_LDFF1_ZPZ_D(sds_le, zss)
-DO_LDFF1_ZPZ_D(sds_le, zd)
+/* Stores with a vector index.  */
 
-DO_LDFF1_ZPZ_D(sds_be, zsu)
-DO_LDFF1_ZPZ_D(sds_be, zss)
-DO_LDFF1_ZPZ_D(sds_be, zd)
+static void sve_probe_st1_zs_pages(CPUARMState *env, void *vg, void *vm,
+                                   target_ulong base, uint32_t desc,
+                                   int msize, zreg_off_fn *off_fn,
+                                   bool mte, uintptr_t ra)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    intptr_t i, oprsz = simd_oprsz(desc);
 
-DO_LDFF1_ZPZ_D(dd_le, zsu)
-DO_LDFF1_ZPZ_D(dd_le, zss)
-DO_LDFF1_ZPZ_D(dd_le, zd)
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
 
-DO_LDFF1_ZPZ_D(dd_be, zsu)
-DO_LDFF1_ZPZ_D(dd_be, zss)
-DO_LDFF1_ZPZ_D(dd_be, zd)
+        do {
+            if (likely(pg & 1)) {
+                target_ulong tagged_addr = base + (off_fn(vm, i) << scale);
+                target_ulong clean_addr = tagged_addr;
+
+                if (mte) {
+                    clean_addr = sve_mte_clean_addr(env, oi, tagged_addr,
+                                                    true, msize);
+                }
+                sve_probe_write_addr(env, clean_addr, msize, oi, ra);
+            }
+            i += 4, pg >>= 4;
+        } while (i & 15);
+    }
+}
 
-/* Stores with a vector index.  */
+static void sve_probe_st1_zd_pages(CPUARMState *env, void *vg, void *vm,
+                                   target_ulong base, uint32_t desc,
+                                   int msize, zreg_off_fn *off_fn,
+                                   bool mte, uintptr_t ra)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+
+    for (i = 0; i < oprsz; i++) {
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(i));
+
+        if (likely(pg & 1)) {
+            intptr_t reg_off = i * 8;
+            target_ulong tagged_addr =
+                base + (off_fn(vm, reg_off) << scale);
+            target_ulong clean_addr = tagged_addr;
+
+            if (mte) {
+                clean_addr = sve_mte_clean_addr(env, oi, tagged_addr,
+                                                true, msize);
+            }
+            sve_probe_write_addr(env, clean_addr, msize, oi, ra);
+        }
+    }
+}
 
 static void sve_st1_zs(CPUARMState *env, void *vd, void *vg, void *vm,
                        target_ulong base, uint32_t desc, uintptr_t ra,
@@ -5281,8 +8958,12 @@ static void sve_st1_zs(CPUARMState *env, void *vd, void *vg, void *vm,
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
     const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
     intptr_t i, oprsz = simd_oprsz(desc);
 
+    sve_probe_st1_zs_pages(env, vg, vm, base, desc, msize, off_fn, false, ra);
+
     set_helper_retaddr(ra);
     for (i = 0; i < oprsz; ) {
         uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
@@ -5303,8 +8984,12 @@ static void sve_st1_zd(CPUARMState *env, void *vd, void *vg, void *vm,
 {
     const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
     const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
     intptr_t i, oprsz = simd_oprsz(desc) / 8;
 
+    sve_probe_st1_zd_pages(env, vg, vm, base, desc, msize, off_fn, false, ra);
+
     set_helper_retaddr(ra);
     for (i = 0; i < oprsz; i++) {
         uint8_t pg = *(uint8_t *)((char *)vg + H1(i));
@@ -5316,6 +9001,91 @@ static void sve_st1_zd(CPUARMState *env, void *vd, void *vg, void *vm,
     clear_helper_retaddr();
 }
 
+static void sve_st1_zs_mte(CPUARMState *env, void *vd, void *vg, void *vm,
+                           target_ulong base, uint32_t desc, uintptr_t ra,
+                           zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
+    intptr_t i, oprsz = simd_oprsz(desc);
+
+    sve_probe_st1_zs_pages(env, vg, vm, base, desc, msize, off_fn, true, ra);
+
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+
+        do {
+            if (likely(pg & 1)) {
+                target_ulong tagged_addr = base + (off_fn(vm, i) << scale);
+
+                sve_mte_check_addr(env, oi, tagged_addr, true, msize, ra);
+            }
+            i += 4, pg >>= 4;
+        } while (i & 15);
+    }
+
+    for (i = 0; i < oprsz; ) {
+        uint16_t pg = *(uint16_t *)((char *)vg + H1_2(i >> 3));
+
+        do {
+            if (likely(pg & 1)) {
+                target_ulong tagged_addr = base + (off_fn(vm, i) << scale);
+                target_ulong clean_addr =
+                    sve_mte_clean_addr(env, oi, tagged_addr,
+                                       true, msize);
+
+                tlb_fn(env, vd, i, clean_addr, oi, ra);
+            }
+            i += 4, pg >>= 4;
+        } while (i & 15);
+    }
+    clear_helper_retaddr();
+}
+
+static void sve_st1_zd_mte(CPUARMState *env, void *vd, void *vg, void *vm,
+                           target_ulong base, uint32_t desc, uintptr_t ra,
+                           zreg_off_fn *off_fn, sve_ld1_tlb_fn *tlb_fn)
+{
+    const TCGMemOpIdx oi = extract32(desc, SIMD_DATA_SHIFT, MEMOPIDX_SHIFT);
+    const int scale = extract32(desc, SIMD_DATA_SHIFT + MEMOPIDX_SHIFT, 2);
+    const int msz = get_memop(oi) & MO_SIZE;
+    const int msize = 1 << msz;
+    intptr_t i, oprsz = simd_oprsz(desc) / 8;
+
+    sve_probe_st1_zd_pages(env, vg, vm, base, desc, msize, off_fn, true, ra);
+
+    set_helper_retaddr(ra);
+    for (i = 0; i < oprsz; i++) {
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(i));
+
+        if (likely(pg & 1)) {
+            intptr_t reg_off = i * 8;
+            target_ulong tagged_addr =
+                base + (off_fn(vm, reg_off) << scale);
+
+            sve_mte_check_addr(env, oi, tagged_addr, true, msize, ra);
+        }
+    }
+
+    for (i = 0; i < oprsz; i++) {
+        uint8_t pg = *(uint8_t *)((char *)vg + H1(i));
+
+        if (likely(pg & 1)) {
+            intptr_t reg_off = i * 8;
+            target_ulong tagged_addr =
+                base + (off_fn(vm, reg_off) << scale);
+            target_ulong clean_addr =
+                sve_mte_clean_addr(env, oi, tagged_addr, true, msize);
+
+            tlb_fn(env, vd, reg_off, clean_addr, oi, ra);
+        }
+    }
+    clear_helper_retaddr();
+}
+
 #define DO_ST1_ZPZ_S(MEM, OFS) \
 void QEMU_FLATTEN HELPER(sve_st##MEM##_##OFS) \
     (CPUARMState *env, void *vd, void *vg, void *vm,         \
@@ -5323,6 +9093,13 @@ void QEMU_FLATTEN HELPER(sve_st##MEM##_##OFS) \
 {                                                            \
     sve_st1_zs(env, vd, vg, vm, base, desc, GETPC(),         \
               off_##OFS##_s, sve_st1##MEM##_tlb);            \
+}                                                            \
+void QEMU_FLATTEN HELPER(sve_st##MEM##_##OFS##_mte)          \
+    (CPUARMState *env, void *vd, void *vg, void *vm,         \
+     target_ulong base, uint32_t desc)                       \
+{                                                            \
+    sve_st1_zs_mte(env, vd, vg, vm, base, desc, GETPC(),     \
+                   off_##OFS##_s, sve_st1##MEM##_tlb);       \
 }
 
 #define DO_ST1_ZPZ_D(MEM, OFS) \
@@ -5332,6 +9109,13 @@ void QEMU_FLATTEN HELPER(sve_st##MEM##_##OFS) \
 {                                                            \
     sve_st1_zd(env, vd, vg, vm, base, desc, GETPC(),         \
                off_##OFS##_d, sve_st1##MEM##_tlb);           \
+}                                                            \
+void QEMU_FLATTEN HELPER(sve_st##MEM##_##OFS##_mte)          \
+    (CPUARMState *env, void *vd, void *vg, void *vm,         \
+     target_ulong base, uint32_t desc)                       \
+{                                                            \
+    sve_st1_zd_mte(env, vd, vg, vm, base, desc, GETPC(),     \
+                   off_##OFS##_d, sve_st1##MEM##_tlb);       \
 }
 
 DO_ST1_ZPZ_S(bs, zsu)
diff --git a/qemu/target/arm/tlb_helper.c b/qemu/target/arm/tlb_helper.c
index e19d6c17a3..605171ae43 100644
--- a/qemu/target/arm/tlb_helper.c
+++ b/qemu/target/arm/tlb_helper.c
@@ -153,6 +153,7 @@ bool arm_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
     target_ulong page_size;
     int prot, ret;
     MemTxAttrs attrs = { 0 };
+    ARMCacheAttrs cacheattrs = { 0 };
     ARMMMUFaultInfo fi = { 0 };
 
     /*
@@ -163,8 +164,11 @@ bool arm_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
      */
     ret = get_phys_addr(&cpu->env, address, access_type,
                         core_to_arm_mmu_idx(&cpu->env, mmu_idx),
-                        &phys_addr, &attrs, &prot, &page_size, &fi, NULL);
+                        &phys_addr, &attrs, &prot, &page_size, &fi,
+                        &cacheattrs);
     if (likely(!ret)) {
+        attrs.target_tlb_bit1 = cacheattrs.attrs == 0xf0;
+
         /*
          * Map a single [sub]page. Regions smaller than our declared
          * target page size are handled specially, so for those we
diff --git a/qemu/target/arm/translate-a64.c b/qemu/target/arm/translate-a64.c
index 922976536e..cc87b3dc84 100644
--- a/qemu/target/arm/translate-a64.c
+++ b/qemu/target/arm/translate-a64.c
@@ -223,19 +223,73 @@ static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
  */
 static TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
 {
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
     TCGv_i64 clean = new_tmp_a64(s);
-    /*
-     * In order to get the correct value in the FAR_ELx register,
-     * we must present the memory subsystem with the "dirty" address
-     * including the TBI.  In system mode we can make this work via
-     * the TLB, dropping the TBI during translation.  But for user-only
-     * mode we don't have that option, and must remove the top byte now.
-     */
-    tcg_gen_mov_i64(tcg_ctx, clean, addr);
+
+    gen_top_byte_ignore(s, clean, addr, s->tbid);
     return clean;
 }
 
+static void gen_address_with_allocation_tag0(TCGContext *tcg_ctx,
+                                             TCGv_i64 dst, TCGv_i64 src)
+{
+    tcg_gen_andi_i64(tcg_ctx, dst, src, ~MAKE_64BIT_MASK(56, 4));
+}
+
+static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
+                                      bool is_write, bool tag_checked,
+                                      int log2_size, bool is_unpriv,
+                                      int core_idx)
+{
+    if (tag_checked && s->mte_active[is_unpriv]) {
+        TCGContext *tcg_ctx = s->uc->tcg_ctx;
+        TCGv_i64 ret = new_tmp_a64(s);
+        TCGv_i32 tcg_desc;
+        int desc = 0;
+
+        FIELD_DP32(desc, MTEDESC, MIDX, core_idx, desc);
+        FIELD_DP32(desc, MTEDESC, TBI, s->tbid, desc);
+        FIELD_DP32(desc, MTEDESC, TCMA, s->tcma, desc);
+        FIELD_DP32(desc, MTEDESC, WRITE, is_write, desc);
+        FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << log2_size) - 1, desc);
+
+        tcg_desc = tcg_const_i32(tcg_ctx, desc);
+        gen_helper_mte_check(tcg_ctx, ret, tcg_ctx->cpu_env, tcg_desc, addr);
+        tcg_temp_free_i32(tcg_ctx, tcg_desc);
+        return ret;
+    }
+    return clean_data_tbi(s, addr);
+}
+
+static TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
+                               bool tag_checked, int log2_size)
+{
+    return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, log2_size,
+                                 false, get_mem_index(s));
+}
+
+TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
+                        bool tag_checked, int size)
+{
+    if (tag_checked && s->mte_active[0]) {
+        TCGContext *tcg_ctx = s->uc->tcg_ctx;
+        TCGv_i64 ret = new_tmp_a64(s);
+        TCGv_i32 tcg_desc;
+        int desc = 0;
+
+        FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s), desc);
+        FIELD_DP32(desc, MTEDESC, TBI, s->tbid, desc);
+        FIELD_DP32(desc, MTEDESC, TCMA, s->tcma, desc);
+        FIELD_DP32(desc, MTEDESC, WRITE, is_write, desc);
+        FIELD_DP32(desc, MTEDESC, SIZEM1, size - 1, desc);
+
+        tcg_desc = tcg_const_i32(tcg_ctx, desc);
+        gen_helper_mte_check(tcg_ctx, ret, tcg_ctx->cpu_env, tcg_desc, addr);
+        tcg_temp_free_i32(tcg_ctx, tcg_desc);
+        return ret;
+    }
+    return clean_data_tbi(s, addr);
+}
+
 typedef struct DisasCompare64 {
     TCGCond cond;
     TCGv_i64 value;
@@ -637,6 +691,19 @@ static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
                        is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
 }
 
+static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
+                             int rm, int ra, int data,
+                             gen_helper_gvec_4 *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, rd),
+                       vec_full_reg_offset(s, rn),
+                       vec_full_reg_offset(s, rm),
+                       vec_full_reg_offset(s, ra),
+                       is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
+}
+
 /* Expand a 3-operand + env pointer operation using
  * an out-of-line helper.
  */
@@ -666,6 +733,21 @@ static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
     tcg_temp_free_ptr(tcg_ctx, fpst);
 }
 
+static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
+                              int rm, int ra, bool is_fp16, int data,
+                              gen_helper_gvec_4_ptr *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr fpst = get_fpstatus_ptr(tcg_ctx, is_fp16);
+
+    tcg_gen_gvec_4_ptr(tcg_ctx, vec_full_reg_offset(s, rd),
+                       vec_full_reg_offset(s, rn),
+                       vec_full_reg_offset(s, rm),
+                       vec_full_reg_offset(s, ra), fpst,
+                       is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
+    tcg_temp_free_ptr(tcg_ctx, fpst);
+}
+
 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
  * than the 32 bit equivalent.
  */
@@ -854,14 +936,17 @@ static void gen_adc_CC(TCGContext *tcg_ctx, int sf, TCGv_i64 dest, TCGv_i64 t0,
  * Store from GPR register to memory.
  */
 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
-                             TCGv_i64 tcg_addr, int size, int memidx,
+                             TCGv_i64 tcg_addr, MemOp memop, int memidx,
                              bool iss_valid,
                              unsigned int iss_srt,
                              bool iss_sf, bool iss_ar)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int size = memop & MO_SIZE;
+
     g_assert(size <= 3);
-    tcg_gen_qemu_st_i64(tcg_ctx, source, tcg_addr, memidx, s->be_data + size);
+    tcg_gen_qemu_st_i64(tcg_ctx, source, tcg_addr, memidx,
+                        s->be_data | memop);
 
     if (iss_valid) {
         uint32_t syn;
@@ -878,12 +963,12 @@ static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
 }
 
 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
-                      TCGv_i64 tcg_addr, int size,
+                      TCGv_i64 tcg_addr, MemOp memop,
                       bool iss_valid,
                       unsigned int iss_srt,
                       bool iss_sf, bool iss_ar)
 {
-    do_gpr_st_memidx(s, source, tcg_addr, size, get_mem_index(s),
+    do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s),
                      iss_valid, iss_srt, iss_sf, iss_ar);
 }
 
@@ -892,21 +977,22 @@ static void do_gpr_st(DisasContext *s, TCGv_i64 source,
  */
 static void do_gpr_ld_memidx(DisasContext *s,
                              TCGv_i64 dest, TCGv_i64 tcg_addr,
-                             int size, bool is_signed,
+                             MemOp memop, bool is_signed,
                              bool extend, int memidx,
                              bool iss_valid, unsigned int iss_srt,
                              bool iss_sf, bool iss_ar)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    MemOp memop = s->be_data + size;
+    int size = memop & MO_SIZE;
+    MemOp mop = s->be_data | memop;
 
     g_assert(size <= 3);
 
     if (is_signed) {
-        memop += MO_SIGN;
+        mop |= MO_SIGN;
     }
 
-    tcg_gen_qemu_ld_i64(tcg_ctx, dest, tcg_addr, memidx, memop);
+    tcg_gen_qemu_ld_i64(tcg_ctx, dest, tcg_addr, memidx, mop);
 
     if (extend && is_signed) {
         g_assert(size < 3);
@@ -929,11 +1015,11 @@ static void do_gpr_ld_memidx(DisasContext *s,
 
 static void do_gpr_ld(DisasContext *s,
                       TCGv_i64 dest, TCGv_i64 tcg_addr,
-                      int size, bool is_signed, bool extend,
+                      MemOp memop, bool is_signed, bool extend,
                       bool iss_valid, unsigned int iss_srt,
                       bool iss_sf, bool iss_ar)
 {
-    do_gpr_ld_memidx(s, dest, tcg_addr, size, is_signed, extend,
+    do_gpr_ld_memidx(s, dest, tcg_addr, memop, is_signed, extend,
                      get_mem_index(s),
                      iss_valid, iss_srt, iss_sf, iss_ar);
 }
@@ -1154,7 +1240,7 @@ static void do_vec_ld(DisasContext *s, int destidx, int element,
  * unallocated-encoding checks (otherwise the syndrome information
  * for the resulting exception will be incorrect).
  */
-static inline bool fp_access_check(DisasContext *s)
+static bool fp_access_check_only(DisasContext *s)
 {
     assert(!s->fp_access_checked);
     s->fp_access_checked = true;
@@ -1168,11 +1254,30 @@ static inline bool fp_access_check(DisasContext *s)
     return false;
 }
 
+static bool nonstreaming_check(DisasContext *s)
+{
+    if (s->sme_trap_nonstreaming && s->is_nonstreaming) {
+        gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
+                           syn_smetrap(SME_ET_Streaming, false),
+                           default_exception_el(s));
+        return false;
+    }
+    return true;
+}
+
+static inline bool fp_access_check(DisasContext *s)
+{
+    return fp_access_check_only(s) && nonstreaming_check(s);
+}
+
 /* Check that SVE access is enabled.  If it is, return true.
  * If not, emit code to generate an appropriate exception and return false.
  */
 bool sve_access_check(DisasContext *s)
 {
+    if (dc_isar_feature(aa64_sme, s) && s->pstate_sm) {
+        return sme_enabled_check(s) && nonstreaming_check(s);
+    }
     if (s->sve_excp_el) {
         gen_exception_insn(s, s->pc_curr, EXCP_UDEF, syn_sve_access_trap(),
                            s->sve_excp_el);
@@ -1181,6 +1286,38 @@ bool sve_access_check(DisasContext *s)
     return fp_access_check(s);
 }
 
+bool sme_enabled_check(DisasContext *s)
+{
+    if (s->sme_excp_el &&
+        (!s->fp_excp_el || s->sme_excp_el <= s->fp_excp_el)) {
+        gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
+                           syn_smetrap(SME_ET_AccessTrap, false),
+                           s->sme_excp_el);
+        return false;
+    }
+    return fp_access_check_only(s);
+}
+
+bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req)
+{
+    if (!sme_enabled_check(s)) {
+        return false;
+    }
+    if ((req & R_SVCR_SM_MASK) && !s->pstate_sm) {
+        gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
+                           syn_smetrap(SME_ET_NotStreaming, false),
+                           default_exception_el(s));
+        return false;
+    }
+    if ((req & R_SVCR_ZA_MASK) && !s->pstate_za) {
+        gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
+                           syn_smetrap(SME_ET_InactiveZA, false),
+                           default_exception_el(s));
+        return false;
+    }
+    return true;
+}
+
 /*
  * This utility function is for doing register extension with an
  * optional shift. You will likely want to pass a temporary for the
@@ -1427,6 +1564,7 @@ static void handle_hint(DisasContext *s, uint32_t insn,
         break;
     case 4: // 0b00100: /* SEV */
     case 5: // 0b00101: /* SEVL */
+    case 6: // 0b00110: /* DGH */
         /* we treat all as NOP at least for now */
         break;
     case 7: // 0b00111: /* XPACLRI */
@@ -1684,6 +1822,38 @@ static void handle_msr_i(DisasContext *s, uint32_t insn,
         tcg_temp_free_i32(tcg_ctx, t1);
         break;
 
+    case 0x1b: /* SVCR* */
+        if (!dc_isar_feature(aa64_sme, s) || crm < 2 || crm > 7) {
+            goto do_unallocated;
+        }
+        t1 = tcg_const_i32(tcg_ctx, (crm & 1) * 3);
+        {
+            TCGv_i32 mask = tcg_const_i32(tcg_ctx, (crm >> 1) & 3);
+
+            gen_helper_set_svcr(tcg_ctx, tcg_ctx->cpu_env, t1, mask);
+            tcg_temp_free_i32(tcg_ctx, mask);
+        }
+        tcg_temp_free_i32(tcg_ctx, t1);
+        break;
+
+    case 0x1c: /* TCO */
+        if (dc_isar_feature(aa64_mte, s)) {
+            if (crm & 1) {
+                set_pstate_bits(tcg_ctx, PSTATE_TCO);
+            } else {
+                clear_pstate_bits(tcg_ctx, PSTATE_TCO);
+            }
+            t1 = tcg_const_i32(tcg_ctx, s->current_el);
+            gen_helper_rebuild_hflags_a64(tcg_ctx, tcg_ctx->cpu_env, t1);
+            tcg_temp_free_i32(tcg_ctx, t1);
+            s->base.is_jmp = DISAS_UPDATE;
+        } else if (dc_isar_feature(aa64_mte_insn_reg, s)) {
+            s->base.is_jmp = DISAS_NEXT;
+        } else {
+            goto do_unallocated;
+        }
+        break;
+
     case 0x1e: /* DAIFSet */
         t1 = tcg_const_i32(tcg_ctx, crm);
         gen_helper_msr_i_daifset(tcg_ctx, tcg_ctx->cpu_env, t1);
@@ -1769,7 +1939,7 @@ static TCGLabel *gen_hook_sys(DisasContext *s, uint32_t insn, struct hook *hk)
     gen_helper_uc_hooksys64(tcg_ctx, tcg_skip, tcg_ctx->cpu_env, tcg_insn, tcg_hk);
 
     tcg_gen_brcondi_i32(tcg_ctx, TCG_COND_NE, tcg_skip, 0, label);
-    
+
     tcg_temp_free_i32(tcg_ctx, tcg_skip);
     tcg_temp_free_i32(tcg_ctx, tcg_insn);
     tcg_temp_free_ptr(tcg_ctx, tcg_hk);
@@ -1916,10 +2086,58 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
         return;
     case ARM_CP_DC_ZVA:
         /* Writes clear the aligned block of memory which rt points into. */
-        tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
+        if (s->mte_active[0]) {
+            TCGv_i32 tcg_desc;
+            int desc = 0;
+
+            FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s), desc);
+            FIELD_DP32(desc, MTEDESC, TBI, s->tbid, desc);
+            FIELD_DP32(desc, MTEDESC, TCMA, s->tcma, desc);
+            tcg_rt = new_tmp_a64(s);
+            tcg_desc = tcg_const_i32(tcg_ctx, desc);
+            gen_helper_mte_check_zva(tcg_ctx, tcg_rt, tcg_ctx->cpu_env,
+                                     tcg_desc, cpu_reg(s, rt));
+            tcg_temp_free_i32(tcg_ctx, tcg_desc);
+        } else {
+            tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
+        }
         gen_helper_dc_zva(tcg_ctx, tcg_ctx->cpu_env, tcg_rt);
         may_gen_set_label(s, label);
         return;
+    case ARM_CP_DC_GVA:
+        {
+            TCGv_i64 clean_addr, tag;
+
+            tcg_rt = cpu_reg(s, rt);
+            clean_addr = clean_data_tbi(s, tcg_rt);
+            gen_helper_dc_gva_probe(tcg_ctx, tcg_ctx->cpu_env, clean_addr);
+            if (s->ata) {
+                tag = tcg_temp_new_i64(tcg_ctx);
+                tcg_gen_shri_i64(tcg_ctx, tag, tcg_rt, 56);
+                gen_helper_stzgm_tags(tcg_ctx, tcg_ctx->cpu_env, clean_addr,
+                                      tag);
+                tcg_temp_free_i64(tcg_ctx, tag);
+            }
+        }
+        may_gen_set_label(s, label);
+        return;
+    case ARM_CP_DC_GZVA:
+        {
+            TCGv_i64 clean_addr, tag;
+
+            tcg_rt = cpu_reg(s, rt);
+            clean_addr = clean_data_tbi(s, tcg_rt);
+            gen_helper_dc_zva(tcg_ctx, tcg_ctx->cpu_env, clean_addr);
+            if (s->ata) {
+                tag = tcg_temp_new_i64(tcg_ctx);
+                tcg_gen_shri_i64(tcg_ctx, tag, tcg_rt, 56);
+                gen_helper_stzgm_tags(tcg_ctx, tcg_ctx->cpu_env, clean_addr,
+                                      tag);
+                tcg_temp_free_i64(tcg_ctx, tag);
+            }
+        }
+        may_gen_set_label(s, label);
+        return;
     default:
         break;
     }
@@ -1929,6 +2147,9 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
     } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
         may_gen_set_label(s, label);
         return;
+    } else if ((ri->type & ARM_CP_SME) && !sme_enabled_check(s)) {
+        may_gen_set_label(s, label);
+        return;
     }
 
     tcg_rt = cpu_reg(s, rt);
@@ -2499,7 +2720,7 @@ static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
     if (rn == 31) {
         gen_check_sp_alignment(s);
     }
-    clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
+    clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, size);
     tcg_gen_atomic_cmpxchg_i64(tcg_ctx, tcg_rs, clean_addr, tcg_rs, tcg_rt, memidx,
                                size | MO_ALIGN | s->be_data);
 }
@@ -2518,7 +2739,8 @@ static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
     if (rn == 31) {
         gen_check_sp_alignment(s);
     }
-    clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
+    clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31,
+                                size + 1);
 
     if (size == 2) {
         TCGv_i64 cmp = tcg_temp_new_i64(tcg_ctx);
@@ -2644,7 +2866,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
         if (is_lasr) {
             tcg_gen_mb(tcg_ctx, TCG_MO_ALL | TCG_BAR_STRL);
         }
-        clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
+        clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31,
+                                    size);
         gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, false);
         return;
 
@@ -2653,7 +2876,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
         if (rn == 31) {
             gen_check_sp_alignment(s);
         }
-        clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
+        clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), false, rn != 31,
+                                    size);
         s->is_ldex = true;
         gen_load_exclusive(s, rt, rt2, clean_addr, size, false);
         if (is_lasr) {
@@ -2673,8 +2897,9 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
             gen_check_sp_alignment(s);
         }
         tcg_gen_mb(tcg_ctx, TCG_MO_ALL | TCG_BAR_STRL);
-        clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
-        do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt,
+        clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31,
+                                    size);
+        do_gpr_st(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, true, rt,
                   disas_ldst_compute_iss_sf(size, false, 0), is_lasr);
         return;
 
@@ -2689,9 +2914,11 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
         if (rn == 31) {
             gen_check_sp_alignment(s);
         }
-        clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
-        do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, false, true, rt,
-                  disas_ldst_compute_iss_sf(size, false, 0), is_lasr);
+        clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), false, rn != 31,
+                                    size);
+        do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, false,
+                  false, true, rt, disas_ldst_compute_iss_sf(size, false, 0),
+                  is_lasr);
         tcg_gen_mb(tcg_ctx, TCG_MO_ALL | TCG_BAR_LDAQ);
         return;
 
@@ -2703,7 +2930,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
             if (is_lasr) {
                 tcg_gen_mb(tcg_ctx, TCG_MO_ALL | TCG_BAR_STRL);
             }
-            clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
+            clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true,
+                                        rn != 31, size);
             gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, true);
             return;
         }
@@ -2721,7 +2949,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
             if (rn == 31) {
                 gen_check_sp_alignment(s);
             }
-            clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
+            clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), false,
+                                        rn != 31, size);
             s->is_ldex = true;
             gen_load_exclusive(s, rt, rt2, clean_addr, size, true);
             if (is_lasr) {
@@ -2825,7 +3054,7 @@ static void disas_ld_lit(DisasContext *s, uint32_t insn)
  * +-----+-------+---+---+-------+---+-------+-------+------+------+
  *
  * opc: LDP/STP/LDNP/STNP        00 -> 32 bit, 10 -> 64 bit
- *      LDPSW                    01
+ *      LDPSW/STGP               01
  *      LDP/STP/LDNP/STNP (SIMD) 00 -> 32 bit, 01 -> 64 bit, 10 -> 128 bit
  *   V: 0 -> GPR, 1 -> Vector
  * idx: 00 -> signed offset with non-temporal hint, 01 -> post-index,
@@ -2851,6 +3080,7 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn)
     bool is_signed = false;
     bool postindex = false;
     bool wback = false;
+    bool set_tag = false;
 
     TCGv_i64 clean_addr, dirty_addr;
 
@@ -2863,6 +3093,13 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn)
 
     if (is_vector) {
         size = 2 + opc;
+    } else if (opc == 1 && !is_load) {
+        if (!dc_isar_feature(aa64_mte_insn_reg, s) || index == 0) {
+            unallocated_encoding(s);
+            return;
+        }
+        size = 3;
+        set_tag = true;
     } else {
         size = 2 + extract32(opc, 1, 1);
         is_signed = extract32(opc, 0, 1);
@@ -2903,7 +3140,7 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn)
         return;
     }
 
-    offset <<= size;
+    offset <<= (set_tag ? LOG2_TAG_GRANULE : size);
 
     if (rn == 31) {
         gen_check_sp_alignment(s);
@@ -2913,7 +3150,17 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn)
     if (!postindex) {
         tcg_gen_addi_i64(tcg_ctx, dirty_addr, dirty_addr, offset);
     }
-    clean_addr = clean_data_tbi(s, dirty_addr);
+
+    if (set_tag) {
+        if (s->ata) {
+            gen_helper_stg(tcg_ctx, tcg_ctx->cpu_env, dirty_addr, dirty_addr);
+        } else {
+            gen_helper_stg_stub(tcg_ctx, tcg_ctx->cpu_env, dirty_addr);
+        }
+    }
+
+    clean_addr = gen_mte_checkN(s, dirty_addr, !is_load,
+                                (wback || rn != 31) && !set_tag, 2 << size);
 
     if (is_vector) {
         if (is_load) {
@@ -2995,6 +3242,7 @@ static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn,
     bool iss_valid = !is_vector;
     bool post_index;
     bool writeback;
+    int memidx;
 
     TCGv_i64 clean_addr, dirty_addr;
 
@@ -3052,7 +3300,10 @@ static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn,
     if (!post_index) {
         tcg_gen_addi_i64(tcg_ctx, dirty_addr, dirty_addr, imm9);
     }
-    clean_addr = clean_data_tbi(s, dirty_addr);
+    memidx = is_unpriv ? get_a64_user_mem_index(s) : get_mem_index(s);
+    clean_addr = gen_mte_check1_mmuidx(s, dirty_addr, is_store,
+                                       writeback || rn != 31, size,
+                                       is_unpriv, memidx);
 
     if (is_vector) {
         if (is_store) {
@@ -3062,7 +3313,6 @@ static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn,
         }
     } else {
         TCGv_i64 tcg_rt = cpu_reg(s, rt);
-        int memidx = is_unpriv ? get_a64_user_mem_index(s) : get_mem_index(s);
         bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
 
         if (is_store) {
@@ -3160,7 +3410,7 @@ static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn,
     ext_and_shift_reg(tcg_ctx, tcg_rm, tcg_rm, opt, shift ? size : 0);
 
     tcg_gen_add_i64(tcg_ctx, dirty_addr, dirty_addr, tcg_rm);
-    clean_addr = clean_data_tbi(s, dirty_addr);
+    clean_addr = gen_mte_check1(s, dirty_addr, is_store, true, size);
 
     if (is_vector) {
         if (is_store) {
@@ -3246,7 +3496,7 @@ static void disas_ldst_reg_unsigned_imm(DisasContext *s, uint32_t insn,
     dirty_addr = read_cpu_reg_sp(s, rn, 1);
     offset = imm12 << size;
     tcg_gen_addi_i64(tcg_ctx, dirty_addr, dirty_addr, offset);
-    clean_addr = clean_data_tbi(s, dirty_addr);
+    clean_addr = gen_mte_check1(s, dirty_addr, is_store, rn != 31, size);
 
     if (is_vector) {
         if (is_store) {
@@ -3290,8 +3540,9 @@ static void disas_ldst_atomic(DisasContext *s, uint32_t insn,
     int o3_opc = extract32(insn, 12, 4);
     bool r = extract32(insn, 22, 1);
     bool a = extract32(insn, 23, 1);
-    TCGv_i64 tcg_rs, clean_addr;
+    TCGv_i64 tcg_rs, tcg_rt, clean_addr;
     AtomicThreeOpFn *fn;
+    MemOp mop = s->be_data | size | MO_ALIGN;
 
     if (is_vector || !dc_isar_feature(aa64_atomics, s)) {
         unallocated_encoding(s);
@@ -3312,9 +3563,11 @@ static void disas_ldst_atomic(DisasContext *s, uint32_t insn,
         break;
     case 004: /* LDSMAX */
         fn = tcg_gen_atomic_fetch_smax_i64;
+        mop |= MO_SIGN;
         break;
     case 005: /* LDSMIN */
         fn = tcg_gen_atomic_fetch_smin_i64;
+        mop |= MO_SIGN;
         break;
     case 006: /* LDUMAX */
         fn = tcg_gen_atomic_fetch_umax_i64;
@@ -3340,7 +3593,7 @@ static void disas_ldst_atomic(DisasContext *s, uint32_t insn,
     if (rn == 31) {
         gen_check_sp_alignment(s);
     }
-    clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
+    clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), false, rn != 31, size);
 
     if (o3_opc == 014) {
         /*
@@ -3357,6 +3610,7 @@ static void disas_ldst_atomic(DisasContext *s, uint32_t insn,
     }
 
     tcg_rs = read_cpu_reg(s, rs, true);
+    tcg_rt = cpu_reg(s, rt);
 
     if (o3_opc == 1) { /* LDCLR */
         tcg_gen_not_i64(tcg_ctx, tcg_rs, tcg_rs);
@@ -3365,8 +3619,25 @@ static void disas_ldst_atomic(DisasContext *s, uint32_t insn,
     /* The tcg atomic primitives are all full barriers.  Therefore we
      * can ignore the Acquire and Release bits of this instruction.
      */
-    fn(tcg_ctx, cpu_reg(s, rt), clean_addr, tcg_rs, get_mem_index(s),
-       s->be_data | size | MO_ALIGN);
+    fn(tcg_ctx, tcg_rt, clean_addr, tcg_rs, get_mem_index(s), mop);
+
+    if (mop & MO_SIGN) {
+        switch (size) {
+        case MO_8:
+            tcg_gen_ext8u_i64(tcg_ctx, tcg_rt, tcg_rt);
+            break;
+        case MO_16:
+            tcg_gen_ext16u_i64(tcg_ctx, tcg_rt, tcg_rt);
+            break;
+        case MO_32:
+            tcg_gen_ext32u_i64(tcg_ctx, tcg_rt, tcg_rt);
+            break;
+        case MO_64:
+            break;
+        default:
+            g_assert_not_reached();
+        }
+    }
 }
 
 /*
@@ -3418,7 +3689,8 @@ static void disas_ldst_pac(DisasContext *s, uint32_t insn,
     tcg_gen_addi_i64(tcg_ctx, dirty_addr, dirty_addr, offset);
 
     /* Note that "clean" and "dirty" here refer to TBI not PAC.  */
-    clean_addr = clean_data_tbi(s, dirty_addr);
+    clean_addr = gen_mte_check1(s, dirty_addr, false, is_wback || rn != 31,
+                                size);
 
     tcg_rt = cpu_reg(s, rt);
     do_gpr_ld(s, tcg_rt, clean_addr, size, /* is_signed */ false,
@@ -3501,14 +3773,15 @@ static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn)
     if (is_store) {
         /* Store-Release semantics */
         tcg_gen_mb(tcg_ctx, TCG_MO_ALL | TCG_BAR_STRL);
-        do_gpr_st(s, cpu_reg(s, rt), clean_addr, size, true, rt, iss_sf, true);
+        do_gpr_st(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, true, rt,
+                  iss_sf, true);
     } else {
         /*
          * Load-AcquirePC semantics; we implement as the slightly more
          * restrictive Load-Acquire.
          */
-        do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, is_signed, extend,
-                  true, rt, iss_sf, true);
+        do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, is_signed,
+                  extend, true, rt, iss_sf, true);
         tcg_gen_mb(tcg_ctx, TCG_MO_ALL | TCG_BAR_LDAQ);
     }
 }
@@ -3586,6 +3859,7 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn)
     int elements; /* elements per vector */
     int rpt;    /* num iterations */
     int selem;  /* structure elements */
+    int total;
     int r;
 
     if (extract32(insn, 31, 1) || extract32(insn, 21, 1)) {
@@ -3652,6 +3926,8 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn)
         endian = MO_LE;
     }
 
+    total = rpt * selem * (is_q ? 16 : 8);
+
     /* Consecutive little-endian elements from a single register
      * can be promoted to a larger little-endian operation.
      */
@@ -3662,7 +3938,8 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn)
     elements = (is_q ? 16 : 8) / ebytes;
 
     tcg_rn = cpu_reg_sp(s, rn);
-    clean_addr = clean_data_tbi(s, tcg_rn);
+    clean_addr = gen_mte_checkN(s, tcg_rn, is_store,
+                                is_postidx || rn != 31, total);
     tcg_ebytes = tcg_const_i64(tcg_ctx, ebytes);
 
     for (r = 0; r < rpt; r++) {
@@ -3806,7 +4083,8 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn)
     }
 
     tcg_rn = cpu_reg_sp(s, rn);
-    clean_addr = clean_data_tbi(s, tcg_rn);
+    clean_addr = gen_mte_checkN(s, tcg_rn, !is_load,
+                                is_postidx || rn != 31, selem << scale);
     tcg_ebytes = tcg_const_i64(tcg_ctx, ebytes);
 
     for (xs = 0; xs < selem; xs++) {
@@ -3842,6 +4120,189 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn)
     }
 }
 
+/*
+ * Load/Store memory tags
+ *
+ *  31 30 29         24     22  21     12    10      5      0
+ * +-----+-------------+-----+---+------+-----+------+------+
+ * | 1 1 | 0 1 1 0 0 1 | op1 | 1 | imm9 | op2 |  Rn  |  Rt  |
+ * +-----+-------------+-----+---+------+-----+------+------+
+ */
+static void disas_ldst_tag(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int rt = extract32(insn, 0, 5);
+    int rn = extract32(insn, 5, 5);
+    int64_t offset = sextract64(insn, 12, 9) << LOG2_TAG_GRANULE;
+    int op2 = extract32(insn, 10, 2);
+    int op1 = extract32(insn, 22, 2);
+    bool is_load = false;
+    bool is_pair = false;
+    bool is_zero = false;
+    bool is_mult = false;
+    int index = 0;
+    TCGv_i64 addr, clean_addr, tcg_rt;
+
+    if (extract32(insn, 30, 2) != 3) {
+        goto do_unallocated;
+    }
+
+    switch (op1) {
+    case 0:
+        if (op2 == 0) {
+            if (s->current_el == 0 || offset != 0) {
+                goto do_unallocated;
+            }
+            is_mult = true;
+            is_zero = true;
+        } else {
+            index = op2 - 2;
+        }
+        break;
+    case 1:
+        if (op2 == 0) {
+            is_load = true;
+        } else {
+            is_zero = true;
+            index = op2 - 2;
+        }
+        break;
+    case 2:
+        if (op2 == 0) {
+            if (s->current_el == 0 || offset != 0) {
+                goto do_unallocated;
+            }
+            is_mult = true;
+        } else {
+            is_pair = true;
+            index = op2 - 2;
+        }
+        break;
+    case 3:
+        if (op2 == 0) {
+            if (s->current_el == 0 || offset != 0) {
+                goto do_unallocated;
+            }
+            is_mult = true;
+            is_load = true;
+        } else {
+            is_pair = true;
+            is_zero = true;
+            index = op2 - 2;
+        }
+        break;
+    default:
+    do_unallocated:
+        unallocated_encoding(s);
+        return;
+    }
+
+    if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
+        goto do_unallocated;
+    }
+
+    if (rn == 31) {
+        gen_check_sp_alignment(s);
+    }
+
+    addr = read_cpu_reg_sp(s, rn, true);
+    if (index >= 0) {
+        tcg_gen_addi_i64(tcg_ctx, addr, addr, offset);
+    }
+
+    if (is_mult) {
+        tcg_rt = cpu_reg(s, rt);
+        if (is_zero) {
+            if (s->ata) {
+                gen_helper_stzgm_tags(tcg_ctx, tcg_ctx->cpu_env, addr,
+                                      tcg_rt);
+            }
+            clean_addr = clean_data_tbi(s, addr);
+            gen_helper_dc_zva(tcg_ctx, tcg_ctx->cpu_env, clean_addr);
+        } else if (s->ata) {
+            if (is_load) {
+                gen_helper_ldgm(tcg_ctx, tcg_rt, tcg_ctx->cpu_env, addr);
+            } else {
+                gen_helper_stgm(tcg_ctx, tcg_ctx->cpu_env, addr, tcg_rt);
+            }
+        } else {
+            TCGv_i32 tcg_desc;
+            int tag_transfer_size = 4 << 6;
+            int desc = 0;
+
+            FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s), desc);
+            FIELD_DP32(desc, MTEDESC, WRITE, !is_load, desc);
+            FIELD_DP32(desc, MTEDESC, SIZEM1, tag_transfer_size - 1, desc);
+            clean_addr = clean_data_tbi(s, addr);
+            tcg_gen_andi_i64(tcg_ctx, clean_addr, clean_addr,
+                             -tag_transfer_size);
+            tcg_desc = tcg_const_i32(tcg_ctx, desc);
+            gen_helper_mte_probe_data(tcg_ctx, tcg_ctx->cpu_env, clean_addr,
+                                      tcg_desc);
+            tcg_temp_free_i32(tcg_ctx, tcg_desc);
+            if (is_load) {
+                tcg_gen_movi_i64(tcg_ctx, tcg_rt, 0);
+            }
+        }
+        return;
+    } else if (is_load) {
+        tcg_gen_andi_i64(tcg_ctx, addr, addr, -TAG_GRANULE);
+        tcg_rt = cpu_reg(s, rt);
+        if (s->ata) {
+            gen_helper_ldg(tcg_ctx, tcg_rt, tcg_ctx->cpu_env, addr, tcg_rt);
+        } else {
+            TCGv_i32 tcg_desc;
+            int desc = 0;
+
+            clean_addr = clean_data_tbi(s, addr);
+            FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s), desc);
+            FIELD_DP32(desc, MTEDESC, SIZEM1, 1 - 1, desc);
+            tcg_desc = tcg_const_i32(tcg_ctx, desc);
+            gen_helper_mte_probe_data(tcg_ctx, tcg_ctx->cpu_env, clean_addr,
+                                      tcg_desc);
+            tcg_temp_free_i32(tcg_ctx, tcg_desc);
+            gen_address_with_allocation_tag0(tcg_ctx, tcg_rt, tcg_rt);
+        }
+    } else {
+        tcg_rt = cpu_reg_sp(s, rt);
+        if (s->ata) {
+            if (is_pair) {
+                gen_helper_st2g(tcg_ctx, tcg_ctx->cpu_env, addr, tcg_rt);
+            } else {
+                gen_helper_stg(tcg_ctx, tcg_ctx->cpu_env, addr, tcg_rt);
+            }
+        } else if (is_pair) {
+            gen_helper_st2g_stub(tcg_ctx, tcg_ctx->cpu_env, addr);
+        } else {
+            gen_helper_stg_stub(tcg_ctx, tcg_ctx->cpu_env, addr);
+        }
+    }
+
+    if (is_zero) {
+        TCGv_i64 tcg_zero = tcg_const_i64(tcg_ctx, 0);
+        int mem_index = get_mem_index(s);
+        int i, n = (1 + is_pair) << LOG2_TAG_GRANULE;
+
+        clean_addr = clean_data_tbi(s, addr);
+        gen_address_with_allocation_tag0(tcg_ctx, clean_addr, clean_addr);
+        tcg_gen_qemu_st_i64(tcg_ctx, tcg_zero, clean_addr, mem_index,
+                            MO_Q | MO_ALIGN_16);
+        for (i = 8; i < n; i += 8) {
+            tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 8);
+            tcg_gen_qemu_st_i64(tcg_ctx, tcg_zero, clean_addr, mem_index,
+                                MO_Q);
+        }
+        tcg_temp_free_i64(tcg_ctx, tcg_zero);
+    }
+
+    if (index != 0) {
+        if (index < 0) {
+            tcg_gen_addi_i64(tcg_ctx, addr, addr, offset);
+        }
+        tcg_gen_mov_i64(tcg_ctx, cpu_reg_sp(s, rn), addr);
+    }
+}
+
 /* Loads and stores */
 static void disas_ldst(DisasContext *s, uint32_t insn)
 {
@@ -3872,9 +4333,12 @@ static void disas_ldst(DisasContext *s, uint32_t insn)
     case 0x0d: /* AdvSIMD load/store single structure */
         disas_ldst_single_struct(s, insn);
         break;
-    case 0x19: /* LDAPR/STLR (unscaled immediate) */
-        if (extract32(insn, 10, 2) != 0 ||
-            extract32(insn, 21, 1) != 0) {
+    case 0x19:
+        if (extract32(insn, 21, 1) != 0) {
+            disas_ldst_tag(s, insn);
+            break;
+        }
+        if (extract32(insn, 10, 2) != 0) {
             unallocated_encoding(s);
             break;
         }
@@ -3980,6 +4444,55 @@ static void disas_add_sub_imm(DisasContext *s, uint32_t insn)
     tcg_temp_free_i64(tcg_ctx, tcg_result);
 }
 
+/*
+ * Add/subtract (immediate, with tags)
+ *
+ *  31 30 29 28         23 22 21     16 14      10 9   5 4   0
+ * +--+--+--+-------------+--+---------+--+-------+-----+-----+
+ * |sf|op| S| 1 0 0 0 1 1 |o2|  uimm6  |o3| uimm4 |  Rn | Rd  |
+ * +--+--+--+-------------+--+---------+--+-------+-----+-----+
+ *
+ *    op: 0 -> add, 1 -> sub
+ */
+static void disas_add_sub_imm_with_tags(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int rd = extract32(insn, 0, 5);
+    int rn = extract32(insn, 5, 5);
+    int uimm4 = extract32(insn, 10, 4);
+    int uimm6 = extract32(insn, 16, 6);
+    bool sub_op = extract32(insn, 30, 1);
+    TCGv_i64 tcg_rn;
+    TCGv_i64 tcg_rd;
+    int imm;
+
+    if ((insn & 0xa040c000u) != 0x80000000u ||
+        !dc_isar_feature(aa64_mte_insn_reg, s)) {
+        unallocated_encoding(s);
+        return;
+    }
+
+    imm = uimm6 << LOG2_TAG_GRANULE;
+    if (sub_op) {
+        imm = -imm;
+    }
+
+    tcg_rn = cpu_reg_sp(s, rn);
+    tcg_rd = cpu_reg_sp(s, rd);
+    if (s->ata) {
+        TCGv_i32 tcg_imm = tcg_const_i32(tcg_ctx, imm);
+        TCGv_i32 tcg_tag = tcg_const_i32(tcg_ctx, uimm4);
+
+        gen_helper_addsubg(tcg_ctx, tcg_rd, tcg_ctx->cpu_env, tcg_rn,
+                           tcg_imm, tcg_tag);
+        tcg_temp_free_i32(tcg_ctx, tcg_tag);
+        tcg_temp_free_i32(tcg_ctx, tcg_imm);
+    } else {
+        tcg_gen_addi_i64(tcg_ctx, tcg_rd, tcg_rn, imm);
+        gen_address_with_allocation_tag0(tcg_ctx, tcg_rd, tcg_rd);
+    }
+}
+
 /* The input should be a value in the bottom e bits (with higher
  * bits zero); returns that value replicated into every element
  * of size e in a 64 bit integer.
@@ -4343,9 +4856,12 @@ static void disas_data_proc_imm(DisasContext *s, uint32_t insn)
     case 0x20: case 0x21: /* PC-rel. addressing */
         disas_pc_rel_adr(s, insn);
         break;
-    case 0x22: case 0x23: /* Add/subtract (immediate) */
+    case 0x22: /* Add/subtract (immediate) */
         disas_add_sub_imm(s, insn);
         break;
+    case 0x23: /* Add/subtract (immediate, with tags) */
+        disas_add_sub_imm_with_tags(s, insn);
+        break;
     case 0x24: /* Logical (immediate) */
         disas_logic_imm(s, insn);
         break;
@@ -5460,25 +5976,71 @@ static void handle_crc32(DisasContext *s,
 static void disas_data_proc_2src(DisasContext *s, uint32_t insn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    unsigned int sf, rm, opcode, rn, rd;
+    unsigned int sf, rm, opcode, rn, rd, setflag;
     sf = extract32(insn, 31, 1);
+    setflag = extract32(insn, 29, 1);
     rm = extract32(insn, 16, 5);
     opcode = extract32(insn, 10, 6);
     rn = extract32(insn, 5, 5);
     rd = extract32(insn, 0, 5);
 
-    if (extract32(insn, 29, 1)) {
+    if (setflag && opcode != 0) {
         unallocated_encoding(s);
         return;
     }
 
     switch (opcode) {
+    case 0: /* SUBP(S) */
+        if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
+            goto do_unallocated;
+        } else {
+            TCGv_i64 tcg_n, tcg_m, tcg_d;
+
+            tcg_n = read_cpu_reg_sp(s, rn, true);
+            tcg_m = read_cpu_reg_sp(s, rm, true);
+            tcg_gen_sextract_i64(tcg_ctx, tcg_n, tcg_n, 0, 56);
+            tcg_gen_sextract_i64(tcg_ctx, tcg_m, tcg_m, 0, 56);
+            tcg_d = cpu_reg(s, rd);
+
+            if (setflag) {
+                gen_sub_CC(tcg_ctx, true, tcg_d, tcg_n, tcg_m);
+            } else {
+                tcg_gen_sub_i64(tcg_ctx, tcg_d, tcg_n, tcg_m);
+            }
+        }
+        break;
     case 2: /* UDIV */
         handle_div(s, false, sf, rm, rn, rd);
         break;
     case 3: /* SDIV */
         handle_div(s, true, sf, rm, rn, rd);
         break;
+    case 4: /* IRG */
+        if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
+            goto do_unallocated;
+        }
+        if (s->ata) {
+            gen_helper_irg(tcg_ctx, cpu_reg_sp(s, rd), tcg_ctx->cpu_env,
+                           cpu_reg_sp(s, rn), cpu_reg(s, rm));
+        } else {
+            gen_address_with_allocation_tag0(tcg_ctx, cpu_reg_sp(s, rd),
+                                             cpu_reg_sp(s, rn));
+        }
+        break;
+    case 5: /* GMI */
+        if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
+            goto do_unallocated;
+        } else {
+            TCGv_i64 one = tcg_const_i64(tcg_ctx, 1);
+            TCGv_i64 t = tcg_temp_new_i64(tcg_ctx);
+
+            tcg_gen_extract_i64(tcg_ctx, t, cpu_reg_sp(s, rn), 56, 4);
+            tcg_gen_shl_i64(tcg_ctx, t, one, t);
+            tcg_gen_or_i64(tcg_ctx, cpu_reg(s, rd), cpu_reg(s, rm), t);
+            tcg_temp_free_i64(tcg_ctx, t);
+            tcg_temp_free_i64(tcg_ctx, one);
+        }
+        break;
     case 8: /* LSLV */
         handle_shift_reg(s, A64_SHIFT_TYPE_LSL, sf, rm, rn, rd);
         break;
@@ -5939,6 +6501,9 @@ static void handle_fp_1src_single(DisasContext *s, int opcode, int rd, int rn)
     case 0x3: /* FSQRT */
         gen_helper_vfp_sqrts(tcg_ctx, tcg_res, tcg_op, tcg_ctx->cpu_env);
         goto done;
+    case 0x6: /* BFCVT */
+        gen_fpst = gen_helper_bfcvt;
+        break;
     case 0x8: /* FRINTN */
     case 0x9: /* FRINTP */
     case 0xa: /* FRINTM */
@@ -6183,6 +6748,17 @@ static void disas_fp_1src(DisasContext *s, uint32_t insn)
         break;
     }
 
+    case 0x6:
+        if (type != 1 || !dc_isar_feature(aa64_bf16, s)) {
+            unallocated_encoding(s);
+            return;
+        }
+        if (!fp_access_check(s)) {
+            return;
+        }
+        handle_fp_1src_single(s, opcode, rd, rn);
+        break;
+
     case 0x10: /* FRINT{32,64}{X,Z} */
     case 0x11: /* FRINT{32,64}{X,Z} */
     case 0x12: /* FRINT{32,64}{X,Z} */
@@ -10083,6 +10659,14 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar,
                 tcg_temp_free_i32(tcg_ctx, ahp);
             }
             break;
+        case 0x36: /* BFCVTN, BFCVTN2 */
+            {
+                TCGv_ptr fpst = get_fpstatus_ptr(tcg_ctx, false);
+
+                gen_helper_bfcvt_pair(tcg_ctx, tcg_res[pass], tcg_op, fpst);
+                tcg_temp_free_ptr(tcg_ctx, fpst);
+            }
+            break;
         case 0x56:  /* FCVTXN, FCVTXN2 */
             /* 64 bit to 32 bit float conversion
              * with von Neumann rounding (round to odd)
@@ -12022,6 +12606,22 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
         }
         feature = dc_isar_feature(aa64_dp, s);
         break;
+    case 0x03: /* USDOT */
+        if (size != MO_32) {
+            unallocated_encoding(s);
+            return;
+        }
+        feature = dc_isar_feature(aa64_i8mm, s);
+        break;
+    case 0x04: /* SMMLA */
+    case 0x14: /* UMMLA */
+    case 0x05: /* USMMLA */
+        if (!is_q || size != MO_32) {
+            unallocated_encoding(s);
+            return;
+        }
+        feature = dc_isar_feature(aa64_i8mm, s);
+        break;
     case 0x18: /* FCMLA, #0 */
     case 0x19: /* FCMLA, #90 */
     case 0x1a: /* FCMLA, #180 */
@@ -12036,6 +12636,24 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
         }
         feature = dc_isar_feature(aa64_fcma, s);
         break;
+    case 0x1d: /* BFMMLA */
+        if (!is_q || size != MO_16) {
+            unallocated_encoding(s);
+            return;
+        }
+        feature = dc_isar_feature(aa64_bf16, s);
+        break;
+    case 0x1f:
+        switch (size) {
+        case 1: /* BFDOT */
+        case 3: /* BFMLAL{B,T} */
+            feature = dc_isar_feature(aa64_bf16, s);
+            break;
+        default:
+            unallocated_encoding(s);
+            return;
+        }
+        break;
     default:
         unallocated_encoding(s);
         return;
@@ -12076,10 +12694,46 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
         return;
 
     case 0x2: /* SDOT / UDOT */
-        gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0,
+        gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
                          u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
         return;
 
+    case 0x3: /* USDOT */
+        gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
+                         gen_helper_gvec_usdot_b);
+        return;
+
+    case 0x4: /* SMMLA, UMMLA */
+        gen_gvec_op4_ool(s, true, rd, rn, rm, rd, 0,
+                         u ? gen_helper_gvec_ummla_b
+                         : gen_helper_gvec_smmla_b);
+        return;
+
+    case 0x5: /* USMMLA */
+        gen_gvec_op4_ool(s, true, rd, rn, rm, rd, 0,
+                         gen_helper_gvec_usmmla_b);
+        return;
+
+    case 0xd: /* BFMMLA */
+        gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
+                         gen_helper_gvec_bfmmla);
+        return;
+
+    case 0xf:
+        switch (size) {
+        case 1: /* BFDOT */
+            gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
+                             gen_helper_gvec_bfdot);
+            break;
+        case 3: /* BFMLAL{B,T} */
+            gen_gvec_op4_fpst(s, true, rd, rn, rm, rd, false, is_q,
+                              gen_helper_gvec_bfmlal);
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        return;
+
     case 0x8: /* FCMLA, #0 */
     case 0x9: /* FCMLA, #90 */
     case 0xa: /* FCMLA, #180 */
@@ -12583,6 +13237,16 @@ static void disas_simd_two_reg_misc(DisasContext *s, uint32_t insn)
             }
             handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
             return;
+        case 0x36: /* BFCVTN, BFCVTN2 */
+            if (size != 2 || !dc_isar_feature(aa64_bf16, s)) {
+                unallocated_encoding(s);
+                return;
+            }
+            if (!fp_access_check(s)) {
+                return;
+            }
+            handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
+            return;
         case 0x17: /* FCVTL, FCVTL2 */
             if (!fp_access_check(s)) {
                 return;
@@ -13278,6 +13942,35 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
             return;
         }
         break;
+    case 0x0f:
+        switch (size) {
+        case 0: /* SUDOT */
+        case 2: /* USDOT */
+            if (is_scalar || !dc_isar_feature(aa64_i8mm, s)) {
+                unallocated_encoding(s);
+                return;
+            }
+            size = MO_32;
+            break;
+        case 1: /* BFDOT */
+            if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
+                unallocated_encoding(s);
+                return;
+            }
+            size = MO_32;
+            break;
+        case 3: /* BFMLAL{B,T} */
+            if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
+                unallocated_encoding(s);
+                return;
+            }
+            size = MO_16;
+            break;
+        default:
+            unallocated_encoding(s);
+            return;
+        }
+        break;
     case 0x11: /* FCMLA #0 */
     case 0x13: /* FCMLA #90 */
     case 0x15: /* FCMLA #180 */
@@ -13388,10 +14081,31 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn)
     switch (16 * u + opcode) {
     case 0x0e: /* SDOT */
     case 0x1e: /* UDOT */
-        gen_gvec_op3_ool(s, is_q, rd, rn, rm, index,
+        gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
                          u ? gen_helper_gvec_udot_idx_b
                          : gen_helper_gvec_sdot_idx_b);
         return;
+    case 0x0f:
+        switch (extract32(insn, 22, 2)) {
+        case 0: /* SUDOT */
+            gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+                             gen_helper_gvec_sudot_idx_b);
+            return;
+        case 1: /* BFDOT */
+            gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+                             gen_helper_gvec_bfdot_idx);
+            return;
+        case 2: /* USDOT */
+            gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
+                             gen_helper_gvec_usdot_idx_b);
+            return;
+        case 3: /* BFMLAL{B,T} */
+            gen_gvec_op4_fpst(s, true, rd, rn, rm, rd, false,
+                              (index << 1) | is_q,
+                              gen_helper_gvec_bfmlal_idx);
+            return;
+        }
+        g_assert_not_reached();
     case 0x11: /* FCMLA #0 */
     case 0x13: /* FCMLA #90 */
     case 0x15: /* FCMLA #180 */
@@ -14422,6 +15136,55 @@ static void disas_data_proc_simd_fp(DisasContext *s, uint32_t insn)
     }
 }
 
+typedef struct SMEFa64Pattern {
+    uint32_t mask;
+    uint32_t value;
+} SMEFa64Pattern;
+
+static bool sme_fa64_match_any(uint32_t insn, const SMEFa64Pattern *patterns,
+                               size_t count)
+{
+    size_t i;
+
+    for (i = 0; i < count; i++) {
+        if ((insn & patterns[i].mask) == patterns[i].value) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static void disas_sme_fa64(DisasContext *s, uint32_t insn)
+{
+    static const SMEFa64Pattern ok[] = {
+        { 0xbffffc00, 0x0e012c00 },
+        { 0xbffffc00, 0x0e022c00 },
+        { 0xfffffc00, 0x4e042c00 },
+        { 0xfffffc00, 0x0e013c00 },
+        { 0xfffffc00, 0x0e023c00 },
+        { 0xfffffc00, 0x0e043c00 },
+        { 0xfffffc00, 0x4e083c00 },
+        { 0xff20dc00, 0x5e20dc00 },
+        { 0xff60dc00, 0x5e401c00 },
+        { 0xdfbfdc00, 0x5ea1d800 },
+        { 0xdfffdc00, 0x5ef9d800 },
+    };
+    static const SMEFa64Pattern fail[] = {
+        { 0x9e000000, 0x0e000000 },
+        { 0xde000000, 0x5e000000 },
+        { 0xbe000000, 0x0c000000 },
+        { 0xff000000, 0xce000000 },
+        { 0xfffffc00, 0x1e7e0000 },
+    };
+
+    if (sme_fa64_match_any(insn, ok, sizeof(ok) / sizeof(ok[0]))) {
+        return;
+    }
+    if (sme_fa64_match_any(insn, fail, sizeof(fail) / sizeof(fail[0]))) {
+        s->is_nonstreaming = true;
+    }
+}
+
 /**
  * is_guarded_page:
  * @env: The cpu environment
@@ -14526,6 +15289,10 @@ static void disas_a64_insn(CPUARMState *env, DisasContext *s)
     }
 
     s->fp_access_checked = false;
+    s->is_nonstreaming = false;
+    if (s->sme_trap_nonstreaming) {
+        disas_sme_fa64(s, insn);
+    }
 
     if (dc_isar_feature(aa64_bti, s)) {
         if (s->base.num_insns == 1) {
@@ -14566,7 +15333,12 @@ static void disas_a64_insn(CPUARMState *env, DisasContext *s)
     }
 
     switch (extract32(insn, 25, 4)) {
-    case 0x0: case 0x1: case 0x3: /* UNALLOCATED */
+    case 0x0:
+        if ((insn & (1U << 31)) && disas_sme(s, insn)) {
+            break;
+        }
+        /* fall through */
+    case 0x1: case 0x3: /* UNALLOCATED */
         unallocated_encoding(s);
         break;
     case 0x2:
@@ -14619,6 +15391,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
     CPUARMState *env = cpu->env_ptr;
     ARMCPU *arm_cpu = env_archcpu(env);
     uint32_t tb_flags = dc->base.tb->flags;
+    uint32_t tb_flags2 = dc->base.tb->cs_base;
     int bound, core_mmu_idx;
 
     // unicorn handle
@@ -14641,12 +15414,22 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
     dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx);
     dc->tbii = FIELD_EX32(tb_flags, TBFLAG_A64, TBII);
     dc->tbid = FIELD_EX32(tb_flags, TBFLAG_A64, TBID);
+    dc->tcma = FIELD_EX32(tb_flags, TBFLAG_A64, TCMA);
     dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx);
     dc->user = (dc->current_el == 0);
     dc->fp_excp_el = FIELD_EX32(tb_flags, TBFLAG_ANY, FPEXC_EL);
     dc->sve_excp_el = FIELD_EX32(tb_flags, TBFLAG_A64, SVEEXC_EL);
+    dc->sme_excp_el = FIELD_EX32(tb_flags2, TBFLAG_A64_2, SMEEXC_EL);
     dc->sve_len = (FIELD_EX32(tb_flags, TBFLAG_A64, ZCR_LEN) + 1) * 16;
+    dc->svl = (FIELD_EX32(tb_flags2, TBFLAG_A64_2, SVL) + 1) * 16;
+    dc->pstate_sm = FIELD_EX32(tb_flags2, TBFLAG_A64_2, PSTATE_SM);
+    dc->pstate_za = FIELD_EX32(tb_flags2, TBFLAG_A64_2, PSTATE_ZA);
+    dc->sme_trap_nonstreaming =
+        FIELD_EX32(tb_flags2, TBFLAG_A64_2, SME_TRAP_NONSTREAMING);
     dc->pauth_active = FIELD_EX32(tb_flags, TBFLAG_A64, PAUTH_ACTIVE);
+    dc->ata = FIELD_EX32(tb_flags, TBFLAG_A64, ATA);
+    dc->mte_active[0] = FIELD_EX32(tb_flags, TBFLAG_A64, MTE_ACTIVE);
+    dc->mte_active[1] = FIELD_EX32(tb_flags, TBFLAG_A64, MTE0_ACTIVE);
     dc->bt = FIELD_EX32(tb_flags, TBFLAG_A64, BT);
     dc->btype = FIELD_EX32(tb_flags, TBFLAG_A64, BTYPE);
     dc->unpriv = FIELD_EX32(tb_flags, TBFLAG_A64, UNPRIV);
diff --git a/qemu/target/arm/translate-a64.h b/qemu/target/arm/translate-a64.h
index 6092d1b02c..ee911735ea 100644
--- a/qemu/target/arm/translate-a64.h
+++ b/qemu/target/arm/translate-a64.h
@@ -36,11 +36,20 @@ TCGv_i64 cpu_reg(DisasContext *s, int reg);
 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg);
 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf);
 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf);
+TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
+                        bool tag_checked, int size);
 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v);
 TCGv_ptr get_fpstatus_ptr(TCGContext *tcg_ctx, bool);
 bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn,
                             unsigned int imms, unsigned int immr);
 bool sve_access_check(DisasContext *s);
+bool sme_enabled_check(DisasContext *s);
+bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req);
+bool disas_sme(DisasContext *, uint32_t);
+void gen_sve_ldr(DisasContext *s, TCGv_ptr base, int vofs,
+                 int len, int rn, int imm);
+void gen_sve_str(DisasContext *s, TCGv_ptr base, int vofs,
+                 int len, int rn, int imm);
 
 /* We should have at some point before trying to access an FP register
  * done the necessary access check, so assert that
@@ -112,7 +121,7 @@ static inline TCGv_ptr vec_full_reg_ptr(DisasContext *s, int regno)
 /* Return the byte size of the "whole" vector register, VL / 8.  */
 static inline int vec_full_reg_size(DisasContext *s)
 {
-    return s->sve_len;
+    return s->pstate_sm ? s->svl : s->sve_len;
 }
 
 bool disas_sve(DisasContext *, uint32_t);
diff --git a/qemu/target/arm/translate-sme.c b/qemu/target/arm/translate-sme.c
new file mode 100644
index 0000000000..075887b52f
--- /dev/null
+++ b/qemu/target/arm/translate-sme.c
@@ -0,0 +1,824 @@
+/*
+ * AArch64 SME translation.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "tcg/tcg-op.h"
+#include "tcg/tcg-op-gvec.h"
+#include "tcg/tcg-gvec-desc.h"
+#include "translate.h"
+#include "internals.h"
+#include "exec/helper-gen.h"
+#include "translate-a64.h"
+
+typedef struct arg_ZERO {
+    int imm;
+} arg_ZERO;
+
+typedef struct arg_MOVA {
+    int esz;
+    int rs;
+    int pg;
+    int zr;
+    int za_imm;
+    bool v;
+    bool to_vec;
+} arg_MOVA;
+
+typedef struct arg_ADDA {
+    int esz;
+    int zad;
+    int zn;
+    int pm;
+    int pn;
+    bool vertical;
+} arg_ADDA;
+
+typedef struct arg_LDSTR {
+    int rv;
+    int rn;
+    int imm;
+    bool store;
+} arg_LDSTR;
+
+typedef struct arg_LDST1 {
+    int esz;
+    int rs;
+    int pg;
+    int rn;
+    int rm;
+    int za_imm;
+    bool vertical;
+    bool store;
+} arg_LDST1;
+
+typedef struct arg_OP {
+    int esz;
+    int zad;
+    int zn;
+    int zm;
+    int pm;
+    int pn;
+    int kind;
+    bool sub;
+} arg_OP;
+
+enum {
+    SME_FP_OP_FMOPA_S,
+    SME_FP_OP_FMOPA_D,
+    SME_FP_OP_BFMOPA,
+    SME_FP_OP_FMOPA_H,
+};
+
+static inline int pred_full_reg_offset(DisasContext *s, int regno)
+{
+    return offsetof(CPUARMState, vfp.pregs[regno]);
+}
+
+static inline TCGv_ptr pred_full_reg_ptr(DisasContext *s, int regno)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr ret = tcg_temp_new_ptr(tcg_ctx);
+
+    tcg_gen_addi_ptr(tcg_ctx, ret, tcg_ctx->cpu_env,
+                     pred_full_reg_offset(s, regno));
+    return ret;
+}
+
+static TCGv_i64 sme_clean_data_tbi(DisasContext *s, TCGv_i64 addr)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i64 clean = tcg_temp_new_i64(tcg_ctx);
+
+    if (s->tbid == 0) {
+        tcg_gen_mov_i64(tcg_ctx, clean, addr);
+    } else if (!regime_has_2_ranges(s->mmu_idx)) {
+        tcg_gen_extract_i64(tcg_ctx, clean, addr, 0, 56);
+    } else {
+        tcg_gen_sextract_i64(tcg_ctx, clean, addr, 0, 56);
+
+        if (s->tbid != 3) {
+            TCGv_i64 zero = tcg_const_i64(tcg_ctx, 0);
+
+            tcg_gen_movcond_i64(tcg_ctx,
+                                s->tbid == 1 ? TCG_COND_GE : TCG_COND_LT,
+                                clean, clean, zero, clean, addr);
+            tcg_temp_free_i64(tcg_ctx, zero);
+        }
+    }
+    return clean;
+}
+
+static inline int streaming_vec_reg_size(DisasContext *s)
+{
+    return s->svl;
+}
+
+static inline int streaming_pred_reg_size(DisasContext *s)
+{
+    return s->svl >> 3;
+}
+
+static inline bool sme_za_enabled_check(DisasContext *s)
+{
+    return sme_enabled_check_with_svcr(s, R_SVCR_ZA_MASK);
+}
+
+static inline bool sme_smza_enabled_check(DisasContext *s)
+{
+    return sme_enabled_check_with_svcr(s, (unsigned)R_SVCR_SM_MASK |
+                                          (unsigned)R_SVCR_ZA_MASK);
+}
+
+static TCGv_ptr get_tile_rowcol(DisasContext *s, int esz, int rs,
+                                int tile_index, bool vertical)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int tile = tile_index >> (4 - esz);
+    int index = esz == MO_128 ? 0 : extract32(tile_index, 0, 4 - esz);
+    int pos, len, offset;
+    TCGv_i32 tmp;
+    TCGv_ptr addr;
+
+    tmp = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_extrl_i64_i32(tcg_ctx, tmp, cpu_reg(s, rs));
+    tcg_gen_addi_i32(tcg_ctx, tmp, tmp, index);
+
+    len = ctz32(streaming_vec_reg_size(s)) - esz;
+    if (!len) {
+        tcg_gen_movi_i32(tcg_ctx, tmp, 0);
+    } else if (vertical) {
+        pos = esz;
+        tcg_gen_deposit_z_i32(tcg_ctx, tmp, tmp, pos, len);
+#ifdef HOST_WORDS_BIGENDIAN
+        if (esz < MO_64) {
+            tcg_gen_xori_i32(tcg_ctx, tmp, tmp, 8 - (1 << esz));
+        }
+#endif
+    } else {
+        pos = esz + ctz32(sizeof(ARMVectorReg));
+        tcg_gen_deposit_z_i32(tcg_ctx, tmp, tmp, pos, len);
+    }
+
+    offset = tile * sizeof(ARMVectorReg) + offsetof(CPUARMState, zarray);
+    tcg_gen_addi_i32(tcg_ctx, tmp, tmp, offset);
+
+    addr = tcg_temp_new_ptr(tcg_ctx);
+    tcg_gen_ext_i32_ptr(tcg_ctx, addr, tmp);
+    tcg_temp_free_i32(tcg_ctx, tmp);
+    tcg_gen_add_ptr(tcg_ctx, addr, addr, tcg_ctx->cpu_env);
+
+    return addr;
+}
+
+static TCGv_ptr get_tile(DisasContext *s, int tile_index)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr addr = tcg_temp_new_ptr(tcg_ctx);
+    int offset;
+
+    offset = tile_index * sizeof(ARMVectorReg) + offsetof(CPUARMState, zarray);
+    tcg_gen_addi_ptr(tcg_ctx, addr, tcg_ctx->cpu_env, offset);
+    return addr;
+}
+
+static bool trans_ZERO(DisasContext *s, arg_ZERO *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm;
+    TCGv_i32 svl;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (sme_za_enabled_check(s)) {
+        imm = tcg_const_i32(tcg_ctx, a->imm);
+        svl = tcg_const_i32(tcg_ctx, streaming_vec_reg_size(s));
+        gen_helper_sme_zero(tcg_ctx, tcg_ctx->cpu_env, imm, svl);
+        tcg_temp_free_i32(tcg_ctx, imm);
+        tcg_temp_free_i32(tcg_ctx, svl);
+    }
+    return true;
+}
+
+static bool trans_MOVA(DisasContext *s, arg_MOVA *a)
+{
+    static gen_helper_gvec_4 * const h_fns[5] = {
+        gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
+        gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d,
+        gen_helper_sve_sel_zpzz_q
+    };
+    static gen_helper_gvec_3 * const cz_fns[5] = {
+        gen_helper_sme_mova_cz_b, gen_helper_sme_mova_cz_h,
+        gen_helper_sme_mova_cz_s, gen_helper_sme_mova_cz_d,
+        gen_helper_sme_mova_cz_q,
+    };
+    static gen_helper_gvec_3 * const zc_fns[5] = {
+        gen_helper_sme_mova_zc_b, gen_helper_sme_mova_zc_h,
+        gen_helper_sme_mova_zc_s, gen_helper_sme_mova_zc_d,
+        gen_helper_sme_mova_zc_q,
+    };
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr t_za, t_zr, t_pg;
+    TCGv_i32 t_desc;
+    int svl;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (!sme_smza_enabled_check(s)) {
+        return true;
+    }
+
+    t_za = get_tile_rowcol(s, a->esz, a->rs, a->za_imm, a->v);
+    t_zr = vec_full_reg_ptr(s, a->zr);
+    t_pg = pred_full_reg_ptr(s, a->pg);
+    svl = streaming_vec_reg_size(s);
+    t_desc = tcg_const_i32(tcg_ctx, simd_desc(svl, svl, 0));
+
+    if (a->v) {
+        if (a->to_vec) {
+            zc_fns[a->esz](tcg_ctx, t_zr, t_za, t_pg, t_desc);
+        } else {
+            cz_fns[a->esz](tcg_ctx, t_za, t_zr, t_pg, t_desc);
+        }
+    } else {
+        if (a->to_vec) {
+            h_fns[a->esz](tcg_ctx, t_zr, t_za, t_zr, t_pg, t_desc);
+        } else {
+            h_fns[a->esz](tcg_ctx, t_za, t_zr, t_za, t_pg, t_desc);
+        }
+    }
+
+    tcg_temp_free_i32(tcg_ctx, t_desc);
+    tcg_temp_free_ptr(tcg_ctx, t_za);
+    tcg_temp_free_ptr(tcg_ctx, t_zr);
+    tcg_temp_free_ptr(tcg_ctx, t_pg);
+    return true;
+}
+
+static bool trans_ADDA(DisasContext *s, arg_ADDA *a)
+{
+    static gen_helper_gvec_4 * const h_fns[2] = {
+        gen_helper_sme_addha_s, gen_helper_sme_addha_d
+    };
+    static gen_helper_gvec_4 * const v_fns[2] = {
+        gen_helper_sme_addva_s, gen_helper_sme_addva_d
+    };
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr za, zn, pn, pm;
+    TCGv_i32 desc;
+    int svl = streaming_vec_reg_size(s);
+    int size_index = a->esz == MO_64;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (a->esz == MO_64 && !dc_isar_feature(aa64_sme_i16i64, s)) {
+        return false;
+    }
+    if (!sme_smza_enabled_check(s)) {
+        return true;
+    }
+
+    za = get_tile(s, a->zad);
+    zn = vec_full_reg_ptr(s, a->zn);
+    pn = pred_full_reg_ptr(s, a->pn);
+    pm = pred_full_reg_ptr(s, a->pm);
+    desc = tcg_const_i32(tcg_ctx, simd_desc(svl, svl, 0));
+
+    if (a->vertical) {
+        v_fns[size_index](tcg_ctx, za, zn, pn, pm, desc);
+    } else {
+        h_fns[size_index](tcg_ctx, za, zn, pn, pm, desc);
+    }
+
+    tcg_temp_free_i32(tcg_ctx, desc);
+    tcg_temp_free_ptr(tcg_ctx, za);
+    tcg_temp_free_ptr(tcg_ctx, zn);
+    tcg_temp_free_ptr(tcg_ctx, pn);
+    tcg_temp_free_ptr(tcg_ctx, pm);
+    return true;
+}
+
+static bool trans_LDST1(DisasContext *s, arg_LDST1 *a)
+{
+    typedef void GenLdSt1(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr,
+                          TCGv_i64, TCGv_i32);
+
+    static GenLdSt1 * const fns[5][2][2][2][2] = {
+        {
+            {
+                {
+                    { gen_helper_sme_ld1b_h, gen_helper_sme_st1b_h },
+                    { gen_helper_sme_ld1b_h_mte,
+                      gen_helper_sme_st1b_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1b_v, gen_helper_sme_st1b_v },
+                    { gen_helper_sme_ld1b_v_mte,
+                      gen_helper_sme_st1b_v_mte },
+                },
+            },
+            {
+                {
+                    { gen_helper_sme_ld1b_h, gen_helper_sme_st1b_h },
+                    { gen_helper_sme_ld1b_h_mte,
+                      gen_helper_sme_st1b_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1b_v, gen_helper_sme_st1b_v },
+                    { gen_helper_sme_ld1b_v_mte,
+                      gen_helper_sme_st1b_v_mte },
+                },
+            },
+        },
+        {
+            {
+                {
+                    { gen_helper_sme_ld1h_le_h, gen_helper_sme_st1h_le_h },
+                    { gen_helper_sme_ld1h_le_h_mte,
+                      gen_helper_sme_st1h_le_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1h_le_v, gen_helper_sme_st1h_le_v },
+                    { gen_helper_sme_ld1h_le_v_mte,
+                      gen_helper_sme_st1h_le_v_mte },
+                },
+            },
+            {
+                {
+                    { gen_helper_sme_ld1h_be_h, gen_helper_sme_st1h_be_h },
+                    { gen_helper_sme_ld1h_be_h_mte,
+                      gen_helper_sme_st1h_be_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1h_be_v, gen_helper_sme_st1h_be_v },
+                    { gen_helper_sme_ld1h_be_v_mte,
+                      gen_helper_sme_st1h_be_v_mte },
+                },
+            },
+        },
+        {
+            {
+                {
+                    { gen_helper_sme_ld1s_le_h, gen_helper_sme_st1s_le_h },
+                    { gen_helper_sme_ld1s_le_h_mte,
+                      gen_helper_sme_st1s_le_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1s_le_v, gen_helper_sme_st1s_le_v },
+                    { gen_helper_sme_ld1s_le_v_mte,
+                      gen_helper_sme_st1s_le_v_mte },
+                },
+            },
+            {
+                {
+                    { gen_helper_sme_ld1s_be_h, gen_helper_sme_st1s_be_h },
+                    { gen_helper_sme_ld1s_be_h_mte,
+                      gen_helper_sme_st1s_be_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1s_be_v, gen_helper_sme_st1s_be_v },
+                    { gen_helper_sme_ld1s_be_v_mte,
+                      gen_helper_sme_st1s_be_v_mte },
+                },
+            },
+        },
+        {
+            {
+                {
+                    { gen_helper_sme_ld1d_le_h, gen_helper_sme_st1d_le_h },
+                    { gen_helper_sme_ld1d_le_h_mte,
+                      gen_helper_sme_st1d_le_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1d_le_v, gen_helper_sme_st1d_le_v },
+                    { gen_helper_sme_ld1d_le_v_mte,
+                      gen_helper_sme_st1d_le_v_mte },
+                },
+            },
+            {
+                {
+                    { gen_helper_sme_ld1d_be_h, gen_helper_sme_st1d_be_h },
+                    { gen_helper_sme_ld1d_be_h_mte,
+                      gen_helper_sme_st1d_be_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1d_be_v, gen_helper_sme_st1d_be_v },
+                    { gen_helper_sme_ld1d_be_v_mte,
+                      gen_helper_sme_st1d_be_v_mte },
+                },
+            },
+        },
+        {
+            {
+                {
+                    { gen_helper_sme_ld1q_le_h, gen_helper_sme_st1q_le_h },
+                    { gen_helper_sme_ld1q_le_h_mte,
+                      gen_helper_sme_st1q_le_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1q_le_v, gen_helper_sme_st1q_le_v },
+                    { gen_helper_sme_ld1q_le_v_mte,
+                      gen_helper_sme_st1q_le_v_mte },
+                },
+            },
+            {
+                {
+                    { gen_helper_sme_ld1q_be_h, gen_helper_sme_st1q_be_h },
+                    { gen_helper_sme_ld1q_be_h_mte,
+                      gen_helper_sme_st1q_be_h_mte },
+                },
+                {
+                    { gen_helper_sme_ld1q_be_v, gen_helper_sme_st1q_be_v },
+                    { gen_helper_sme_ld1q_be_v_mte,
+                      gen_helper_sme_st1q_be_v_mte },
+                },
+            },
+        },
+    };
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr za, pg;
+    TCGv_i64 addr;
+    TCGv_i64 clean_addr;
+    TCGv_i32 desc;
+    uint32_t desc_data = 0;
+    int svl = streaming_vec_reg_size(s);
+    int be = s->be_data == MO_BE;
+    int mte = s->mte_active[0];
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (!sme_smza_enabled_check(s)) {
+        return true;
+    }
+
+    za = get_tile_rowcol(s, a->esz, a->rs, a->za_imm, a->vertical);
+    pg = pred_full_reg_ptr(s, a->pg);
+    addr = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_shli_i64(tcg_ctx, addr, cpu_reg(s, a->rm), a->esz);
+    tcg_gen_add_i64(tcg_ctx, addr, addr, cpu_reg_sp(s, a->rn));
+    if (mte) {
+        desc_data = deposit32(desc_data, R_MTEDESC_MIDX_SHIFT,
+                              R_MTEDESC_MIDX_LENGTH, get_mem_index(s));
+        desc_data = deposit32(desc_data, R_MTEDESC_TBI_SHIFT,
+                              R_MTEDESC_TBI_LENGTH, s->tbid);
+        desc_data = deposit32(desc_data, R_MTEDESC_TCMA_SHIFT,
+                              R_MTEDESC_TCMA_LENGTH, s->tcma);
+        desc_data = deposit32(desc_data, R_MTEDESC_WRITE_SHIFT,
+                              R_MTEDESC_WRITE_LENGTH, a->store);
+        desc_data = deposit32(desc_data, R_MTEDESC_SIZEM1_SHIFT,
+                              R_MTEDESC_SIZEM1_LENGTH,
+                              (1 << a->esz) - 1);
+        clean_addr = addr;
+    } else {
+        desc_data = get_mem_index(s);
+        clean_addr = sme_clean_data_tbi(s, addr);
+        tcg_temp_free_i64(tcg_ctx, addr);
+    }
+    desc = tcg_const_i32(tcg_ctx, simd_desc(svl, svl, desc_data));
+
+    fns[a->esz][be][a->vertical][mte][a->store](tcg_ctx, tcg_ctx->cpu_env,
+                                                za, pg, clean_addr, desc);
+
+    tcg_temp_free_i32(tcg_ctx, desc);
+    tcg_temp_free_i64(tcg_ctx, clean_addr);
+    tcg_temp_free_ptr(tcg_ctx, za);
+    tcg_temp_free_ptr(tcg_ctx, pg);
+    return true;
+}
+
+static bool trans_LDSTR(DisasContext *s, arg_LDSTR *a)
+{
+    TCGv_ptr base;
+    int svl = streaming_vec_reg_size(s);
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (!sme_za_enabled_check(s)) {
+        return true;
+    }
+
+    base = get_tile_rowcol(s, MO_8, a->rv, a->imm, false);
+    if (a->store) {
+        gen_sve_str(s, base, 0, svl, a->rn, a->imm * svl);
+    } else {
+        gen_sve_ldr(s, base, 0, svl, a->rn, a->imm * svl);
+    }
+    tcg_temp_free_ptr(s->uc->tcg_ctx, base);
+    return true;
+}
+
+static bool trans_OUTPROD(DisasContext *s, arg_OP *a)
+{
+    static gen_helper_gvec_5 * const fns_s[4] = {
+        gen_helper_sme_smopa_s, gen_helper_sme_sumopa_s,
+        gen_helper_sme_usmopa_s, gen_helper_sme_umopa_s,
+    };
+    static gen_helper_gvec_5 * const fns_d[4] = {
+        gen_helper_sme_smopa_d, gen_helper_sme_sumopa_d,
+        gen_helper_sme_usmopa_d, gen_helper_sme_umopa_d,
+    };
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr za, zn, zm, pn, pm;
+    TCGv_i32 desc;
+    int svl = streaming_vec_reg_size(s);
+    gen_helper_gvec_5 *fn;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (a->esz == MO_64 && !dc_isar_feature(aa64_sme_i16i64, s)) {
+        return false;
+    }
+    if (!sme_smza_enabled_check(s)) {
+        return true;
+    }
+
+    fn = a->esz == MO_64 ? fns_d[a->kind] : fns_s[a->kind];
+    za = get_tile(s, a->zad);
+    zn = vec_full_reg_ptr(s, a->zn);
+    zm = vec_full_reg_ptr(s, a->zm);
+    pn = pred_full_reg_ptr(s, a->pn);
+    pm = pred_full_reg_ptr(s, a->pm);
+    desc = tcg_const_i32(tcg_ctx, simd_desc(svl, svl, a->sub));
+
+    fn(tcg_ctx, za, zn, zm, pn, pm, desc);
+
+    tcg_temp_free_i32(tcg_ctx, desc);
+    tcg_temp_free_ptr(tcg_ctx, za);
+    tcg_temp_free_ptr(tcg_ctx, zn);
+    tcg_temp_free_ptr(tcg_ctx, zm);
+    tcg_temp_free_ptr(tcg_ctx, pn);
+    tcg_temp_free_ptr(tcg_ctx, pm);
+    return true;
+}
+
+static bool trans_FPOUTPROD(DisasContext *s, arg_OP *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr za, zn, zm, pn, pm;
+    TCGv_i32 desc;
+    int svl = streaming_vec_reg_size(s);
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (a->kind == SME_FP_OP_FMOPA_D &&
+        !dc_isar_feature(aa64_sme_f64f64, s)) {
+        return false;
+    }
+    if (!sme_smza_enabled_check(s)) {
+        return true;
+    }
+
+    za = get_tile(s, a->zad);
+    zn = vec_full_reg_ptr(s, a->zn);
+    zm = vec_full_reg_ptr(s, a->zm);
+    pn = pred_full_reg_ptr(s, a->pn);
+    pm = pred_full_reg_ptr(s, a->pm);
+    desc = tcg_const_i32(tcg_ctx, simd_desc(svl, svl, a->sub));
+
+    switch (a->kind) {
+    case SME_FP_OP_FMOPA_S:
+    {
+        TCGv_ptr fpst = get_fpstatus_ptr(tcg_ctx, false);
+
+        gen_helper_sme_fmopa_s(tcg_ctx, za, zn, zm, pn, pm, fpst, desc);
+        tcg_temp_free_ptr(tcg_ctx, fpst);
+        break;
+    }
+    case SME_FP_OP_FMOPA_D:
+    {
+        TCGv_ptr fpst = get_fpstatus_ptr(tcg_ctx, false);
+
+        gen_helper_sme_fmopa_d(tcg_ctx, za, zn, zm, pn, pm, fpst, desc);
+        tcg_temp_free_ptr(tcg_ctx, fpst);
+        break;
+    }
+    case SME_FP_OP_FMOPA_H:
+        gen_helper_sme_fmopa_h(tcg_ctx, za, zn, zm, pn, pm,
+                               tcg_ctx->cpu_env, desc);
+        break;
+    case SME_FP_OP_BFMOPA:
+        gen_helper_sme_bfmopa(tcg_ctx, za, zn, zm, pn, pm, desc);
+        break;
+    }
+
+    tcg_temp_free_i32(tcg_ctx, desc);
+    tcg_temp_free_ptr(tcg_ctx, za);
+    tcg_temp_free_ptr(tcg_ctx, zn);
+    tcg_temp_free_ptr(tcg_ctx, zm);
+    tcg_temp_free_ptr(tcg_ctx, pn);
+    tcg_temp_free_ptr(tcg_ctx, pm);
+    return true;
+}
+
+static bool decode_mova(uint32_t insn, arg_MOVA *a)
+{
+    int group;
+    int qbit;
+
+    if (extract32(insn, 24, 8) != 0xc0) {
+        return false;
+    }
+    group = extract32(insn, 17, 5);
+    if (group != 0 && group != 1) {
+        return false;
+    }
+
+    a->esz = extract32(insn, 22, 2);
+    qbit = extract32(insn, 16, 1);
+    if (qbit) {
+        if (a->esz != MO_64) {
+            return false;
+        }
+        a->esz = MO_128;
+    }
+
+    a->v = extract32(insn, 15, 1);
+    a->rs = extract32(insn, 13, 2) + 12;
+    a->pg = extract32(insn, 10, 3);
+    a->to_vec = group == 1;
+    if (a->to_vec) {
+        if (extract32(insn, 9, 1)) {
+            return false;
+        }
+        a->za_imm = extract32(insn, 5, 4);
+        a->zr = extract32(insn, 0, 5);
+    } else {
+        if (extract32(insn, 4, 1)) {
+            return false;
+        }
+        a->zr = extract32(insn, 5, 5);
+        a->za_imm = extract32(insn, 0, 4);
+    }
+    return true;
+}
+
+static bool decode_adda(uint32_t insn, arg_ADDA *a)
+{
+    if ((insn & 0xfffe001c) == 0xc0900000) {
+        a->esz = MO_32;
+        a->zad = extract32(insn, 0, 2);
+    } else if ((insn & 0xfffe0018) == 0xc0d00000) {
+        a->esz = MO_64;
+        a->zad = extract32(insn, 0, 3);
+    } else {
+        return false;
+    }
+
+    a->vertical = extract32(insn, 16, 1);
+    a->pm = extract32(insn, 13, 3);
+    a->pn = extract32(insn, 10, 3);
+    a->zn = extract32(insn, 5, 5);
+    return true;
+}
+
+static bool decode_ldst1(uint32_t insn, arg_LDST1 *a)
+{
+    if ((insn & 0xff000010) == 0xe0000000) {
+        a->esz = extract32(insn, 22, 2);
+    } else if ((insn & 0xffc00010) == 0xe1c00000) {
+        a->esz = MO_128;
+    } else {
+        return false;
+    }
+
+    a->store = extract32(insn, 21, 1);
+    a->rm = extract32(insn, 16, 5);
+    a->vertical = extract32(insn, 15, 1);
+    a->pg = extract32(insn, 10, 3);
+    a->rn = extract32(insn, 5, 5);
+    a->rs = extract32(insn, 13, 2) + 12;
+    a->za_imm = extract32(insn, 0, 4);
+    return true;
+}
+
+static bool decode_ldstr(uint32_t insn, arg_LDSTR *a)
+{
+    if ((insn & 0xffff9c10) != 0xe1000000 &&
+        (insn & 0xffff9c10) != 0xe1200000) {
+        return false;
+    }
+
+    a->rv = extract32(insn, 13, 2) + 12;
+    a->rn = extract32(insn, 5, 5);
+    a->imm = extract32(insn, 0, 4);
+    a->store = extract32(insn, 21, 1);
+    return true;
+}
+
+static bool decode_fp_op(uint32_t insn, arg_OP *a)
+{
+    if ((insn & 0xffe0000c) == 0x80800000) {
+        a->kind = SME_FP_OP_FMOPA_S;
+        a->esz = MO_32;
+        a->zad = extract32(insn, 0, 2);
+    } else if ((insn & 0xffe00008) == 0x80c00000) {
+        a->kind = SME_FP_OP_FMOPA_D;
+        a->esz = MO_64;
+        a->zad = extract32(insn, 0, 3);
+    } else if ((insn & 0xffe0000c) == 0x81800000) {
+        a->kind = SME_FP_OP_BFMOPA;
+        a->esz = MO_32;
+        a->zad = extract32(insn, 0, 2);
+    } else if ((insn & 0xffe0000c) == 0x81a00000) {
+        a->kind = SME_FP_OP_FMOPA_H;
+        a->esz = MO_32;
+        a->zad = extract32(insn, 0, 2);
+    } else {
+        return false;
+    }
+
+    a->zm = extract32(insn, 16, 5);
+    a->pm = extract32(insn, 13, 3);
+    a->pn = extract32(insn, 10, 3);
+    a->zn = extract32(insn, 5, 5);
+    a->sub = extract32(insn, 4, 1);
+    return true;
+}
+
+static bool decode_int_op(uint32_t insn, arg_OP *a)
+{
+    if ((insn & 0xfec0000c) == 0xa0800000) {
+        a->esz = MO_32;
+        a->zad = extract32(insn, 0, 2);
+    } else if ((insn & 0xfec00008) == 0xa0c00000) {
+        a->esz = MO_64;
+        a->zad = extract32(insn, 0, 3);
+    } else {
+        return false;
+    }
+
+    a->kind = (extract32(insn, 24, 1) << 1) | extract32(insn, 21, 1);
+    a->zm = extract32(insn, 16, 5);
+    a->pm = extract32(insn, 13, 3);
+    a->pn = extract32(insn, 10, 3);
+    a->zn = extract32(insn, 5, 5);
+    a->sub = extract32(insn, 4, 1);
+    return true;
+}
+
+bool disas_sme(DisasContext *s, uint32_t insn)
+{
+    if ((insn & 0xffffff00) == 0xc0080000) {
+        arg_ZERO a = { .imm = extract32(insn, 0, 8) };
+        return trans_ZERO(s, &a);
+    }
+
+    {
+        arg_MOVA a;
+
+        if (decode_mova(insn, &a)) {
+            return trans_MOVA(s, &a);
+        }
+    }
+
+    {
+        arg_ADDA a;
+
+        if (decode_adda(insn, &a)) {
+            return trans_ADDA(s, &a);
+        }
+    }
+
+    {
+        arg_LDST1 a;
+
+        if (decode_ldst1(insn, &a)) {
+            return trans_LDST1(s, &a);
+        }
+    }
+
+    {
+        arg_LDSTR a;
+
+        if (decode_ldstr(insn, &a)) {
+            return trans_LDSTR(s, &a);
+        }
+    }
+
+    {
+        arg_OP a;
+
+        if (decode_fp_op(insn, &a)) {
+            return trans_FPOUTPROD(s, &a);
+        }
+    }
+
+    {
+        arg_OP a;
+
+        if (decode_int_op(insn, &a)) {
+            return trans_OUTPROD(s, &a);
+        }
+    }
+
+    return false;
+}
diff --git a/qemu/target/arm/translate-sve.c b/qemu/target/arm/translate-sve.c
index ff2f8ff323..3bf61d7494 100644
--- a/qemu/target/arm/translate-sve.c
+++ b/qemu/target/arm/translate-sve.c
@@ -43,7 +43,13 @@ typedef void gen_helper_gvec_flags_4(TCGContext *, TCGv_i32, TCGv_ptr, TCGv_ptr,
 
 typedef void gen_helper_gvec_mem(TCGContext *, TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32);
 typedef void gen_helper_gvec_mem_scatter(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr,
-                                         TCGv_ptr, TCGv_i64, TCGv_i32);
+                                          TCGv_ptr, TCGv_i64, TCGv_i32);
+
+static bool sve_nonstreaming_access_check(DisasContext *s)
+{
+    s->is_nonstreaming = true;
+    return sve_access_check(s);
+}
 
 /*
  * Helpers for extracting complex instruction fields.
@@ -60,13 +66,27 @@ static int tszimm_esz(DisasContext *s, int x)
 
 static int tszimm_shr(DisasContext *s, int x)
 {
-    return (16 << tszimm_esz(s, x)) - x;
+    /*
+     * We won't use the tszimm_shr() value if tszimm_esz() returns -1; the
+     * translator will reject the invalid tsz encoding before using imm.
+     */
+    int esz = tszimm_esz(s, x);
+
+    if (esz < 0) {
+        return esz;
+    }
+    return (16 << esz) - x;
 }
 
 /* See e.g. LSL (immediate, predicated).  */
 static int tszimm_shl(DisasContext *s, int x)
 {
-    return x - (8 << tszimm_esz(s, x));
+    int esz = tszimm_esz(s, x);
+
+    if (esz < 0) {
+        return esz;
+    }
+    return x - (8 << esz);
 }
 
 static inline int plus1(DisasContext *s, int x)
@@ -115,7 +135,17 @@ static inline int pred_full_reg_offset(DisasContext *s, int regno)
 /* Return the byte size of the whole predicate register, VL / 64.  */
 static inline int pred_full_reg_size(DisasContext *s)
 {
-    return s->sve_len >> 3;
+    return (s->pstate_sm ? s->svl : s->sve_len) >> 3;
+}
+
+static inline int streaming_vec_reg_size(DisasContext *s)
+{
+    return s->svl;
+}
+
+static inline int streaming_pred_reg_size(DisasContext *s)
+{
+    return s->svl >> 3;
 }
 
 /* Round up the size of a register to a size allowed by
@@ -298,175 +328,1669 @@ static bool trans_BIC_zzz(DisasContext *s, arg_rrr_esz *a)
     return do_vector3_z(s, tcg_gen_gvec_andc, 0, a->rd, a->rn, a->rm);
 }
 
-/*
- *** SVE Integer Arithmetic - Unpredicated Group
- */
+static bool do_zzzz_ool(DisasContext *s, arg_rprrr_esz *a,
+                        gen_helper_gvec_4 *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
 
-static bool trans_ADD_zzz(DisasContext *s, arg_rrr_esz *a)
+    if (fn == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, 0, fn);
+    }
+    return true;
+}
+
+static bool trans_EOR3(DisasContext *s, arg_rprrr_esz *a)
 {
-    return do_vector3_z(s, tcg_gen_gvec_add, a->esz, a->rd, a->rn, a->rm);
+    return do_zzzz_ool(s, a, gen_helper_sve2_eor3);
 }
 
-static bool trans_SUB_zzz(DisasContext *s, arg_rrr_esz *a)
+static bool trans_BSL(DisasContext *s, arg_rprrr_esz *a)
 {
-    return do_vector3_z(s, tcg_gen_gvec_sub, a->esz, a->rd, a->rn, a->rm);
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_bitsel(tcg_ctx, 0, vec_full_reg_offset(s, a->rd),
+                            vec_full_reg_offset(s, a->ra),
+                            vec_full_reg_offset(s, a->rn),
+                            vec_full_reg_offset(s, a->rm), vsz, vsz);
+    }
+    return true;
 }
 
-static bool trans_SQADD_zzz(DisasContext *s, arg_rrr_esz *a)
+static bool trans_BCAX(DisasContext *s, arg_rprrr_esz *a)
 {
-    return do_vector3_z(s, tcg_gen_gvec_ssadd, a->esz, a->rd, a->rn, a->rm);
+    return do_zzzz_ool(s, a, gen_helper_sve2_bcax);
 }
 
-static bool trans_SQSUB_zzz(DisasContext *s, arg_rrr_esz *a)
+static bool trans_BSL1N(DisasContext *s, arg_rprrr_esz *a)
 {
-    return do_vector3_z(s, tcg_gen_gvec_sssub, a->esz, a->rd, a->rn, a->rm);
+    return do_zzzz_ool(s, a, gen_helper_sve2_bsl1n);
 }
 
-static bool trans_UQADD_zzz(DisasContext *s, arg_rrr_esz *a)
+static bool trans_BSL2N(DisasContext *s, arg_rprrr_esz *a)
 {
-    return do_vector3_z(s, tcg_gen_gvec_usadd, a->esz, a->rd, a->rn, a->rm);
+    return do_zzzz_ool(s, a, gen_helper_sve2_bsl2n);
 }
 
-static bool trans_UQSUB_zzz(DisasContext *s, arg_rrr_esz *a)
+static bool trans_NBSL(DisasContext *s, arg_rprrr_esz *a)
 {
-    return do_vector3_z(s, tcg_gen_gvec_ussub, a->esz, a->rd, a->rn, a->rm);
+    return do_zzzz_ool(s, a, gen_helper_sve2_nbsl);
 }
 
-/*
- *** SVE Integer Arithmetic - Binary Predicated Group
- */
+static bool trans_XAR(DisasContext *s, arg_rrri_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_xar_b,
+        gen_helper_sve2_xar_h,
+        gen_helper_sve2_xar_s,
+        gen_helper_sve2_xar_d,
+    };
+    int esize;
+    int shift;
 
-static bool do_zpzz_ool(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4 *fn)
+    if (a->esz < 0) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        esize = 8 << a->esz;
+        shift = a->imm & (esize - 1);
+        if (shift == 0) {
+            tcg_gen_gvec_xor(tcg_ctx, a->esz,
+                             vec_full_reg_offset(s, a->rd),
+                             vec_full_reg_offset(s, a->rn),
+                             vec_full_reg_offset(s, a->rm), vsz, vsz);
+        } else {
+            tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                               vec_full_reg_offset(s, a->rn),
+                               vec_full_reg_offset(s, a->rm),
+                               vsz, vsz, shift, fns[a->esz]);
+        }
+    }
+    return true;
+}
+
+static bool do_pmull(DisasContext *s, arg_rrr_esz *a, int sel)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    unsigned vsz = vec_full_reg_size(s);
-    if (fn == NULL) {
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h,
+        NULL,                    gen_helper_sve2_pmull_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
         return false;
     }
+    if (a->esz == 0) {
+        if (!dc_isar_feature(aa64_sve2_pmull128, s)) {
+            return false;
+        }
+        s->is_nonstreaming = true;
+    }
     if (sve_access_check(s)) {
-        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
                            vec_full_reg_offset(s, a->rm),
-                           pred_full_reg_offset(s, a->pg),
-                           vsz, vsz, 0, fn);
+                           vsz, vsz, sel, fns[a->esz]);
     }
     return true;
 }
 
-/* Select active elememnts from Zn and inactive elements from Zm,
- * storing the result in Zd.
- */
-static void do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz)
+static bool trans_PMULLB(DisasContext *s, arg_rrr_esz *a)
 {
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    static gen_helper_gvec_4 * const fns[4] = {
-        gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
-        gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d
-    };
-    unsigned vsz = vec_full_reg_size(s);
-    tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, rd),
-                       vec_full_reg_offset(s, rn),
-                       vec_full_reg_offset(s, rm),
-                       pred_full_reg_offset(s, pg),
-                       vsz, vsz, 0, fns[esz]);
+    return do_pmull(s, a, 0);
 }
 
-#define DO_ZPZZ(NAME, name) \
-static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a)         \
-{                                                                         \
-    static gen_helper_gvec_4 * const fns[4] = {                           \
-        gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h,   \
-        gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d,   \
-    };                                                                    \
-    return do_zpzz_ool(s, a, fns[a->esz]);                                \
+static bool trans_PMULLT(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_pmull(s, a, 1);
 }
 
-DO_ZPZZ(AND, and)
-DO_ZPZZ(EOR, eor)
-DO_ZPZZ(ORR, orr)
-DO_ZPZZ(BIC, bic)
+static bool trans_MUL_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
 
-DO_ZPZZ(ADD, add)
-DO_ZPZZ(SUB, sub)
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
 
-DO_ZPZZ(SMAX, smax)
-DO_ZPZZ(UMAX, umax)
-DO_ZPZZ(SMIN, smin)
-DO_ZPZZ(UMIN, umin)
-DO_ZPZZ(SABD, sabd)
-DO_ZPZZ(UABD, uabd)
+        tcg_gen_gvec_mul(tcg_ctx, a->esz, vec_full_reg_offset(s, a->rd),
+                         vec_full_reg_offset(s, a->rn),
+                         vec_full_reg_offset(s, a->rm), vsz, vsz);
+    }
+    return true;
+}
 
-DO_ZPZZ(MUL, mul)
-DO_ZPZZ(SMULH, smulh)
-DO_ZPZZ(UMULH, umulh)
+static bool do_mulh_zzz(DisasContext *s, arg_rrr_esz *a,
+                        gen_helper_gvec_3 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
 
-DO_ZPZZ(ASR, asr)
-DO_ZPZZ(LSR, lsr)
-DO_ZPZZ(LSL, lsl)
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
 
-static bool trans_SDIV_zpzz(DisasContext *s, arg_rprr_esz *a)
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_SMULH_zzz(DisasContext *s, arg_rrr_esz *a)
 {
-    static gen_helper_gvec_4 * const fns[4] = {
-        NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_smulh_zzz_b,
+        gen_helper_sve2_smulh_zzz_h,
+        gen_helper_sve2_smulh_zzz_s,
+        gen_helper_sve2_smulh_zzz_d,
     };
-    return do_zpzz_ool(s, a, fns[a->esz]);
+
+    return do_mulh_zzz(s, a, fns);
 }
 
-static bool trans_UDIV_zpzz(DisasContext *s, arg_rprr_esz *a)
+static bool trans_UMULH_zzz(DisasContext *s, arg_rrr_esz *a)
 {
-    static gen_helper_gvec_4 * const fns[4] = {
-        NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_umulh_zzz_b,
+        gen_helper_sve2_umulh_zzz_h,
+        gen_helper_sve2_umulh_zzz_s,
+        gen_helper_sve2_umulh_zzz_d,
     };
-    return do_zpzz_ool(s, a, fns[a->esz]);
+
+    return do_mulh_zzz(s, a, fns);
 }
 
-static bool trans_SEL_zpzz(DisasContext *s, arg_rprr_esz *a)
+static bool trans_SQDMULH_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_sqdmulh_b,
+        gen_helper_sve2_sqdmulh_h,
+        gen_helper_sve2_sqdmulh_s,
+        gen_helper_sve2_sqdmulh_d,
+    };
+
+    return do_mulh_zzz(s, a, fns);
+}
+
+static bool trans_SQRDMULH_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_sqrdmulh_b,
+        gen_helper_sve2_sqrdmulh_h,
+        gen_helper_sve2_sqrdmulh_s,
+        gen_helper_sve2_sqrdmulh_d,
+    };
+
+    return do_mulh_zzz(s, a, fns);
+}
+
+static bool trans_PMUL_zzz(DisasContext *s, arg_rrr_esz *a)
 {
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || a->esz != 0) {
+        return false;
+    }
     if (sve_access_check(s)) {
-        do_sel_z(s, a->rd, a->rn, a->rm, a->pg, a->esz);
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, gen_helper_gvec_pmul_b);
     }
     return true;
 }
 
-#undef DO_ZPZZ
+static bool do_mul_zzx(DisasContext *s, arg_rrx_esz *a,
+                       gen_helper_gvec_3 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
 
-/*
- *** SVE Integer Arithmetic - Unary Predicated Group
- */
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
 
-static bool do_zpz_ool(DisasContext *s, arg_rpr_esz *a, gen_helper_gvec_3 *fn)
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, a->index, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_MUL_zzx(DisasContext *s, arg_rrx_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_mul_idx_h,
+        gen_helper_sve2_mul_idx_s,
+        gen_helper_sve2_mul_idx_d,
+    };
+
+    return do_mul_zzx(s, a, fns);
+}
+
+static bool trans_SQDMULH_zzx(DisasContext *s, arg_rrx_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_sqdmulh_idx_h,
+        gen_helper_sve2_sqdmulh_idx_s,
+        gen_helper_sve2_sqdmulh_idx_d,
+    };
+
+    return do_mul_zzx(s, a, fns);
+}
+
+static bool trans_SQRDMULH_zzx(DisasContext *s, arg_rrx_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_sqrdmulh_idx_h,
+        gen_helper_sve2_sqrdmulh_idx_s,
+        gen_helper_sve2_sqrdmulh_idx_d,
+    };
+
+    return do_mul_zzx(s, a, fns);
+}
+
+static bool do_sqrdmla_zzx(DisasContext *s, arg_rrxr_esz *a,
+                           gen_helper_gvec_4 * const fns[4])
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    if (fn == NULL) {
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
         return false;
     }
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
-                           pred_full_reg_offset(s, a->pg),
-                           vsz, vsz, 0, fn);
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, a->index, fns[a->esz]);
     }
     return true;
 }
 
-#define DO_ZPZ(NAME, name) \
-static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a)           \
-{                                                                   \
-    static gen_helper_gvec_3 * const fns[4] = {                     \
-        gen_helper_sve_##name##_b, gen_helper_sve_##name##_h,       \
-        gen_helper_sve_##name##_s, gen_helper_sve_##name##_d,       \
-    };                                                              \
-    return do_zpz_ool(s, a, fns[a->esz]);                           \
+static bool trans_SQRDMLAH_zzxz(DisasContext *s, arg_rrxr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_sqrdmlah_idx_h,
+        gen_helper_sve2_sqrdmlah_idx_s,
+        gen_helper_sve2_sqrdmlah_idx_d,
+    };
+
+    return do_sqrdmla_zzx(s, a, fns);
+}
+
+static bool trans_SQRDMLSH_zzxz(DisasContext *s, arg_rrxr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_sqrdmlsh_idx_h,
+        gen_helper_sve2_sqrdmlsh_idx_s,
+        gen_helper_sve2_sqrdmlsh_idx_d,
+    };
+
+    return do_sqrdmla_zzx(s, a, fns);
+}
+
+static bool do_widen_mul_zzx(DisasContext *s, arg_rrx_esz *a,
+                             gen_helper_gvec_3 * const fns[4], int top)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, (a->index << 1) | top,
+                           fns[a->esz]);
+    }
+    return true;
+}
+
+#define DO_WIDEN_MUL_ZZX(NAME, helper, TOP)                             \
+static bool trans_##NAME(DisasContext *s, arg_rrx_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_3 * const fns[4] = {                         \
+        NULL, NULL,                                                     \
+        gen_helper_sve2_##helper##_s,                                   \
+        gen_helper_sve2_##helper##_d,                                   \
+    };                                                                  \
+                                                                        \
+    return do_widen_mul_zzx(s, a, fns, TOP);                            \
+}
+
+DO_WIDEN_MUL_ZZX(SMULLB_zzx, smull_idx, 0)
+DO_WIDEN_MUL_ZZX(SMULLT_zzx, smull_idx, 1)
+DO_WIDEN_MUL_ZZX(UMULLB_zzx, umull_idx, 0)
+DO_WIDEN_MUL_ZZX(UMULLT_zzx, umull_idx, 1)
+DO_WIDEN_MUL_ZZX(SQDMULLB_zzx, sqdmull_idx, 0)
+DO_WIDEN_MUL_ZZX(SQDMULLT_zzx, sqdmull_idx, 1)
+
+#undef DO_WIDEN_MUL_ZZX
+
+static bool do_widen_acc_zzx(DisasContext *s, arg_rrx_esz *a,
+                             gen_helper_gvec_4 * const fns[4], int top)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->rd), vsz, vsz,
+                           (a->index << 1) | top, fns[a->esz]);
+    }
+    return true;
+}
+
+#define DO_WIDEN_ACC_ZZX(NAME, helper, TOP)                             \
+static bool trans_##NAME(DisasContext *s, arg_rrx_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_4 * const fns[4] = {                         \
+        NULL, NULL,                                                     \
+        gen_helper_sve2_##helper##_s,                                   \
+        gen_helper_sve2_##helper##_d,                                   \
+    };                                                                  \
+                                                                        \
+    return do_widen_acc_zzx(s, a, fns, TOP);                            \
+}
+
+DO_WIDEN_ACC_ZZX(SQDMLALB_zzxw, sqdmlal_idx, 0)
+DO_WIDEN_ACC_ZZX(SQDMLALT_zzxw, sqdmlal_idx, 1)
+DO_WIDEN_ACC_ZZX(SQDMLSLB_zzxw, sqdmlsl_idx, 0)
+DO_WIDEN_ACC_ZZX(SQDMLSLT_zzxw, sqdmlsl_idx, 1)
+DO_WIDEN_ACC_ZZX(SMLALB_zzxw, smlal_idx, 0)
+DO_WIDEN_ACC_ZZX(SMLALT_zzxw, smlal_idx, 1)
+DO_WIDEN_ACC_ZZX(UMLALB_zzxw, umlal_idx, 0)
+DO_WIDEN_ACC_ZZX(UMLALT_zzxw, umlal_idx, 1)
+DO_WIDEN_ACC_ZZX(SMLSLB_zzxw, smlsl_idx, 0)
+DO_WIDEN_ACC_ZZX(SMLSLT_zzxw, smlsl_idx, 1)
+DO_WIDEN_ACC_ZZX(UMLSLB_zzxw, umlsl_idx, 0)
+DO_WIDEN_ACC_ZZX(UMLSLT_zzxw, umlsl_idx, 1)
+
+#undef DO_WIDEN_ACC_ZZX
+
+static bool do_widen_acc_zzzw(DisasContext *s, arg_rprrr_esz *a,
+                              gen_helper_gvec_4 * const fns[4], int data)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra), vsz, vsz,
+                           data, fns[a->esz]);
+    }
+    return true;
+}
+
+#define DO_WIDEN_ACC_ZZZW(NAME, helper, DATA)                           \
+static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a)             \
+{                                                                       \
+    static gen_helper_gvec_4 * const fns[4] = {                         \
+        NULL,                                                           \
+        gen_helper_sve2_##helper##_h,                                   \
+        gen_helper_sve2_##helper##_s,                                   \
+        gen_helper_sve2_##helper##_d,                                   \
+    };                                                                  \
+                                                                        \
+    return do_widen_acc_zzzw(s, a, fns, DATA);                          \
+}
+
+DO_WIDEN_ACC_ZZZW(SQDMLALB_zzzw, sqdmlal_zzzw, 0)
+DO_WIDEN_ACC_ZZZW(SQDMLALT_zzzw, sqdmlal_zzzw, 3)
+DO_WIDEN_ACC_ZZZW(SQDMLALBT, sqdmlal_zzzw, 2)
+DO_WIDEN_ACC_ZZZW(SQDMLSLB_zzzw, sqdmlsl_zzzw, 0)
+DO_WIDEN_ACC_ZZZW(SQDMLSLT_zzzw, sqdmlsl_zzzw, 3)
+DO_WIDEN_ACC_ZZZW(SQDMLSLBT, sqdmlsl_zzzw, 2)
+DO_WIDEN_ACC_ZZZW(SMLALB_zzzw, smlal_zzzw, 0)
+DO_WIDEN_ACC_ZZZW(SMLALT_zzzw, smlal_zzzw, 3)
+DO_WIDEN_ACC_ZZZW(UMLALB_zzzw, umlal_zzzw, 0)
+DO_WIDEN_ACC_ZZZW(UMLALT_zzzw, umlal_zzzw, 3)
+DO_WIDEN_ACC_ZZZW(SMLSLB_zzzw, smlsl_zzzw, 0)
+DO_WIDEN_ACC_ZZZW(SMLSLT_zzzw, smlsl_zzzw, 3)
+DO_WIDEN_ACC_ZZZW(UMLSLB_zzzw, umlsl_zzzw, 0)
+DO_WIDEN_ACC_ZZZW(UMLSLT_zzzw, umlsl_zzzw, 3)
+DO_WIDEN_ACC_ZZZW(SABALB, sabal, 0)
+DO_WIDEN_ACC_ZZZW(SABALT, sabal, 3)
+DO_WIDEN_ACC_ZZZW(UABALB, uabal, 0)
+DO_WIDEN_ACC_ZZZW(UABALT, uabal, 3)
+
+#undef DO_WIDEN_ACC_ZZZW
+
+static bool do_sqrdmla_zzzz(DisasContext *s, arg_rprrr_esz *a,
+                            gen_helper_gvec_4 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_SQRDMLAH_zzzz(DisasContext *s, arg_rprrr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        gen_helper_sve2_sqrdmlah_b,
+        gen_helper_sve2_sqrdmlah_h,
+        gen_helper_sve2_sqrdmlah_s,
+        gen_helper_sve2_sqrdmlah_d,
+    };
+
+    return do_sqrdmla_zzzz(s, a, fns);
+}
+
+static bool trans_SQRDMLSH_zzzz(DisasContext *s, arg_rprrr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        gen_helper_sve2_sqrdmlsh_b,
+        gen_helper_sve2_sqrdmlsh_h,
+        gen_helper_sve2_sqrdmlsh_s,
+        gen_helper_sve2_sqrdmlsh_d,
+    };
+
+    return do_sqrdmla_zzzz(s, a, fns);
+}
+
+static bool do_sve2_zzzz_rot(DisasContext *s, arg_rprrr_rot_esz *a,
+                             gen_helper_gvec_4 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, a->rot, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_CMLA_zzzz(DisasContext *s, arg_rprrr_rot_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        gen_helper_sve2_cmla_zzzz_b,
+        gen_helper_sve2_cmla_zzzz_h,
+        gen_helper_sve2_cmla_zzzz_s,
+        gen_helper_sve2_cmla_zzzz_d,
+    };
+
+    return do_sve2_zzzz_rot(s, a, fns);
+}
+
+static bool trans_CDOT_zzzz(DisasContext *s, arg_rprrr_rot_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL,
+        NULL,
+        gen_helper_sve2_cdot_zzzz_s,
+        gen_helper_sve2_cdot_zzzz_d,
+    };
+
+    return do_sve2_zzzz_rot(s, a, fns);
+}
+
+static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_rprrr_rot_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        gen_helper_sve2_sqrdcmlah_zzzz_b,
+        gen_helper_sve2_sqrdcmlah_zzzz_h,
+        gen_helper_sve2_sqrdcmlah_zzzz_s,
+        gen_helper_sve2_sqrdcmlah_zzzz_d,
+    };
+
+    return do_sve2_zzzz_rot(s, a, fns);
+}
+
+static bool do_sve2_rrxr_rot(DisasContext *s, arg_disas_sve43 *a,
+                             gen_helper_gvec_4 *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || fn == NULL) {
+        return false;
+    }
+    tcg_debug_assert(a->rd == a->ra);
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, (a->index << 2) | a->rot, fn);
+    }
+    return true;
+}
+
+static bool trans_CMLA_zzxz_h(DisasContext *s, arg_disas_sve43 *a)
+{
+    return do_sve2_rrxr_rot(s, a, gen_helper_sve2_cmla_idx_h);
+}
+
+static bool trans_CMLA_zzxz_s(DisasContext *s, arg_disas_sve43 *a)
+{
+    return do_sve2_rrxr_rot(s, a, gen_helper_sve2_cmla_idx_s);
+}
+
+static bool trans_SQRDCMLAH_zzxz_h(DisasContext *s, arg_disas_sve43 *a)
+{
+    return do_sve2_rrxr_rot(s, a, gen_helper_sve2_sqrdcmlah_idx_h);
+}
+
+static bool trans_SQRDCMLAH_zzxz_s(DisasContext *s, arg_disas_sve43 *a)
+{
+    return do_sve2_rrxr_rot(s, a, gen_helper_sve2_sqrdcmlah_idx_s);
+}
+
+static bool trans_CDOT_zzxw_s(DisasContext *s, arg_disas_sve43 *a)
+{
+    return do_sve2_rrxr_rot(s, a, gen_helper_sve2_cdot_idx_s);
+}
+
+static bool trans_CDOT_zzxw_d(DisasContext *s, arg_disas_sve43 *a)
+{
+    return do_sve2_rrxr_rot(s, a, gen_helper_sve2_cdot_idx_d);
+}
+
+static bool do_sve2_aba(DisasContext *s, arg_rrr_esz *a,
+                        gen_helper_gvec_3 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_SABA(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_gvec_saba_b,
+        gen_helper_gvec_saba_h,
+        gen_helper_gvec_saba_s,
+        gen_helper_gvec_saba_d,
+    };
+
+    return do_sve2_aba(s, a, fns);
+}
+
+static bool trans_UABA(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_gvec_uaba_b,
+        gen_helper_gvec_uaba_h,
+        gen_helper_gvec_uaba_s,
+        gen_helper_gvec_uaba_d,
+    };
+
+    return do_sve2_aba(s, a, fns);
+}
+
+static bool do_widen_zzz(DisasContext *s, arg_rrr_esz *a,
+                         gen_helper_gvec_3 * const fns[4], int data)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, data, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_SADDLB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_saddl_h,
+        gen_helper_sve2_saddl_s,
+        gen_helper_sve2_saddl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_SADDLT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_saddl_h,
+        gen_helper_sve2_saddl_s,
+        gen_helper_sve2_saddl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 3);
+}
+
+static bool trans_UADDLB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_uaddl_h,
+        gen_helper_sve2_uaddl_s,
+        gen_helper_sve2_uaddl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_UADDLT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_uaddl_h,
+        gen_helper_sve2_uaddl_s,
+        gen_helper_sve2_uaddl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 3);
+}
+
+static bool trans_SSUBLB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_ssubl_h,
+        gen_helper_sve2_ssubl_s,
+        gen_helper_sve2_ssubl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_SSUBLT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_ssubl_h,
+        gen_helper_sve2_ssubl_s,
+        gen_helper_sve2_ssubl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 3);
+}
+
+static bool trans_USUBLB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_usubl_h,
+        gen_helper_sve2_usubl_s,
+        gen_helper_sve2_usubl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_USUBLT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_usubl_h,
+        gen_helper_sve2_usubl_s,
+        gen_helper_sve2_usubl_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 3);
+}
+
+#define DO_WIDEN_ZZZ_TRANS(NAME, helper, DATA)                          \
+static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_3 * const fns[4] = {                         \
+        NULL,                                                           \
+        gen_helper_sve2_##helper##_h,                                   \
+        gen_helper_sve2_##helper##_s,                                   \
+        gen_helper_sve2_##helper##_d,                                   \
+    };                                                                  \
+                                                                        \
+    return do_widen_zzz(s, a, fns, DATA);                               \
+}
+
+DO_WIDEN_ZZZ_TRANS(SADDLBT, saddl, 2)
+DO_WIDEN_ZZZ_TRANS(SSUBLBT, ssubl, 2)
+DO_WIDEN_ZZZ_TRANS(SSUBLTB, ssubl, 1)
+DO_WIDEN_ZZZ_TRANS(SABDLB, sabdl, 0)
+DO_WIDEN_ZZZ_TRANS(SABDLT, sabdl, 3)
+DO_WIDEN_ZZZ_TRANS(UABDLB, uabdl, 0)
+DO_WIDEN_ZZZ_TRANS(UABDLT, uabdl, 3)
+DO_WIDEN_ZZZ_TRANS(SMULLB_zzz, smull_zzz, 0)
+DO_WIDEN_ZZZ_TRANS(SMULLT_zzz, smull_zzz, 3)
+DO_WIDEN_ZZZ_TRANS(UMULLB_zzz, umull_zzz, 0)
+DO_WIDEN_ZZZ_TRANS(UMULLT_zzz, umull_zzz, 3)
+DO_WIDEN_ZZZ_TRANS(SQDMULLB_zzz, sqdmull_zzz, 0)
+DO_WIDEN_ZZZ_TRANS(SQDMULLT_zzz, sqdmull_zzz, 3)
+
+#undef DO_WIDEN_ZZZ_TRANS
+
+static bool trans_SADDWB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_saddw_h,
+        gen_helper_sve2_saddw_s,
+        gen_helper_sve2_saddw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_SADDWT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_saddw_h,
+        gen_helper_sve2_saddw_s,
+        gen_helper_sve2_saddw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 1);
+}
+
+static bool trans_UADDWB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_uaddw_h,
+        gen_helper_sve2_uaddw_s,
+        gen_helper_sve2_uaddw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_UADDWT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_uaddw_h,
+        gen_helper_sve2_uaddw_s,
+        gen_helper_sve2_uaddw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 1);
+}
+
+static bool trans_SSUBWB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_ssubw_h,
+        gen_helper_sve2_ssubw_s,
+        gen_helper_sve2_ssubw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_SSUBWT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_ssubw_h,
+        gen_helper_sve2_ssubw_s,
+        gen_helper_sve2_ssubw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 1);
+}
+
+static bool trans_USUBWB(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_usubw_h,
+        gen_helper_sve2_usubw_s,
+        gen_helper_sve2_usubw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 0);
+}
+
+static bool trans_USUBWT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_usubw_h,
+        gen_helper_sve2_usubw_s,
+        gen_helper_sve2_usubw_d,
+    };
+
+    return do_widen_zzz(s, a, fns, 1);
+}
+
+static bool do_shll(DisasContext *s, arg_rri_esz *a,
+                    gen_helper_gvec_2 * const fns[3], int sel)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || a->esz < 0 || a->esz > 2) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_2_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn), vsz, vsz,
+                           (a->imm << 1) | sel, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_SSHLLB(DisasContext *s, arg_rri_esz *a)
+{
+    static gen_helper_gvec_2 * const fns[3] = {
+        gen_helper_sve2_sshll_h,
+        gen_helper_sve2_sshll_s,
+        gen_helper_sve2_sshll_d,
+    };
+
+    return do_shll(s, a, fns, 0);
+}
+
+static bool trans_SSHLLT(DisasContext *s, arg_rri_esz *a)
+{
+    static gen_helper_gvec_2 * const fns[3] = {
+        gen_helper_sve2_sshll_h,
+        gen_helper_sve2_sshll_s,
+        gen_helper_sve2_sshll_d,
+    };
+
+    return do_shll(s, a, fns, 1);
+}
+
+static bool trans_USHLLB(DisasContext *s, arg_rri_esz *a)
+{
+    static gen_helper_gvec_2 * const fns[3] = {
+        gen_helper_sve2_ushll_h,
+        gen_helper_sve2_ushll_s,
+        gen_helper_sve2_ushll_d,
+    };
+
+    return do_shll(s, a, fns, 0);
+}
+
+static bool trans_USHLLT(DisasContext *s, arg_rri_esz *a)
+{
+    static gen_helper_gvec_2 * const fns[3] = {
+        gen_helper_sve2_ushll_h,
+        gen_helper_sve2_ushll_s,
+        gen_helper_sve2_ushll_d,
+    };
+
+    return do_shll(s, a, fns, 1);
+}
+
+static bool do_narrow_zzz(DisasContext *s, arg_rrr_esz *a,
+                          gen_helper_gvec_3 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || fns[a->esz] == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
+#define DO_SVE2_NARROW(NAME, name)                                      \
+static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_3 * const fns[4] = {                         \
+        NULL,                                                           \
+        gen_helper_sve2_##name##_h,                                     \
+        gen_helper_sve2_##name##_s,                                     \
+        gen_helper_sve2_##name##_d,                                     \
+    };                                                                  \
+                                                                        \
+    return do_narrow_zzz(s, a, fns);                                    \
+}
+
+DO_SVE2_NARROW(ADDHNB, addhnb)
+DO_SVE2_NARROW(ADDHNT, addhnt)
+DO_SVE2_NARROW(RADDHNB, raddhnb)
+DO_SVE2_NARROW(RADDHNT, raddhnt)
+DO_SVE2_NARROW(SUBHNB, subhnb)
+DO_SVE2_NARROW(SUBHNT, subhnt)
+DO_SVE2_NARROW(RSUBHNB, rsubhnb)
+DO_SVE2_NARROW(RSUBHNT, rsubhnt)
+
+#undef DO_SVE2_NARROW
+
+static bool do_xtn(DisasContext *s, arg_rri_esz *a,
+                   gen_helper_gvec_2 * const fns[3])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) ||
+        a->esz < 0 || a->esz > 2 || a->imm != 0) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_2_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
+#define DO_XTN_TRANS(NAME, name)                                        \
+static bool trans_##NAME(DisasContext *s, arg_rri_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_2 * const fns[3] = {                         \
+        gen_helper_sve2_##name##_h,                                     \
+        gen_helper_sve2_##name##_s,                                     \
+        gen_helper_sve2_##name##_d,                                     \
+    };                                                                  \
+                                                                        \
+    return do_xtn(s, a, fns);                                           \
+}
+
+DO_XTN_TRANS(SQXTNB, sqxtnb)
+DO_XTN_TRANS(SQXTNT, sqxtnt)
+DO_XTN_TRANS(UQXTNB, uqxtnb)
+DO_XTN_TRANS(UQXTNT, uqxtnt)
+DO_XTN_TRANS(SQXTUNB, sqxtunb)
+DO_XTN_TRANS(SQXTUNT, sqxtunt)
+
+#undef DO_XTN_TRANS
+
+static bool do_shr_narrow(DisasContext *s, arg_rri_esz *a,
+                          gen_helper_gvec_2 * const fns[3])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || a->esz < 0 || a->esz > 2) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_2_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vsz, vsz, a->imm, fns[a->esz]);
+    }
+    return true;
+}
+
+#define DO_SHR_NARROW_TRANS(NAME, name)                                 \
+static bool trans_##NAME(DisasContext *s, arg_rri_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_2 * const fns[3] = {                         \
+        gen_helper_sve2_##name##_h,                                     \
+        gen_helper_sve2_##name##_s,                                     \
+        gen_helper_sve2_##name##_d,                                     \
+    };                                                                  \
+                                                                        \
+    return do_shr_narrow(s, a, fns);                                    \
+}
+
+DO_SHR_NARROW_TRANS(SQSHRUNB, sqshrunb)
+DO_SHR_NARROW_TRANS(SQSHRUNT, sqshrunt)
+DO_SHR_NARROW_TRANS(SQRSHRUNB, sqrshrunb)
+DO_SHR_NARROW_TRANS(SQRSHRUNT, sqrshrunt)
+DO_SHR_NARROW_TRANS(SHRNB, shrnb)
+DO_SHR_NARROW_TRANS(SHRNT, shrnt)
+DO_SHR_NARROW_TRANS(RSHRNB, rshrnb)
+DO_SHR_NARROW_TRANS(RSHRNT, rshrnt)
+DO_SHR_NARROW_TRANS(SQSHRNB, sqshrnb)
+DO_SHR_NARROW_TRANS(SQSHRNT, sqshrnt)
+DO_SHR_NARROW_TRANS(SQRSHRNB, sqrshrnb)
+DO_SHR_NARROW_TRANS(SQRSHRNT, sqrshrnt)
+DO_SHR_NARROW_TRANS(UQSHRNB, uqshrnb)
+DO_SHR_NARROW_TRANS(UQSHRNT, uqshrnt)
+DO_SHR_NARROW_TRANS(UQRSHRNB, uqrshrnb)
+DO_SHR_NARROW_TRANS(UQRSHRNT, uqrshrnt)
+
+#undef DO_SHR_NARROW_TRANS
+
+static bool do_sra_zzi(DisasContext *s, arg_rri_esz *a,
+                       gen_helper_gvec_2 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || a->esz < 0 || a->esz > 3) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_2_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vsz, vsz, a->imm, fns[a->esz]);
+    }
+    return true;
+}
+
+#define DO_SRA_ZZI_TRANS(NAME, name)                                    \
+static bool trans_##NAME(DisasContext *s, arg_rri_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_2 * const fns[4] = {                         \
+        gen_helper_sve2_##name##_b,                                     \
+        gen_helper_sve2_##name##_h,                                     \
+        gen_helper_sve2_##name##_s,                                     \
+        gen_helper_sve2_##name##_d,                                     \
+    };                                                                  \
+                                                                        \
+    return do_sra_zzi(s, a, fns);                                      \
+}
+
+DO_SRA_ZZI_TRANS(SSRA, ssra)
+DO_SRA_ZZI_TRANS(USRA, usra)
+DO_SRA_ZZI_TRANS(SRSRA, srsra)
+DO_SRA_ZZI_TRANS(URSRA, ursra)
+
+#undef DO_SRA_ZZI_TRANS
+
+static bool do_shift_insert_zzi(DisasContext *s, arg_rri_esz *a,
+                                gen_helper_gvec_2 * const fns[4],
+                                bool insert_left)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || a->esz < 0 || a->esz > 3) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned bits = 8u << a->esz;
+        unsigned vsz = vec_full_reg_size(s);
+        uint32_t rd_ofs = vec_full_reg_offset(s, a->rd);
+        uint32_t rn_ofs = vec_full_reg_offset(s, a->rn);
+
+        if (insert_left && a->imm == 0) {
+            tcg_gen_gvec_mov(tcg_ctx, a->esz, rd_ofs, rn_ofs, vsz, vsz);
+        } else if (!insert_left && a->imm == bits) {
+            return true;
+        } else {
+            tcg_gen_gvec_2_ool(tcg_ctx, rd_ofs, rn_ofs, vsz, vsz,
+                               a->imm, fns[a->esz]);
+        }
+    }
+    return true;
+}
+
+#define DO_SHIFT_INSERT_TRANS(NAME, name, LEFT)                         \
+static bool trans_##NAME(DisasContext *s, arg_rri_esz *a)               \
+{                                                                       \
+    static gen_helper_gvec_2 * const fns[4] = {                         \
+        gen_helper_sve2_##name##_b,                                     \
+        gen_helper_sve2_##name##_h,                                     \
+        gen_helper_sve2_##name##_s,                                     \
+        gen_helper_sve2_##name##_d,                                     \
+    };                                                                  \
+                                                                        \
+    return do_shift_insert_zzi(s, a, fns, LEFT);                       \
+}
+
+DO_SHIFT_INSERT_TRANS(SRI, sri, false)
+DO_SHIFT_INSERT_TRANS(SLI, sli, true)
+
+#undef DO_SHIFT_INSERT_TRANS
+
+static bool do_eoril(DisasContext *s, arg_rrr_esz *a, int data)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_eoril_b,
+        gen_helper_sve2_eoril_h,
+        gen_helper_sve2_eoril_s,
+        gen_helper_sve2_eoril_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, data, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_EORBT(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_eoril(s, a, 2);
+}
+
+static bool trans_EORTB(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_eoril(s, a, 1);
+}
+
+static bool do_bitperm(DisasContext *s, arg_rrr_esz *a,
+                       gen_helper_gvec_3 * const fns[4])
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2_bitperm, s)) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_BEXT(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_bext_b,
+        gen_helper_sve2_bext_h,
+        gen_helper_sve2_bext_s,
+        gen_helper_sve2_bext_d,
+    };
+
+    return do_bitperm(s, a, fns);
+}
+
+static bool trans_BDEP(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_bdep_b,
+        gen_helper_sve2_bdep_h,
+        gen_helper_sve2_bdep_s,
+        gen_helper_sve2_bdep_d,
+    };
+
+    return do_bitperm(s, a, fns);
+}
+
+static bool trans_BGRP(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_bgrp_b,
+        gen_helper_sve2_bgrp_h,
+        gen_helper_sve2_bgrp_s,
+        gen_helper_sve2_bgrp_d,
+    };
+
+    return do_bitperm(s, a, fns);
+}
+
+static bool do_cadd(DisasContext *s, arg_rrr_esz *a,
+                    gen_helper_gvec_3 * const fns[4], int data)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, data, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_CADD_rot90(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_cadd_b,
+        gen_helper_sve2_cadd_h,
+        gen_helper_sve2_cadd_s,
+        gen_helper_sve2_cadd_d,
+    };
+
+    return do_cadd(s, a, fns, 0);
+}
+
+static bool trans_CADD_rot270(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_cadd_b,
+        gen_helper_sve2_cadd_h,
+        gen_helper_sve2_cadd_s,
+        gen_helper_sve2_cadd_d,
+    };
+
+    return do_cadd(s, a, fns, 1);
+}
+
+static bool trans_SQCADD_rot90(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_sqcadd_b,
+        gen_helper_sve2_sqcadd_h,
+        gen_helper_sve2_sqcadd_s,
+        gen_helper_sve2_sqcadd_d,
+    };
+
+    return do_cadd(s, a, fns, 0);
+}
+
+static bool trans_SQCADD_rot270(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_sqcadd_b,
+        gen_helper_sve2_sqcadd_h,
+        gen_helper_sve2_sqcadd_s,
+        gen_helper_sve2_sqcadd_d,
+    };
+
+    return do_cadd(s, a, fns, 1);
+}
+
+static bool do_adcl(DisasContext *s, arg_rprrr_esz *a, int sel)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_4 * const fns[2] = {
+        gen_helper_sve2_adcl_s,
+        gen_helper_sve2_adcl_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        int data = (a->esz & 2) | sel;
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, data, fns[a->esz & 1]);
+    }
+    return true;
+}
+
+static bool trans_ADCLB(DisasContext *s, arg_rprrr_esz *a)
+{
+    return do_adcl(s, a, 0);
+}
+
+static bool trans_ADCLT(DisasContext *s, arg_rprrr_esz *a)
+{
+    return do_adcl(s, a, 1);
+}
+
+/*
+ *** SVE Integer Arithmetic - Unpredicated Group
+ */
+
+static bool trans_ADD_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_vector3_z(s, tcg_gen_gvec_add, a->esz, a->rd, a->rn, a->rm);
+}
+
+static bool trans_SUB_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_vector3_z(s, tcg_gen_gvec_sub, a->esz, a->rd, a->rn, a->rm);
+}
+
+static bool trans_SQADD_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_vector3_z(s, tcg_gen_gvec_ssadd, a->esz, a->rd, a->rn, a->rm);
+}
+
+static bool trans_SQSUB_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_vector3_z(s, tcg_gen_gvec_sssub, a->esz, a->rd, a->rn, a->rm);
+}
+
+static bool trans_UQADD_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_vector3_z(s, tcg_gen_gvec_usadd, a->esz, a->rd, a->rn, a->rm);
+}
+
+static bool trans_UQSUB_zzz(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_vector3_z(s, tcg_gen_gvec_ussub, a->esz, a->rd, a->rn, a->rm);
+}
+
+/*
+ *** SVE Integer Arithmetic - Binary Predicated Group
+ */
+
+static bool do_zpzz_ool(DisasContext *s, arg_rprr_esz *a, gen_helper_gvec_4 *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    unsigned vsz = vec_full_reg_size(s);
+    if (fn == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           pred_full_reg_offset(s, a->pg),
+                           vsz, vsz, 0, fn);
+    }
+    return true;
+}
+
+/* Select active elememnts from Zn and inactive elements from Zm,
+ * storing the result in Zd.
+ */
+static void do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_4 * const fns[4] = {
+        gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
+        gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d
+    };
+    unsigned vsz = vec_full_reg_size(s);
+    tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, rd),
+                       vec_full_reg_offset(s, rn),
+                       vec_full_reg_offset(s, rm),
+                       pred_full_reg_offset(s, pg),
+                       vsz, vsz, 0, fns[esz]);
+}
+
+#define DO_ZPZZ(NAME, name) \
+static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a)         \
+{                                                                         \
+    static gen_helper_gvec_4 * const fns[4] = {                           \
+        gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h,   \
+        gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d,   \
+    };                                                                    \
+    return do_zpzz_ool(s, a, fns[a->esz]);                                \
+}
+
+DO_ZPZZ(AND, and)
+DO_ZPZZ(EOR, eor)
+DO_ZPZZ(ORR, orr)
+DO_ZPZZ(BIC, bic)
+
+DO_ZPZZ(ADD, add)
+DO_ZPZZ(SUB, sub)
+
+DO_ZPZZ(SMAX, smax)
+DO_ZPZZ(UMAX, umax)
+DO_ZPZZ(SMIN, smin)
+DO_ZPZZ(UMIN, umin)
+DO_ZPZZ(SABD, sabd)
+DO_ZPZZ(UABD, uabd)
+
+static bool trans_SADALP_zpzz(DisasContext *s, arg_rprr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_sadalp_zpzz_h,
+        gen_helper_sve2_sadalp_zpzz_s,
+        gen_helper_sve2_sadalp_zpzz_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpzz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_UADALP_zpzz(DisasContext *s, arg_rprr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL,
+        gen_helper_sve2_uadalp_zpzz_h,
+        gen_helper_sve2_uadalp_zpzz_s,
+        gen_helper_sve2_uadalp_zpzz_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpzz_ool(s, a, fns[a->esz]);
+}
+
+#define DO_ZPZZ_SVE2(NAME, name)                                        \
+static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a)              \
+{                                                                       \
+    static gen_helper_gvec_4 * const fns[4] = {                         \
+        gen_helper_sve2_##name##_zpzz_b,                                \
+        gen_helper_sve2_##name##_zpzz_h,                                \
+        gen_helper_sve2_##name##_zpzz_s,                                \
+        gen_helper_sve2_##name##_zpzz_d,                                \
+    };                                                                  \
+                                                                        \
+    if (!dc_isar_feature(aa64_sve2, s)) {                               \
+        return false;                                                   \
+    }                                                                   \
+    return do_zpzz_ool(s, a, fns[a->esz]);                              \
+}
+
+DO_ZPZZ_SVE2(SHADD, shadd)
+DO_ZPZZ_SVE2(UHADD, uhadd)
+DO_ZPZZ_SVE2(SHSUB, shsub)
+DO_ZPZZ_SVE2(UHSUB, uhsub)
+DO_ZPZZ_SVE2(SRHADD, srhadd)
+DO_ZPZZ_SVE2(URHADD, urhadd)
+DO_ZPZZ_SVE2(ADDP, addp)
+DO_ZPZZ_SVE2(SMAXP, smaxp)
+DO_ZPZZ_SVE2(UMAXP, umaxp)
+DO_ZPZZ_SVE2(SMINP, sminp)
+DO_ZPZZ_SVE2(UMINP, uminp)
+DO_ZPZZ_SVE2(SRSHL, srshl)
+DO_ZPZZ_SVE2(URSHL, urshl)
+DO_ZPZZ_SVE2(SQSHL, sqshl)
+DO_ZPZZ_SVE2(UQSHL, uqshl)
+DO_ZPZZ_SVE2(SQRSHL, sqrshl)
+DO_ZPZZ_SVE2(UQRSHL, uqrshl)
+DO_ZPZZ_SVE2(SQADD_zpzz, sqadd)
+DO_ZPZZ_SVE2(UQADD_zpzz, uqadd)
+DO_ZPZZ_SVE2(SQSUB_zpzz, sqsub)
+DO_ZPZZ_SVE2(UQSUB_zpzz, uqsub)
+DO_ZPZZ_SVE2(SUQADD, suqadd)
+DO_ZPZZ_SVE2(USQADD, usqadd)
+
+DO_ZPZZ(MUL, mul)
+DO_ZPZZ(SMULH, smulh)
+DO_ZPZZ(UMULH, umulh)
+
+DO_ZPZZ(ASR, asr)
+DO_ZPZZ(LSR, lsr)
+DO_ZPZZ(LSL, lsl)
+
+static bool trans_SDIV_zpzz(DisasContext *s, arg_rprr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d
+    };
+    return do_zpzz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_UDIV_zpzz(DisasContext *s, arg_rprr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d
+    };
+    return do_zpzz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_SEL_zpzz(DisasContext *s, arg_rprr_esz *a)
+{
+    if (sve_access_check(s)) {
+        do_sel_z(s, a->rd, a->rn, a->rm, a->pg, a->esz);
+    }
+    return true;
+}
+
+#undef DO_ZPZZ
+#undef DO_ZPZZ_SVE2
+
+/*
+ *** SVE Integer Arithmetic - Unary Predicated Group
+ */
+
+static bool do_zpz_ool(DisasContext *s, arg_rpr_esz *a, gen_helper_gvec_3 *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (fn == NULL) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           pred_full_reg_offset(s, a->pg),
+                           vsz, vsz, 0, fn);
+    }
+    return true;
+}
+
+#define DO_ZPZ(NAME, name) \
+static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a)           \
+{                                                                   \
+    static gen_helper_gvec_3 * const fns[4] = {                     \
+        gen_helper_sve_##name##_b, gen_helper_sve_##name##_h,       \
+        gen_helper_sve_##name##_s, gen_helper_sve_##name##_d,       \
+    };                                                              \
+    return do_zpz_ool(s, a, fns[a->esz]);                           \
+}
+
+DO_ZPZ(CLS, cls)
+DO_ZPZ(CLZ, clz)
+DO_ZPZ(CNT_zpz, cnt_zpz)
+DO_ZPZ(CNOT, cnot)
+DO_ZPZ(NOT_zpz, not_zpz)
+DO_ZPZ(ABS, abs)
+DO_ZPZ(NEG, neg)
+
+static bool trans_SQABS(DisasContext *s, arg_rpr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_sqabs_b,
+        gen_helper_sve2_sqabs_h,
+        gen_helper_sve2_sqabs_s,
+        gen_helper_sve2_sqabs_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_SQNEG(DisasContext *s, arg_rpr_esz *a)
+{
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_sqneg_b,
+        gen_helper_sve2_sqneg_h,
+        gen_helper_sve2_sqneg_s,
+        gen_helper_sve2_sqneg_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ool(s, a, fns[a->esz]);
 }
 
-DO_ZPZ(CLS, cls)
-DO_ZPZ(CLZ, clz)
-DO_ZPZ(CNT_zpz, cnt_zpz)
-DO_ZPZ(CNOT, cnot)
-DO_ZPZ(NOT_zpz, not_zpz)
-DO_ZPZ(ABS, abs)
-DO_ZPZ(NEG, neg)
+static bool trans_URECPE(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ool(s, a, a->esz == 2 ? gen_helper_sve2_urecpe_s : NULL);
+}
+
+static bool trans_URSQRTE(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ool(s, a, a->esz == 2 ? gen_helper_sve2_ursqrte_s : NULL);
+}
 
 static bool trans_FABS(DisasContext *s, arg_rpr_esz *a)
 {
@@ -971,6 +2495,23 @@ static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a)
     return true;
 }
 
+static bool trans_ADDSVL(DisasContext *s, arg_ADDSVL *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (sme_enabled_check(s)) {
+        TCGv_i64 rd = cpu_reg_sp(s, a->rd);
+        TCGv_i64 rn = cpu_reg_sp(s, a->rn);
+
+        tcg_gen_addi_i64(tcg_ctx, rd, rn,
+                         a->imm * streaming_vec_reg_size(s));
+    }
+    return true;
+}
+
 static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -982,6 +2523,23 @@ static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a)
     return true;
 }
 
+static bool trans_ADDSPL(DisasContext *s, arg_ADDSPL *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (sme_enabled_check(s)) {
+        TCGv_i64 rd = cpu_reg_sp(s, a->rd);
+        TCGv_i64 rn = cpu_reg_sp(s, a->rn);
+
+        tcg_gen_addi_i64(tcg_ctx, rd, rn,
+                         a->imm * streaming_pred_reg_size(s));
+    }
+    return true;
+}
+
 static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -992,6 +2550,22 @@ static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
     return true;
 }
 
+static bool trans_RDSVL(DisasContext *s, arg_RDSVL *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (sme_enabled_check(s)) {
+        TCGv_i64 reg = cpu_reg(s, a->rd);
+
+        tcg_gen_movi_i64(tcg_ctx, reg,
+                         a->imm * streaming_vec_reg_size(s));
+    }
+    return true;
+}
+
 /*
  *** SVE Compute Vector Address Group
  */
@@ -999,7 +2573,7 @@ static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
 static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    if (sve_access_check(s)) {
+    if (sve_nonstreaming_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
         tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
@@ -1045,7 +2619,7 @@ static bool trans_FEXPA(DisasContext *s, arg_rr_esz *a)
     if (a->esz == 0) {
         return false;
     }
-    if (sve_access_check(s)) {
+    if (sve_nonstreaming_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
         tcg_gen_gvec_2_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
@@ -1066,7 +2640,7 @@ static bool trans_FTSSEL(DisasContext *s, arg_rrr_esz *a)
     if (a->esz == 0) {
         return false;
     }
-    if (sve_access_check(s)) {
+    if (sve_nonstreaming_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
         tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
@@ -1526,6 +3100,7 @@ static bool trans_PTRUE(DisasContext *s, arg_PTRUE *a)
 static bool trans_SETFFR(DisasContext *s, arg_SETFFR *a)
 {
     /* Note pat == 31 is #all, to set all elements.  */
+    s->is_nonstreaming = true;
     return do_predset(s, 0, FFR_PRED_NUM, 31, false);
 }
 
@@ -1544,16 +3119,19 @@ static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a)
         .rd = a->rd, .pg = a->pg, .s = a->s,
         .rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM,
     };
+    s->is_nonstreaming = true;
     return trans_AND_pppp(s, &alt_a);
 }
 
 static bool trans_RDFFR(DisasContext *s, arg_RDFFR *a)
 {
+    s->is_nonstreaming = true;
     return do_mov_p(s, a->rd, FFR_PRED_NUM);
 }
 
 static bool trans_WRFFR(DisasContext *s, arg_WRFFR *a)
 {
+    s->is_nonstreaming = true;
     return do_mov_p(s, FFR_PRED_NUM, a->rn);
 }
 
@@ -1597,6 +3175,60 @@ static bool trans_PNEXT(DisasContext *s, arg_rr_esz *a)
     return do_pfirst_pnext(s, a, gen_helper_sve_pnext);
 }
 
+static bool trans_PSEL(DisasContext *s, arg_PSEL *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vl = vec_full_reg_size(s);
+    int pl = pred_gvec_reg_size(s);
+    int elements = vl >> a->esz;
+    TCGv_i64 tmp, didx, dbit;
+    TCGv_ptr ptr;
+
+    if (!dc_isar_feature(aa64_sme, s)) {
+        return false;
+    }
+    if (!sve_access_check(s)) {
+        return true;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    dbit = tcg_temp_new_i64(tcg_ctx);
+    didx = tcg_temp_new_i64(tcg_ctx);
+    ptr = tcg_temp_new_ptr(tcg_ctx);
+
+    tcg_gen_addi_i64(tcg_ctx, tmp, cpu_reg(s, a->rv), a->imm);
+    if (is_power_of_2(elements)) {
+        tcg_gen_andi_i64(tcg_ctx, tmp, tmp, elements - 1);
+    } else {
+        tcg_gen_remu_i64(tcg_ctx, tmp, tmp,
+                         tcg_constant_i64(tcg_ctx, elements));
+    }
+
+    tcg_gen_shli_i64(tcg_ctx, tmp, tmp, a->esz);
+    tcg_gen_andi_i64(tcg_ctx, dbit, tmp, 7);
+    tcg_gen_shri_i64(tcg_ctx, didx, tmp, 3);
+#ifdef HOST_WORDS_BIGENDIAN
+    tcg_gen_xori_i64(tcg_ctx, didx, didx, 7);
+#endif
+
+    tcg_gen_trunc_i64_ptr(tcg_ctx, ptr, didx);
+    tcg_gen_add_ptr(tcg_ctx, ptr, ptr, tcg_ctx->cpu_env);
+    tcg_gen_ld8u_i64(tcg_ctx, tmp, ptr, pred_full_reg_offset(s, a->pm));
+
+    tcg_gen_shr_i64(tcg_ctx, tmp, tmp, dbit);
+    tcg_gen_andi_i64(tcg_ctx, tmp, tmp, 1);
+    tcg_gen_neg_i64(tcg_ctx, tmp, tmp);
+
+    tcg_gen_gvec_ands(tcg_ctx, MO_64, pred_full_reg_offset(s, a->pd),
+                      pred_full_reg_offset(s, a->pn), tmp, pl, pl);
+
+    tcg_temp_free_i64(tcg_ctx, tmp);
+    tcg_temp_free_i64(tcg_ctx, dbit);
+    tcg_temp_free_i64(tcg_ctx, didx);
+    tcg_temp_free_ptr(tcg_ctx, ptr);
+    return true;
+}
+
 /*
  *** SVE Element Count Group
  */
@@ -2024,7 +3656,7 @@ static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a)
  *** SVE Permute Extract Group
  */
 
-static bool trans_EXT(DisasContext *s, arg_EXT *a)
+static bool do_ext(DisasContext *s, int rd, int rn, int rm, int imm)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     if (!sve_access_check(s)) {
@@ -2032,11 +3664,11 @@ static bool trans_EXT(DisasContext *s, arg_EXT *a)
     }
 
     unsigned vsz = vec_full_reg_size(s);
-    unsigned n_ofs = a->imm >= vsz ? 0 : a->imm;
+    unsigned n_ofs = imm >= vsz ? 0 : imm;
     unsigned n_siz = vsz - n_ofs;
-    unsigned d = vec_full_reg_offset(s, a->rd);
-    unsigned n = vec_full_reg_offset(s, a->rn);
-    unsigned m = vec_full_reg_offset(s, a->rm);
+    unsigned d = vec_full_reg_offset(s, rd);
+    unsigned n = vec_full_reg_offset(s, rn);
+    unsigned m = vec_full_reg_offset(s, rm);
 
     /* Use host vector move insns if we have appropriate sizes
      * and no unfortunate overlap.
@@ -2055,6 +3687,16 @@ static bool trans_EXT(DisasContext *s, arg_EXT *a)
     return true;
 }
 
+static bool trans_EXT(DisasContext *s, arg_EXT *a)
+{
+    return do_ext(s, a->rd, a->rn, a->rm, a->imm);
+}
+
+static bool trans_EXT_sve2(DisasContext *s, arg_EXT_sve2 *a)
+{
+    return do_ext(s, a->rd, a->rn, (a->rn + 1) % 32, a->imm);
+}
+
 /*
  *** SVE Permute - Unpredicated Group
  */
@@ -2172,6 +3814,45 @@ static bool trans_TBL(DisasContext *s, arg_rrr_esz *a)
     return true;
 }
 
+static bool trans_TBL_sve2(DisasContext *s, arg_rrr_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_4 * const fns[4] = {
+        gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h,
+        gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d
+    };
+
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, (a->rn + 1) % 32),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
+static bool trans_TBX(DisasContext *s, arg_rrr_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_3 * const fns[4] = {
+        gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h,
+        gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d
+    };
+
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, fns[a->esz]);
+    }
+    return true;
+}
+
 static bool trans_UNPK(DisasContext *s, arg_UNPK *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -2331,11 +4012,10 @@ static bool do_zip(DisasContext *s, arg_rrr_esz *a, bool high)
 
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        unsigned high_ofs = high ? vsz / 2 : 0;
         tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
-                           vec_full_reg_offset(s, a->rn) + high_ofs,
-                           vec_full_reg_offset(s, a->rm) + high_ofs,
-                           vsz, vsz, 0, fns[a->esz]);
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, high ? vsz / 2 : 0, fns[a->esz]);
     }
     return true;
 }
@@ -2364,6 +4044,40 @@ static bool trans_ZIP2_z(DisasContext *s, arg_rrr_esz *a)
     return do_zip(s, a, true);
 }
 
+static bool do_interleave_q(DisasContext *s, arg_rrr_esz *a,
+                            gen_helper_gvec_3 *fn, int data)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_f64mm, s)) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        if (vsz < 32) {
+            unallocated_encoding(s);
+        } else {
+            tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                               vec_full_reg_offset(s, a->rn),
+                               vec_full_reg_offset(s, a->rm),
+                               vsz, vsz, data, fn);
+        }
+    }
+    return true;
+}
+
+static bool trans_ZIP1_q(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_interleave_q(s, a, gen_helper_sve2_zip_q, 0);
+}
+
+static bool trans_ZIP2_q(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_interleave_q(s, a, gen_helper_sve2_zip_q,
+                           QEMU_ALIGN_DOWN(vec_full_reg_size(s), 32) / 2);
+}
+
 static gen_helper_gvec_3 * const uzp_fns[4] = {
     gen_helper_sve_uzp_b, gen_helper_sve_uzp_h,
     gen_helper_sve_uzp_s, gen_helper_sve_uzp_d,
@@ -2379,6 +4093,16 @@ static bool trans_UZP2_z(DisasContext *s, arg_rrr_esz *a)
     return do_zzz_data_ool(s, a, 1 << a->esz, uzp_fns[a->esz]);
 }
 
+static bool trans_UZP1_q(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_interleave_q(s, a, gen_helper_sve2_uzp_q, 0);
+}
+
+static bool trans_UZP2_q(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_interleave_q(s, a, gen_helper_sve2_uzp_q, 16);
+}
+
 static gen_helper_gvec_3 * const trn_fns[4] = {
     gen_helper_sve_trn_b, gen_helper_sve_trn_h,
     gen_helper_sve_trn_s, gen_helper_sve_trn_d,
@@ -2394,6 +4118,16 @@ static bool trans_TRN2_z(DisasContext *s, arg_rrr_esz *a)
     return do_zzz_data_ool(s, a, 1 << a->esz, trn_fns[a->esz]);
 }
 
+static bool trans_TRN1_q(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_interleave_q(s, a, gen_helper_sve2_trn_q, 0);
+}
+
+static bool trans_TRN2_q(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_interleave_q(s, a, gen_helper_sve2_trn_q, 16);
+}
+
 /*
  *** SVE Permute Vector - Predicated Group
  */
@@ -2403,6 +4137,7 @@ static bool trans_COMPACT(DisasContext *s, arg_rpr_esz *a)
     static gen_helper_gvec_3 * const fns[4] = {
         NULL, NULL, gen_helper_sve_compact_s, gen_helper_sve_compact_d
     };
+    s->is_nonstreaming = true;
     return do_zpz_ool(s, a, fns[a->esz]);
 }
 
@@ -2817,6 +4552,25 @@ static bool trans_SPLICE(DisasContext *s, arg_rprr_esz *a)
     return true;
 }
 
+static bool trans_SPLICE_sve2(DisasContext *s, arg_rpr_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, (a->rn + 1) % 32),
+                           pred_full_reg_offset(s, a->pg),
+                           vsz, vsz, a->esz, gen_helper_sve_splice);
+    }
+    return true;
+}
+
 /*
  *** SVE Integer Compare - Vectors Group
  */
@@ -2903,6 +4657,138 @@ DO_PPZW(CMPLS, cmpls)
 
 #undef DO_PPZW
 
+static bool trans_MATCH(DisasContext *s, arg_rprr_esz *a)
+{
+    static gen_helper_gvec_flags_4 * const fns[4] = {
+        gen_helper_sve2_match_ppzz_b,
+        gen_helper_sve2_match_ppzz_h,
+        NULL,
+        NULL,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    s->is_nonstreaming = true;
+    return do_ppzz_flags(s, a, fns[a->esz]);
+}
+
+static bool trans_NMATCH(DisasContext *s, arg_rprr_esz *a)
+{
+    static gen_helper_gvec_flags_4 * const fns[4] = {
+        gen_helper_sve2_nmatch_ppzz_b,
+        gen_helper_sve2_nmatch_ppzz_h,
+        NULL,
+        NULL,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    s->is_nonstreaming = true;
+    return do_ppzz_flags(s, a, fns[a->esz]);
+}
+
+static bool trans_HISTCNT(DisasContext *s, arg_rprr_esz *a)
+{
+    static gen_helper_gvec_4 * const fns[4] = {
+        NULL,
+        NULL,
+        gen_helper_sve2_histcnt_s,
+        gen_helper_sve2_histcnt_d,
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    s->is_nonstreaming = true;
+    return do_zpzz_ool(s, a, fns[a->esz]);
+}
+
+static bool trans_HISTSEG(DisasContext *s, arg_rrr_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2, s) || a->esz != 0) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, 0, gen_helper_sve2_histseg);
+    }
+    return true;
+}
+
+static bool trans_AESMC(DisasContext *s, arg_rri *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve2_aes, s)) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_2_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rd),
+                           vsz, vsz, a->imm, gen_helper_crypto_sve_aesmc);
+    }
+    return true;
+}
+
+static bool do_sve2_crypto_zzz(DisasContext *s, arg_rrr_esz *a,
+                               gen_helper_gvec_3 *fn, int data, bool feature)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!feature || a->esz != 0) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vsz, vsz, data, fn);
+    }
+    return true;
+}
+
+static bool trans_AESE(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_sve2_crypto_zzz(s, a, gen_helper_crypto_sve_aese, 0,
+                              dc_isar_feature(aa64_sve2_aes, s));
+}
+
+static bool trans_AESD(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_sve2_crypto_zzz(s, a, gen_helper_crypto_sve_aese, 1,
+                              dc_isar_feature(aa64_sve2_aes, s));
+}
+
+static bool trans_SM4E(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_sve2_crypto_zzz(s, a, gen_helper_crypto_sve_sm4e, 0,
+                              dc_isar_feature(aa64_sve2_sm4, s));
+}
+
+static bool trans_SM4EKEY(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_sve2_crypto_zzz(s, a, gen_helper_crypto_sve_sm4ekey, 0,
+                              dc_isar_feature(aa64_sve2_sm4, s));
+}
+
+static bool trans_RAX1(DisasContext *s, arg_rrr_esz *a)
+{
+    return do_sve2_crypto_zzz(s, a, gen_helper_crypto_rax1, 0,
+                              dc_isar_feature(aa64_sve2_sha3, s));
+}
+
 /*
  *** SVE Integer Compare - Immediate Groups
  */
@@ -3341,205 +5227,530 @@ static bool trans_FDUP(DisasContext *s, arg_FDUP *a)
         imm = vfp_expand_imm(a->esz, a->imm);
         imm = dup_const(a->esz, imm);
 
-        tcg_gen_gvec_dup64i(tcg_ctx, dofs, vsz, vsz, imm);
-    }
-    return true;
+        tcg_gen_gvec_dup64i(tcg_ctx, dofs, vsz, vsz, imm);
+    }
+    return true;
+}
+
+static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        int dofs = vec_full_reg_offset(s, a->rd);
+
+        tcg_gen_gvec_dup64i(tcg_ctx, dofs, vsz, vsz, dup_const(a->esz, a->imm));
+    }
+    return true;
+}
+
+static bool trans_ADD_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        tcg_gen_gvec_addi(tcg_ctx, a->esz, vec_full_reg_offset(s, a->rd),
+                          vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
+    }
+    return true;
+}
+
+static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    a->imm = -a->imm;
+    return trans_ADD_zzi(s, a);
+}
+
+static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
+    static const GVecGen2s op[4] = {
+        { .fni8 = tcg_gen_vec_sub8_i64,
+          .fniv = tcg_gen_sub_vec,
+          .fno = gen_helper_sve_subri_b,
+          .opt_opc = vecop_list,
+          .vece = MO_8,
+          .scalar_first = true },
+        { .fni8 = tcg_gen_vec_sub16_i64,
+          .fniv = tcg_gen_sub_vec,
+          .fno = gen_helper_sve_subri_h,
+          .opt_opc = vecop_list,
+          .vece = MO_16,
+          .scalar_first = true },
+        { .fni4 = tcg_gen_sub_i32,
+          .fniv = tcg_gen_sub_vec,
+          .fno = gen_helper_sve_subri_s,
+          .opt_opc = vecop_list,
+          .vece = MO_32,
+          .scalar_first = true },
+        { .fni8 = tcg_gen_sub_i64,
+          .fniv = tcg_gen_sub_vec,
+          .fno = gen_helper_sve_subri_d,
+          .opt_opc = vecop_list,
+          .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+          .vece = MO_64,
+          .scalar_first = true }
+    };
+
+    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        TCGv_i64 c = tcg_const_i64(tcg_ctx, a->imm);
+        tcg_gen_gvec_2s(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                        vec_full_reg_offset(s, a->rn),
+                        vsz, vsz, c, &op[a->esz]);
+        tcg_temp_free_i64(tcg_ctx, c);
+    }
+    return true;
+}
+
+static bool trans_MUL_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        tcg_gen_gvec_muli(tcg_ctx, a->esz, vec_full_reg_offset(s, a->rd),
+                          vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
+    }
+    return true;
+}
+
+static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        TCGv_i64 val = tcg_const_i64(tcg_ctx, a->imm);
+        do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, u, d);
+        tcg_temp_free_i64(tcg_ctx, val);
+    }
+    return true;
+}
+
+static bool trans_SQADD_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    return do_zzi_sat(s, a, false, false);
+}
+
+static bool trans_UQADD_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    return do_zzi_sat(s, a, true, false);
+}
+
+static bool trans_SQSUB_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    return do_zzi_sat(s, a, false, true);
+}
+
+static bool trans_UQSUB_zzi(DisasContext *s, arg_rri_esz *a)
+{
+    return do_zzi_sat(s, a, true, true);
+}
+
+static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        TCGv_i64 c = tcg_const_i64(tcg_ctx, a->imm);
+
+        tcg_gen_gvec_2i_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                            vec_full_reg_offset(s, a->rn),
+                            c, vsz, vsz, 0, fn);
+        tcg_temp_free_i64(tcg_ctx, c);
+    }
+    return true;
+}
+
+#define DO_ZZI(NAME, name) \
+static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a)         \
+{                                                                       \
+    static gen_helper_gvec_2i * const fns[4] = {                        \
+        gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h,         \
+        gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d,         \
+    };                                                                  \
+    return do_zzi_ool(s, a, fns[a->esz]);                               \
+}
+
+DO_ZZI(SMAX, smax)
+DO_ZZI(UMAX, umax)
+DO_ZZI(SMIN, smin)
+DO_ZZI(UMIN, umin)
+
+#undef DO_ZZI
+
+static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_4 * const fns[2][2] = {
+        { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
+        { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
+    };
+
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, 0, fns[a->u][a->sz]);
+    }
+    return true;
+}
+
+static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_4 * const fns[2][2] = {
+        { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h },
+        { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h }
+    };
+
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, a->index, fns[a->u][a->sz]);
+    }
+    return true;
+}
+
+static bool trans_USDOT_zzzz(DisasContext *s, arg_rprrr_esz *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_i8mm, s) || a->esz != 2) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, 0, gen_helper_gvec_usdot_b);
+    }
+    return true;
+}
+
+static bool do_i8mm_dot_zzxw(DisasContext *s, arg_rrxr_esz *a,
+                             gen_helper_gvec_4 *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_i8mm, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, a->index, fn);
+    }
+    return true;
+}
+
+static bool trans_USDOT_zzxw_s(DisasContext *s, arg_rrxr_esz *a)
+{
+    return do_i8mm_dot_zzxw(s, a, gen_helper_gvec_usdot_idx_b);
+}
+
+static bool trans_SUDOT_zzxw_s(DisasContext *s, arg_rrxr_esz *a)
+{
+    return do_i8mm_dot_zzxw(s, a, gen_helper_gvec_sudot_idx_b);
+}
+
+static bool do_i8mm_mmla(DisasContext *s, arg_rprrr_esz *a,
+                         gen_helper_gvec_4 *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_i8mm, s)) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, 0, fn);
+    }
+    return true;
+}
+
+static bool trans_SMMLA(DisasContext *s, arg_rprrr_esz *a)
+{
+    return do_i8mm_mmla(s, a, gen_helper_gvec_smmla_b);
+}
+
+static bool trans_USMMLA(DisasContext *s, arg_rprrr_esz *a)
+{
+    return do_i8mm_mmla(s, a, gen_helper_gvec_usmmla_b);
 }
 
-static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a)
+static bool trans_UMMLA(DisasContext *s, arg_rprrr_esz *a)
+{
+    return do_i8mm_mmla(s, a, gen_helper_gvec_ummla_b);
+}
+
+static bool trans_BFDOT_zzzz(DisasContext *s, arg_rprrr_esz *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+
+    if (!dc_isar_feature(aa64_sve_bf16, s)) {
         return false;
     }
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        int dofs = vec_full_reg_offset(s, a->rd);
 
-        tcg_gen_gvec_dup64i(tcg_ctx, dofs, vsz, vsz, dup_const(a->esz, a->imm));
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, 0, gen_helper_gvec_bfdot);
     }
     return true;
 }
 
-static bool trans_ADD_zzi(DisasContext *s, arg_rri_esz *a)
+static bool trans_BFDOT_zzxz(DisasContext *s, arg_rrxr_esz *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+
+    if (!dc_isar_feature(aa64_sve_bf16, s)) {
         return false;
     }
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        tcg_gen_gvec_addi(tcg_ctx, a->esz, vec_full_reg_offset(s, a->rd),
-                          vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, a->index, gen_helper_gvec_bfdot_idx);
     }
     return true;
 }
 
-static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a)
-{
-    a->imm = -a->imm;
-    return trans_ADD_zzi(s, a);
-}
-
-static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a)
+static bool trans_BFMMLA(DisasContext *s, arg_rprrr_esz *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
-    static const GVecGen2s op[4] = {
-        { .fni8 = tcg_gen_vec_sub8_i64,
-          .fniv = tcg_gen_sub_vec,
-          .fno = gen_helper_sve_subri_b,
-          .opt_opc = vecop_list,
-          .vece = MO_8,
-          .scalar_first = true },
-        { .fni8 = tcg_gen_vec_sub16_i64,
-          .fniv = tcg_gen_sub_vec,
-          .fno = gen_helper_sve_subri_h,
-          .opt_opc = vecop_list,
-          .vece = MO_16,
-          .scalar_first = true },
-        { .fni4 = tcg_gen_sub_i32,
-          .fniv = tcg_gen_sub_vec,
-          .fno = gen_helper_sve_subri_s,
-          .opt_opc = vecop_list,
-          .vece = MO_32,
-          .scalar_first = true },
-        { .fni8 = tcg_gen_sub_i64,
-          .fniv = tcg_gen_sub_vec,
-          .fno = gen_helper_sve_subri_d,
-          .opt_opc = vecop_list,
-          .prefer_i64 = TCG_TARGET_REG_BITS == 64,
-          .vece = MO_64,
-          .scalar_first = true }
-    };
 
-    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+    if (!dc_isar_feature(aa64_sve_bf16, s)) {
         return false;
     }
-    if (sve_access_check(s)) {
+    if (sve_nonstreaming_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        TCGv_i64 c = tcg_const_i64(tcg_ctx, a->imm);
-        tcg_gen_gvec_2s(tcg_ctx, vec_full_reg_offset(s, a->rd),
-                        vec_full_reg_offset(s, a->rn),
-                        vsz, vsz, c, &op[a->esz]);
-        tcg_temp_free_i64(tcg_ctx, c);
+
+        tcg_gen_gvec_4_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra),
+                           vsz, vsz, 0, gen_helper_gvec_bfmmla);
     }
     return true;
 }
 
-static bool trans_MUL_zzi(DisasContext *s, arg_rri_esz *a)
+static bool do_FMMLA(DisasContext *s, arg_rprrr_esz *a,
+                     gen_helper_gvec_4_ptr *fn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    if (sve_access_check(s)) {
+
+    if (sve_nonstreaming_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        tcg_gen_gvec_muli(tcg_ctx, a->esz, vec_full_reg_offset(s, a->rd),
-                          vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
+        TCGv_ptr status = get_fpstatus_ptr(tcg_ctx, false);
+
+        tcg_gen_gvec_4_ptr(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra), status,
+                           vsz, vsz, 0, fn);
+        tcg_temp_free_ptr(tcg_ctx, status);
     }
     return true;
 }
 
-static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d)
+static bool trans_FMMLA_s(DisasContext *s, arg_rprrr_esz *a)
 {
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    if (a->esz == 0 && extract32(s->insn, 13, 1)) {
+    if (!dc_isar_feature(aa64_sve_f32mm, s)) {
         return false;
     }
-    if (sve_access_check(s)) {
-        TCGv_i64 val = tcg_const_i64(tcg_ctx, a->imm);
-        do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, u, d);
-        tcg_temp_free_i64(tcg_ctx, val);
-    }
-    return true;
+    return do_FMMLA(s, a, gen_helper_fmmla_s);
 }
 
-static bool trans_SQADD_zzi(DisasContext *s, arg_rri_esz *a)
+static bool trans_FMMLA_d(DisasContext *s, arg_rprrr_esz *a)
 {
-    return do_zzi_sat(s, a, false, false);
+    if (!dc_isar_feature(aa64_sve_f64mm, s)) {
+        return false;
+    }
+    return do_FMMLA(s, a, gen_helper_fmmla_d);
 }
 
-static bool trans_UQADD_zzi(DisasContext *s, arg_rri_esz *a)
+static bool do_BFMLAL_zzzw(DisasContext *s, arg_rprrr_esz *a, bool sel)
 {
-    return do_zzi_sat(s, a, true, false);
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_bf16, s)) {
+        return false;
+    }
+    if (sve_access_check(s)) {
+        unsigned vsz = vec_full_reg_size(s);
+        TCGv_ptr status = get_fpstatus_ptr(tcg_ctx, false);
+
+        tcg_gen_gvec_4_ptr(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra), status,
+                           vsz, vsz, sel, gen_helper_gvec_bfmlal);
+        tcg_temp_free_ptr(tcg_ctx, status);
+    }
+    return true;
 }
 
-static bool trans_SQSUB_zzi(DisasContext *s, arg_rri_esz *a)
+static bool trans_BFMLALB_zzzw(DisasContext *s, arg_BFMLALB_zzzw *a)
 {
-    return do_zzi_sat(s, a, false, true);
+    return do_BFMLAL_zzzw(s, a, false);
 }
 
-static bool trans_UQSUB_zzi(DisasContext *s, arg_rri_esz *a)
+static bool trans_BFMLALT_zzzw(DisasContext *s, arg_BFMLALT_zzzw *a)
 {
-    return do_zzi_sat(s, a, true, true);
+    return do_BFMLAL_zzzw(s, a, true);
 }
 
-static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn)
+static bool do_BFMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sel)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_bf16, s)) {
+        return false;
+    }
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        TCGv_i64 c = tcg_const_i64(tcg_ctx, a->imm);
+        TCGv_ptr status = get_fpstatus_ptr(tcg_ctx, false);
 
-        tcg_gen_gvec_2i_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
-                            vec_full_reg_offset(s, a->rn),
-                            c, vsz, vsz, 0, fn);
-        tcg_temp_free_i64(tcg_ctx, c);
+        tcg_gen_gvec_4_ptr(tcg_ctx, vec_full_reg_offset(s, a->rd),
+                           vec_full_reg_offset(s, a->rn),
+                           vec_full_reg_offset(s, a->rm),
+                           vec_full_reg_offset(s, a->ra), status,
+                           vsz, vsz, (a->index << 1) | sel,
+                           gen_helper_gvec_bfmlal_idx);
+        tcg_temp_free_ptr(tcg_ctx, status);
     }
     return true;
 }
 
-#define DO_ZZI(NAME, name) \
-static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a)         \
-{                                                                       \
-    static gen_helper_gvec_2i * const fns[4] = {                        \
-        gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h,         \
-        gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d,         \
-    };                                                                  \
-    return do_zzi_ool(s, a, fns[a->esz]);                               \
+static bool trans_BFMLALB_zzxw(DisasContext *s, arg_BFMLALB_zzxw *a)
+{
+    return do_BFMLAL_zzxw(s, a, false);
 }
 
-DO_ZZI(SMAX, smax)
-DO_ZZI(UMAX, umax)
-DO_ZZI(SMIN, smin)
-DO_ZZI(UMIN, umin)
-
-#undef DO_ZZI
+static bool trans_BFMLALT_zzxw(DisasContext *s, arg_BFMLALT_zzxw *a)
+{
+    return do_BFMLAL_zzxw(s, a, true);
+}
 
-static bool trans_DOT_zzz(DisasContext *s, arg_DOT_zzz *a)
+static bool do_FMLAL_zzzw(DisasContext *s, arg_rprrr_esz *a,
+                          bool sub, bool sel)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    static gen_helper_gvec_3 * const fns[2][2] = {
-        { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
-        { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
-    };
 
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+
+        tcg_gen_gvec_4_ptr(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
                            vec_full_reg_offset(s, a->rm),
-                           vsz, vsz, 0, fns[a->u][a->sz]);
+                           vec_full_reg_offset(s, a->ra),
+                           tcg_ctx->cpu_env, vsz, vsz,
+                           (sel << 1) | sub,
+                           gen_helper_sve2_fmlal_zzzw_s);
     }
     return true;
 }
 
-static bool trans_DOT_zzx(DisasContext *s, arg_DOT_zzx *a)
+static bool trans_FMLALB_zzzw(DisasContext *s, arg_FMLALB_zzzw *a)
+{
+    return do_FMLAL_zzzw(s, a, false, false);
+}
+
+static bool trans_FMLALT_zzzw(DisasContext *s, arg_FMLALT_zzzw *a)
+{
+    return do_FMLAL_zzzw(s, a, false, true);
+}
+
+static bool trans_FMLSLB_zzzw(DisasContext *s, arg_FMLSLB_zzzw *a)
+{
+    return do_FMLAL_zzzw(s, a, true, false);
+}
+
+static bool trans_FMLSLT_zzzw(DisasContext *s, arg_FMLSLT_zzzw *a)
+{
+    return do_FMLAL_zzzw(s, a, true, true);
+}
+
+static bool do_FMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a,
+                          bool sub, bool sel)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    static gen_helper_gvec_3 * const fns[2][2] = {
-        { gen_helper_gvec_sdot_idx_b, gen_helper_gvec_sdot_idx_h },
-        { gen_helper_gvec_udot_idx_b, gen_helper_gvec_udot_idx_h }
-    };
 
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
-        tcg_gen_gvec_3_ool(tcg_ctx, vec_full_reg_offset(s, a->rd),
+
+        tcg_gen_gvec_4_ptr(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
                            vec_full_reg_offset(s, a->rm),
-                           vsz, vsz, a->index, fns[a->u][a->sz]);
+                           vec_full_reg_offset(s, a->ra),
+                           tcg_ctx->cpu_env, vsz, vsz,
+                           (a->index << 2) | (sel << 1) | sub,
+                           gen_helper_sve2_fmlal_zzxw_s);
     }
     return true;
 }
 
+static bool trans_FMLALB_zzxw(DisasContext *s, arg_FMLALB_zzxw *a)
+{
+    return do_FMLAL_zzxw(s, a, false, false);
+}
+
+static bool trans_FMLALT_zzxw(DisasContext *s, arg_FMLALT_zzxw *a)
+{
+    return do_FMLAL_zzxw(s, a, false, true);
+}
+
+static bool trans_FMLSLB_zzxw(DisasContext *s, arg_FMLSLB_zzxw *a)
+{
+    return do_FMLAL_zzxw(s, a, true, false);
+}
+
+static bool trans_FMLSLT_zzxw(DisasContext *s, arg_FMLSLT_zzxw *a)
+{
+    return do_FMLAL_zzxw(s, a, true, true);
+}
+
 
 /*
  *** SVE Floating Point Multiply-Add Indexed Group
@@ -3759,7 +5970,7 @@ static bool trans_FTMAD(DisasContext *s, arg_FTMAD *a)
     if (a->esz == 0) {
         return false;
     }
-    if (sve_access_check(s)) {
+    if (sve_nonstreaming_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
         TCGv_ptr status = get_fpstatus_ptr(tcg_ctx, a->esz == MO_16);
         tcg_gen_gvec_3_ptr(tcg_ctx, vec_full_reg_offset(s, a->rd),
@@ -3793,7 +6004,7 @@ static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a)
     if (a->esz == 0) {
         return false;
     }
-    if (!sve_access_check(s)) {
+    if (!sve_nonstreaming_access_check(s)) {
         return true;
     }
 
@@ -3854,12 +6065,25 @@ static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a)           \
 DO_FP3(FADD_zzz, fadd)
 DO_FP3(FSUB_zzz, fsub)
 DO_FP3(FMUL_zzz, fmul)
-DO_FP3(FTSMUL, ftsmul)
 DO_FP3(FRECPS, recps)
 DO_FP3(FRSQRTS, rsqrts)
 
 #undef DO_FP3
 
+static bool trans_FTSMUL(DisasContext *s, arg_rrr_esz *a)
+{
+    static gen_helper_gvec_3_ptr * const fns[4] = {
+        NULL, gen_helper_gvec_ftsmul_h,
+        gen_helper_gvec_ftsmul_s, gen_helper_gvec_ftsmul_d,
+    };
+
+    if (fns[a->esz] == NULL) {
+        return false;
+    }
+    s->is_nonstreaming = true;
+    return do_zzz_fp(s, a, fns[a->esz]);
+}
+
 /*
  *** SVE Floating Point Arithmetic - Predicated Group
  */
@@ -3908,6 +6132,27 @@ DO_FP3(FMULX, fmulx)
 
 #undef DO_FP3
 
+#define DO_SVE2_FP_PAIR(NAME, name)                                    \
+static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a)             \
+{                                                                      \
+    static gen_helper_gvec_4_ptr * const fns[4] = {                    \
+        NULL, gen_helper_sve2_##name##_h,                              \
+        gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d         \
+    };                                                                 \
+    if (!dc_isar_feature(aa64_sve2, s)) {                              \
+        return false;                                                  \
+    }                                                                  \
+    return do_zpzz_fp(s, a, fns[a->esz]);                              \
+}
+
+DO_SVE2_FP_PAIR(FADDP, faddp_zpzz)
+DO_SVE2_FP_PAIR(FMAXNMP, fmaxnmp_zpzz)
+DO_SVE2_FP_PAIR(FMINNMP, fminnmp_zpzz)
+DO_SVE2_FP_PAIR(FMAXP, fmaxp_zpzz)
+DO_SVE2_FP_PAIR(FMINP, fminp_zpzz)
+
+#undef DO_SVE2_FP_PAIR
+
 typedef void gen_helper_sve_fp2scalar(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv_ptr,
                                       TCGv_i64, TCGv_ptr, TCGv_i32);
 
@@ -4209,6 +6454,72 @@ static bool trans_FCVT_sd(DisasContext *s, arg_rpr_esz *a)
     return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_fcvt_sd);
 }
 
+static bool trans_BFCVT(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve_bf16, s)) {
+        return false;
+    }
+    return do_zpz_ptr(s, a->rd, a->rn, a->pg, false, gen_helper_sve_bfcvt);
+}
+
+static bool trans_FCVTNT_sh(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ptr(s, a->rd, a->rn, a->pg, false,
+                      gen_helper_sve2_fcvtnt_sh);
+}
+
+static bool trans_FCVTNT_ds(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ptr(s, a->rd, a->rn, a->pg, false,
+                      gen_helper_sve2_fcvtnt_ds);
+}
+
+static bool trans_BFCVTNT(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve_bf16, s)) {
+        return false;
+    }
+    return do_zpz_ptr(s, a->rd, a->rn, a->pg, false,
+                      gen_helper_sve_bfcvtnt);
+}
+
+static bool trans_FCVTLT_hs(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ptr(s, a->rd, a->rn, a->pg, false,
+                      gen_helper_sve2_fcvtlt_hs);
+}
+
+static bool trans_FCVTLT_sd(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_ptr(s, a->rd, a->rn, a->pg, false,
+                      gen_helper_sve2_fcvtlt_sd);
+}
+
+static bool trans_FLOGB(DisasContext *s, arg_rpr_esz *a)
+{
+    static gen_helper_gvec_3_ptr * const fns[4] = {
+        NULL, gen_helper_flogb_h, gen_helper_flogb_s, gen_helper_flogb_d
+    };
+
+    if (!dc_isar_feature(aa64_sve2, s) || a->esz == 0) {
+        return false;
+    }
+    return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16,
+                      fns[a->esz]);
+}
+
 static bool trans_FCVTZS_hh(DisasContext *s, arg_rpr_esz *a)
 {
     return do_zpz_ptr(s, a->rd, a->rn, a->pg, true, gen_helper_sve_fcvtzs_hh);
@@ -4307,23 +6618,24 @@ static bool trans_FRINTX(DisasContext *s, arg_rpr_esz *a)
     return do_zpz_ptr(s, a->rd, a->rn, a->pg, a->esz == MO_16, fns[a->esz - 1]);
 }
 
-static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
+static bool do_zpz_fp_rmode(DisasContext *s, arg_rpr_esz *a, int mode,
+                            bool is_fp16, gen_helper_gvec_3_ptr *fn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    if (a->esz == 0) {
+    if (fn == NULL) {
         return false;
     }
     if (sve_access_check(s)) {
         unsigned vsz = vec_full_reg_size(s);
         TCGv_i32 tmode = tcg_const_i32(tcg_ctx, mode);
-        TCGv_ptr status = get_fpstatus_ptr(tcg_ctx, a->esz == MO_16);
+        TCGv_ptr status = get_fpstatus_ptr(tcg_ctx, is_fp16);
 
         gen_helper_set_rmode(tcg_ctx, tmode, tmode, status);
 
         tcg_gen_gvec_3_ptr(tcg_ctx, vec_full_reg_offset(s, a->rd),
                            vec_full_reg_offset(s, a->rn),
                            pred_full_reg_offset(s, a->pg),
-                           status, vsz, vsz, 0, frint_fns[a->esz - 1]);
+                           status, vsz, vsz, 0, fn);
 
         gen_helper_set_rmode(tcg_ctx, tmode, tmode, status);
         tcg_temp_free_i32(tcg_ctx, tmode);
@@ -4332,6 +6644,15 @@ static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
     return true;
 }
 
+static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, int mode)
+{
+    if (a->esz == 0) {
+        return false;
+    }
+    return do_zpz_fp_rmode(s, a, mode, a->esz == MO_16,
+                           frint_fns[a->esz - 1]);
+}
+
 static bool trans_FRINTN(DisasContext *s, arg_rpr_esz *a)
 {
     return do_frint_mode(s, a, float_round_nearest_even);
@@ -4357,6 +6678,24 @@ static bool trans_FRINTA(DisasContext *s, arg_rpr_esz *a)
     return do_frint_mode(s, a, float_round_ties_away);
 }
 
+static bool trans_FCVTX_ds(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_fp_rmode(s, a, float_round_to_odd, false,
+                           gen_helper_sve_fcvt_ds);
+}
+
+static bool trans_FCVTXNT_ds(DisasContext *s, arg_rpr_esz *a)
+{
+    if (!dc_isar_feature(aa64_sve2, s)) {
+        return false;
+    }
+    return do_zpz_fp_rmode(s, a, float_round_to_odd, false,
+                           gen_helper_sve2_fcvtnt_ds);
+}
+
 static bool trans_FRECPX(DisasContext *s, arg_rpr_esz *a)
 {
     static gen_helper_gvec_3_ptr * const fns[3] = {
@@ -4457,20 +6796,34 @@ static bool trans_UCVTF_dd(DisasContext *s, arg_rpr_esz *a)
  *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
  */
 
-/* Subroutine loading a vector register at VOFS of LEN bytes.
+static void gen_mov_ptr(TCGContext *tcg_ctx, TCGv_ptr ret, TCGv_ptr arg)
+{
+#if UINTPTR_MAX == UINT32_MAX
+    tcg_gen_mov_i32(tcg_ctx, (TCGv_i32)ret, (TCGv_i32)arg);
+#else
+    tcg_gen_mov_i64(tcg_ctx, (TCGv_i64)ret, (TCGv_i64)arg);
+#endif
+}
+
+/* Subroutine loading a vector register at BASE + VOFS of LEN bytes.
  * The load should begin at the address Rn + IMM.
  */
 
-static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
+void gen_sve_ldr(DisasContext *s, TCGv_ptr base, int vofs,
+                 int len, int rn, int imm)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     int len_align = QEMU_ALIGN_DOWN(len, 8);
     int len_remain = len % 8;
     int nparts = len / 8 + ctpop8(len_remain);
     int midx = get_mem_index(s);
-    TCGv_i64 addr, t0, t1;
+    TCGv_i64 dirty_addr, clean_addr, t0, t1;
+    TCGv_ptr base_local = NULL;
 
-    addr = tcg_temp_new_i64(tcg_ctx);
+    dirty_addr = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_addi_i64(tcg_ctx, dirty_addr, cpu_reg_sp(s, rn), imm);
+    clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len);
+    tcg_temp_free_i64(tcg_ctx, dirty_addr);
     t0 = tcg_temp_new_i64(tcg_ctx);
 
     /* Note that unpredicated load/store of vector/predicate registers
@@ -4485,13 +6838,23 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
         int i;
 
         for (i = 0; i < len_align; i += 8) {
-            tcg_gen_addi_i64(tcg_ctx, addr, cpu_reg_sp(s, rn), imm + i);
-            tcg_gen_qemu_ld_i64(tcg_ctx, t0, addr, midx, MO_LEQ);
-            tcg_gen_st_i64(tcg_ctx, t0, tcg_ctx->cpu_env, vofs + i);
+            tcg_gen_qemu_ld_i64(tcg_ctx, t0, clean_addr, midx, MO_LEQ);
+            tcg_gen_st_i64(tcg_ctx, t0, base, vofs + i);
+            tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 8);
         }
     } else {
         TCGLabel *loop = gen_new_label(tcg_ctx);
         TCGv_ptr tp, i = tcg_const_local_ptr(tcg_ctx, 0);
+        TCGv_i64 clean_addr_local = tcg_temp_local_new_i64(tcg_ctx);
+
+        tcg_gen_mov_i64(tcg_ctx, clean_addr_local, clean_addr);
+        clean_addr = clean_addr_local;
+
+        if (base != tcg_ctx->cpu_env) {
+            base_local = tcg_temp_local_new_ptr(tcg_ctx);
+            gen_mov_ptr(tcg_ctx, base_local, base);
+            base = base_local;
+        }
 
         gen_set_label(tcg_ctx, loop);
 
@@ -4499,14 +6862,11 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
          * the stack each iteration.  Instead, re-compute values other
          * than the loop counter.
          */
-        tp = tcg_temp_new_ptr(tcg_ctx);
-        tcg_gen_addi_ptr(tcg_ctx, tp, i, imm);
-        tcg_gen_extu_ptr_i64(tcg_ctx, addr, tp);
-        tcg_gen_add_i64(tcg_ctx, addr, addr, cpu_reg_sp(s, rn));
+        tcg_gen_qemu_ld_i64(tcg_ctx, t0, clean_addr, midx, MO_LEQ);
+        tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 8);
 
-        tcg_gen_qemu_ld_i64(tcg_ctx, t0, addr, midx, MO_LEQ);
-
-        tcg_gen_add_ptr(tcg_ctx, tp, tcg_ctx->cpu_env, i);
+        tp = tcg_temp_new_ptr(tcg_ctx);
+        tcg_gen_add_ptr(tcg_ctx, tp, base, i);
         tcg_gen_addi_ptr(tcg_ctx, i, i, 8);
         tcg_gen_st_i64(tcg_ctx, t0, tp, vofs);
         tcg_temp_free_ptr(tcg_ctx, tp);
@@ -4519,20 +6879,19 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
      * Note that we still store the entire 64-bit unit into cpu_env.
      */
     if (len_remain) {
-        tcg_gen_addi_i64(tcg_ctx, addr, cpu_reg_sp(s, rn), imm + len_align);
-
         switch (len_remain) {
         case 2:
         case 4:
         case 8:
-            tcg_gen_qemu_ld_i64(tcg_ctx, t0, addr, midx, MO_LE | ctz32(len_remain));
+            tcg_gen_qemu_ld_i64(tcg_ctx, t0, clean_addr, midx,
+                                MO_LE | ctz32(len_remain));
             break;
 
         case 6:
             t1 = tcg_temp_new_i64(tcg_ctx);
-            tcg_gen_qemu_ld_i64(tcg_ctx, t0, addr, midx, MO_LEUL);
-            tcg_gen_addi_i64(tcg_ctx, addr, addr, 4);
-            tcg_gen_qemu_ld_i64(tcg_ctx, t1, addr, midx, MO_LEUW);
+            tcg_gen_qemu_ld_i64(tcg_ctx, t0, clean_addr, midx, MO_LEUL);
+            tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 4);
+            tcg_gen_qemu_ld_i64(tcg_ctx, t1, clean_addr, midx, MO_LEUW);
             tcg_gen_deposit_i64(tcg_ctx, t0, t0, t1, 32, 32);
             tcg_temp_free_i64(tcg_ctx, t1);
             break;
@@ -4540,23 +6899,30 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
         default:
             g_assert_not_reached();
         }
-        tcg_gen_st_i64(tcg_ctx, t0, tcg_ctx->cpu_env, vofs + len_align);
+        tcg_gen_st_i64(tcg_ctx, t0, base, vofs + len_align);
+    }
+    if (base_local != NULL) {
+        tcg_temp_free_ptr(tcg_ctx, base_local);
     }
-    tcg_temp_free_i64(tcg_ctx, addr);
     tcg_temp_free_i64(tcg_ctx, t0);
 }
 
 /* Similarly for stores.  */
-static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
+void gen_sve_str(DisasContext *s, TCGv_ptr base, int vofs,
+                 int len, int rn, int imm)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     int len_align = QEMU_ALIGN_DOWN(len, 8);
     int len_remain = len % 8;
     int nparts = len / 8 + ctpop8(len_remain);
     int midx = get_mem_index(s);
-    TCGv_i64 addr, t0;
+    TCGv_i64 dirty_addr, clean_addr, t0;
+    TCGv_ptr base_local = NULL;
 
-    addr = tcg_temp_new_i64(tcg_ctx);
+    dirty_addr = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_addi_i64(tcg_ctx, dirty_addr, cpu_reg_sp(s, rn), imm);
+    clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len);
+    tcg_temp_free_i64(tcg_ctx, dirty_addr);
     t0 = tcg_temp_new_i64(tcg_ctx);
 
     /* Note that unpredicated load/store of vector/predicate registers
@@ -4571,30 +6937,38 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
         int i;
 
         for (i = 0; i < len_align; i += 8) {
-            tcg_gen_ld_i64(tcg_ctx, t0, tcg_ctx->cpu_env, vofs + i);
-            tcg_gen_addi_i64(tcg_ctx, addr, cpu_reg_sp(s, rn), imm + i);
-            tcg_gen_qemu_st_i64(tcg_ctx, t0, addr, midx, MO_LEQ);
+            tcg_gen_ld_i64(tcg_ctx, t0, base, vofs + i);
+            tcg_gen_qemu_st_i64(tcg_ctx, t0, clean_addr, midx, MO_LEQ);
+            tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 8);
         }
     } else {
         TCGLabel *loop = gen_new_label(tcg_ctx);
         TCGv_ptr t2, i = tcg_const_local_ptr(tcg_ctx, 0);
+        TCGv_i64 clean_addr_local = tcg_temp_local_new_i64(tcg_ctx);
+
+        tcg_gen_mov_i64(tcg_ctx, clean_addr_local, clean_addr);
+        clean_addr = clean_addr_local;
+
+        if (base != tcg_ctx->cpu_env) {
+            base_local = tcg_temp_local_new_ptr(tcg_ctx);
+            gen_mov_ptr(tcg_ctx, base_local, base);
+            base = base_local;
+        }
 
         gen_set_label(tcg_ctx, loop);
 
         t2 = tcg_temp_new_ptr(tcg_ctx);
-        tcg_gen_add_ptr(tcg_ctx, t2, tcg_ctx->cpu_env, i);
+        tcg_gen_add_ptr(tcg_ctx, t2, base, i);
         tcg_gen_ld_i64(tcg_ctx, t0, t2, vofs);
 
         /* Minimize the number of local temps that must be re-read from
          * the stack each iteration.  Instead, re-compute values other
          * than the loop counter.
          */
-        tcg_gen_addi_ptr(tcg_ctx, t2, i, imm);
-        tcg_gen_extu_ptr_i64(tcg_ctx, addr, t2);
-        tcg_gen_add_i64(tcg_ctx, addr, addr, cpu_reg_sp(s, rn));
         tcg_temp_free_ptr(tcg_ctx, t2);
 
-        tcg_gen_qemu_st_i64(tcg_ctx, t0, addr, midx, MO_LEQ);
+        tcg_gen_qemu_st_i64(tcg_ctx, t0, clean_addr, midx, MO_LEQ);
+        tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 8);
 
         tcg_gen_addi_ptr(tcg_ctx, i, i, 8);
 
@@ -4604,28 +6978,30 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
 
     /* Predicate register stores can be any multiple of 2.  */
     if (len_remain) {
-        tcg_gen_ld_i64(tcg_ctx, t0, tcg_ctx->cpu_env, vofs + len_align);
-        tcg_gen_addi_i64(tcg_ctx, addr, cpu_reg_sp(s, rn), imm + len_align);
+        tcg_gen_ld_i64(tcg_ctx, t0, base, vofs + len_align);
 
         switch (len_remain) {
         case 2:
         case 4:
         case 8:
-            tcg_gen_qemu_st_i64(tcg_ctx, t0, addr, midx, MO_LE | ctz32(len_remain));
+            tcg_gen_qemu_st_i64(tcg_ctx, t0, clean_addr, midx,
+                                MO_LE | ctz32(len_remain));
             break;
 
         case 6:
-            tcg_gen_qemu_st_i64(tcg_ctx, t0, addr, midx, MO_LEUL);
-            tcg_gen_addi_i64(tcg_ctx, addr, addr, 4);
+            tcg_gen_qemu_st_i64(tcg_ctx, t0, clean_addr, midx, MO_LEUL);
+            tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 4);
             tcg_gen_shri_i64(tcg_ctx, t0, t0, 32);
-            tcg_gen_qemu_st_i64(tcg_ctx, t0, addr, midx, MO_LEUW);
+            tcg_gen_qemu_st_i64(tcg_ctx, t0, clean_addr, midx, MO_LEUW);
             break;
 
         default:
             g_assert_not_reached();
         }
     }
-    tcg_temp_free_i64(tcg_ctx, addr);
+    if (base_local != NULL) {
+        tcg_temp_free_ptr(tcg_ctx, base_local);
+    }
     tcg_temp_free_i64(tcg_ctx, t0);
 }
 
@@ -4634,7 +7010,8 @@ static bool trans_LDR_zri(DisasContext *s, arg_rri *a)
     if (sve_access_check(s)) {
         int size = vec_full_reg_size(s);
         int off = vec_full_reg_offset(s, a->rd);
-        do_ldr(s, off, size, a->rn, a->imm * size);
+        gen_sve_ldr(s, s->uc->tcg_ctx->cpu_env, off, size, a->rn,
+                    a->imm * size);
     }
     return true;
 }
@@ -4644,7 +7021,8 @@ static bool trans_LDR_pri(DisasContext *s, arg_rri *a)
     if (sve_access_check(s)) {
         int size = pred_full_reg_size(s);
         int off = pred_full_reg_offset(s, a->rd);
-        do_ldr(s, off, size, a->rn, a->imm * size);
+        gen_sve_ldr(s, s->uc->tcg_ctx->cpu_env, off, size, a->rn,
+                    a->imm * size);
     }
     return true;
 }
@@ -4654,7 +7032,8 @@ static bool trans_STR_zri(DisasContext *s, arg_rri *a)
     if (sve_access_check(s)) {
         int size = vec_full_reg_size(s);
         int off = vec_full_reg_offset(s, a->rd);
-        do_str(s, off, size, a->rn, a->imm * size);
+        gen_sve_str(s, s->uc->tcg_ctx->cpu_env, off, size, a->rn,
+                    a->imm * size);
     }
     return true;
 }
@@ -4664,7 +7043,8 @@ static bool trans_STR_pri(DisasContext *s, arg_rri *a)
     if (sve_access_check(s)) {
         int size = pred_full_reg_size(s);
         int off = pred_full_reg_offset(s, a->rd);
-        do_str(s, off, size, a->rn, a->imm * size);
+        gen_sve_str(s, s->uc->tcg_ctx->cpu_env, off, size, a->rn,
+                    a->imm * size);
     }
     return true;
 }
@@ -4696,8 +7076,33 @@ static TCGMemOpIdx sve_memopidx(DisasContext *s, int dtype)
     return make_memop_idx(s->be_data | dtype_mop[dtype], get_mem_index(s));
 }
 
+static TCGv_i64 sve_clean_data_tbi(DisasContext *s, TCGv_i64 addr)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i64 clean = new_tmp_a64(s);
+
+    if (s->tbid == 0) {
+        tcg_gen_mov_i64(tcg_ctx, clean, addr);
+    } else if (!regime_has_2_ranges(s->mmu_idx)) {
+        tcg_gen_extract_i64(tcg_ctx, clean, addr, 0, 56);
+    } else {
+        tcg_gen_sextract_i64(tcg_ctx, clean, addr, 0, 56);
+
+        if (s->tbid != 3) {
+            TCGv_i64 tcg_zero = tcg_const_i64(tcg_ctx, 0);
+
+            tcg_gen_movcond_i64(tcg_ctx,
+                                s->tbid == 1 ? TCG_COND_GE : TCG_COND_LT,
+                                clean, clean, tcg_zero, clean, addr);
+            tcg_temp_free_i64(tcg_ctx, tcg_zero);
+        }
+    }
+
+    return clean;
+}
+
 static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
-                       int dtype, gen_helper_gvec_mem *fn)
+                       int dtype, bool mte, gen_helper_gvec_mem *fn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     unsigned vsz = vec_full_reg_size(s);
@@ -4715,6 +7120,10 @@ static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
     t_desc = tcg_const_i32(tcg_ctx, desc);
     t_pg = tcg_temp_new_ptr(tcg_ctx);
 
+    if (!mte) {
+        addr = sve_clean_data_tbi(s, addr);
+    }
+
     tcg_gen_addi_ptr(tcg_ctx, t_pg, tcg_ctx->cpu_env, pred_full_reg_offset(s, pg));
     fn(tcg_ctx, tcg_ctx->cpu_env, t_pg, addr, t_desc);
 
@@ -4776,13 +7185,82 @@ static void do_ld_zpa(DisasContext *s, int zt, int pg,
           { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r,
             gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } }
     };
-    gen_helper_gvec_mem *fn = fns[s->be_data == MO_BE][dtype][nreg];
+    static gen_helper_gvec_mem * const mte_fns[2][16][4] = {
+        /* Little-endian */
+        { { gen_helper_sve_ld1bb_r_mte, gen_helper_sve_ld2bb_r_mte,
+            gen_helper_sve_ld3bb_r_mte, gen_helper_sve_ld4bb_r_mte },
+          { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
+
+          { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1hh_le_r_mte,
+            gen_helper_sve_ld2hh_le_r_mte,
+            gen_helper_sve_ld3hh_le_r_mte,
+            gen_helper_sve_ld4hh_le_r_mte },
+          { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL },
+
+          { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1ss_le_r_mte,
+            gen_helper_sve_ld2ss_le_r_mte,
+            gen_helper_sve_ld3ss_le_r_mte,
+            gen_helper_sve_ld4ss_le_r_mte },
+          { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL },
+
+          { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1dd_le_r_mte,
+            gen_helper_sve_ld2dd_le_r_mte,
+            gen_helper_sve_ld3dd_le_r_mte,
+            gen_helper_sve_ld4dd_le_r_mte } },
+
+        /* Big-endian */
+        { { gen_helper_sve_ld1bb_r_mte, gen_helper_sve_ld2bb_r_mte,
+            gen_helper_sve_ld3bb_r_mte, gen_helper_sve_ld4bb_r_mte },
+          { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
+
+          { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1hh_be_r_mte,
+            gen_helper_sve_ld2hh_be_r_mte,
+            gen_helper_sve_ld3hh_be_r_mte,
+            gen_helper_sve_ld4hh_be_r_mte },
+          { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL },
+
+          { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1ss_be_r_mte,
+            gen_helper_sve_ld2ss_be_r_mte,
+            gen_helper_sve_ld3ss_be_r_mte,
+            gen_helper_sve_ld4ss_be_r_mte },
+          { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL },
+
+          { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
+          { gen_helper_sve_ld1dd_be_r_mte,
+            gen_helper_sve_ld2dd_be_r_mte,
+            gen_helper_sve_ld3dd_be_r_mte,
+            gen_helper_sve_ld4dd_be_r_mte } }
+    };
+    gen_helper_gvec_mem *fn;
+
+    if (s->mte_active[0]) {
+        fn = mte_fns[s->be_data == MO_BE][dtype][nreg];
+    } else {
+        fn = fns[s->be_data == MO_BE][dtype][nreg];
+    }
 
     /* While there are holes in the table, they are not
      * accessible via the instruction encoding.
      */
     assert(fn != NULL);
-    do_mem_zpa(s, zt, pg, addr, dtype, fn);
+    do_mem_zpa(s, zt, pg, addr, dtype, s->mte_active[0], fn);
 }
 
 static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a)
@@ -4862,13 +7340,61 @@ static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a)
           gen_helper_sve_ldff1bhs_r,
           gen_helper_sve_ldff1dd_be_r },
     };
+    static gen_helper_gvec_mem * const mte_fns[2][16] = {
+        /* Little-endian */
+        { gen_helper_sve_ldff1bb_r_mte,
+          gen_helper_sve_ldff1bhu_r_mte,
+          gen_helper_sve_ldff1bsu_r_mte,
+          gen_helper_sve_ldff1bdu_r_mte,
+
+          gen_helper_sve_ldff1sds_le_r_mte,
+          gen_helper_sve_ldff1hh_le_r_mte,
+          gen_helper_sve_ldff1hsu_le_r_mte,
+          gen_helper_sve_ldff1hdu_le_r_mte,
+
+          gen_helper_sve_ldff1hds_le_r_mte,
+          gen_helper_sve_ldff1hss_le_r_mte,
+          gen_helper_sve_ldff1ss_le_r_mte,
+          gen_helper_sve_ldff1sdu_le_r_mte,
+
+          gen_helper_sve_ldff1bds_r_mte,
+          gen_helper_sve_ldff1bss_r_mte,
+          gen_helper_sve_ldff1bhs_r_mte,
+          gen_helper_sve_ldff1dd_le_r_mte },
+
+        /* Big-endian */
+        { gen_helper_sve_ldff1bb_r_mte,
+          gen_helper_sve_ldff1bhu_r_mte,
+          gen_helper_sve_ldff1bsu_r_mte,
+          gen_helper_sve_ldff1bdu_r_mte,
+
+          gen_helper_sve_ldff1sds_be_r_mte,
+          gen_helper_sve_ldff1hh_be_r_mte,
+          gen_helper_sve_ldff1hsu_be_r_mte,
+          gen_helper_sve_ldff1hdu_be_r_mte,
+
+          gen_helper_sve_ldff1hds_be_r_mte,
+          gen_helper_sve_ldff1hss_be_r_mte,
+          gen_helper_sve_ldff1ss_be_r_mte,
+          gen_helper_sve_ldff1sdu_be_r_mte,
+
+          gen_helper_sve_ldff1bds_r_mte,
+          gen_helper_sve_ldff1bss_r_mte,
+          gen_helper_sve_ldff1bhs_r_mte,
+          gen_helper_sve_ldff1dd_be_r_mte },
+    };
+    gen_helper_gvec_mem *fn;
 
-    if (sve_access_check(s)) {
+    if (sve_nonstreaming_access_check(s)) {
         TCGv_i64 addr = new_tmp_a64(s);
         tcg_gen_shli_i64(tcg_ctx, addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
         tcg_gen_add_i64(tcg_ctx, addr, addr, cpu_reg_sp(s, a->rn));
-        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype,
-                   fns[s->be_data == MO_BE][a->dtype]);
+        if (s->mte_active[0]) {
+            fn = mte_fns[s->be_data == MO_BE][a->dtype];
+        } else {
+            fn = fns[s->be_data == MO_BE][a->dtype];
+        }
+        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, s->mte_active[0], fn);
     }
     return true;
 }
@@ -4919,16 +7445,64 @@ static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a)
           gen_helper_sve_ldnf1bhs_r,
           gen_helper_sve_ldnf1dd_be_r },
     };
+    static gen_helper_gvec_mem * const mte_fns[2][16] = {
+        /* Little-endian */
+        { gen_helper_sve_ldnf1bb_r_mte,
+          gen_helper_sve_ldnf1bhu_r_mte,
+          gen_helper_sve_ldnf1bsu_r_mte,
+          gen_helper_sve_ldnf1bdu_r_mte,
+
+          gen_helper_sve_ldnf1sds_le_r_mte,
+          gen_helper_sve_ldnf1hh_le_r_mte,
+          gen_helper_sve_ldnf1hsu_le_r_mte,
+          gen_helper_sve_ldnf1hdu_le_r_mte,
+
+          gen_helper_sve_ldnf1hds_le_r_mte,
+          gen_helper_sve_ldnf1hss_le_r_mte,
+          gen_helper_sve_ldnf1ss_le_r_mte,
+          gen_helper_sve_ldnf1sdu_le_r_mte,
+
+          gen_helper_sve_ldnf1bds_r_mte,
+          gen_helper_sve_ldnf1bss_r_mte,
+          gen_helper_sve_ldnf1bhs_r_mte,
+          gen_helper_sve_ldnf1dd_le_r_mte },
+
+        /* Big-endian */
+        { gen_helper_sve_ldnf1bb_r_mte,
+          gen_helper_sve_ldnf1bhu_r_mte,
+          gen_helper_sve_ldnf1bsu_r_mte,
+          gen_helper_sve_ldnf1bdu_r_mte,
+
+          gen_helper_sve_ldnf1sds_be_r_mte,
+          gen_helper_sve_ldnf1hh_be_r_mte,
+          gen_helper_sve_ldnf1hsu_be_r_mte,
+          gen_helper_sve_ldnf1hdu_be_r_mte,
+
+          gen_helper_sve_ldnf1hds_be_r_mte,
+          gen_helper_sve_ldnf1hss_be_r_mte,
+          gen_helper_sve_ldnf1ss_be_r_mte,
+          gen_helper_sve_ldnf1sdu_be_r_mte,
+
+          gen_helper_sve_ldnf1bds_r_mte,
+          gen_helper_sve_ldnf1bss_r_mte,
+          gen_helper_sve_ldnf1bhs_r_mte,
+          gen_helper_sve_ldnf1dd_be_r_mte },
+    };
+    gen_helper_gvec_mem *fn;
 
-    if (sve_access_check(s)) {
+    if (sve_nonstreaming_access_check(s)) {
         int vsz = vec_full_reg_size(s);
         int elements = vsz >> dtype_esz[a->dtype];
         int off = (a->imm * elements) << dtype_msz(a->dtype);
         TCGv_i64 addr = new_tmp_a64(s);
 
         tcg_gen_addi_i64(tcg_ctx, addr, cpu_reg_sp(s, a->rn), off);
-        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype,
-                   fns[s->be_data == MO_BE][a->dtype]);
+        if (s->mte_active[0]) {
+            fn = mte_fns[s->be_data == MO_BE][a->dtype];
+        } else {
+            fn = fns[s->be_data == MO_BE][a->dtype];
+        }
+        do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, s->mte_active[0], fn);
     }
     return true;
 }
@@ -5014,6 +7588,118 @@ static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a)
     return true;
 }
 
+static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static gen_helper_gvec_mem * const fns[2][4] = {
+        { gen_helper_sve_ld1bb_r,    gen_helper_sve_ld1hh_le_r,
+          gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld1dd_le_r },
+        { gen_helper_sve_ld1bb_r,    gen_helper_sve_ld1hh_be_r,
+          gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld1dd_be_r },
+    };
+    static gen_helper_gvec_mem * const mte_fns[2][4] = {
+        { gen_helper_sve_ld1bb_r_mte,    gen_helper_sve_ld1hh_le_r_mte,
+          gen_helper_sve_ld1ss_le_r_mte, gen_helper_sve_ld1dd_le_r_mte },
+        { gen_helper_sve_ld1bb_r_mte,    gen_helper_sve_ld1hh_be_r_mte,
+          gen_helper_sve_ld1ss_be_r_mte, gen_helper_sve_ld1dd_be_r_mte },
+    };
+    unsigned vsz = vec_full_reg_size(s);
+    unsigned vsz_r32;
+    TCGv_ptr t_pg;
+    TCGv_i32 t_desc;
+    gen_helper_gvec_mem *fn;
+    int desc, poff;
+    unsigned dofs;
+
+    if (vsz < 32) {
+        unallocated_encoding(s);
+        return;
+    }
+
+    desc = sve_memopidx(s, dtype);
+    desc |= zt << MEMOPIDX_SHIFT;
+    desc = simd_desc(32, 32, desc);
+    t_desc = tcg_const_i32(tcg_ctx, desc);
+
+    poff = pred_full_reg_offset(s, pg);
+    if (vsz > 32) {
+        TCGv_i64 tmp = tcg_temp_new_i64(tcg_ctx);
+
+#ifdef HOST_WORDS_BIGENDIAN
+        poff += 4;
+#endif
+        tcg_gen_ld32u_i64(tcg_ctx, tmp, tcg_ctx->cpu_env, poff);
+
+        poff = offsetof(CPUARMState, vfp.preg_tmp);
+        tcg_gen_st_i64(tcg_ctx, tmp, tcg_ctx->cpu_env, poff);
+        tcg_temp_free_i64(tcg_ctx, tmp);
+    }
+
+    t_pg = tcg_temp_new_ptr(tcg_ctx);
+    if (s->mte_active[0]) {
+        fn = mte_fns[s->be_data == MO_BE][dtype_msz(dtype)];
+    } else {
+        addr = sve_clean_data_tbi(s, addr);
+        fn = fns[s->be_data == MO_BE][dtype_msz(dtype)];
+    }
+    tcg_gen_addi_ptr(tcg_ctx, t_pg, tcg_ctx->cpu_env, poff);
+    fn(tcg_ctx, tcg_ctx->cpu_env, t_pg, addr, t_desc);
+
+    tcg_temp_free_ptr(tcg_ctx, t_pg);
+    tcg_temp_free_i32(tcg_ctx, t_desc);
+
+    dofs = vec_full_reg_offset(s, zt);
+    vsz_r32 = QEMU_ALIGN_DOWN(vsz, 32);
+    if (vsz >= 64) {
+        unsigned off;
+
+        for (off = 32; off < vsz_r32; off += 32) {
+            tcg_gen_gvec_dup_mem(tcg_ctx, 4, dofs + off, dofs, 16, 16);
+            tcg_gen_gvec_dup_mem(tcg_ctx, 4, dofs + off + 16, dofs + 16,
+                                 16, 16);
+        }
+    }
+    vsz -= vsz_r32;
+    if (vsz) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, dofs + vsz_r32, vsz, vsz, 0);
+    }
+}
+
+static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_f64mm, s)) {
+        return false;
+    }
+    if (a->rm == 31) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        TCGv_i64 addr = new_tmp_a64(s);
+        tcg_gen_shli_i64(tcg_ctx, addr, cpu_reg(s, a->rm),
+                         dtype_msz(a->dtype));
+        tcg_gen_add_i64(tcg_ctx, addr, addr, cpu_reg_sp(s, a->rn));
+        do_ldro(s, a->rd, a->pg, addr, a->dtype);
+    }
+    return true;
+}
+
+static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (!dc_isar_feature(aa64_sve_f64mm, s)) {
+        return false;
+    }
+    if (sve_nonstreaming_access_check(s)) {
+        TCGv_i64 addr = new_tmp_a64(s);
+        tcg_gen_addi_i64(tcg_ctx, addr, cpu_reg_sp(s, a->rn), a->imm * 32);
+        do_ldro(s, a->rd, a->pg, addr, a->dtype);
+    }
+    return true;
+}
+
 /* Load and broadcast element.  */
 static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a)
 {
@@ -5121,19 +7807,81 @@ static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
             gen_helper_sve_st4ss_be_r,
             gen_helper_sve_st4dd_be_r } },
     };
+    static gen_helper_gvec_mem * const mte_fn_single[2][4][4] = {
+        { { gen_helper_sve_st1bb_r_mte,
+            gen_helper_sve_st1bh_r_mte,
+            gen_helper_sve_st1bs_r_mte,
+            gen_helper_sve_st1bd_r_mte },
+          { NULL,
+            gen_helper_sve_st1hh_le_r_mte,
+            gen_helper_sve_st1hs_le_r_mte,
+            gen_helper_sve_st1hd_le_r_mte },
+          { NULL, NULL,
+            gen_helper_sve_st1ss_le_r_mte,
+            gen_helper_sve_st1sd_le_r_mte },
+          { NULL, NULL, NULL,
+            gen_helper_sve_st1dd_le_r_mte } },
+        { { gen_helper_sve_st1bb_r_mte,
+            gen_helper_sve_st1bh_r_mte,
+            gen_helper_sve_st1bs_r_mte,
+            gen_helper_sve_st1bd_r_mte },
+          { NULL,
+            gen_helper_sve_st1hh_be_r_mte,
+            gen_helper_sve_st1hs_be_r_mte,
+            gen_helper_sve_st1hd_be_r_mte },
+          { NULL, NULL,
+            gen_helper_sve_st1ss_be_r_mte,
+            gen_helper_sve_st1sd_be_r_mte },
+          { NULL, NULL, NULL,
+            gen_helper_sve_st1dd_be_r_mte } },
+    };
+    static gen_helper_gvec_mem * const mte_fn_multiple[2][3][4] = {
+        { { gen_helper_sve_st2bb_r_mte,
+            gen_helper_sve_st2hh_le_r_mte,
+            gen_helper_sve_st2ss_le_r_mte,
+            gen_helper_sve_st2dd_le_r_mte },
+          { gen_helper_sve_st3bb_r_mte,
+            gen_helper_sve_st3hh_le_r_mte,
+            gen_helper_sve_st3ss_le_r_mte,
+            gen_helper_sve_st3dd_le_r_mte },
+          { gen_helper_sve_st4bb_r_mte,
+            gen_helper_sve_st4hh_le_r_mte,
+            gen_helper_sve_st4ss_le_r_mte,
+            gen_helper_sve_st4dd_le_r_mte } },
+        { { gen_helper_sve_st2bb_r_mte,
+            gen_helper_sve_st2hh_be_r_mte,
+            gen_helper_sve_st2ss_be_r_mte,
+            gen_helper_sve_st2dd_be_r_mte },
+          { gen_helper_sve_st3bb_r_mte,
+            gen_helper_sve_st3hh_be_r_mte,
+            gen_helper_sve_st3ss_be_r_mte,
+            gen_helper_sve_st3dd_be_r_mte },
+          { gen_helper_sve_st4bb_r_mte,
+            gen_helper_sve_st4hh_be_r_mte,
+            gen_helper_sve_st4ss_be_r_mte,
+            gen_helper_sve_st4dd_be_r_mte } },
+    };
     gen_helper_gvec_mem *fn;
     int be = s->be_data == MO_BE;
 
     if (nreg == 0) {
         /* ST1 */
-        fn = fn_single[be][msz][esz];
+        if (s->mte_active[0]) {
+            fn = mte_fn_single[be][msz][esz];
+        } else {
+            fn = fn_single[be][msz][esz];
+        }
     } else {
         /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
         assert(msz == esz);
-        fn = fn_multiple[be][nreg - 1][msz];
+        if (s->mte_active[0]) {
+            fn = mte_fn_multiple[be][nreg - 1][msz];
+        } else {
+            fn = fn_multiple[be][nreg - 1][msz];
+        }
     }
     assert(fn != NULL);
-    do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), fn);
+    do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), s->mte_active[0], fn);
 }
 
 static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a)
@@ -5367,21 +8115,207 @@ static gen_helper_gvec_mem_scatter * const gather_load_fn64[2][2][3][2][4] = {
             gen_helper_sve_ldffdd_be_zd, } } } },
 };
 
+/* Indexed by [be][xs][u][msz].  */
+static gen_helper_gvec_mem_scatter * const
+gather_ld_mte_fn32[2][2][2][3] = {
+    /* Little-endian */
+    { { { gen_helper_sve_ldbss_zsu_mte,
+          gen_helper_sve_ldhss_le_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldbsu_zsu_mte,
+          gen_helper_sve_ldhsu_le_zsu_mte,
+          gen_helper_sve_ldss_le_zsu_mte, } },
+      { { gen_helper_sve_ldbss_zss_mte,
+          gen_helper_sve_ldhss_le_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldbsu_zss_mte,
+          gen_helper_sve_ldhsu_le_zss_mte,
+          gen_helper_sve_ldss_le_zss_mte, } } },
+
+    /* Big-endian */
+    { { { gen_helper_sve_ldbss_zsu_mte,
+          gen_helper_sve_ldhss_be_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldbsu_zsu_mte,
+          gen_helper_sve_ldhsu_be_zsu_mte,
+          gen_helper_sve_ldss_be_zsu_mte, } },
+      { { gen_helper_sve_ldbss_zss_mte,
+          gen_helper_sve_ldhss_be_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldbsu_zss_mte,
+          gen_helper_sve_ldhsu_be_zss_mte,
+          gen_helper_sve_ldss_be_zss_mte, } } },
+};
+
+/* Indexed by [be][xs][u][msz].  */
+static gen_helper_gvec_mem_scatter * const
+gather_ld_mte_fn64[2][3][2][4] = {
+    /* Little-endian */
+    { { { gen_helper_sve_ldbds_zsu_mte,
+          gen_helper_sve_ldhds_le_zsu_mte,
+          gen_helper_sve_ldsds_le_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldbdu_zsu_mte,
+          gen_helper_sve_ldhdu_le_zsu_mte,
+          gen_helper_sve_ldsdu_le_zsu_mte,
+          gen_helper_sve_lddd_le_zsu_mte, } },
+      { { gen_helper_sve_ldbds_zss_mte,
+          gen_helper_sve_ldhds_le_zss_mte,
+          gen_helper_sve_ldsds_le_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldbdu_zss_mte,
+          gen_helper_sve_ldhdu_le_zss_mte,
+          gen_helper_sve_ldsdu_le_zss_mte,
+          gen_helper_sve_lddd_le_zss_mte, } },
+      { { gen_helper_sve_ldbds_zd_mte,
+          gen_helper_sve_ldhds_le_zd_mte,
+          gen_helper_sve_ldsds_le_zd_mte,
+          NULL, },
+        { gen_helper_sve_ldbdu_zd_mte,
+          gen_helper_sve_ldhdu_le_zd_mte,
+          gen_helper_sve_ldsdu_le_zd_mte,
+          gen_helper_sve_lddd_le_zd_mte, } } },
+
+    /* Big-endian */
+    { { { gen_helper_sve_ldbds_zsu_mte,
+          gen_helper_sve_ldhds_be_zsu_mte,
+          gen_helper_sve_ldsds_be_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldbdu_zsu_mte,
+          gen_helper_sve_ldhdu_be_zsu_mte,
+          gen_helper_sve_ldsdu_be_zsu_mte,
+          gen_helper_sve_lddd_be_zsu_mte, } },
+      { { gen_helper_sve_ldbds_zss_mte,
+          gen_helper_sve_ldhds_be_zss_mte,
+          gen_helper_sve_ldsds_be_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldbdu_zss_mte,
+          gen_helper_sve_ldhdu_be_zss_mte,
+          gen_helper_sve_ldsdu_be_zss_mte,
+          gen_helper_sve_lddd_be_zss_mte, } },
+      { { gen_helper_sve_ldbds_zd_mte,
+          gen_helper_sve_ldhds_be_zd_mte,
+          gen_helper_sve_ldsds_be_zd_mte,
+          NULL, },
+        { gen_helper_sve_ldbdu_zd_mte,
+          gen_helper_sve_ldhdu_be_zd_mte,
+          gen_helper_sve_ldsdu_be_zd_mte,
+          gen_helper_sve_lddd_be_zd_mte, } } },
+};
+
+/* Indexed by [be][xs][u][msz].  */
+static gen_helper_gvec_mem_scatter * const
+gather_ldff_mte_fn32[2][2][2][3] = {
+    /* Little-endian */
+    { { { gen_helper_sve_ldffbss_zsu_mte,
+          gen_helper_sve_ldffhss_le_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldffbsu_zsu_mte,
+          gen_helper_sve_ldffhsu_le_zsu_mte,
+          gen_helper_sve_ldffss_le_zsu_mte, } },
+      { { gen_helper_sve_ldffbss_zss_mte,
+          gen_helper_sve_ldffhss_le_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldffbsu_zss_mte,
+          gen_helper_sve_ldffhsu_le_zss_mte,
+          gen_helper_sve_ldffss_le_zss_mte, } } },
+
+    /* Big-endian */
+    { { { gen_helper_sve_ldffbss_zsu_mte,
+          gen_helper_sve_ldffhss_be_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldffbsu_zsu_mte,
+          gen_helper_sve_ldffhsu_be_zsu_mte,
+          gen_helper_sve_ldffss_be_zsu_mte, } },
+      { { gen_helper_sve_ldffbss_zss_mte,
+          gen_helper_sve_ldffhss_be_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldffbsu_zss_mte,
+          gen_helper_sve_ldffhsu_be_zss_mte,
+          gen_helper_sve_ldffss_be_zss_mte, } } },
+};
+
+/* Indexed by [be][xs][u][msz].  */
+static gen_helper_gvec_mem_scatter * const
+gather_ldff_mte_fn64[2][3][2][4] = {
+    /* Little-endian */
+    { { { gen_helper_sve_ldffbds_zsu_mte,
+          gen_helper_sve_ldffhds_le_zsu_mte,
+          gen_helper_sve_ldffsds_le_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldffbdu_zsu_mte,
+          gen_helper_sve_ldffhdu_le_zsu_mte,
+          gen_helper_sve_ldffsdu_le_zsu_mte,
+          gen_helper_sve_ldffdd_le_zsu_mte, } },
+      { { gen_helper_sve_ldffbds_zss_mte,
+          gen_helper_sve_ldffhds_le_zss_mte,
+          gen_helper_sve_ldffsds_le_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldffbdu_zss_mte,
+          gen_helper_sve_ldffhdu_le_zss_mte,
+          gen_helper_sve_ldffsdu_le_zss_mte,
+          gen_helper_sve_ldffdd_le_zss_mte, } },
+      { { gen_helper_sve_ldffbds_zd_mte,
+          gen_helper_sve_ldffhds_le_zd_mte,
+          gen_helper_sve_ldffsds_le_zd_mte,
+          NULL, },
+        { gen_helper_sve_ldffbdu_zd_mte,
+          gen_helper_sve_ldffhdu_le_zd_mte,
+          gen_helper_sve_ldffsdu_le_zd_mte,
+          gen_helper_sve_ldffdd_le_zd_mte, } } },
+
+    /* Big-endian */
+    { { { gen_helper_sve_ldffbds_zsu_mte,
+          gen_helper_sve_ldffhds_be_zsu_mte,
+          gen_helper_sve_ldffsds_be_zsu_mte,
+          NULL, },
+        { gen_helper_sve_ldffbdu_zsu_mte,
+          gen_helper_sve_ldffhdu_be_zsu_mte,
+          gen_helper_sve_ldffsdu_be_zsu_mte,
+          gen_helper_sve_ldffdd_be_zsu_mte, } },
+      { { gen_helper_sve_ldffbds_zss_mte,
+          gen_helper_sve_ldffhds_be_zss_mte,
+          gen_helper_sve_ldffsds_be_zss_mte,
+          NULL, },
+        { gen_helper_sve_ldffbdu_zss_mte,
+          gen_helper_sve_ldffhdu_be_zss_mte,
+          gen_helper_sve_ldffsdu_be_zss_mte,
+          gen_helper_sve_ldffdd_be_zss_mte, } },
+      { { gen_helper_sve_ldffbds_zd_mte,
+          gen_helper_sve_ldffhds_be_zd_mte,
+          gen_helper_sve_ldffsds_be_zd_mte,
+          NULL, },
+        { gen_helper_sve_ldffbdu_zd_mte,
+          gen_helper_sve_ldffhdu_be_zd_mte,
+          gen_helper_sve_ldffsdu_be_zd_mte,
+          gen_helper_sve_ldffdd_be_zd_mte, } } },
+};
+
 static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
 {
     gen_helper_gvec_mem_scatter *fn = NULL;
     int be = s->be_data == MO_BE;
 
-    if (!sve_access_check(s)) {
+    if (!sve_nonstreaming_access_check(s)) {
         return true;
     }
 
     switch (a->esz) {
     case MO_32:
-        fn = gather_load_fn32[be][a->ff][a->xs][a->u][a->msz];
+        if (s->mte_active[0]) {
+            fn = a->ff ? gather_ldff_mte_fn32[be][a->xs][a->u][a->msz]
+                       : gather_ld_mte_fn32[be][a->xs][a->u][a->msz];
+        } else {
+            fn = gather_load_fn32[be][a->ff][a->xs][a->u][a->msz];
+        }
         break;
     case MO_64:
-        fn = gather_load_fn64[be][a->ff][a->xs][a->u][a->msz];
+        if (s->mte_active[0]) {
+            fn = a->ff ? gather_ldff_mte_fn64[be][a->xs][a->u][a->msz]
+                       : gather_ld_mte_fn64[be][a->xs][a->u][a->msz];
+        } else {
+            fn = gather_load_fn64[be][a->ff][a->xs][a->u][a->msz];
+        }
         break;
     }
     assert(fn != NULL);
@@ -5401,16 +8335,26 @@ static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
     if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {
         return false;
     }
-    if (!sve_access_check(s)) {
+    if (!sve_nonstreaming_access_check(s)) {
         return true;
     }
 
     switch (a->esz) {
     case MO_32:
-        fn = gather_load_fn32[be][a->ff][0][a->u][a->msz];
+        if (s->mte_active[0]) {
+            fn = a->ff ? gather_ldff_mte_fn32[be][0][a->u][a->msz]
+                       : gather_ld_mte_fn32[be][0][a->u][a->msz];
+        } else {
+            fn = gather_load_fn32[be][a->ff][0][a->u][a->msz];
+        }
         break;
     case MO_64:
-        fn = gather_load_fn64[be][a->ff][2][a->u][a->msz];
+        if (s->mte_active[0]) {
+            fn = a->ff ? gather_ldff_mte_fn64[be][2][a->u][a->msz]
+                       : gather_ld_mte_fn64[be][2][a->u][a->msz];
+        } else {
+            fn = gather_load_fn64[be][a->ff][2][a->u][a->msz];
+        }
         break;
     }
     assert(fn != NULL);
@@ -5424,6 +8368,41 @@ static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
     return true;
 }
 
+static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a)
+{
+    gen_helper_gvec_mem_scatter *fn = NULL;
+    int be = s->be_data == MO_BE;
+
+    if (a->esz < a->msz + !a->u) {
+        return false;
+    }
+    if (!sve_nonstreaming_access_check(s)) {
+        return true;
+    }
+
+    switch (a->esz) {
+    case MO_32:
+        if (s->mte_active[0]) {
+            fn = gather_ld_mte_fn32[be][0][a->u][a->msz];
+        } else {
+            fn = gather_load_fn32[be][0][0][a->u][a->msz];
+        }
+        break;
+    case MO_64:
+        if (s->mte_active[0]) {
+            fn = gather_ld_mte_fn64[be][2][a->u][a->msz];
+        } else {
+            fn = gather_load_fn64[be][0][2][a->u][a->msz];
+        }
+        break;
+    }
+    assert(fn != NULL);
+
+    do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
+               cpu_reg(s, a->rm), a->msz, fn);
+    return true;
+}
+
 /* Indexed by [be][xs][msz].  */
 static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][3] = {
     /* Little-endian */
@@ -5442,6 +8421,24 @@ static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][3] = {
         gen_helper_sve_stss_be_zss, } },
 };
 
+/* Indexed by [be][xs][msz].  */
+static gen_helper_gvec_mem_scatter * const scatter_store_mte_fn32[2][2][3] = {
+    /* Little-endian */
+    { { gen_helper_sve_stbs_zsu_mte,
+        gen_helper_sve_sths_le_zsu_mte,
+        gen_helper_sve_stss_le_zsu_mte, },
+      { gen_helper_sve_stbs_zss_mte,
+        gen_helper_sve_sths_le_zss_mte,
+        gen_helper_sve_stss_le_zss_mte, } },
+    /* Big-endian */
+    { { gen_helper_sve_stbs_zsu_mte,
+        gen_helper_sve_sths_be_zsu_mte,
+        gen_helper_sve_stss_be_zsu_mte, },
+      { gen_helper_sve_stbs_zss_mte,
+        gen_helper_sve_sths_be_zss_mte,
+        gen_helper_sve_stss_be_zss_mte, } },
+};
+
 /* Note that we overload xs=2 to indicate 64-bit offset.  */
 static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][3][4] = {
     /* Little-endian */
@@ -5472,6 +8469,36 @@ static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][3][4] = {
         gen_helper_sve_stdd_be_zd, } },
 };
 
+/* Note that we overload xs=2 to indicate 64-bit offset.  */
+static gen_helper_gvec_mem_scatter * const scatter_store_mte_fn64[2][3][4] = {
+    /* Little-endian */
+    { { gen_helper_sve_stbd_zsu_mte,
+        gen_helper_sve_sthd_le_zsu_mte,
+        gen_helper_sve_stsd_le_zsu_mte,
+        gen_helper_sve_stdd_le_zsu_mte, },
+      { gen_helper_sve_stbd_zss_mte,
+        gen_helper_sve_sthd_le_zss_mte,
+        gen_helper_sve_stsd_le_zss_mte,
+        gen_helper_sve_stdd_le_zss_mte, },
+      { gen_helper_sve_stbd_zd_mte,
+        gen_helper_sve_sthd_le_zd_mte,
+        gen_helper_sve_stsd_le_zd_mte,
+        gen_helper_sve_stdd_le_zd_mte, } },
+    /* Big-endian */
+    { { gen_helper_sve_stbd_zsu_mte,
+        gen_helper_sve_sthd_be_zsu_mte,
+        gen_helper_sve_stsd_be_zsu_mte,
+        gen_helper_sve_stdd_be_zsu_mte, },
+      { gen_helper_sve_stbd_zss_mte,
+        gen_helper_sve_sthd_be_zss_mte,
+        gen_helper_sve_stsd_be_zss_mte,
+        gen_helper_sve_stdd_be_zss_mte, },
+      { gen_helper_sve_stbd_zd_mte,
+        gen_helper_sve_sthd_be_zd_mte,
+        gen_helper_sve_stsd_be_zd_mte,
+        gen_helper_sve_stdd_be_zd_mte, } },
+};
+
 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
 {
     gen_helper_gvec_mem_scatter *fn = NULL;
@@ -5480,15 +8507,23 @@ static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
     if (a->esz < a->msz || (a->msz == 0 && a->scale)) {
         return false;
     }
-    if (!sve_access_check(s)) {
+    if (!sve_nonstreaming_access_check(s)) {
         return true;
     }
     switch (a->esz) {
     case MO_32:
-        fn = scatter_store_fn32[be][a->xs][a->msz];
+        if (s->mte_active[0]) {
+            fn = scatter_store_mte_fn32[be][a->xs][a->msz];
+        } else {
+            fn = scatter_store_fn32[be][a->xs][a->msz];
+        }
         break;
     case MO_64:
-        fn = scatter_store_fn64[be][a->xs][a->msz];
+        if (s->mte_active[0]) {
+            fn = scatter_store_mte_fn64[be][a->xs][a->msz];
+        } else {
+            fn = scatter_store_fn64[be][a->xs][a->msz];
+        }
         break;
     default:
         g_assert_not_reached();
@@ -5508,16 +8543,24 @@ static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
     if (a->esz < a->msz) {
         return false;
     }
-    if (!sve_access_check(s)) {
+    if (!sve_nonstreaming_access_check(s)) {
         return true;
     }
 
     switch (a->esz) {
     case MO_32:
-        fn = scatter_store_fn32[be][0][a->msz];
+        if (s->mte_active[0]) {
+            fn = scatter_store_mte_fn32[be][0][a->msz];
+        } else {
+            fn = scatter_store_fn32[be][0][a->msz];
+        }
         break;
     case MO_64:
-        fn = scatter_store_fn64[be][2][a->msz];
+        if (s->mte_active[0]) {
+            fn = scatter_store_mte_fn64[be][2][a->msz];
+        } else {
+            fn = scatter_store_fn64[be][2][a->msz];
+        }
         break;
     }
     assert(fn != NULL);
@@ -5531,13 +8574,71 @@ static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
     return true;
 }
 
+static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a)
+{
+    gen_helper_gvec_mem_scatter *fn = NULL;
+    int be = s->be_data == MO_BE;
+
+    if (a->esz < a->msz) {
+        return false;
+    }
+    if (!sve_nonstreaming_access_check(s)) {
+        return true;
+    }
+
+    switch (a->esz) {
+    case MO_32:
+        if (s->mte_active[0]) {
+            fn = scatter_store_mte_fn32[be][0][a->msz];
+        } else {
+            fn = scatter_store_fn32[be][0][a->msz];
+        }
+        break;
+    case MO_64:
+        if (s->mte_active[0]) {
+            fn = scatter_store_mte_fn64[be][2][a->msz];
+        } else {
+            fn = scatter_store_fn64[be][2][a->msz];
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    assert(fn != NULL);
+
+    do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
+               cpu_reg(s, a->rm), a->msz, fn);
+    return true;
+}
+
 /*
  * Prefetches
  */
 
+static bool sve_prf_is_nonstreaming(uint32_t insn)
+{
+    uint32_t op = extract32(insn, 25, 7);
+    bool scalar_offsets;
+    bool vector_imm;
+
+    if (op != 0x42 && op != 0x62) {
+        return false;
+    }
+
+    scalar_offsets = extract32(insn, 23, 2) == 0 &&
+                     extract32(insn, 21, 1);
+    vector_imm = extract32(insn, 21, 2) == 0 &&
+                 extract32(insn, 13, 3) == 7;
+
+    return scalar_offsets || vector_imm;
+}
+
 static bool trans_PRF(DisasContext *s, arg_PRF *a)
 {
     /* Prefetch is a nop within QEMU.  */
+    if (sve_prf_is_nonstreaming(s->insn)) {
+        s->is_nonstreaming = true;
+    }
     (void)sve_access_check(s);
     return true;
 }
diff --git a/qemu/target/arm/translate-vfp.inc.c b/qemu/target/arm/translate-vfp.inc.c
index 4773efb9b2..27dde2e537 100644
--- a/qemu/target/arm/translate-vfp.inc.c
+++ b/qemu/target/arm/translate-vfp.inc.c
@@ -92,8 +92,11 @@ static inline long vfp_f16_offset(unsigned reg, bool top)
  * The ignore_vfp_enabled argument specifies that we should ignore
  * whether VFP is enabled via FPEXC[EN]: this should be true for FMXR/FMRX
  * accesses to FPSID, FPEXC, MVFR0, MVFR1, MVFR2, and false for all other insns.
+ * skip_context_update is M-profile only and skips the ownership/new-context
+ * parts of ExecuteFPCheck() after PreserveFPState().
  */
-static bool full_vfp_access_check(DisasContext *s, bool ignore_vfp_enabled)
+static bool full_vfp_access_check(DisasContext *s, bool ignore_vfp_enabled,
+                                  bool skip_context_update)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
 
@@ -137,6 +140,14 @@ static bool full_vfp_access_check(DisasContext *s, bool ignore_vfp_enabled)
              * any further FP insns in this TB.
              */
             s->v7m_lspact = false;
+            if (skip_context_update || !s->v7m_new_fp_ctxt_needed) {
+                s->mve_no_pred = false;
+                s->base.is_jmp = DISAS_UPDATE;
+            }
+        }
+
+        if (skip_context_update) {
+            return true;
         }
 
         /* Update ownership of FP context: set FPCCR.S to match current state */
@@ -165,6 +176,12 @@ static bool full_vfp_access_check(DisasContext *s, bool ignore_vfp_enabled)
             fpscr = load_cpu_field(tcg_ctx, v7m.fpdscr[s->v8m_secure]);
             gen_helper_vfp_set_fpscr(tcg_ctx, tcg_ctx->cpu_env, fpscr);
             tcg_temp_free_i32(tcg_ctx, fpscr);
+            if (dc_isar_feature(aa32_mve, s)) {
+                TCGv_i32 vpr = tcg_const_i32(tcg_ctx, 0);
+
+                store_cpu_field(tcg_ctx, vpr, v7m.vpr);
+                s->mve_no_pred = true;
+            }
             /*
              * We don't need to arrange to end the TB, because the only
              * parts of FPSCR which we cache in the TB flags are the VECLEN
@@ -191,7 +208,150 @@ static bool full_vfp_access_check(DisasContext *s, bool ignore_vfp_enabled)
  */
 static bool vfp_access_check(DisasContext *s)
 {
-    return full_vfp_access_check(s, false);
+    return full_vfp_access_check(s, false, false);
+}
+
+static bool m_profile_vfp_access_check(DisasContext *s,
+                                       bool skip_context_update)
+{
+    return full_vfp_access_check(s, false, skip_context_update);
+}
+
+static void m_profile_fp_inactive_ns_branch(DisasContext *s,
+                                            bool branch_if_inactive,
+                                            TCGLabel *label)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 aspen;
+    TCGv_i32 fpca;
+
+    aspen = load_cpu_field(tcg_ctx, v7m.fpccr[M_REG_NS]);
+    fpca = load_cpu_field(tcg_ctx, v7m.control[M_REG_S]);
+    tcg_gen_andi_i32(tcg_ctx, aspen, aspen, R_V7M_FPCCR_ASPEN_MASK);
+    tcg_gen_xori_i32(tcg_ctx, aspen, aspen, R_V7M_FPCCR_ASPEN_MASK);
+    tcg_gen_andi_i32(tcg_ctx, fpca, fpca, R_V7M_CONTROL_FPCA_MASK);
+    tcg_gen_or_i32(tcg_ctx, fpca, fpca, aspen);
+    tcg_gen_brcondi_i32(tcg_ctx, branch_if_inactive ? TCG_COND_EQ :
+                        TCG_COND_NE, fpca, 0, label);
+    tcg_temp_free_i32(tcg_ctx, aspen);
+    tcg_temp_free_i32(tcg_ctx, fpca);
+}
+
+static TCGv_i32 m_profile_fpcxt_value(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 control;
+    TCGv_i32 sfpa;
+    TCGv_i32 value;
+
+    value = tcg_temp_new_i32(tcg_ctx);
+    sfpa = tcg_temp_new_i32(tcg_ctx);
+    gen_helper_vfp_get_fpscr(tcg_ctx, value, tcg_ctx->cpu_env);
+    tcg_gen_andi_i32(tcg_ctx, value, value, ~FPCR_NZCV_MASK);
+    control = load_cpu_field(tcg_ctx, v7m.control[M_REG_S]);
+    tcg_gen_andi_i32(tcg_ctx, sfpa, control, R_V7M_CONTROL_SFPA_MASK);
+    tcg_gen_shli_i32(tcg_ctx, sfpa, sfpa,
+                     31 - R_V7M_CONTROL_SFPA_SHIFT);
+    tcg_gen_or_i32(tcg_ctx, value, value, sfpa);
+    tcg_temp_free_i32(tcg_ctx, control);
+    tcg_temp_free_i32(tcg_ctx, sfpa);
+    return value;
+}
+
+static void m_profile_fpcxt_write_value(DisasContext *s, TCGv_i32 value)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 control;
+    TCGv_i32 sfpa;
+
+    sfpa = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_shri_i32(tcg_ctx, sfpa, value, 31);
+    control = load_cpu_field(tcg_ctx, v7m.control[M_REG_S]);
+    tcg_gen_deposit_i32(tcg_ctx, control, control, sfpa,
+                        R_V7M_CONTROL_SFPA_SHIFT, 1);
+    store_cpu_field(tcg_ctx, control, v7m.control[M_REG_S]);
+    tcg_gen_andi_i32(tcg_ctx, value, value, ~FPCR_NZCV_MASK);
+    gen_helper_vfp_set_fpscr(tcg_ctx, tcg_ctx->cpu_env, value);
+    tcg_temp_free_i32(tcg_ctx, value);
+    tcg_temp_free_i32(tcg_ctx, sfpa);
+    s->base.is_jmp = DISAS_UPDATE;
+}
+
+static void m_profile_fpcxt_s_read_side_effect(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 control;
+    TCGv_i32 fpscr;
+
+    control = load_cpu_field(tcg_ctx, v7m.control[M_REG_S]);
+    tcg_gen_andi_i32(tcg_ctx, control, control, ~R_V7M_CONTROL_SFPA_MASK);
+    store_cpu_field(tcg_ctx, control, v7m.control[M_REG_S]);
+    fpscr = load_cpu_field(tcg_ctx, v7m.fpdscr[M_REG_NS]);
+    gen_helper_vfp_set_fpscr(tcg_ctx, tcg_ctx->cpu_env, fpscr);
+    tcg_temp_free_i32(tcg_ctx, fpscr);
+    s->base.is_jmp = DISAS_UPDATE;
+}
+
+static void m_profile_fpcxt_ns_read_active_side_effect(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 control;
+    TCGv_i32 sfpa;
+    TCGv_i32 fpscr;
+    TCGv_i32 fpdscr;
+    TCGv_i32 zero;
+
+    control = load_cpu_field(tcg_ctx, v7m.control[M_REG_S]);
+    sfpa = tcg_temp_new_i32(tcg_ctx);
+    fpscr = tcg_temp_new_i32(tcg_ctx);
+    zero = tcg_const_i32(tcg_ctx, 0);
+    gen_helper_vfp_get_fpscr(tcg_ctx, fpscr, tcg_ctx->cpu_env);
+    tcg_gen_andi_i32(tcg_ctx, sfpa, control, R_V7M_CONTROL_SFPA_MASK);
+    fpdscr = load_cpu_field(tcg_ctx, v7m.fpdscr[M_REG_NS]);
+    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_EQ, fpscr, sfpa, zero,
+                        fpdscr, fpscr);
+    gen_helper_vfp_set_fpscr(tcg_ctx, tcg_ctx->cpu_env, fpscr);
+    tcg_temp_free_i32(tcg_ctx, control);
+    tcg_temp_free_i32(tcg_ctx, sfpa);
+    tcg_temp_free_i32(tcg_ctx, fpscr);
+    tcg_temp_free_i32(tcg_ctx, fpdscr);
+    tcg_temp_free_i32(tcg_ctx, zero);
+}
+
+static bool m_profile_fpcxt_ns_gpr(DisasContext *s, arg_VMSR_VMRS *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGLabel *lab_active = gen_new_label(tcg_ctx);
+    TCGLabel *lab_end = gen_new_label(tcg_ctx);
+    TCGv_i32 value;
+
+    m_profile_fp_inactive_ns_branch(s, false, lab_active);
+    if (a->l) {
+        value = load_cpu_field(tcg_ctx, v7m.fpdscr[M_REG_NS]);
+        store_reg(s, a->rt, value);
+        tcg_gen_br(tcg_ctx, lab_end);
+    } else {
+        tcg_gen_br(tcg_ctx, lab_end);
+    }
+
+    gen_set_label(tcg_ctx, lab_active);
+    if (!m_profile_vfp_access_check(s, true)) {
+        s->base.is_jmp = DISAS_NEXT;
+        gen_set_label(tcg_ctx, lab_end);
+        return true;
+    }
+    if (a->l) {
+        value = m_profile_fpcxt_value(s);
+        store_reg(s, a->rt, value);
+        m_profile_fpcxt_ns_read_active_side_effect(s);
+    } else {
+        value = load_reg(s, a->rt);
+        m_profile_fpcxt_write_value(s, value);
+    }
+
+    gen_set_label(tcg_ctx, lab_end);
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
 }
 
 static bool trans_VSEL(DisasContext *s, arg_VSEL *a)
@@ -643,19 +803,40 @@ static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a)
     TCGv_i32 tmp = 0;
     bool ignore_vfp_enabled = false;
 
-    if (!dc_isar_feature(aa32_fpsp_v2, s)) {
+    if (!dc_isar_feature(aa32_fpsp_v2, s) &&
+        !dc_isar_feature(aa32_mve, s)) {
         return false;
     }
 
     if (arm_dc_feature(s, ARM_FEATURE_M)) {
-        /*
-         * The only M-profile VFP vmrs/vmsr sysreg is FPSCR.
-         * Accesses to R15 are UNPREDICTABLE; we choose to undef.
-         * (FPSCR -> r15 is a special case which writes to the PSR flags.)
-         */
+        /* Accesses to R15 are UNPREDICTABLE, except FPSCR -> APSR.NZCV. */
         if (a->rt == 15 && (!a->l || a->reg != ARM_VFP_FPSCR)) {
             return false;
         }
+        switch (a->reg) {
+        case ARM_VFP_FPSCR:
+            break;
+        case ARM_VFP_FPSCR_NZCVQC:
+            if (!arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+                return false;
+            }
+            break;
+        case ARM_VFP_VPR:
+        case ARM_VFP_P0:
+            if (!dc_isar_feature(aa32_mve, s)) {
+                unallocated_encoding(s);
+                return true;
+            }
+            break;
+        case ARM_VFP_FPCXT_NS:
+        case ARM_VFP_FPCXT_S:
+            if (!arm_dc_feature(s, ARM_FEATURE_V8_1M) || !s->v8m_secure) {
+                return false;
+            }
+            break;
+        default:
+            return false;
+        }
     }
 
     switch (a->reg) {
@@ -684,6 +865,26 @@ static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a)
         break;
     case ARM_VFP_FPSCR:
         break;
+    case ARM_VFP_FPSCR_NZCVQC:
+        if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+            !arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+            return false;
+        }
+        break;
+    case ARM_VFP_VPR:
+    case ARM_VFP_P0:
+        if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+            !dc_isar_feature(aa32_mve, s)) {
+            return false;
+        }
+        break;
+    case ARM_VFP_FPCXT_NS:
+    case ARM_VFP_FPCXT_S:
+        if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+            !arm_dc_feature(s, ARM_FEATURE_V8_1M) || !s->v8m_secure) {
+            return false;
+        }
+        break;
     case ARM_VFP_FPEXC:
         if (IS_USER(s)) {
             return false;
@@ -701,7 +902,11 @@ static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a)
         return false;
     }
 
-    if (!full_vfp_access_check(s, ignore_vfp_enabled)) {
+    if (arm_dc_feature(s, ARM_FEATURE_M) && a->reg == ARM_VFP_FPCXT_NS) {
+        return m_profile_fpcxt_ns_gpr(s, a);
+    }
+
+    if (!full_vfp_access_check(s, ignore_vfp_enabled, false)) {
         return true;
     }
 
@@ -738,6 +943,25 @@ static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a)
                 gen_helper_vfp_get_fpscr(tcg_ctx, tmp, tcg_ctx->cpu_env);
             }
             break;
+        case ARM_VFP_FPSCR_NZCVQC:
+            tmp = tcg_temp_new_i32(tcg_ctx);
+            gen_helper_vfp_get_fpscr(tcg_ctx, tmp, tcg_ctx->cpu_env);
+            tcg_gen_andi_i32(tcg_ctx, tmp, tmp,
+                             dc_isar_feature(aa32_mve, s) ?
+                             FPCR_NZCVQC_MASK : FPCR_NZCV_MASK);
+            break;
+        case ARM_VFP_VPR:
+            tmp = load_cpu_field(tcg_ctx, v7m.vpr);
+            break;
+        case ARM_VFP_P0:
+            tmp = load_cpu_field(tcg_ctx, v7m.vpr);
+            tcg_gen_extract_i32(tcg_ctx, tmp, tmp,
+                                R_V7M_VPR_P0_SHIFT,
+                                R_V7M_VPR_P0_LENGTH);
+            break;
+        case ARM_VFP_FPCXT_S:
+            tmp = m_profile_fpcxt_value(s);
+            break;
         default:
             g_assert_not_reached();
         }
@@ -749,6 +973,9 @@ static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a)
         } else {
             store_reg(s, a->rt, tmp);
         }
+        if (a->reg == ARM_VFP_FPCXT_S) {
+            m_profile_fpcxt_s_read_side_effect(s);
+        }
     } else {
         /* VMSR, move gp register to VFP special register */
         switch (a->reg) {
@@ -764,6 +991,51 @@ static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a)
             tcg_temp_free_i32(tcg_ctx, tmp);
             gen_lookup_tb(s);
             break;
+        case ARM_VFP_FPSCR_NZCVQC:
+        {
+            TCGv_i32 fpscr;
+
+            tmp = load_reg(s, a->rt);
+            if (dc_isar_feature(aa32_mve, s)) {
+                TCGv_i32 qc = tcg_temp_new_i32(tcg_ctx);
+
+                tcg_gen_andi_i32(tcg_ctx, qc, tmp, FPCR_QC);
+                tcg_gen_gvec_dup_i32(tcg_ctx, MO_32,
+                                      offsetof(CPUARMState, vfp.qc),
+                                      16, 16, qc);
+                tcg_temp_free_i32(tcg_ctx, qc);
+            }
+            tcg_gen_andi_i32(tcg_ctx, tmp, tmp, FPCR_NZCV_MASK);
+            fpscr = load_cpu_field(tcg_ctx, vfp.xregs[ARM_VFP_FPSCR]);
+            tcg_gen_andi_i32(tcg_ctx, fpscr, fpscr, ~FPCR_NZCV_MASK);
+            tcg_gen_or_i32(tcg_ctx, fpscr, fpscr, tmp);
+            store_cpu_field(tcg_ctx, fpscr, vfp.xregs[ARM_VFP_FPSCR]);
+            tcg_temp_free_i32(tcg_ctx, tmp);
+            break;
+        }
+        case ARM_VFP_VPR:
+            tmp = load_reg(s, a->rt);
+            store_cpu_field(tcg_ctx, tmp, v7m.vpr);
+            gen_lookup_tb(s);
+            break;
+        case ARM_VFP_P0:
+        {
+            TCGv_i32 vpr;
+
+            tmp = load_reg(s, a->rt);
+            vpr = load_cpu_field(tcg_ctx, v7m.vpr);
+            tcg_gen_deposit_i32(tcg_ctx, vpr, vpr, tmp,
+                                R_V7M_VPR_P0_SHIFT,
+                                R_V7M_VPR_P0_LENGTH);
+            store_cpu_field(tcg_ctx, vpr, v7m.vpr);
+            tcg_temp_free_i32(tcg_ctx, tmp);
+            gen_lookup_tb(s);
+            break;
+        }
+        case ARM_VFP_FPCXT_S:
+            tmp = load_reg(s, a->rt);
+            m_profile_fpcxt_write_value(s, tmp);
+            break;
         case ARM_VFP_FPEXC:
             /*
              * TODO: VFP subarchitecture support.
@@ -787,6 +1059,482 @@ static bool trans_VMSR_VMRS(DisasContext *s, arg_VMSR_VMRS *a)
     return true;
 }
 
+static void m_profile_clear_eci_state(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 zero;
+
+    if (s->eci) {
+        zero = tcg_const_i32(tcg_ctx, 0);
+        store_cpu_field(tcg_ctx, zero, condexec_bits);
+        s->eci = 0;
+    }
+}
+
+static bool trans_m_profile_vlldm_vlstm(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 fptr;
+    int rn;
+    bool is_load;
+    bool t2;
+
+    if ((insn & 0xffe0ff7f) != 0xec200a00) {
+        return false;
+    }
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !arm_dc_feature(s, ARM_FEATURE_V8)) {
+        return false;
+    }
+
+    is_load = extract32(insn, 20, 1);
+    rn = extract32(insn, 16, 4);
+    t2 = extract32(insn, 7, 1);
+
+    if (t2) {
+        if (!arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+            unallocated_encoding(s);
+            return true;
+        }
+    } else if (dc_isar_feature(aa32_simd_r32, s)) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    if (!s->v8m_secure) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    s->eci_handled = true;
+
+    if (!dc_isar_feature(aa32_vfp, s)) {
+        m_profile_clear_eci_state(s);
+        return true;
+    }
+
+    fptr = load_reg(s, rn);
+    if (is_load) {
+        gen_helper_v7m_vlldm(tcg_ctx, tcg_ctx->cpu_env, fptr);
+    } else {
+        gen_helper_v7m_vlstm(tcg_ctx, tcg_ctx->cpu_env, fptr);
+    }
+    tcg_temp_free_i32(tcg_ctx, fptr);
+
+    m_profile_clear_eci_state(s);
+
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_m_profile_vscclrm(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 aspen;
+    TCGv_i32 sfpa;
+    TCGv_i32 zero;
+    int btmreg;
+    int topreg;
+    int vd;
+    int imm;
+    bool dp;
+
+    if ((insn & 0xffbf0f01) == 0xec9f0b00) {
+        dp = true;
+        vd = (extract32(insn, 22, 1) << 4) | extract32(insn, 12, 4);
+        imm = extract32(insn, 1, 7);
+    } else if ((insn & 0xffbf0f00) == 0xec9f0a00) {
+        dp = false;
+        vd = (extract32(insn, 12, 4) << 1) | extract32(insn, 22, 1);
+        imm = extract32(insn, 0, 8);
+    } else {
+        return false;
+    }
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M)) {
+        return false;
+    }
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M_SECURITY) ||
+        !arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M_MAIN) || !s->v8m_secure) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    s->eci_handled = true;
+
+    if (!dc_isar_feature(aa32_vfp_simd, s)) {
+        m_profile_clear_eci_state(s);
+        return true;
+    }
+
+    aspen = load_cpu_field(tcg_ctx, v7m.fpccr[M_REG_S]);
+    sfpa = load_cpu_field(tcg_ctx, v7m.control[M_REG_S]);
+    tcg_gen_andi_i32(tcg_ctx, aspen, aspen, R_V7M_FPCCR_ASPEN_MASK);
+    tcg_gen_xori_i32(tcg_ctx, aspen, aspen, R_V7M_FPCCR_ASPEN_MASK);
+    tcg_gen_andi_i32(tcg_ctx, sfpa, sfpa, R_V7M_CONTROL_SFPA_MASK);
+    tcg_gen_or_i32(tcg_ctx, sfpa, sfpa, aspen);
+    tcg_temp_free_i32(tcg_ctx, aspen);
+    arm_gen_condlabel(s);
+    tcg_gen_brcondi_i32(tcg_ctx, TCG_COND_EQ, sfpa, 0, s->condlabel);
+    tcg_temp_free_i32(tcg_ctx, sfpa);
+
+    if (s->fp_excp_el != 0) {
+        gen_exception_insn(s, s->pc_curr, EXCP_NOCP, syn_uncategorized(),
+                           s->fp_excp_el);
+        return true;
+    }
+
+    btmreg = vd;
+    topreg = vd + imm - 1;
+    if (dp) {
+        btmreg *= 2;
+        topreg = topreg * 2 + 1;
+    }
+
+    if (topreg > 63 || (topreg > 31 && !(topreg & 1))) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    if (topreg > 31 && !dc_isar_feature(aa32_simd_r32, s)) {
+        topreg = 31;
+    }
+
+    if (!vfp_access_check(s)) {
+        return true;
+    }
+
+    zero = tcg_const_i32(tcg_ctx, 0);
+    for (; btmreg <= topreg; btmreg++) {
+        neon_store_reg32(tcg_ctx, zero, btmreg);
+    }
+    tcg_temp_free_i32(tcg_ctx, zero);
+
+    if (dc_isar_feature(aa32_mve, s)) {
+        zero = tcg_const_i32(tcg_ctx, 0);
+        store_cpu_field(tcg_ctx, zero, v7m.vpr);
+    }
+
+    m_profile_clear_eci_state(s);
+    return true;
+}
+
+static bool m_profile_fp_sysreg_check(DisasContext *s, int reg)
+{
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !arm_dc_feature(s, ARM_FEATURE_V8_1M) ||
+        (!dc_isar_feature(aa32_fpsp_v2, s) &&
+         !dc_isar_feature(aa32_mve, s))) {
+        return false;
+    }
+
+    switch (reg) {
+    case ARM_VFP_FPSCR:
+        return true;
+    case ARM_VFP_FPSCR_NZCVQC:
+        return true;
+    case ARM_VFP_VPR:
+    case ARM_VFP_P0:
+        return dc_isar_feature(aa32_mve, s);
+    case ARM_VFP_FPCXT_NS:
+    case ARM_VFP_FPCXT_S:
+        return s->v8m_secure;
+    default:
+        return false;
+    }
+}
+
+static TCGv_i32 m_profile_fp_sysreg_read(DisasContext *s, int reg)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 tmp;
+
+    switch (reg) {
+    case ARM_VFP_FPSCR:
+        tmp = tcg_temp_new_i32(tcg_ctx);
+        gen_helper_vfp_get_fpscr(tcg_ctx, tmp, tcg_ctx->cpu_env);
+        return tmp;
+    case ARM_VFP_FPSCR_NZCVQC:
+        tmp = tcg_temp_new_i32(tcg_ctx);
+        gen_helper_vfp_get_fpscr(tcg_ctx, tmp, tcg_ctx->cpu_env);
+        tcg_gen_andi_i32(tcg_ctx, tmp, tmp,
+                         dc_isar_feature(aa32_mve, s) ?
+                         FPCR_NZCVQC_MASK : FPCR_NZCV_MASK);
+        return tmp;
+    case ARM_VFP_VPR:
+        return load_cpu_field(tcg_ctx, v7m.vpr);
+    case ARM_VFP_P0:
+        tmp = load_cpu_field(tcg_ctx, v7m.vpr);
+        tcg_gen_extract_i32(tcg_ctx, tmp, tmp,
+                            R_V7M_VPR_P0_SHIFT, R_V7M_VPR_P0_LENGTH);
+        return tmp;
+    case ARM_VFP_FPCXT_NS:
+    case ARM_VFP_FPCXT_S:
+        return m_profile_fpcxt_value(s);
+    default:
+        g_assert_not_reached();
+    }
+
+    return NULL;
+}
+
+static void m_profile_fp_sysreg_write(DisasContext *s, int reg, TCGv_i32 tmp)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    switch (reg) {
+    case ARM_VFP_FPSCR:
+        gen_helper_vfp_set_fpscr(tcg_ctx, tcg_ctx->cpu_env, tmp);
+        tcg_temp_free_i32(tcg_ctx, tmp);
+        gen_lookup_tb(s);
+        break;
+    case ARM_VFP_FPSCR_NZCVQC:
+    {
+        TCGv_i32 fpscr;
+
+        if (dc_isar_feature(aa32_mve, s)) {
+            TCGv_i32 qc = tcg_temp_new_i32(tcg_ctx);
+
+            tcg_gen_andi_i32(tcg_ctx, qc, tmp, FPCR_QC);
+            tcg_gen_gvec_dup_i32(tcg_ctx, MO_32,
+                                  offsetof(CPUARMState, vfp.qc),
+                                  16, 16, qc);
+            tcg_temp_free_i32(tcg_ctx, qc);
+        }
+        tcg_gen_andi_i32(tcg_ctx, tmp, tmp, FPCR_NZCV_MASK);
+        fpscr = load_cpu_field(tcg_ctx, vfp.xregs[ARM_VFP_FPSCR]);
+        tcg_gen_andi_i32(tcg_ctx, fpscr, fpscr, ~FPCR_NZCV_MASK);
+        tcg_gen_or_i32(tcg_ctx, fpscr, fpscr, tmp);
+        store_cpu_field(tcg_ctx, fpscr, vfp.xregs[ARM_VFP_FPSCR]);
+        tcg_temp_free_i32(tcg_ctx, tmp);
+        break;
+    }
+    case ARM_VFP_VPR:
+        store_cpu_field(tcg_ctx, tmp, v7m.vpr);
+        gen_lookup_tb(s);
+        break;
+    case ARM_VFP_P0:
+    {
+        TCGv_i32 vpr;
+
+        vpr = load_cpu_field(tcg_ctx, v7m.vpr);
+        tcg_gen_deposit_i32(tcg_ctx, vpr, vpr, tmp,
+                            R_V7M_VPR_P0_SHIFT, R_V7M_VPR_P0_LENGTH);
+        store_cpu_field(tcg_ctx, vpr, v7m.vpr);
+        tcg_temp_free_i32(tcg_ctx, tmp);
+        gen_lookup_tb(s);
+        break;
+    }
+    case ARM_VFP_FPCXT_NS:
+    case ARM_VFP_FPCXT_S:
+        m_profile_fpcxt_write_value(s, tmp);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static void m_profile_sysreg_mem_writeback(DisasContext *s, int rn,
+                                           int32_t offset)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 wb;
+
+    wb = load_reg(s, rn);
+    if (offset) {
+        tcg_gen_addi_i32(tcg_ctx, wb, wb, offset);
+    }
+    store_reg(s, rn, wb);
+}
+
+static TCGv_i32 m_profile_sysreg_mem_addr(DisasContext *s, int rn,
+                                          int32_t offset, bool p, bool w)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 addr;
+
+    addr = load_reg(s, rn);
+    if (p) {
+        tcg_gen_addi_i32(tcg_ctx, addr, addr, offset);
+    }
+
+    if (s->v8m_stackcheck && rn == 13 && w) {
+        gen_helper_v8m_stackcheck(tcg_ctx, tcg_ctx->cpu_env, addr);
+    }
+
+    return addr;
+}
+
+static bool trans_m_profile_fpcxt_ns_mem(DisasContext *s, int rn,
+                                         int32_t offset, bool p, bool w,
+                                         bool load)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGLabel *lab_active = gen_new_label(tcg_ctx);
+    TCGLabel *lab_end = gen_new_label(tcg_ctx);
+    TCGv_i32 addr;
+    TCGv_i32 value;
+
+    m_profile_fp_inactive_ns_branch(s, false, lab_active);
+    if (load) {
+        if (w) {
+            addr = m_profile_sysreg_mem_addr(s, rn, offset, p, w);
+            m_profile_sysreg_mem_writeback(s, rn, offset);
+            tcg_temp_free_i32(tcg_ctx, addr);
+        }
+    } else {
+        addr = m_profile_sysreg_mem_addr(s, rn, offset, p, w);
+        value = load_cpu_field(tcg_ctx, v7m.fpdscr[M_REG_NS]);
+        gen_aa32_st_i32(s, value, addr, get_mem_index(s),
+                        MO_UL | MO_ALIGN | s->be_data);
+        tcg_temp_free_i32(tcg_ctx, value);
+        if (w) {
+            m_profile_sysreg_mem_writeback(s, rn, offset);
+        }
+        tcg_temp_free_i32(tcg_ctx, addr);
+    }
+    tcg_gen_br(tcg_ctx, lab_end);
+
+    gen_set_label(tcg_ctx, lab_active);
+    if (!m_profile_vfp_access_check(s, true)) {
+        s->base.is_jmp = DISAS_NEXT;
+        gen_set_label(tcg_ctx, lab_end);
+        return true;
+    }
+
+    addr = m_profile_sysreg_mem_addr(s, rn, offset, p, w);
+    if (load) {
+        value = tcg_temp_new_i32(tcg_ctx);
+        gen_aa32_ld_i32(s, value, addr, get_mem_index(s),
+                        MO_UL | MO_ALIGN | s->be_data);
+        if (w) {
+            m_profile_sysreg_mem_writeback(s, rn, offset);
+        }
+        m_profile_fpcxt_write_value(s, value);
+    } else {
+        value = m_profile_fpcxt_value(s);
+        gen_aa32_st_i32(s, value, addr, get_mem_index(s),
+                        MO_UL | MO_ALIGN | s->be_data);
+        tcg_temp_free_i32(tcg_ctx, value);
+        if (w) {
+            m_profile_sysreg_mem_writeback(s, rn, offset);
+        }
+        m_profile_fpcxt_ns_read_active_side_effect(s);
+    }
+    tcg_temp_free_i32(tcg_ctx, addr);
+
+    gen_set_label(tcg_ctx, lab_end);
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_m_profile_sysreg_mem(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 addr;
+    TCGv_i32 value = NULL;
+    int32_t offset;
+    int reg;
+    int rn;
+    bool p;
+    bool w;
+    bool load;
+    bool do_access;
+
+    if ((insn & 0xff101f80) == 0xed000f80 ||
+        (insn & 0xff101f80) == 0xed100f80) {
+        p = true;
+        w = extract32(insn, 21, 1);
+    } else if ((insn & 0xff301f80) == 0xec200f80 ||
+               (insn & 0xff301f80) == 0xec300f80) {
+        p = false;
+        w = true;
+    } else {
+        return false;
+    }
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+        return false;
+    }
+
+    rn = extract32(insn, 16, 4);
+    if (rn == 15) {
+        return false;
+    }
+
+    reg = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    if (!m_profile_fp_sysreg_check(s, reg)) {
+        return false;
+    }
+
+    offset = extract32(insn, 0, 7) << 2;
+    if (!extract32(insn, 23, 1)) {
+        offset = -offset;
+    }
+    load = extract32(insn, 20, 1);
+
+    if (reg == ARM_VFP_FPCXT_NS) {
+        return trans_m_profile_fpcxt_ns_mem(s, rn, offset, p, w, load);
+    }
+
+    if (!vfp_access_check(s)) {
+        return true;
+    }
+
+    do_access = !(reg == ARM_VFP_VPR && IS_USER(s));
+    if (!do_access && !w) {
+        return true;
+    }
+
+    addr = load_reg(s, rn);
+    if (p) {
+        tcg_gen_addi_i32(tcg_ctx, addr, addr, offset);
+    }
+
+    if (s->v8m_stackcheck && rn == 13 && w) {
+        gen_helper_v8m_stackcheck(tcg_ctx, tcg_ctx->cpu_env, addr);
+    }
+
+    if (do_access) {
+        if (load) {
+            value = tcg_temp_new_i32(tcg_ctx);
+            gen_aa32_ld_i32(s, value, addr, get_mem_index(s),
+                            MO_UL | MO_ALIGN | s->be_data);
+        } else {
+            value = m_profile_fp_sysreg_read(s, reg);
+            gen_aa32_st_i32(s, value, addr, get_mem_index(s),
+                            MO_UL | MO_ALIGN | s->be_data);
+            tcg_temp_free_i32(tcg_ctx, value);
+            value = NULL;
+        }
+    }
+
+    if (w) {
+        if (!p) {
+            tcg_gen_addi_i32(tcg_ctx, addr, addr, offset);
+        }
+        store_reg(s, rn, addr);
+    } else {
+        tcg_temp_free_i32(tcg_ctx, addr);
+    }
+
+    if (load && do_access) {
+        m_profile_fp_sysreg_write(s, reg, value);
+    } else if (!load && do_access && reg == ARM_VFP_FPCXT_S) {
+        m_profile_fpcxt_s_read_side_effect(s);
+    }
+
+    return true;
+}
+
 static bool trans_VMOV_single(DisasContext *s, arg_VMOV_single *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -2306,6 +3054,32 @@ static bool trans_VCVT_f16_f32(DisasContext *s, arg_VCVT_f16_f32 *a)
     return true;
 }
 
+static bool trans_VCVT_b16_f32(DisasContext *s, arg_VCVT_b16_f32 *a)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr fpst;
+    TCGv_i32 tmp;
+
+    if (!dc_isar_feature(aa32_bf16, s)) {
+        return false;
+    }
+
+    if (!vfp_access_check(s)) {
+        return true;
+    }
+
+    fpst = get_fpstatus_ptr(tcg_ctx, false);
+    tmp = tcg_temp_new_i32(tcg_ctx);
+
+    neon_load_reg32(tcg_ctx, tmp, a->vm);
+    gen_helper_bfcvt(tcg_ctx, tmp, tmp, fpst);
+    tcg_gen_st16_i32(tcg_ctx, tmp, tcg_ctx->cpu_env,
+                     vfp_f16_offset(a->vd, a->t));
+    tcg_temp_free_ptr(tcg_ctx, fpst);
+    tcg_temp_free_i32(tcg_ctx, tmp);
+    return true;
+}
+
 static bool trans_VCVT_f16_f64(DisasContext *s, arg_VCVT_f16_f64 *a)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
diff --git a/qemu/target/arm/translate.c b/qemu/target/arm/translate.c
index 744d8ff709..61cf376aba 100644
--- a/qemu/target/arm/translate.c
+++ b/qemu/target/arm/translate.c
@@ -86,6 +86,76 @@ void arm_translate_init(struct uc_struct *uc)
     a64_translate_init(uc);
 }
 
+static uint64_t asimd_imm_const(uint32_t imm, int cmode, int op)
+{
+    switch (cmode) {
+    case 0:
+    case 1:
+        break;
+    case 2:
+    case 3:
+        imm <<= 8;
+        break;
+    case 4:
+    case 5:
+        imm <<= 16;
+        break;
+    case 6:
+    case 7:
+        imm <<= 24;
+        break;
+    case 8:
+    case 9:
+        imm |= imm << 16;
+        break;
+    case 10:
+    case 11:
+        imm = (imm << 8) | (imm << 24);
+        break;
+    case 12:
+        imm = (imm << 8) | 0xff;
+        break;
+    case 13:
+        imm = (imm << 16) | 0xffff;
+        break;
+    case 14:
+        if (op) {
+            uint64_t imm64 = 0;
+            int n;
+
+            for (n = 0; n < 8; n++) {
+                if (imm & (1 << n)) {
+                    imm64 |= 0xffULL << (n * 8);
+                }
+            }
+            return imm64;
+        }
+        imm |= (imm << 8) | (imm << 16) | (imm << 24);
+        break;
+    case 15:
+        if (op) {
+            uint64_t imm64 = (uint64_t)(imm & 0x3f) << 48;
+
+            if (imm & 0x80) {
+                imm64 |= 0x8000000000000000ULL;
+            }
+            if (imm & 0x40) {
+                imm64 |= 0x3fc0000000000000ULL;
+            } else {
+                imm64 |= 0x4000000000000000ULL;
+            }
+            return imm64;
+        }
+        imm = ((imm & 0x80) << 24) | ((imm & 0x3f) << 19) |
+              ((imm & 0x40) ? (0x1f << 25) : (1 << 30));
+        break;
+    }
+    if (op) {
+        imm = ~imm;
+    }
+    return dup_const(MO_32, imm);
+}
+
 /* Flags for the disas_set_da_iss info argument:
  * lower bits hold the Rt register number, higher bits are flags.
  */
@@ -1344,6 +1414,8 @@ static TCGv_ptr vfp_reg_ptr(TCGContext *tcg_ctx, bool dp, int reg)
 
 #define ARM_CP_RW_BIT   (1 << 20)
 
+static void arm_gen_condlabel(DisasContext *s);
+
 /* Include the VFP decoder */
 #include "translate-vfp.inc.c"
 
@@ -4119,6 +4191,80 @@ const GVecGen2i usra_op[4] = {
       .vece = MO_64, },
 };
 
+static void gen_mve_srshr32_i32(TCGContext *tcg_ctx, TCGv_i32 d,
+                                TCGv_i32 a, int32_t shift)
+{
+    TCGv_i32 t;
+
+    if (shift == 32) {
+        tcg_gen_movi_i32(tcg_ctx, d, 0);
+        return;
+    }
+
+    t = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_extract_i32(tcg_ctx, t, a, shift - 1, 1);
+    tcg_gen_sari_i32(tcg_ctx, d, a, shift);
+    tcg_gen_add_i32(tcg_ctx, d, d, t);
+    tcg_temp_free_i32(tcg_ctx, t);
+}
+
+static void gen_mve_urshr32_i32(TCGContext *tcg_ctx, TCGv_i32 d,
+                                TCGv_i32 a, int32_t shift)
+{
+    TCGv_i32 t;
+
+    if (shift == 32) {
+        tcg_gen_extract_i32(tcg_ctx, d, a, shift - 1, 1);
+        return;
+    }
+
+    t = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_extract_i32(tcg_ctx, t, a, shift - 1, 1);
+    tcg_gen_shri_i32(tcg_ctx, d, a, shift);
+    tcg_gen_add_i32(tcg_ctx, d, d, t);
+    tcg_temp_free_i32(tcg_ctx, t);
+}
+
+static void gen_mve_srshr64_i64(TCGContext *tcg_ctx, TCGv_i64 d,
+                                TCGv_i64 a, int64_t shift)
+{
+    TCGv_i64 t = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_extract_i64(tcg_ctx, t, a, shift - 1, 1);
+    tcg_gen_sari_i64(tcg_ctx, d, a, shift);
+    tcg_gen_add_i64(tcg_ctx, d, d, t);
+    tcg_temp_free_i64(tcg_ctx, t);
+}
+
+static void gen_mve_urshr64_i64(TCGContext *tcg_ctx, TCGv_i64 d,
+                                TCGv_i64 a, int64_t shift)
+{
+    TCGv_i64 t = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_extract_i64(tcg_ctx, t, a, shift - 1, 1);
+    tcg_gen_shri_i64(tcg_ctx, d, a, shift);
+    tcg_gen_add_i64(tcg_ctx, d, d, t);
+    tcg_temp_free_i64(tcg_ctx, t);
+}
+
+static void gen_mve_shl64_i64(TCGContext *tcg_ctx, TCGv_i64 d,
+                              TCGv_i64 a, int64_t shift)
+{
+    tcg_gen_shli_i64(tcg_ctx, d, a, shift);
+}
+
+static void gen_mve_shr64_i64(TCGContext *tcg_ctx, TCGv_i64 d,
+                              TCGv_i64 a, int64_t shift)
+{
+    tcg_gen_shri_i64(tcg_ctx, d, a, shift);
+}
+
+static void gen_mve_sar64_i64(TCGContext *tcg_ctx, TCGv_i64 d,
+                              TCGv_i64 a, int64_t shift)
+{
+    tcg_gen_sari_i64(tcg_ctx, d, a, shift);
+}
+
 static void gen_shr8_ins_i64(TCGContext *tcg_ctx, TCGv_i64 d, TCGv_i64 a, int64_t shift)
 {
     uint64_t mask = dup_const(MO_8, 0xff >> shift);
@@ -6331,6 +6477,35 @@ static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
                 /* Two register misc.  */
                 op = ((insn >> 12) & 0x30) | ((insn >> 7) & 0xf);
                 size = (insn >> 18) & 3;
+                if ((insn & 0xffb30fd0) == 0xf3b20640) {
+                    TCGv_ptr fpst;
+
+                    if (!dc_isar_feature(aa32_bf16, s) ||
+                        size != 1 || (rm & 1)) {
+                        return 1;
+                    }
+                    if (!dc_isar_feature(aa32_simd_r32, s) &&
+                        ((rd | rm) & 0x10)) {
+                        return 1;
+                    }
+                    fpst = get_fpstatus_ptr(tcg_ctx, true);
+                    tmp64 = tcg_temp_new_i64(tcg_ctx);
+                    tmp = tcg_temp_new_i32(tcg_ctx);
+                    tmp2 = tcg_temp_new_i32(tcg_ctx);
+
+                    neon_load_reg64(tcg_ctx, tmp64, rm);
+                    gen_helper_bfcvt_pair(tcg_ctx, tmp, tmp64, fpst);
+                    neon_load_reg64(tcg_ctx, tmp64, rm + 1);
+                    gen_helper_bfcvt_pair(tcg_ctx, tmp2, tmp64, fpst);
+                    neon_store_reg(tcg_ctx, rd, 0, tmp);
+                    neon_store_reg(tcg_ctx, rd, 1, tmp2);
+
+                    tcg_temp_free_i64(tcg_ctx, tmp64);
+                    tcg_temp_free_i32(tcg_ctx, tmp2);
+                    tcg_temp_free_i32(tcg_ctx, tmp);
+                    tcg_temp_free_ptr(tcg_ctx, fpst);
+                    return 0;
+                }
                 /* UNDEF for unknown op values and bad op-size combinations */
                 if ((neon_2rm_sizes[op] & (1 << size)) == 0) {
                     return 1;
@@ -6939,7 +7114,9 @@ static int disas_neon_insn_3same_ext(DisasContext *s, uint32_t insn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     gen_helper_gvec_3 *fn_gvec = NULL;
+    gen_helper_gvec_4 *fn_gvec_acc = NULL;
     gen_helper_gvec_3_ptr *fn_gvec_ptr = NULL;
+    gen_helper_gvec_4_ptr *fn_gvec_acc_ptr = NULL;
     int rd, rn, rm, opr_sz;
     int data = 0;
     int off_rn, off_rm;
@@ -6970,7 +7147,48 @@ static int disas_neon_insn_3same_ext(DisasContext *s, uint32_t insn)
         if (!dc_isar_feature(aa32_dp, s)) {
             return 1;
         }
-        fn_gvec = u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b;
+        fn_gvec_acc = u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b;
+    } else if ((insn & 0xfeb00f10) == 0xfca00d00) {
+        /* VUSDOT -- 1111 1100 1.10 .... .... 1101 .Q.0 .... */
+        if (!dc_isar_feature(aa32_i8mm, s)) {
+            return 1;
+        }
+        fn_gvec_acc = gen_helper_gvec_usdot_b;
+    } else if ((insn & 0xfeb00f40) == 0xfc200c40) {
+        /* V[SU]MMLA -- 1111 1100 0.10 .... .... 1100 .1.U .... */
+        bool u = extract32(insn, 4, 1);
+        if (!dc_isar_feature(aa32_i8mm, s)) {
+            return 1;
+        }
+        fn_gvec_acc = u ? gen_helper_gvec_ummla_b
+                        : gen_helper_gvec_smmla_b;
+    } else if ((insn & 0xfeb00f50) == 0xfca00c40) {
+        /* VUSMMLA -- 1111 1100 1.10 .... .... 1100 .1.0 .... */
+        if (!dc_isar_feature(aa32_i8mm, s)) {
+            return 1;
+        }
+        fn_gvec_acc = gen_helper_gvec_usmmla_b;
+    } else if ((insn & 0xfeb00f10) == 0xfc000d00) {
+        /* VDOT.bf16 -- 1111 1100 0.00 .... .... 1101 .Q.0 .... */
+        if (!dc_isar_feature(aa32_bf16, s)) {
+            return 1;
+        }
+        fn_gvec_acc = gen_helper_gvec_bfdot;
+    } else if ((insn & 0xfeb00f50) == 0xfc000c40) {
+        /* VMMLA.bf16 -- 1111 1100 0.00 .... .... 1100 .1.0 .... */
+        if (!dc_isar_feature(aa32_bf16, s)) {
+            return 1;
+        }
+        q = true;
+        fn_gvec_acc = gen_helper_gvec_bfmmla;
+    } else if ((insn & 0xffb00f10) == 0xfc300810) {
+        /* VFMA.bf16 -- 1111 1100 0.11 .... .... 1000 .T.1 .... */
+        if (!dc_isar_feature(aa32_bf16, s)) {
+            return 1;
+        }
+        data = q;
+        q = true;
+        fn_gvec_acc_ptr = gen_helper_gvec_bfmlal;
     } else if ((insn & 0xff300f10) == 0xfc200810) {
         /* VFM[AS]L -- 1111 1100 S.10 .... .... 1000 .Q.1 .... */
         int is_s = extract32(insn, 23, 1);
@@ -7014,7 +7232,13 @@ static int disas_neon_insn_3same_ext(DisasContext *s, uint32_t insn)
     }
 
     opr_sz = (1 + q) * 8;
-    if (fn_gvec_ptr) {
+    if (fn_gvec_acc_ptr) {
+        TCGv_ptr ptr = get_fpstatus_ptr(tcg_ctx, 1);
+        tcg_gen_gvec_4_ptr(tcg_ctx, vfp_reg_offset(1, rd), off_rn, off_rm,
+                           vfp_reg_offset(1, rd), ptr, opr_sz, opr_sz, data,
+                           fn_gvec_acc_ptr);
+        tcg_temp_free_ptr(tcg_ctx, ptr);
+    } else if (fn_gvec_ptr) {
         TCGv_ptr ptr;
         if (ptr_is_env) {
             ptr = tcg_ctx->cpu_env;
@@ -7026,6 +7250,10 @@ static int disas_neon_insn_3same_ext(DisasContext *s, uint32_t insn)
         if (!ptr_is_env) {
             tcg_temp_free_ptr(tcg_ctx, ptr);
         }
+    } else if (fn_gvec_acc) {
+        tcg_gen_gvec_4_ool(tcg_ctx, vfp_reg_offset(1, rd), off_rn, off_rm,
+                           vfp_reg_offset(1, rd), opr_sz, opr_sz, data,
+                           fn_gvec_acc);
     } else {
         tcg_gen_gvec_3_ool(tcg_ctx, vfp_reg_offset(1, rd), off_rn, off_rm,
                            opr_sz, opr_sz, data, fn_gvec);
@@ -7045,7 +7273,9 @@ static int disas_neon_insn_2reg_scalar_ext(DisasContext *s, uint32_t insn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     gen_helper_gvec_3 *fn_gvec = NULL;
+    gen_helper_gvec_4 *fn_gvec_acc = NULL;
     gen_helper_gvec_3_ptr *fn_gvec_ptr = NULL;
+    gen_helper_gvec_4_ptr *fn_gvec_acc_ptr = NULL;
     int rd, rn, rm, opr_sz, data;
     int off_rn, off_rm;
     bool is_long = false, q = extract32(insn, 6, 1);
@@ -7082,10 +7312,41 @@ static int disas_neon_insn_2reg_scalar_ext(DisasContext *s, uint32_t insn)
         if (!dc_isar_feature(aa32_dp, s)) {
             return 1;
         }
-        fn_gvec = u ? gen_helper_gvec_udot_idx_b : gen_helper_gvec_sdot_idx_b;
+        fn_gvec_acc = u ? gen_helper_gvec_udot_idx_b
+                         : gen_helper_gvec_sdot_idx_b;
         /* rm is just Vm, and index is M.  */
         data = extract32(insn, 5, 1); /* index */
         rm = extract32(insn, 0, 4);
+    } else if ((insn & 0xffb00f00) == 0xfe800d00) {
+        /* V[S/U]DOT -- 1111 1110 1.00 .... .... 1101 .Q.I.U .... */
+        int u = extract32(insn, 4, 1);
+
+        if (!dc_isar_feature(aa32_i8mm, s)) {
+            return 1;
+        }
+        fn_gvec_acc = u ? gen_helper_gvec_sudot_idx_b
+                         : gen_helper_gvec_usdot_idx_b;
+        data = extract32(insn, 5, 1); /* index */
+        rm = extract32(insn, 0, 4);
+    } else if ((insn & 0xffb00f10) == 0xfe000d00) {
+        /* VDOT.bf16 scalar -- 1111 1110 0.00 .... .... 1101 .Q.I0 .... */
+        if (!dc_isar_feature(aa32_bf16, s)) {
+            return 1;
+        }
+        fn_gvec_acc = gen_helper_gvec_bfdot_idx;
+        data = extract32(insn, 5, 1);
+        rm = extract32(insn, 0, 4);
+    } else if ((insn & 0xffb00f10) == 0xfe300810) {
+        /* VFMA.bf16 scalar -- 1111 1110 0.11 .... .... 1000 .T.1I... */
+        int index = (extract32(insn, 5, 1) << 1) | extract32(insn, 3, 1);
+
+        if (!dc_isar_feature(aa32_bf16, s)) {
+            return 1;
+        }
+        data = (index << 1) | q;
+        q = true;
+        rm = extract32(insn, 0, 3);
+        fn_gvec_acc_ptr = gen_helper_gvec_bfmlal_idx;
     } else if ((insn & 0xffa00f10) == 0xfe000810) {
         /* VFM[AS]L -- 1111 1110 0.0S .... .... 1000 .Q.1 .... */
         int is_s = extract32(insn, 20, 1);
@@ -7138,7 +7399,13 @@ static int disas_neon_insn_2reg_scalar_ext(DisasContext *s, uint32_t insn)
     }
 
     opr_sz = (1 + q) * 8;
-    if (fn_gvec_ptr) {
+    if (fn_gvec_acc_ptr) {
+        TCGv_ptr ptr = get_fpstatus_ptr(tcg_ctx, 1);
+        tcg_gen_gvec_4_ptr(tcg_ctx, vfp_reg_offset(1, rd), off_rn, off_rm,
+                           vfp_reg_offset(1, rd), ptr, opr_sz, opr_sz, data,
+                           fn_gvec_acc_ptr);
+        tcg_temp_free_ptr(tcg_ctx, ptr);
+    } else if (fn_gvec_ptr) {
         TCGv_ptr ptr;
         if (ptr_is_env) {
             ptr = tcg_ctx->cpu_env;
@@ -7150,6 +7417,10 @@ static int disas_neon_insn_2reg_scalar_ext(DisasContext *s, uint32_t insn)
         if (!ptr_is_env) {
             tcg_temp_free_ptr(tcg_ctx, ptr);
         }
+    } else if (fn_gvec_acc) {
+        tcg_gen_gvec_4_ool(tcg_ctx, vfp_reg_offset(1, rd), off_rn, off_rm,
+                           vfp_reg_offset(1, rd), opr_sz, opr_sz, data,
+                           fn_gvec_acc);
     } else {
         tcg_gen_gvec_3_ool(tcg_ctx, vfp_reg_offset(1, rd), off_rn, off_rm,
                            opr_sz, opr_sz, data, fn_gvec);
@@ -7835,128 +8106,4765 @@ static int t16_pop_list(DisasContext *s, int x)
     return (x & 0xff) | (x & 0x100) << (15 - 8);
 }
 
-/*
- * Include the generated decoders.
- */
-
-#include "decode-a32.inc.c"
-#include "decode-a32-uncond.inc.c"
-#include "decode-t32.inc.c"
-#include "decode-t16.inc.c"
+static long mve_qreg_offset(unsigned reg)
+{
+    return offsetof(CPUARMState, vfp.zregs[reg].d[0]);
+}
 
-/* Helpers to swap operands for reverse-subtract.  */
-static void gen_rsb(TCGContext *tcg_ctx, TCGv_i32 dst, TCGv_i32 a, TCGv_i32 b)
+static TCGv_ptr mve_qreg_ptr(DisasContext *s, unsigned reg)
 {
-    tcg_gen_sub_i32(tcg_ctx, dst, b, a);
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr ret = tcg_temp_new_ptr(tcg_ctx);
+
+    tcg_gen_addi_ptr(tcg_ctx, ret, tcg_ctx->cpu_env, mve_qreg_offset(reg));
+    return ret;
 }
 
-static void gen_rsb_CC(TCGContext *tcg_ctx, TCGv_i32 dst, TCGv_i32 a, TCGv_i32 b)
+static bool mve_check_qreg_bank(int qmask)
+{
+    return qmask < 8;
+}
+
+typedef void MVEGenTwoOpFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr,
+                           TCGv_ptr);
+typedef void MVEGenOneOpFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr);
+typedef void MVEGenLdStFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_i32);
+typedef void MVEGenLdStSGFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr,
+                            TCGv_i32);
+typedef void MVEGenLdStIlFn(TCGContext *, TCGv_env, TCGv_i32, TCGv_i32);
+typedef void MVEGenOneOpI32Fn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr,
+                              TCGv_i32);
+typedef void MVEGenVIDUPFn(TCGContext *, TCGv_i32, TCGv_env, TCGv_ptr,
+                           TCGv_i32, TCGv_i32);
+typedef void MVEGenVIWDUPFn(TCGContext *, TCGv_i32, TCGv_env, TCGv_ptr,
+                             TCGv_i32, TCGv_i32, TCGv_i32);
+typedef void MVEGenCmpFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr);
+typedef void MVEGenScalarCmpFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_i32);
+typedef void MVEGenTwoOpScalarFn(TCGContext *, TCGv_env, TCGv_ptr,
+                                 TCGv_ptr, TCGv_i32);
+typedef void MVEGenOneOpImmFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_i64);
+typedef void MVEGenTwoOpShiftFn(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr,
+                                TCGv_i32);
+typedef void MVEGenGPRShiftImmFn(TCGContext *, TCGv_i32, TCGv_i32, int32_t);
+typedef void MVEGenGPRWideShiftImmFn(TCGContext *, TCGv_i64, TCGv_i64,
+                                     int64_t);
+typedef void MVEGenGPRShiftFn(TCGContext *, TCGv_i32, TCGv_env, TCGv_i32,
+                              TCGv_i32);
+typedef void MVEGenGPRWideShiftFn(TCGContext *, TCGv_i64, TCGv_env,
+                                  TCGv_i64, TCGv_i32);
+typedef void MVEGenLongDualAccFn(TCGContext *, TCGv_i64, TCGv_env, TCGv_ptr,
+                                 TCGv_ptr, TCGv_i64);
+typedef void MVEGenDualAccFn(TCGContext *, TCGv_i32, TCGv_env, TCGv_ptr,
+                             TCGv_ptr, TCGv_i32);
+typedef void MVEGenVADDVFn(TCGContext *, TCGv_i32, TCGv_env, TCGv_ptr,
+                           TCGv_i32);
+typedef void MVEGenVABAVFn(TCGContext *, TCGv_i32, TCGv_env, TCGv_ptr,
+                            TCGv_ptr, TCGv_i32);
+
+static bool do_mve_2shift(DisasContext *s, uint32_t insn,
+                          MVEGenTwoOpShiftFn *fn, unsigned size,
+                          uint32_t shift, bool negateshift);
+
+static bool mve_eci_check(DisasContext *s)
+{
+    s->eci_handled = true;
+    switch (s->eci) {
+    case ECI_NONE:
+    case ECI_A0:
+    case ECI_A0A1:
+    case ECI_A0A1A2:
+    case ECI_A0A1A2B0:
+        return true;
+    default:
+        gen_exception_insn(s, s->pc_curr, EXCP_INVSTATE,
+                           syn_uncategorized(), default_exception_el(s));
+        return false;
+    }
+}
+
+static void mve_update_eci(DisasContext *s)
 {
-    gen_sub_CC(tcg_ctx, dst, b, a);
+    if (s->eci) {
+        s->eci = (s->eci == ECI_A0A1A2B0) ? ECI_A0 : ECI_NONE;
+    }
 }
 
-static void gen_rsc(TCGContext *tcg_ctx, TCGv_i32 dest, TCGv_i32 a, TCGv_i32 b)
+static void mve_update_and_store_eci(DisasContext *s)
 {
-    gen_sub_carry(tcg_ctx, dest, b, a);
+    if (s->eci) {
+        TCGContext *tcg_ctx = s->uc->tcg_ctx;
+        TCGv_i32 eci;
+
+        mve_update_eci(s);
+        eci = tcg_const_i32(tcg_ctx, s->eci << 4);
+        store_cpu_field(tcg_ctx, eci, condexec_bits);
+    }
 }
 
-static void gen_rsc_CC(TCGContext *tcg_ctx, TCGv_i32 dest, TCGv_i32 a, TCGv_i32 b)
+static bool mve_skip_vmov(DisasContext *s, int vn, int index, int size)
 {
-    gen_sbc_CC(tcg_ctx, dest, b, a);
+    int ofs = (index << size) + ((vn & 1) * 8);
+
+    switch (s->eci) {
+    case ECI_NONE:
+        return false;
+    case ECI_A0:
+        return ofs < 4;
+    case ECI_A0A1:
+        return ofs < 8;
+    case ECI_A0A1A2:
+    case ECI_A0A1A2B0:
+        return ofs < 12;
+    default:
+        g_assert_not_reached();
+    }
+
+    return false;
 }
 
-/*
- * Helpers for the data processing routines.
- *
- * After the computation store the results back.
- * This may be suppressed altogether (STREG_NONE), require a runtime
- * check against the stack limits (STREG_SP_CHECK), or generate an
- * exception return.  Oh, or store into a register.
- *
- * Always return true, indicating success for a trans_* function.
- */
-typedef enum {
-   STREG_NONE,
-   STREG_NORMAL,
-   STREG_SP_CHECK,
-   STREG_EXC_RET,
-} StoreRegKind;
+static bool mve_skip_first_beat(DisasContext *s)
+{
+    switch (s->eci) {
+    case ECI_NONE:
+        return false;
+    case ECI_A0:
+    case ECI_A0A1:
+    case ECI_A0A1A2:
+    case ECI_A0A1A2B0:
+        return true;
+    default:
+        g_assert_not_reached();
+    }
 
-static bool store_reg_kind(DisasContext *s, int rd,
-                            TCGv_i32 val, StoreRegKind kind)
+    return false;
+}
+
+static void gen_mve_vpst(DisasContext *s, uint32_t mask)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    switch (kind) {
-    case STREG_NONE:
-        tcg_temp_free_i32(tcg_ctx, val);
-        return true;
-    case STREG_NORMAL:
-        /* See ALUWritePC: Interworking only from a32 mode. */
-        if (s->thumb) {
-            store_reg(s, rd, val);
-        } else {
-            store_reg_bx(s, rd, val);
-        }
-        return true;
-    case STREG_SP_CHECK:
-        store_sp_checked(s, val);
-        return true;
-    case STREG_EXC_RET:
-        gen_exception_return(s, val);
-        return true;
+    TCGv_i32 vpr = load_cpu_field(tcg_ctx, v7m.vpr);
+    TCGv_i32 tcg_mask;
+    int shift;
+    int width;
+    uint32_t mask_value;
+
+    switch (s->eci) {
+    case ECI_NONE:
+    case ECI_A0:
+        shift = R_V7M_VPR_MASK01_SHIFT;
+        width = R_V7M_VPR_MASK01_LENGTH + R_V7M_VPR_MASK23_LENGTH;
+        mask_value = mask | (mask << 4);
+        break;
+    case ECI_A0A1:
+    case ECI_A0A1A2:
+    case ECI_A0A1A2B0:
+        shift = R_V7M_VPR_MASK23_SHIFT;
+        width = R_V7M_VPR_MASK23_LENGTH;
+        mask_value = mask;
+        break;
+    default:
+        g_assert_not_reached();
+        shift = 0;
+        width = 0;
+        mask_value = 0;
+        break;
     }
-    g_assert_not_reached();
-    // never reach here
-    return true;
+
+    tcg_mask = tcg_const_i32(tcg_ctx, mask_value);
+    tcg_gen_deposit_i32(tcg_ctx, vpr, vpr, tcg_mask, shift, width);
+    tcg_temp_free_i32(tcg_ctx, tcg_mask);
+    store_cpu_field(tcg_ctx, vpr, v7m.vpr);
 }
 
-/*
- * Data Processing (register)
- *
- * Operate, with set flags, one register source,
- * one immediate shifted register source, and a destination.
- */
-static bool op_s_rrr_shi(DisasContext *s, arg_s_rrr_shi *a,
-                         void (*gen)(TCGContext *, TCGv_i32, TCGv_i32, TCGv_i32),
-                         int logic_cc, StoreRegKind kind)
+static bool trans_mve_vctp(DisasContext *s, uint32_t insn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv_i32 tmp1, tmp2;
+    TCGv_i32 rn_shifted;
+    TCGv_i32 masklen;
+    TCGv_i32 limit;
+    TCGv_i32 full_mask;
+    int size;
+    int rn;
 
-    tmp2 = load_reg(s, a->rm);
-    gen_arm_shift_im(tcg_ctx, tmp2, a->shty, a->shim, logic_cc);
-    tmp1 = load_reg(s, a->rn);
+    if ((insn & 0xffc0ffff) != 0xf000e801) {
+        return false;
+    }
 
-    gen(tcg_ctx, tmp1, tmp1, tmp2);
-    tcg_temp_free_i32(tcg_ctx, tmp2);
+    size = extract32(insn, 20, 2);
+    rn = extract32(insn, 16, 4);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) || rn == 13 || rn == 15) {
+        return false;
+    }
 
-    if (logic_cc) {
-        gen_logic_CC(tcg_ctx, tmp1);
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
     }
-    return store_reg_kind(s, a->rd, tmp1, kind);
+
+    rn_shifted = tcg_temp_new_i32(tcg_ctx);
+    masklen = load_reg(s, rn);
+    limit = tcg_const_i32(tcg_ctx, 1 << (4 - size));
+    full_mask = tcg_const_i32(tcg_ctx, 16);
+
+    tcg_gen_shli_i32(tcg_ctx, rn_shifted, masklen, size);
+    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_LEU, masklen,
+                        masklen, limit, rn_shifted, full_mask);
+    gen_helper_mve_vctp(tcg_ctx, tcg_ctx->cpu_env, masklen);
+    tcg_temp_free_i32(tcg_ctx, full_mask);
+    tcg_temp_free_i32(tcg_ctx, limit);
+    tcg_temp_free_i32(tcg_ctx, masklen);
+    tcg_temp_free_i32(tcg_ctx, rn_shifted);
+
+    mve_update_eci(s);
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
 }
 
-static bool op_s_rxr_shi(DisasContext *s, arg_s_rrr_shi *a,
-                         void (*gen)(TCGContext *, TCGv_i32, TCGv_i32),
-                         int logic_cc, StoreRegKind kind)
+static bool trans_mve_vpnot(DisasContext *s, uint32_t insn)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv_i32 tmp;
 
-    tmp = load_reg(s, a->rm);
-    gen_arm_shift_im(tcg_ctx, tmp, a->shty, a->shim, logic_cc);
+    if (insn != 0xfe310f4d) {
+        return false;
+    }
 
-    gen(tcg_ctx, tmp, tmp);
-    if (logic_cc) {
-        gen_logic_CC(tcg_ctx, tmp);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) || !dc_isar_feature(aa32_mve, s)) {
+        return false;
     }
-    return store_reg_kind(s, a->rd, tmp, kind);
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    gen_helper_mve_vpnot(tcg_ctx, tcg_ctx->cpu_env);
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
 }
 
-/*
- * Data-processing (register-shifted register)
- *
- * Operate, with set flags, one register source,
+static bool trans_mve_vpst(DisasContext *s, uint32_t insn)
+{
+    uint32_t mask;
+
+    if ((insn & 0xffbf1fff) != 0xfe310f4d) {
+        return false;
+    }
+
+    mask = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    if (!mask) {
+        return false;
+    }
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) || !dc_isar_feature(aa32_mve, s)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    gen_mve_vpst(s, mask);
+    mve_update_and_store_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_2op_inner(DisasContext *s, uint32_t insn,
+                             MVEGenTwoOpFn *fn, bool reversed);
+
+static bool do_mve_2op(DisasContext *s, uint32_t insn, MVEGenTwoOpFn *fn)
+{
+    return do_mve_2op_inner(s, insn, fn, false);
+}
+
+static bool do_mve_2op_rev(DisasContext *s, uint32_t insn,
+                            MVEGenTwoOpFn *fn)
+{
+    return do_mve_2op_inner(s, insn, fn, true);
+}
+
+static bool do_mve_2op_qdqn(DisasContext *s, uint32_t insn,
+                            MVEGenTwoOpFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_ptr qmptr;
+    int qd;
+    int qm;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s)) {
+        return false;
+    }
+    if (!mve_check_qreg_bank(qd | qm)) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qdptr, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_2op_inner(DisasContext *s, uint32_t insn,
+                             MVEGenTwoOpFn *fn, bool reversed)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_ptr qnptr;
+    TCGv_ptr qmptr;
+    int qd;
+    int qn;
+    int qm;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qn = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (reversed) {
+        int tmp = qn;
+
+        qn = qm;
+        qm = tmp;
+    }
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd | qn | qm)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    qnptr = mve_qreg_ptr(s, qn);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qnptr, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_mve_simple_2op(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpFn *fn = NULL;
+    bool reversed = false;
+
+    switch (insn & 0xffb11f51) {
+    case 0xef000150:
+        fn = gen_helper_mve_vand;
+        break;
+    case 0xef100150:
+        fn = gen_helper_mve_vbic;
+        break;
+    case 0xef200150:
+        fn = gen_helper_mve_vorr;
+        break;
+    case 0xef300150:
+        fn = gen_helper_mve_vorn;
+        break;
+    case 0xff000150:
+        fn = gen_helper_mve_veor;
+        break;
+    default:
+        break;
+    }
+
+    if (!fn) {
+        MVEGenTwoOpFn * const add_fns[] = {
+            gen_helper_mve_vaddb,
+            gen_helper_mve_vaddh,
+            gen_helper_mve_vaddw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const sub_fns[] = {
+            gen_helper_mve_vsubb,
+            gen_helper_mve_vsubh,
+            gen_helper_mve_vsubw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const mul_fns[] = {
+            gen_helper_mve_vmulb,
+            gen_helper_mve_vmulh,
+            gen_helper_mve_vmulw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmulhs_fns[] = {
+            gen_helper_mve_vmulhsb,
+            gen_helper_mve_vmulhsh,
+            gen_helper_mve_vmulhsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmulhu_fns[] = {
+            gen_helper_mve_vmulhub,
+            gen_helper_mve_vmulhuh,
+            gen_helper_mve_vmulhuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vrmulhs_fns[] = {
+            gen_helper_mve_vrmulhsb,
+            gen_helper_mve_vrmulhsh,
+            gen_helper_mve_vrmulhsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vrmulhu_fns[] = {
+            gen_helper_mve_vrmulhub,
+            gen_helper_mve_vrmulhuh,
+            gen_helper_mve_vrmulhuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmullbs_fns[] = {
+            gen_helper_mve_vmullbsb,
+            gen_helper_mve_vmullbsh,
+            gen_helper_mve_vmullbsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmullbu_fns[] = {
+            gen_helper_mve_vmullbub,
+            gen_helper_mve_vmullbuh,
+            gen_helper_mve_vmullbuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmullts_fns[] = {
+            gen_helper_mve_vmulltsb,
+            gen_helper_mve_vmulltsh,
+            gen_helper_mve_vmulltsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmulltu_fns[] = {
+            gen_helper_mve_vmulltub,
+            gen_helper_mve_vmulltuh,
+            gen_helper_mve_vmulltuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmullpb_fns[] = {
+            NULL,
+            gen_helper_mve_vmullpbh,
+            gen_helper_mve_vmullpbw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmullpt_fns[] = {
+            NULL,
+            gen_helper_mve_vmullpth,
+            gen_helper_mve_vmullptw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqdmullb_fns[] = {
+            NULL,
+            gen_helper_mve_vqdmullbh,
+            gen_helper_mve_vqdmullbw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqdmullt_fns[] = {
+            NULL,
+            gen_helper_mve_vqdmullth,
+            gen_helper_mve_vqdmulltw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vcadd90_fns[] = {
+            gen_helper_mve_vcadd90b,
+            gen_helper_mve_vcadd90h,
+            gen_helper_mve_vcadd90w,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vcadd270_fns[] = {
+            gen_helper_mve_vcadd270b,
+            gen_helper_mve_vcadd270h,
+            gen_helper_mve_vcadd270w,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vhcadd90_fns[] = {
+            gen_helper_mve_vhcadd90b,
+            gen_helper_mve_vhcadd90h,
+            gen_helper_mve_vhcadd90w,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vhcadd270_fns[] = {
+            gen_helper_mve_vhcadd270b,
+            gen_helper_mve_vhcadd270h,
+            gen_helper_mve_vhcadd270w,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmaxs_fns[] = {
+            gen_helper_mve_vmaxsb,
+            gen_helper_mve_vmaxsh,
+            gen_helper_mve_vmaxsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmaxu_fns[] = {
+            gen_helper_mve_vmaxub,
+            gen_helper_mve_vmaxuh,
+            gen_helper_mve_vmaxuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vmins_fns[] = {
+            gen_helper_mve_vminsb,
+            gen_helper_mve_vminsh,
+            gen_helper_mve_vminsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vminu_fns[] = {
+            gen_helper_mve_vminub,
+            gen_helper_mve_vminuh,
+            gen_helper_mve_vminuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vabds_fns[] = {
+            gen_helper_mve_vabdsb,
+            gen_helper_mve_vabdsh,
+            gen_helper_mve_vabdsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vabdu_fns[] = {
+            gen_helper_mve_vabdub,
+            gen_helper_mve_vabduh,
+            gen_helper_mve_vabduw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vhadds_fns[] = {
+            gen_helper_mve_vhaddsb,
+            gen_helper_mve_vhaddsh,
+            gen_helper_mve_vhaddsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vhaddu_fns[] = {
+            gen_helper_mve_vhaddub,
+            gen_helper_mve_vhadduh,
+            gen_helper_mve_vhadduw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vrhadds_fns[] = {
+            gen_helper_mve_vrhaddsb,
+            gen_helper_mve_vrhaddsh,
+            gen_helper_mve_vrhaddsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vrhaddu_fns[] = {
+            gen_helper_mve_vrhaddub,
+            gen_helper_mve_vrhadduh,
+            gen_helper_mve_vrhadduw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vhsubs_fns[] = {
+            gen_helper_mve_vhsubsb,
+            gen_helper_mve_vhsubsh,
+            gen_helper_mve_vhsubsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vhsubu_fns[] = {
+            gen_helper_mve_vhsubub,
+            gen_helper_mve_vhsubuh,
+            gen_helper_mve_vhsubuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqadds_fns[] = {
+            gen_helper_mve_vqaddsb,
+            gen_helper_mve_vqaddsh,
+            gen_helper_mve_vqaddsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqaddu_fns[] = {
+            gen_helper_mve_vqaddub,
+            gen_helper_mve_vqadduh,
+            gen_helper_mve_vqadduw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqsubs_fns[] = {
+            gen_helper_mve_vqsubsb,
+            gen_helper_mve_vqsubsh,
+            gen_helper_mve_vqsubsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqsubu_fns[] = {
+            gen_helper_mve_vqsubub,
+            gen_helper_mve_vqsubuh,
+            gen_helper_mve_vqsubuw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqdmulh_fns[] = {
+            gen_helper_mve_vqdmulhb,
+            gen_helper_mve_vqdmulhh,
+            gen_helper_mve_vqdmulhw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqrdmulh_fns[] = {
+            gen_helper_mve_vqrdmulhb,
+            gen_helper_mve_vqrdmulhh,
+            gen_helper_mve_vqrdmulhw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vshls_fns[] = {
+            gen_helper_mve_vshlsb,
+            gen_helper_mve_vshlsh,
+            gen_helper_mve_vshlsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vshlu_fns[] = {
+            gen_helper_mve_vshlub,
+            gen_helper_mve_vshluh,
+            gen_helper_mve_vshluw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vrshls_fns[] = {
+            gen_helper_mve_vrshlsb,
+            gen_helper_mve_vrshlsh,
+            gen_helper_mve_vrshlsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vrshlu_fns[] = {
+            gen_helper_mve_vrshlub,
+            gen_helper_mve_vrshluh,
+            gen_helper_mve_vrshluw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqshls_fns[] = {
+            gen_helper_mve_vqshlsb,
+            gen_helper_mve_vqshlsh,
+            gen_helper_mve_vqshlsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqshlu_fns[] = {
+            gen_helper_mve_vqshlub,
+            gen_helper_mve_vqshluh,
+            gen_helper_mve_vqshluw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqrshls_fns[] = {
+            gen_helper_mve_vqrshlsb,
+            gen_helper_mve_vqrshlsh,
+            gen_helper_mve_vqrshlsw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqrshlu_fns[] = {
+            gen_helper_mve_vqrshlub,
+            gen_helper_mve_vqrshluh,
+            gen_helper_mve_vqrshluw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqdmladh_fns[] = {
+            gen_helper_mve_vqdmladhb,
+            gen_helper_mve_vqdmladhh,
+            gen_helper_mve_vqdmladhw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqdmladhx_fns[] = {
+            gen_helper_mve_vqdmladhxb,
+            gen_helper_mve_vqdmladhxh,
+            gen_helper_mve_vqdmladhxw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqrdmladh_fns[] = {
+            gen_helper_mve_vqrdmladhb,
+            gen_helper_mve_vqrdmladhh,
+            gen_helper_mve_vqrdmladhw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqrdmladhx_fns[] = {
+            gen_helper_mve_vqrdmladhxb,
+            gen_helper_mve_vqrdmladhxh,
+            gen_helper_mve_vqrdmladhxw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqdmlsdh_fns[] = {
+            gen_helper_mve_vqdmlsdhb,
+            gen_helper_mve_vqdmlsdhh,
+            gen_helper_mve_vqdmlsdhw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqdmlsdhx_fns[] = {
+            gen_helper_mve_vqdmlsdhxb,
+            gen_helper_mve_vqdmlsdhxh,
+            gen_helper_mve_vqdmlsdhxw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqrdmlsdh_fns[] = {
+            gen_helper_mve_vqrdmlsdhb,
+            gen_helper_mve_vqrdmlsdhh,
+            gen_helper_mve_vqrdmlsdhw,
+            NULL,
+        };
+        MVEGenTwoOpFn * const vqrdmlsdhx_fns[] = {
+            gen_helper_mve_vqrdmlsdhxb,
+            gen_helper_mve_vqrdmlsdhxh,
+            gen_helper_mve_vqrdmlsdhxw,
+            NULL,
+        };
+        unsigned size = extract32(insn, 20, 2);
+        unsigned psize = extract32(insn, 28, 1) + 1;
+
+        if ((insn & 0xefb11f51) == 0xee310e00) {
+            fn = vmullpb_fns[psize];
+        } else if ((insn & 0xefb11f51) == 0xee311e00) {
+            fn = vmullpt_fns[psize];
+        } else if ((insn & 0xefb11f51) == 0xee300f01) {
+            if (psize == 2) {
+                int qd = (extract32(insn, 22, 1) << 3) |
+                    extract32(insn, 13, 3);
+                int qn = (extract32(insn, 7, 1) << 3) |
+                    extract32(insn, 17, 3);
+                int qm = (extract32(insn, 5, 1) << 3) |
+                    extract32(insn, 1, 3);
+
+                if (qd == qn || qd == qm) {
+                    return false;
+                }
+            }
+            fn = vqdmullb_fns[psize];
+        } else if ((insn & 0xefb11f51) == 0xee301f01) {
+            if (psize == 2) {
+                int qd = (extract32(insn, 22, 1) << 3) |
+                    extract32(insn, 13, 3);
+                int qn = (extract32(insn, 7, 1) << 3) |
+                    extract32(insn, 17, 3);
+                int qm = (extract32(insn, 5, 1) << 3) |
+                    extract32(insn, 1, 3);
+
+                if (qd == qn || qd == qm) {
+                    return false;
+                }
+            }
+            fn = vqdmullt_fns[psize];
+        } else if ((insn & 0xffb11f51) == 0xee300f00) {
+            fn = gen_helper_mve_vadc;
+        } else if ((insn & 0xffb11f51) == 0xee301f00) {
+            fn = mve_skip_first_beat(s) ? gen_helper_mve_vadc :
+                                          gen_helper_mve_vadci;
+        } else if ((insn & 0xffb11f51) == 0xfe300f00) {
+            fn = gen_helper_mve_vsbc;
+        } else if ((insn & 0xffb11f51) == 0xfe301f00) {
+            fn = mve_skip_first_beat(s) ? gen_helper_mve_vsbc :
+                                          gen_helper_mve_vsbci;
+        } else if ((insn & 0xff811f51) == 0xee000f00) {
+            fn = vhcadd90_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee001f00) {
+            fn = vhcadd270_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe000f00) {
+            fn = vcadd90_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe001f00) {
+            fn = vcadd270_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000840) {
+            fn = add_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000840) {
+            fn = sub_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000950) {
+            fn = mul_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee010e01) {
+            fn = vmulhs_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe010e01) {
+            fn = vmulhu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee011e01) {
+            fn = vrmulhs_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe011e01) {
+            fn = vrmulhu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee010e00) {
+            fn = vmullbs_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe010e00) {
+            fn = vmullbu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee011e00) {
+            fn = vmullts_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe011e00) {
+            fn = vmulltu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000640) {
+            fn = vmaxs_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000640) {
+            fn = vmaxu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000650) {
+            fn = vmins_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000650) {
+            fn = vminu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000740) {
+            fn = vabds_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000740) {
+            fn = vabdu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000040) {
+            fn = vhadds_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000040) {
+            fn = vhaddu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000140) {
+            fn = vrhadds_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000140) {
+            fn = vrhaddu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000240) {
+            fn = vhsubs_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000240) {
+            fn = vhsubu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000050) {
+            fn = vqadds_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000050) {
+            fn = vqaddu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000250) {
+            fn = vqsubs_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000250) {
+            fn = vqsubu_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000b40) {
+            fn = vqdmulh_fns[size];
+        } else if ((insn & 0xff811f51) == 0xff000b40) {
+            fn = vqrdmulh_fns[size];
+        } else if ((insn & 0xff811f51) == 0xef000440) {
+            fn = vshls_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xff000440) {
+            fn = vshlu_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xef000540) {
+            fn = vrshls_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xff000540) {
+            fn = vrshlu_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xef000450) {
+            fn = vqshls_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xff000450) {
+            fn = vqshlu_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xef000550) {
+            fn = vqrshls_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xff000550) {
+            fn = vqrshlu_fns[size];
+            reversed = true;
+        } else if ((insn & 0xff811f51) == 0xee000e00) {
+            fn = vqdmladh_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee001e00) {
+            fn = vqdmladhx_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee000e01) {
+            fn = vqrdmladh_fns[size];
+        } else if ((insn & 0xff811f51) == 0xee001e01) {
+            fn = vqrdmladhx_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe000e00) {
+            fn = vqdmlsdh_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe001e00) {
+            fn = vqdmlsdhx_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe000e01) {
+            fn = vqrdmlsdh_fns[size];
+        } else if ((insn & 0xff811f51) == 0xfe001e01) {
+            fn = vqrdmlsdhx_fns[size];
+        }
+    }
+
+    if (!fn) {
+        return false;
+    }
+    return reversed ? do_mve_2op_rev(s, insn, fn) :
+                      do_mve_2op(s, insn, fn);
+}
+
+static bool trans_mve_fp_2op(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpFn * const vfadd_fns[] = {
+        NULL,
+        gen_helper_mve_vfaddh,
+        gen_helper_mve_vfadds,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vfsub_fns[] = {
+        NULL,
+        gen_helper_mve_vfsubh,
+        gen_helper_mve_vfsubs,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vfmul_fns[] = {
+        NULL,
+        gen_helper_mve_vfmulh,
+        gen_helper_mve_vfmuls,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vfabd_fns[] = {
+        NULL,
+        gen_helper_mve_vfabdh,
+        gen_helper_mve_vfabds,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vmaxnm_fns[] = {
+        NULL,
+        gen_helper_mve_vmaxnmh,
+        gen_helper_mve_vmaxnms,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vminnm_fns[] = {
+        NULL,
+        gen_helper_mve_vminnmh,
+        gen_helper_mve_vminnms,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vfcadd90_fns[] = {
+        NULL,
+        gen_helper_mve_vfcadd90h,
+        gen_helper_mve_vfcadd90s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vfcadd270_fns[] = {
+        NULL,
+        gen_helper_mve_vfcadd270h,
+        gen_helper_mve_vfcadd270s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vcmla0_fns[] = {
+        NULL,
+        gen_helper_mve_vcmla0h,
+        gen_helper_mve_vcmla0s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vcmla90_fns[] = {
+        NULL,
+        gen_helper_mve_vcmla90h,
+        gen_helper_mve_vcmla90s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vcmla180_fns[] = {
+        NULL,
+        gen_helper_mve_vcmla180h,
+        gen_helper_mve_vcmla180s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vcmla270_fns[] = {
+        NULL,
+        gen_helper_mve_vcmla270h,
+        gen_helper_mve_vcmla270s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vfma_fns[] = {
+        NULL,
+        gen_helper_mve_vfmah,
+        gen_helper_mve_vfmas,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vfms_fns[] = {
+        NULL,
+        gen_helper_mve_vfmsh,
+        gen_helper_mve_vfmss,
+        NULL,
+    };
+    MVEGenTwoOpFn *fn = NULL;
+    unsigned size;
+    unsigned size_rev;
+
+    if (!dc_isar_feature(aa32_mve_fp, s)) {
+        return false;
+    }
+
+    size = extract32(insn, 20, 1) ? 1 : 2;
+    size_rev = extract32(insn, 20, 1) + 1;
+    switch (insn & 0xffa11f51) {
+    case 0xef000d40:
+        fn = vfadd_fns[size];
+        break;
+    case 0xef200d40:
+        fn = vfsub_fns[size];
+        break;
+    case 0xff000d50:
+        fn = vfmul_fns[size];
+        break;
+    case 0xff200d40:
+        fn = vfabd_fns[size];
+        break;
+    case 0xff000f50:
+        fn = vmaxnm_fns[size];
+        break;
+    case 0xff200f50:
+        fn = vminnm_fns[size];
+        break;
+    case 0xfc800840:
+        fn = vfcadd90_fns[size_rev];
+        break;
+    case 0xfd800840:
+        fn = vfcadd270_fns[size_rev];
+        break;
+    case 0xfc200840:
+        fn = vcmla0_fns[size_rev];
+        break;
+    case 0xfca00840:
+        fn = vcmla90_fns[size_rev];
+        break;
+    case 0xfd200840:
+        fn = vcmla180_fns[size_rev];
+        break;
+    case 0xfda00840:
+        fn = vcmla270_fns[size_rev];
+        break;
+    case 0xef000c50:
+        fn = vfma_fns[size];
+        break;
+    case 0xef200c50:
+        fn = vfms_fns[size];
+        break;
+    default:
+        break;
+    }
+
+    return fn ? do_mve_2op(s, insn, fn) : false;
+}
+
+static bool trans_mve_fp_nma(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpFn *fn;
+
+    if (!dc_isar_feature(aa32_mve_fp, s)) {
+        return false;
+    }
+
+    switch (insn & 0xffbf1fd1) {
+    case 0xee3f0e41:
+        fn = gen_helper_mve_vmaxnmas;
+        break;
+    case 0xfe3f0e41:
+        fn = gen_helper_mve_vmaxnmah;
+        break;
+    case 0xee3f1e41:
+        fn = gen_helper_mve_vminnmas;
+        break;
+    case 0xfe3f1e41:
+        fn = gen_helper_mve_vminnmah;
+        break;
+    default:
+        return false;
+    }
+
+    return do_mve_2op_qdqn(s, insn, fn);
+}
+
+static bool trans_mve_fp_vcmul(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpFn * const vcmul0_fns[] = {
+        NULL,
+        gen_helper_mve_vcmul0h,
+        gen_helper_mve_vcmul0s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vcmul90_fns[] = {
+        NULL,
+        gen_helper_mve_vcmul90h,
+        gen_helper_mve_vcmul90s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vcmul180_fns[] = {
+        NULL,
+        gen_helper_mve_vcmul180h,
+        gen_helper_mve_vcmul180s,
+        NULL,
+    };
+    MVEGenTwoOpFn * const vcmul270_fns[] = {
+        NULL,
+        gen_helper_mve_vcmul270h,
+        gen_helper_mve_vcmul270s,
+        NULL,
+    };
+    MVEGenTwoOpFn *fn = NULL;
+    unsigned psize;
+
+    if (!dc_isar_feature(aa32_mve_fp, s)) {
+        return false;
+    }
+
+    psize = extract32(insn, 28, 1) + 1;
+    switch (insn & 0xefb11f51) {
+    case 0xee300e00:
+        fn = vcmul0_fns[psize];
+        break;
+    case 0xee300e01:
+        fn = vcmul90_fns[psize];
+        break;
+    case 0xee301e00:
+        fn = vcmul180_fns[psize];
+        break;
+    case 0xee301e01:
+        fn = vcmul270_fns[psize];
+        break;
+    default:
+        break;
+    }
+
+    return fn ? do_mve_2op(s, insn, fn) : false;
+}
+
+static bool do_mve_2op_scalar(DisasContext *s, uint32_t insn,
+                              MVEGenTwoOpScalarFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_ptr qnptr;
+    TCGv_i32 rmval;
+    int qd;
+    int qn;
+    int rm;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qn = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    rm = extract32(insn, 0, 4);
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd | qn) || !fn) {
+        return false;
+    }
+    if (rm == 13 || rm == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    qnptr = mve_qreg_ptr(s, qn);
+    rmval = load_reg(s, rm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qnptr, rmval);
+    tcg_temp_free_i32(tcg_ctx, rmval);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_mve_scalar_2op(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpScalarFn * const vadd_fns[] = {
+        gen_helper_mve_vadd_scalarb,
+        gen_helper_mve_vadd_scalarh,
+        gen_helper_mve_vadd_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vsub_fns[] = {
+        gen_helper_mve_vsub_scalarb,
+        gen_helper_mve_vsub_scalarh,
+        gen_helper_mve_vsub_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vmul_fns[] = {
+        gen_helper_mve_vmul_scalarb,
+        gen_helper_mve_vmul_scalarh,
+        gen_helper_mve_vmul_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vhadds_fns[] = {
+        gen_helper_mve_vhadds_scalarb,
+        gen_helper_mve_vhadds_scalarh,
+        gen_helper_mve_vhadds_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vhaddu_fns[] = {
+        gen_helper_mve_vhaddu_scalarb,
+        gen_helper_mve_vhaddu_scalarh,
+        gen_helper_mve_vhaddu_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vhsubs_fns[] = {
+        gen_helper_mve_vhsubs_scalarb,
+        gen_helper_mve_vhsubs_scalarh,
+        gen_helper_mve_vhsubs_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vhsubu_fns[] = {
+        gen_helper_mve_vhsubu_scalarb,
+        gen_helper_mve_vhsubu_scalarh,
+        gen_helper_mve_vhsubu_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqadds_fns[] = {
+        gen_helper_mve_vqadds_scalarb,
+        gen_helper_mve_vqadds_scalarh,
+        gen_helper_mve_vqadds_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqaddu_fns[] = {
+        gen_helper_mve_vqaddu_scalarb,
+        gen_helper_mve_vqaddu_scalarh,
+        gen_helper_mve_vqaddu_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqsubs_fns[] = {
+        gen_helper_mve_vqsubs_scalarb,
+        gen_helper_mve_vqsubs_scalarh,
+        gen_helper_mve_vqsubs_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqsubu_fns[] = {
+        gen_helper_mve_vqsubu_scalarb,
+        gen_helper_mve_vqsubu_scalarh,
+        gen_helper_mve_vqsubu_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqdmulh_fns[] = {
+        gen_helper_mve_vqdmulh_scalarb,
+        gen_helper_mve_vqdmulh_scalarh,
+        gen_helper_mve_vqdmulh_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqrdmulh_fns[] = {
+        gen_helper_mve_vqrdmulh_scalarb,
+        gen_helper_mve_vqrdmulh_scalarh,
+        gen_helper_mve_vqrdmulh_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vmla_fns[] = {
+        gen_helper_mve_vmlab,
+        gen_helper_mve_vmlah,
+        gen_helper_mve_vmlaw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vmlas_fns[] = {
+        gen_helper_mve_vmlasb,
+        gen_helper_mve_vmlash,
+        gen_helper_mve_vmlasw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqdmlah_fns[] = {
+        gen_helper_mve_vqdmlahb,
+        gen_helper_mve_vqdmlahh,
+        gen_helper_mve_vqdmlahw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqrdmlah_fns[] = {
+        gen_helper_mve_vqrdmlahb,
+        gen_helper_mve_vqrdmlahh,
+        gen_helper_mve_vqrdmlahw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqdmlash_fns[] = {
+        gen_helper_mve_vqdmlashb,
+        gen_helper_mve_vqdmlashh,
+        gen_helper_mve_vqdmlashw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqrdmlash_fns[] = {
+        gen_helper_mve_vqrdmlashb,
+        gen_helper_mve_vqrdmlashh,
+        gen_helper_mve_vqrdmlashw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vbrsr_fns[] = {
+        gen_helper_mve_vbrsrb,
+        gen_helper_mve_vbrsrh,
+        gen_helper_mve_vbrsrw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn *fn = NULL;
+    unsigned size = extract32(insn, 20, 2);
+
+    switch (insn & 0xff811f70) {
+    case 0xee000f40:
+        fn = vhadds_fns[size];
+        break;
+    case 0xfe000f40:
+        fn = vhaddu_fns[size];
+        break;
+    case 0xee001f40:
+        fn = vhsubs_fns[size];
+        break;
+    case 0xfe001f40:
+        fn = vhsubu_fns[size];
+        break;
+    case 0xee000f60:
+        fn = vqadds_fns[size];
+        break;
+    case 0xfe000f60:
+        fn = vqaddu_fns[size];
+        break;
+    case 0xee001f60:
+        fn = vqsubs_fns[size];
+        break;
+    case 0xfe001f60:
+        fn = vqsubu_fns[size];
+        break;
+    case 0xee010e60:
+        fn = vqdmulh_fns[size];
+        break;
+    case 0xfe010e60:
+        fn = vqrdmulh_fns[size];
+        break;
+    case 0xee010f40:
+        fn = vadd_fns[size];
+        break;
+    case 0xee011f40:
+        fn = vsub_fns[size];
+        break;
+    case 0xee011e60:
+        fn = vmul_fns[size];
+        break;
+    case 0xee010e40:
+    case 0xfe010e40:
+        fn = vmla_fns[size];
+        break;
+    case 0xee011e40:
+    case 0xfe011e40:
+        fn = vmlas_fns[size];
+        break;
+    case 0xee000e60:
+        fn = vqdmlah_fns[size];
+        break;
+    case 0xee000e40:
+        fn = vqrdmlah_fns[size];
+        break;
+    case 0xee001e60:
+        fn = vqdmlash_fns[size];
+        break;
+    case 0xee001e40:
+        fn = vqrdmlash_fns[size];
+        break;
+    case 0xfe011e60:
+        fn = vbrsr_fns[size];
+        break;
+    default:
+        break;
+    }
+
+    return fn ? do_mve_2op_scalar(s, insn, fn) : false;
+}
+
+static bool trans_mve_fp_scalar_2op(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpScalarFn * const vfadd_fns[] = {
+        NULL,
+        gen_helper_mve_vfadd_scalarh,
+        gen_helper_mve_vfadd_scalars,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vfsub_fns[] = {
+        NULL,
+        gen_helper_mve_vfsub_scalarh,
+        gen_helper_mve_vfsub_scalars,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vfmul_fns[] = {
+        NULL,
+        gen_helper_mve_vfmul_scalarh,
+        gen_helper_mve_vfmul_scalars,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vfma_fns[] = {
+        NULL,
+        gen_helper_mve_vfma_scalarh,
+        gen_helper_mve_vfma_scalars,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vfmas_fns[] = {
+        NULL,
+        gen_helper_mve_vfmas_scalarh,
+        gen_helper_mve_vfmas_scalars,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn *fn = NULL;
+    unsigned size = extract32(insn, 28, 1) ? 1 : 2;
+
+    if (!dc_isar_feature(aa32_mve_fp, s)) {
+        return false;
+    }
+
+    switch (insn & 0xefb11f70) {
+    case 0xee300f40:
+        fn = vfadd_fns[size];
+        break;
+    case 0xee301f40:
+        fn = vfsub_fns[size];
+        break;
+    case 0xee310e60:
+        fn = vfmul_fns[size];
+        break;
+    case 0xee310e40:
+        fn = vfma_fns[size];
+        break;
+    case 0xee311e40:
+        fn = vfmas_fns[size];
+        break;
+    default:
+        break;
+    }
+
+    return fn ? do_mve_2op_scalar(s, insn, fn) : false;
+}
+
+static bool trans_mve_scalar_qdmull(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpScalarFn * const vqdmullb_fns[] = {
+        NULL,
+        gen_helper_mve_vqdmullb_scalarh,
+        gen_helper_mve_vqdmullb_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn * const vqdmullt_fns[] = {
+        NULL,
+        gen_helper_mve_vqdmullt_scalarh,
+        gen_helper_mve_vqdmullt_scalarw,
+        NULL,
+    };
+    MVEGenTwoOpScalarFn *fn = NULL;
+    unsigned size = extract32(insn, 28, 1) + 1;
+
+    if ((insn & 0xefb11f70) == 0xee300f60) {
+        fn = vqdmullb_fns[size];
+    } else if ((insn & 0xefb11f70) == 0xee301f60) {
+        fn = vqdmullt_fns[size];
+    }
+
+    if (fn && size == 2) {
+        int qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+        int qn = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+
+        if (qd == qn) {
+            return false;
+        }
+    }
+
+    return fn ? do_mve_2op_scalar(s, insn, fn) : false;
+}
+
+static bool do_mve_1op(DisasContext *s, uint32_t insn, MVEGenOneOpFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_ptr qmptr;
+    int qd;
+    int qm;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd | qm) || !fn) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_1op_i32(DisasContext *s, uint32_t insn,
+                           MVEGenOneOpI32Fn *fn, uint32_t imm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_ptr qmptr;
+    TCGv_i32 immval;
+    int qd;
+    int qm;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve_fp, s) ||
+        !mve_check_qreg_bank(qd | qm) || !fn) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    qmptr = mve_qreg_ptr(s, qm);
+    immval = tcg_const_i32(tcg_ctx, imm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qmptr, immval);
+    tcg_temp_free_i32(tcg_ctx, immval);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static MVEGenOneOpI32Fn *mve_fp_convert_fn(uint32_t insn, unsigned size,
+                                           uint32_t *imm)
+{
+    switch (insn & 0xffb31fd1) {
+    case 0xffb30640:
+        return size == 1 ? gen_helper_mve_vcvt_sh :
+            size == 2 ? gen_helper_mve_vcvt_sf : NULL;
+    case 0xffb306c0:
+        return size == 1 ? gen_helper_mve_vcvt_uh :
+            size == 2 ? gen_helper_mve_vcvt_uf : NULL;
+    case 0xffb30740:
+        return size == 1 ? gen_helper_mve_vcvt_hs :
+            size == 2 ? gen_helper_mve_vcvt_fs : NULL;
+    case 0xffb307c0:
+        return size == 1 ? gen_helper_mve_vcvt_hu :
+            size == 2 ? gen_helper_mve_vcvt_fu : NULL;
+    case 0xffb30040:
+        *imm = arm_rmode_to_sf(FPROUNDING_TIEAWAY);
+        return size == 1 ? gen_helper_mve_vcvt_rm_sh :
+            size == 2 ? gen_helper_mve_vcvt_rm_ss : NULL;
+    case 0xffb300c0:
+        *imm = arm_rmode_to_sf(FPROUNDING_TIEAWAY);
+        return size == 1 ? gen_helper_mve_vcvt_rm_uh :
+            size == 2 ? gen_helper_mve_vcvt_rm_us : NULL;
+    case 0xffb30140:
+        *imm = arm_rmode_to_sf(FPROUNDING_TIEEVEN);
+        return size == 1 ? gen_helper_mve_vcvt_rm_sh :
+            size == 2 ? gen_helper_mve_vcvt_rm_ss : NULL;
+    case 0xffb301c0:
+        *imm = arm_rmode_to_sf(FPROUNDING_TIEEVEN);
+        return size == 1 ? gen_helper_mve_vcvt_rm_uh :
+            size == 2 ? gen_helper_mve_vcvt_rm_us : NULL;
+    case 0xffb30240:
+        *imm = arm_rmode_to_sf(FPROUNDING_POSINF);
+        return size == 1 ? gen_helper_mve_vcvt_rm_sh :
+            size == 2 ? gen_helper_mve_vcvt_rm_ss : NULL;
+    case 0xffb302c0:
+        *imm = arm_rmode_to_sf(FPROUNDING_POSINF);
+        return size == 1 ? gen_helper_mve_vcvt_rm_uh :
+            size == 2 ? gen_helper_mve_vcvt_rm_us : NULL;
+    case 0xffb30340:
+        *imm = arm_rmode_to_sf(FPROUNDING_NEGINF);
+        return size == 1 ? gen_helper_mve_vcvt_rm_sh :
+            size == 2 ? gen_helper_mve_vcvt_rm_ss : NULL;
+    case 0xffb303c0:
+        *imm = arm_rmode_to_sf(FPROUNDING_NEGINF);
+        return size == 1 ? gen_helper_mve_vcvt_rm_uh :
+            size == 2 ? gen_helper_mve_vcvt_rm_us : NULL;
+    case 0xffb20440:
+        *imm = arm_rmode_to_sf(FPROUNDING_TIEEVEN);
+        return size == 1 ? gen_helper_mve_vrint_rm_h :
+            size == 2 ? gen_helper_mve_vrint_rm_s : NULL;
+    case 0xffb20540:
+        *imm = arm_rmode_to_sf(FPROUNDING_TIEAWAY);
+        return size == 1 ? gen_helper_mve_vrint_rm_h :
+            size == 2 ? gen_helper_mve_vrint_rm_s : NULL;
+    case 0xffb205c0:
+        *imm = arm_rmode_to_sf(FPROUNDING_ZERO);
+        return size == 1 ? gen_helper_mve_vrint_rm_h :
+            size == 2 ? gen_helper_mve_vrint_rm_s : NULL;
+    case 0xffb206c0:
+        *imm = arm_rmode_to_sf(FPROUNDING_NEGINF);
+        return size == 1 ? gen_helper_mve_vrint_rm_h :
+            size == 2 ? gen_helper_mve_vrint_rm_s : NULL;
+    case 0xffb207c0:
+        *imm = arm_rmode_to_sf(FPROUNDING_POSINF);
+        return size == 1 ? gen_helper_mve_vrint_rm_h :
+            size == 2 ? gen_helper_mve_vrint_rm_s : NULL;
+    default:
+        return NULL;
+    }
+}
+
+static bool trans_mve_fp_convert_round(DisasContext *s, uint32_t insn)
+{
+    MVEGenOneOpI32Fn *fn;
+    MVEGenOneOpFn *oneop_fn = NULL;
+    uint32_t imm = 0;
+    unsigned size = extract32(insn, 18, 2);
+
+    switch (insn & 0xffbf1fd1) {
+    case 0xee3f0e01:
+        oneop_fn = gen_helper_mve_vcvtb_sh;
+        break;
+    case 0xee3f1e01:
+        oneop_fn = gen_helper_mve_vcvtt_sh;
+        break;
+    case 0xfe3f0e01:
+        oneop_fn = gen_helper_mve_vcvtb_hs;
+        break;
+    case 0xfe3f1e01:
+        oneop_fn = gen_helper_mve_vcvtt_hs;
+        break;
+    default:
+        break;
+    }
+    if (oneop_fn) {
+        if (!dc_isar_feature(aa32_mve_fp, s)) {
+            return false;
+        }
+        return do_mve_1op(s, insn, oneop_fn);
+    }
+
+    if ((insn & 0xffb31fd1) == 0xffb204c0) {
+        oneop_fn = size == 1 ? gen_helper_mve_vrintx_h :
+            size == 2 ? gen_helper_mve_vrintx_s : NULL;
+        if (!dc_isar_feature(aa32_mve_fp, s)) {
+            return false;
+        }
+        return do_mve_1op(s, insn, oneop_fn);
+    }
+
+    fn = mve_fp_convert_fn(insn, size, &imm);
+    return fn ? do_mve_1op_i32(s, insn, fn, imm) : false;
+}
+
+static MVEGenTwoOpShiftFn *mve_fp_vcvt_fixed_fn(uint32_t insn,
+                                                unsigned *size,
+                                                uint32_t *shift)
+{
+    switch (insn & 0xffb01fd1) {
+    case 0xefb00c50:
+        *size = 1;
+        *shift = 16 - extract32(insn, 16, 4);
+        return gen_helper_mve_vcvt_sh;
+    case 0xffb00c50:
+        *size = 1;
+        *shift = 16 - extract32(insn, 16, 4);
+        return gen_helper_mve_vcvt_uh;
+    case 0xefb00d50:
+        *size = 1;
+        *shift = 16 - extract32(insn, 16, 4);
+        return gen_helper_mve_vcvt_hs;
+    case 0xffb00d50:
+        *size = 1;
+        *shift = 16 - extract32(insn, 16, 4);
+        return gen_helper_mve_vcvt_hu;
+    default:
+        break;
+    }
+
+    switch (insn & 0xffa01fd1) {
+    case 0xefa00e50:
+        *size = 2;
+        *shift = 32 - extract32(insn, 16, 5);
+        return gen_helper_mve_vcvt_sf;
+    case 0xffa00e50:
+        *size = 2;
+        *shift = 32 - extract32(insn, 16, 5);
+        return gen_helper_mve_vcvt_uf;
+    case 0xefa00f50:
+        *size = 2;
+        *shift = 32 - extract32(insn, 16, 5);
+        return gen_helper_mve_vcvt_fs;
+    case 0xffa00f50:
+        *size = 2;
+        *shift = 32 - extract32(insn, 16, 5);
+        return gen_helper_mve_vcvt_fu;
+    default:
+        return NULL;
+    }
+}
+
+static bool trans_mve_fp_vcvt_fixed(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpShiftFn *fn;
+    uint32_t shift = 0;
+    unsigned size = 0;
+
+    fn = mve_fp_vcvt_fixed_fn(insn, &size, &shift);
+    if (!fn) {
+        return false;
+    }
+    if (!dc_isar_feature(aa32_mve_fp, s)) {
+        return false;
+    }
+    return do_mve_2shift(s, insn, fn, size, shift, false);
+}
+
+static bool trans_mve_simple_1op(DisasContext *s, uint32_t insn)
+{
+    MVEGenOneOpFn * const vcls_fns[] = {
+        gen_helper_mve_vclsb,
+        gen_helper_mve_vclsh,
+        gen_helper_mve_vclsw,
+        NULL,
+    };
+    MVEGenOneOpFn * const vclz_fns[] = {
+        gen_helper_mve_vclzb,
+        gen_helper_mve_vclzh,
+        gen_helper_mve_vclzw,
+        NULL,
+    };
+    MVEGenOneOpFn * const vrev16_fns[] = {
+        gen_helper_mve_vrev16b,
+        NULL,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vrev32_fns[] = {
+        gen_helper_mve_vrev32b,
+        gen_helper_mve_vrev32h,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vrev64_fns[] = {
+        gen_helper_mve_vrev64b,
+        gen_helper_mve_vrev64h,
+        gen_helper_mve_vrev64w,
+        NULL,
+    };
+    MVEGenOneOpFn * const vabs_fns[] = {
+        gen_helper_mve_vabsb,
+        gen_helper_mve_vabsh,
+        gen_helper_mve_vabsw,
+        NULL,
+    };
+    MVEGenOneOpFn * const vneg_fns[] = {
+        gen_helper_mve_vnegb,
+        gen_helper_mve_vnegh,
+        gen_helper_mve_vnegw,
+        NULL,
+    };
+    MVEGenOneOpFn * const vfabs_fns[] = {
+        NULL,
+        gen_helper_mve_vfabsh,
+        gen_helper_mve_vfabss,
+        NULL,
+    };
+    MVEGenOneOpFn * const vfneg_fns[] = {
+        NULL,
+        gen_helper_mve_vfnegh,
+        gen_helper_mve_vfnegs,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqabs_fns[] = {
+        gen_helper_mve_vqabsb,
+        gen_helper_mve_vqabsh,
+        gen_helper_mve_vqabsw,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqneg_fns[] = {
+        gen_helper_mve_vqnegb,
+        gen_helper_mve_vqnegh,
+        gen_helper_mve_vqnegw,
+        NULL,
+    };
+    MVEGenOneOpFn * const vmaxa_fns[] = {
+        gen_helper_mve_vmaxab,
+        gen_helper_mve_vmaxah,
+        gen_helper_mve_vmaxaw,
+        NULL,
+    };
+    MVEGenOneOpFn * const vmina_fns[] = {
+        gen_helper_mve_vminab,
+        gen_helper_mve_vminah,
+        gen_helper_mve_vminaw,
+        NULL,
+    };
+    MVEGenOneOpFn *fn = NULL;
+    unsigned size = extract32(insn, 18, 2);
+
+    if ((insn & 0xffbf1fd1) == 0xffb005c0) {
+        return do_mve_1op(s, insn, gen_helper_mve_vmvn);
+    }
+
+    switch (insn & 0xffb31fd1) {
+    case 0xffb00440:
+        fn = vcls_fns[size];
+        break;
+    case 0xffb004c0:
+        fn = vclz_fns[size];
+        break;
+    case 0xffb00140:
+        fn = vrev16_fns[size];
+        break;
+    case 0xffb000c0:
+        fn = vrev32_fns[size];
+        break;
+    case 0xffb00040:
+        fn = vrev64_fns[size];
+        break;
+    case 0xffb10340:
+        fn = vabs_fns[size];
+        break;
+    case 0xffb103c0:
+        fn = vneg_fns[size];
+        break;
+    case 0xffb10740:
+        if (!dc_isar_feature(aa32_mve_fp, s)) {
+            return false;
+        }
+        fn = vfabs_fns[size];
+        break;
+    case 0xffb107c0:
+        if (!dc_isar_feature(aa32_mve_fp, s)) {
+            return false;
+        }
+        fn = vfneg_fns[size];
+        break;
+    case 0xffb00740:
+        fn = vqabs_fns[size];
+        break;
+    case 0xffb007c0:
+        fn = vqneg_fns[size];
+        break;
+    case 0xee330e81:
+        fn = vmaxa_fns[size];
+        break;
+    case 0xee331e81:
+        fn = vmina_fns[size];
+        break;
+    default:
+        break;
+    }
+
+    return fn ? do_mve_1op(s, insn, fn) : false;
+}
+
+static bool trans_mve_movn(DisasContext *s, uint32_t insn)
+{
+    MVEGenOneOpFn * const vmovnb_fns[] = {
+        gen_helper_mve_vmovnbb,
+        gen_helper_mve_vmovnbh,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vmovnt_fns[] = {
+        gen_helper_mve_vmovntb,
+        gen_helper_mve_vmovnth,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqmovnbs_fns[] = {
+        gen_helper_mve_vqmovnbsb,
+        gen_helper_mve_vqmovnbsh,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqmovnts_fns[] = {
+        gen_helper_mve_vqmovntsb,
+        gen_helper_mve_vqmovntsh,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqmovnbu_fns[] = {
+        gen_helper_mve_vqmovnbub,
+        gen_helper_mve_vqmovnbuh,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqmovntu_fns[] = {
+        gen_helper_mve_vqmovntub,
+        gen_helper_mve_vqmovntuh,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqmovunb_fns[] = {
+        gen_helper_mve_vqmovunbb,
+        gen_helper_mve_vqmovunbh,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn * const vqmovunt_fns[] = {
+        gen_helper_mve_vqmovuntb,
+        gen_helper_mve_vqmovunth,
+        NULL,
+        NULL,
+    };
+    MVEGenOneOpFn *fn = NULL;
+    unsigned size = extract32(insn, 18, 2);
+
+    switch (insn & 0xffb31fd1) {
+    case 0xfe310e81:
+        fn = vmovnb_fns[size];
+        break;
+    case 0xfe311e81:
+        fn = vmovnt_fns[size];
+        break;
+    case 0xee330e01:
+        fn = vqmovnbs_fns[size];
+        break;
+    case 0xee331e01:
+        fn = vqmovnts_fns[size];
+        break;
+    case 0xfe330e01:
+        fn = vqmovnbu_fns[size];
+        break;
+    case 0xfe331e01:
+        fn = vqmovntu_fns[size];
+        break;
+    case 0xee310e81:
+        fn = vqmovunb_fns[size];
+        break;
+    case 0xee311e81:
+        fn = vqmovunt_fns[size];
+        break;
+    default:
+        return false;
+    }
+
+    return fn ? do_mve_1op(s, insn, fn) : false;
+}
+
+static bool trans_mve_vmlaldav(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MVEGenLongDualAccFn * const add_s_fns[3][2] = {
+        { NULL, NULL },
+        { gen_helper_mve_vmlaldavsh, gen_helper_mve_vmlaldavxsh },
+        { gen_helper_mve_vmlaldavsw, gen_helper_mve_vmlaldavxsw },
+    };
+    MVEGenLongDualAccFn * const add_u_fns[3][2] = {
+        { NULL, NULL },
+        { gen_helper_mve_vmlaldavuh, NULL },
+        { gen_helper_mve_vmlaldavuw, NULL },
+    };
+    MVEGenLongDualAccFn * const sub_s_fns[3][2] = {
+        { NULL, NULL },
+        { gen_helper_mve_vmlsldavsh, gen_helper_mve_vmlsldavxsh },
+        { gen_helper_mve_vmlsldavsw, gen_helper_mve_vmlsldavxsw },
+    };
+    MVEGenLongDualAccFn * const round_add_s_fns[] = {
+        gen_helper_mve_vrmlaldavhsw,
+        gen_helper_mve_vrmlaldavhxsw,
+    };
+    MVEGenLongDualAccFn * const round_add_u_fns[] = {
+        gen_helper_mve_vrmlaldavhuw,
+        NULL,
+    };
+    MVEGenLongDualAccFn * const round_sub_s_fns[] = {
+        gen_helper_mve_vrmlsldavhsw,
+        gen_helper_mve_vrmlsldavhxsw,
+    };
+    MVEGenLongDualAccFn *fn = NULL;
+    TCGv_i64 rda;
+    TCGv_i32 rdalo;
+    TCGv_i32 rdahi;
+    TCGv_ptr qnptr;
+    TCGv_ptr qmptr;
+    unsigned size;
+    unsigned rdalo_reg;
+    unsigned rdahi_reg;
+    unsigned qn;
+    unsigned qm;
+    bool accum;
+    bool xchg;
+
+    size = extract32(insn, 16, 1) + 1;
+    xchg = extract32(insn, 12, 1);
+    if ((insn & 0xff800f51) == 0xee800e00) {
+        fn = add_s_fns[size][xchg];
+    } else if ((insn & 0xff800f51) == 0xfe800e00) {
+        fn = add_u_fns[size][xchg];
+    } else if ((insn & 0xff800f51) == 0xee800e01) {
+        fn = sub_s_fns[size][xchg];
+    } else if ((insn & 0xff810f51) == 0xee800f00) {
+        fn = round_add_s_fns[xchg];
+    } else if ((insn & 0xff810f51) == 0xfe800f00) {
+        fn = round_add_u_fns[xchg];
+    } else if ((insn & 0xff810f51) == 0xfe800e01) {
+        fn = round_sub_s_fns[xchg];
+    } else {
+        return false;
+    }
+
+    rdalo_reg = extract32(insn, 13, 3) << 1;
+    rdahi_reg = (extract32(insn, 20, 3) << 1) | 1;
+    qn = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    qm = extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qn | qm) || !fn ||
+        rdahi_reg == 13 || rdahi_reg == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    accum = extract32(insn, 5, 1);
+    if (accum || mve_skip_first_beat(s)) {
+        rda = tcg_temp_new_i64(tcg_ctx);
+        rdalo = load_reg(s, rdalo_reg);
+        rdahi = load_reg(s, rdahi_reg);
+        tcg_gen_concat_i32_i64(tcg_ctx, rda, rdalo, rdahi);
+        tcg_temp_free_i32(tcg_ctx, rdalo);
+        tcg_temp_free_i32(tcg_ctx, rdahi);
+    } else {
+        rda = tcg_const_i64(tcg_ctx, 0);
+    }
+
+    qnptr = mve_qreg_ptr(s, qn);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, rda, tcg_ctx->cpu_env, qnptr, qmptr, rda);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+
+    rdalo = tcg_temp_new_i32(tcg_ctx);
+    rdahi = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_extrl_i64_i32(tcg_ctx, rdalo, rda);
+    tcg_gen_extrh_i64_i32(tcg_ctx, rdahi, rda);
+    store_reg(s, rdalo_reg, rdalo);
+    store_reg(s, rdahi_reg, rdahi);
+    tcg_temp_free_i64(tcg_ctx, rda);
+
+    mve_update_eci(s);
+    return true;
+}
+
+static bool trans_mve_vmladav(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MVEGenDualAccFn * const add_s_fns[3][2] = {
+        { gen_helper_mve_vmladavsb, gen_helper_mve_vmladavsxb },
+        { gen_helper_mve_vmladavsh, gen_helper_mve_vmladavsxh },
+        { gen_helper_mve_vmladavsw, gen_helper_mve_vmladavsxw },
+    };
+    MVEGenDualAccFn * const add_u_fns[3][2] = {
+        { gen_helper_mve_vmladavub, NULL },
+        { gen_helper_mve_vmladavuh, NULL },
+        { gen_helper_mve_vmladavuw, NULL },
+    };
+    MVEGenDualAccFn * const sub_s_fns[3][2] = {
+        { gen_helper_mve_vmlsdavb, gen_helper_mve_vmlsdavxb },
+        { gen_helper_mve_vmlsdavh, gen_helper_mve_vmlsdavxh },
+        { gen_helper_mve_vmlsdavw, gen_helper_mve_vmlsdavxw },
+    };
+    MVEGenDualAccFn *fn = NULL;
+    TCGv_i32 rda;
+    TCGv_ptr qnptr;
+    TCGv_ptr qmptr;
+    unsigned size;
+    unsigned rda_reg;
+    unsigned qn;
+    unsigned qm;
+    bool accum;
+    bool xchg;
+
+    xchg = extract32(insn, 12, 1);
+    if ((insn & 0xfff00f51) == 0xeef00e00) {
+        size = extract32(insn, 16, 1) + 1;
+        fn = add_s_fns[size][xchg];
+    } else if ((insn & 0xfff00f51) == 0xfef00e00) {
+        size = extract32(insn, 16, 1) + 1;
+        fn = add_u_fns[size][xchg];
+    } else if ((insn & 0xfff00f51) == 0xeef00e01) {
+        size = extract32(insn, 16, 1) + 1;
+        fn = sub_s_fns[size][xchg];
+    } else if ((insn & 0xfff10f50) == 0xeef00f00) {
+        size = 0;
+        fn = add_s_fns[size][xchg];
+    } else if ((insn & 0xfff10f50) == 0xfef00f00) {
+        size = 0;
+        fn = add_u_fns[size][xchg];
+    } else if ((insn & 0xfff10f51) == 0xfef00e01) {
+        size = 0;
+        fn = sub_s_fns[size][xchg];
+    } else {
+        return false;
+    }
+
+    rda_reg = extract32(insn, 13, 3) << 1;
+    qn = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    qm = extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qn) || !fn) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    accum = extract32(insn, 5, 1);
+    rda = (accum || mve_skip_first_beat(s)) ?
+        load_reg(s, rda_reg) : tcg_const_i32(tcg_ctx, 0);
+    qnptr = mve_qreg_ptr(s, qn);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, rda, tcg_ctx->cpu_env, qnptr, qmptr, rda);
+    store_reg(s, rda_reg, rda);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+
+    mve_update_eci(s);
+    return true;
+}
+
+static bool trans_mve_vaddv(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MVEGenVADDVFn * const vaddv_fns[3][2] = {
+        { gen_helper_mve_vaddvsb, gen_helper_mve_vaddvub },
+        { gen_helper_mve_vaddvsh, gen_helper_mve_vaddvuh },
+        { gen_helper_mve_vaddvsw, gen_helper_mve_vaddvuw },
+    };
+    TCGv_i32 rda;
+    TCGv_ptr qmptr;
+    unsigned size;
+    unsigned rda_reg;
+    unsigned qm;
+    bool accum;
+    bool is_unsigned;
+
+    if ((insn & 0xeff31fd1) != 0xeef10f00) {
+        return false;
+    }
+
+    size = extract32(insn, 18, 2);
+    if (size == 3 ||
+        !arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    rda_reg = extract32(insn, 13, 3) << 1;
+    qm = extract32(insn, 1, 3);
+    is_unsigned = extract32(insn, 28, 1);
+    accum = extract32(insn, 5, 1);
+
+    rda = (accum || mve_skip_first_beat(s)) ?
+        load_reg(s, rda_reg) : tcg_const_i32(tcg_ctx, 0);
+    qmptr = mve_qreg_ptr(s, qm);
+    vaddv_fns[size][is_unsigned](tcg_ctx, rda, tcg_ctx->cpu_env, qmptr,
+                                 rda);
+    store_reg(s, rda_reg, rda);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+
+    mve_update_eci(s);
+    return true;
+}
+
+static bool trans_mve_vmaxv(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MVEGenVADDVFn * const vmaxvs_fns[] = {
+        gen_helper_mve_vmaxvsb,
+        gen_helper_mve_vmaxvsh,
+        gen_helper_mve_vmaxvsw,
+        NULL,
+    };
+    MVEGenVADDVFn * const vmaxvu_fns[] = {
+        gen_helper_mve_vmaxvub,
+        gen_helper_mve_vmaxvuh,
+        gen_helper_mve_vmaxvuw,
+        NULL,
+    };
+    MVEGenVADDVFn * const vmaxav_fns[] = {
+        gen_helper_mve_vmaxavb,
+        gen_helper_mve_vmaxavh,
+        gen_helper_mve_vmaxavw,
+        NULL,
+    };
+    MVEGenVADDVFn * const vminvs_fns[] = {
+        gen_helper_mve_vminvsb,
+        gen_helper_mve_vminvsh,
+        gen_helper_mve_vminvsw,
+        NULL,
+    };
+    MVEGenVADDVFn * const vminvu_fns[] = {
+        gen_helper_mve_vminvub,
+        gen_helper_mve_vminvuh,
+        gen_helper_mve_vminvuw,
+        NULL,
+    };
+    MVEGenVADDVFn * const vminav_fns[] = {
+        gen_helper_mve_vminavb,
+        gen_helper_mve_vminavh,
+        gen_helper_mve_vminavw,
+        NULL,
+    };
+    MVEGenVADDVFn *fn = NULL;
+    TCGv_i32 rda;
+    TCGv_ptr qmptr;
+    unsigned size;
+    unsigned rda_reg;
+    unsigned qm;
+
+    size = extract32(insn, 18, 2);
+    if ((insn & 0xfff30fd1) == 0xeee20f00) {
+        fn = vmaxvs_fns[size];
+    } else if ((insn & 0xfff30fd1) == 0xfee20f00) {
+        fn = vmaxvu_fns[size];
+    } else if ((insn & 0xfff30fd1) == 0xeee00f00) {
+        fn = vmaxav_fns[size];
+    } else if ((insn & 0xfff30fd1) == 0xeee20f80) {
+        fn = vminvs_fns[size];
+    } else if ((insn & 0xfff30fd1) == 0xfee20f80) {
+        fn = vminvu_fns[size];
+    } else if ((insn & 0xfff30fd1) == 0xeee00f80) {
+        fn = vminav_fns[size];
+    } else {
+        return false;
+    }
+
+    rda_reg = extract32(insn, 12, 4);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qm) || !fn ||
+        rda_reg == 13 || rda_reg == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    rda = load_reg(s, rda_reg);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, rda, tcg_ctx->cpu_env, qmptr, rda);
+    store_reg(s, rda_reg, rda);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+
+    mve_update_eci(s);
+    return true;
+}
+
+static bool trans_mve_vmaxnmv(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MVEGenVADDVFn *fn = NULL;
+    TCGv_i32 rda;
+    TCGv_ptr qmptr;
+    unsigned rda_reg;
+    unsigned qm;
+
+    if ((insn & 0xffff0fd1) == 0xeeee0f00) {
+        fn = gen_helper_mve_vmaxnmvs;
+    } else if ((insn & 0xffff0fd1) == 0xfeee0f00) {
+        fn = gen_helper_mve_vmaxnmvh;
+    } else if ((insn & 0xffff0fd1) == 0xeeec0f00) {
+        fn = gen_helper_mve_vmaxnmavs;
+    } else if ((insn & 0xffff0fd1) == 0xfeec0f00) {
+        fn = gen_helper_mve_vmaxnmavh;
+    } else if ((insn & 0xffff0fd1) == 0xeeee0f80) {
+        fn = gen_helper_mve_vminnmvs;
+    } else if ((insn & 0xffff0fd1) == 0xfeee0f80) {
+        fn = gen_helper_mve_vminnmvh;
+    } else if ((insn & 0xffff0fd1) == 0xeeec0f80) {
+        fn = gen_helper_mve_vminnmavs;
+    } else if ((insn & 0xffff0fd1) == 0xfeec0f80) {
+        fn = gen_helper_mve_vminnmavh;
+    } else {
+        return false;
+    }
+
+    rda_reg = extract32(insn, 12, 4);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve_fp, s) ||
+        !mve_check_qreg_bank(qm) ||
+        rda_reg == 13 || rda_reg == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    rda = load_reg(s, rda_reg);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, rda, tcg_ctx->cpu_env, qmptr, rda);
+    store_reg(s, rda_reg, rda);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+
+    mve_update_eci(s);
+    return true;
+}
+
+static bool trans_mve_vaddlv(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i64 rda;
+    TCGv_i32 rdalo;
+    TCGv_i32 rdahi;
+    TCGv_ptr qmptr;
+    unsigned rdalo_reg;
+    unsigned rdahi_reg;
+    unsigned qm;
+    bool accum;
+    bool is_unsigned;
+
+    if ((insn & 0xef8f1fd1) != 0xee890f00) {
+        return false;
+    }
+
+    rdahi_reg = (extract32(insn, 20, 3) << 1) | 1;
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        rdahi_reg == 13 || rdahi_reg == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    rdalo_reg = extract32(insn, 13, 3) << 1;
+    qm = extract32(insn, 1, 3);
+    is_unsigned = extract32(insn, 28, 1);
+    accum = extract32(insn, 5, 1);
+
+    if (accum || mve_skip_first_beat(s)) {
+        rda = tcg_temp_new_i64(tcg_ctx);
+        rdalo = load_reg(s, rdalo_reg);
+        rdahi = load_reg(s, rdahi_reg);
+        tcg_gen_concat_i32_i64(tcg_ctx, rda, rdalo, rdahi);
+        tcg_temp_free_i32(tcg_ctx, rdalo);
+        tcg_temp_free_i32(tcg_ctx, rdahi);
+    } else {
+        rda = tcg_const_i64(tcg_ctx, 0);
+    }
+
+    qmptr = mve_qreg_ptr(s, qm);
+    if (is_unsigned) {
+        gen_helper_mve_vaddlv_u(tcg_ctx, rda, tcg_ctx->cpu_env, qmptr, rda);
+    } else {
+        gen_helper_mve_vaddlv_s(tcg_ctx, rda, tcg_ctx->cpu_env, qmptr, rda);
+    }
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+
+    rdalo = tcg_temp_new_i32(tcg_ctx);
+    rdahi = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_extrl_i64_i32(tcg_ctx, rdalo, rda);
+    tcg_gen_extrh_i64_i32(tcg_ctx, rdahi, rda);
+    store_reg(s, rdalo_reg, rdalo);
+    store_reg(s, rdahi_reg, rdahi);
+    tcg_temp_free_i64(tcg_ctx, rda);
+
+    mve_update_eci(s);
+    return true;
+}
+
+static bool trans_mve_vabav(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MVEGenVABAVFn * const vabavs_fns[] = {
+        gen_helper_mve_vabavsb,
+        gen_helper_mve_vabavsh,
+        gen_helper_mve_vabavsw,
+        NULL,
+    };
+    MVEGenVABAVFn * const vabavu_fns[] = {
+        gen_helper_mve_vabavub,
+        gen_helper_mve_vabavuh,
+        gen_helper_mve_vabavuw,
+        NULL,
+    };
+    MVEGenVABAVFn *fn;
+    TCGv_i32 rda;
+    TCGv_ptr qnptr;
+    TCGv_ptr qmptr;
+    unsigned size;
+    unsigned rda_reg;
+    unsigned qn;
+    unsigned qm;
+    bool is_unsigned;
+
+    if ((insn & 0xefc10f51) != 0xee800f01) {
+        return false;
+    }
+
+    size = extract32(insn, 20, 2);
+    rda_reg = extract32(insn, 12, 4);
+    qn = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    is_unsigned = extract32(insn, 28, 1);
+    fn = is_unsigned ? vabavu_fns[size] : vabavs_fns[size];
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qn | qm) || !fn ||
+        rda_reg == 13 || rda_reg == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qnptr = mve_qreg_ptr(s, qn);
+    qmptr = mve_qreg_ptr(s, qm);
+    rda = load_reg(s, rda_reg);
+    fn(tcg_ctx, rda, tcg_ctx->cpu_env, qnptr, qmptr, rda);
+    store_reg(s, rda_reg, rda);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+
+    mve_update_eci(s);
+    return true;
+}
+
+static bool do_mve_ldst_reg(DisasContext *s, uint32_t insn, MVEGenLdStFn *fn,
+                             bool p, bool w, unsigned msize, int qd, int rn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 addr;
+    TCGv_ptr qreg;
+    int32_t offset;
+    bool add;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd) || !fn || rn == 15 || (rn == 13 && w)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    add = extract32(insn, 23, 1);
+    offset = extract32(insn, 0, 7) << msize;
+    if (!add) {
+        offset = -offset;
+    }
+
+    addr = load_reg(s, rn);
+    if (p) {
+        tcg_gen_addi_i32(tcg_ctx, addr, addr, offset);
+    }
+
+    qreg = mve_qreg_ptr(s, qd);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qreg, addr);
+    tcg_temp_free_ptr(tcg_ctx, qreg);
+
+    if (w) {
+        if (!p) {
+            tcg_gen_addi_i32(tcg_ctx, addr, addr, offset);
+        }
+        store_reg(s, rn, addr);
+    } else {
+        tcg_temp_free_i32(tcg_ctx, addr);
+    }
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_ldst(DisasContext *s, uint32_t insn, MVEGenLdStFn *fn,
+                        bool p, bool w, unsigned msize)
+{
+    int qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    int rn = extract32(insn, 16, 4);
+
+    return do_mve_ldst_reg(s, insn, fn, p, w, msize, qd, rn);
+}
+
+static bool trans_mve_vldst_wn(DisasContext *s, uint32_t insn)
+{
+    MVEGenLdStFn * const ldst_fns[3][2][2] = {
+        {
+            { gen_helper_mve_vstrb_h, gen_helper_mve_vldrb_sh },
+            { NULL, gen_helper_mve_vldrb_uh },
+        },
+        {
+            { gen_helper_mve_vstrb_w, gen_helper_mve_vldrb_sw },
+            { NULL, gen_helper_mve_vldrb_uw },
+        },
+        {
+            { gen_helper_mve_vstrh_w, gen_helper_mve_vldrh_sw },
+            { NULL, gen_helper_mve_vldrh_uw },
+        },
+    };
+    MVEGenLdStFn *fn;
+    unsigned group;
+    unsigned msize;
+    int qd;
+    int rn;
+    bool p;
+    bool w;
+
+    switch (insn & 0xef681f80) {
+    case 0xec200e80:
+        group = 0;
+        msize = 0;
+        p = false;
+        w = true;
+        break;
+    case 0xec200f00:
+        group = 1;
+        msize = 0;
+        p = false;
+        w = true;
+        break;
+    case 0xec280f00:
+        group = 2;
+        msize = 1;
+        p = false;
+        w = true;
+        break;
+    default:
+        switch (insn & 0xef481f80) {
+        case 0xed000e80:
+            group = 0;
+            msize = 0;
+            p = true;
+            w = extract32(insn, 21, 1);
+            break;
+        case 0xed000f00:
+            group = 1;
+            msize = 0;
+            p = true;
+            w = extract32(insn, 21, 1);
+            break;
+        case 0xed080f00:
+            group = 2;
+            msize = 1;
+            p = true;
+            w = extract32(insn, 21, 1);
+            break;
+        default:
+            return false;
+        }
+        break;
+    }
+
+    fn = ldst_fns[group][extract32(insn, 28, 1)][extract32(insn, 20, 1)];
+    qd = extract32(insn, 13, 3);
+    rn = extract32(insn, 16, 3);
+    return do_mve_ldst_reg(s, insn, fn, p, w, msize, qd, rn);
+}
+
+static bool trans_mve_vldr_vstr(DisasContext *s, uint32_t insn)
+{
+    MVEGenLdStFn * const ldst_fns[3][2] = {
+        { gen_helper_mve_vstrb, gen_helper_mve_vldrb },
+        { gen_helper_mve_vstrh, gen_helper_mve_vldrh },
+        { gen_helper_mve_vstrw, gen_helper_mve_vldrw },
+    };
+    MVEGenLdStFn *fn;
+    unsigned msize;
+    bool p;
+    bool w;
+
+    switch (insn & 0xff201f80) {
+    case 0xec201e00:
+        msize = 0;
+        p = false;
+        w = true;
+        break;
+    case 0xec201e80:
+        msize = 1;
+        p = false;
+        w = true;
+        break;
+    case 0xec201f00:
+        msize = 2;
+        p = false;
+        w = true;
+        break;
+    default:
+        switch (insn & 0xff001f80) {
+        case 0xed001e00:
+            msize = 0;
+            p = true;
+            w = extract32(insn, 21, 1);
+            break;
+        case 0xed001e80:
+            msize = 1;
+            p = true;
+            w = extract32(insn, 21, 1);
+            break;
+        case 0xed001f00:
+            msize = 2;
+            p = true;
+            w = extract32(insn, 21, 1);
+            break;
+        default:
+            return false;
+        }
+        break;
+    }
+
+    fn = ldst_fns[msize][extract32(insn, 20, 1)];
+    return do_mve_ldst(s, insn, fn, p, w, msize);
+}
+
+static bool do_mve_ldst_sg(DisasContext *s, MVEGenLdStSGFn *fn,
+                           int qd, int qm, int rn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 addr;
+    TCGv_ptr qdptr;
+    TCGv_ptr qmptr;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd | qm) || !fn || rn == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    addr = load_reg(s, rn);
+    qdptr = mve_qreg_ptr(s, qd);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qmptr, addr);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+    tcg_temp_free_i32(tcg_ctx, addr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_mve_vldst_sg(DisasContext *s, uint32_t insn)
+{
+    MVEGenLdStSGFn * const s_load_fns[2][4][4] = {
+        {
+            { NULL, gen_helper_mve_vldrb_sg_sh,
+              gen_helper_mve_vldrb_sg_sw, NULL },
+            { NULL, NULL, gen_helper_mve_vldrh_sg_sw, NULL },
+            { NULL, NULL, NULL, NULL },
+            { NULL, NULL, NULL, NULL },
+        },
+        {
+            { NULL, NULL, NULL, NULL },
+            { NULL, NULL, gen_helper_mve_vldrh_sg_os_sw, NULL },
+            { NULL, NULL, NULL, NULL },
+            { NULL, NULL, NULL, NULL },
+        },
+    };
+    MVEGenLdStSGFn * const u_load_fns[2][4][4] = {
+        {
+            { gen_helper_mve_vldrb_sg_ub, gen_helper_mve_vldrb_sg_uh,
+              gen_helper_mve_vldrb_sg_uw, NULL },
+            { NULL, gen_helper_mve_vldrh_sg_uh,
+              gen_helper_mve_vldrh_sg_uw, NULL },
+            { NULL, NULL, gen_helper_mve_vldrw_sg_uw, NULL },
+            { NULL, NULL, NULL, gen_helper_mve_vldrd_sg_ud },
+        },
+        {
+            { NULL, NULL, NULL, NULL },
+            { NULL, gen_helper_mve_vldrh_sg_os_uh,
+              gen_helper_mve_vldrh_sg_os_uw, NULL },
+            { NULL, NULL, gen_helper_mve_vldrw_sg_os_uw, NULL },
+            { NULL, NULL, NULL, gen_helper_mve_vldrd_sg_os_ud },
+        },
+    };
+    MVEGenLdStSGFn * const store_fns[2][4][4] = {
+        {
+            { gen_helper_mve_vstrb_sg_ub, gen_helper_mve_vstrb_sg_uh,
+              gen_helper_mve_vstrb_sg_uw, NULL },
+            { NULL, gen_helper_mve_vstrh_sg_uh,
+              gen_helper_mve_vstrh_sg_uw, NULL },
+            { NULL, NULL, gen_helper_mve_vstrw_sg_uw, NULL },
+            { NULL, NULL, NULL, gen_helper_mve_vstrd_sg_ud },
+        },
+        {
+            { NULL, NULL, NULL, NULL },
+            { NULL, gen_helper_mve_vstrh_sg_os_uh,
+              gen_helper_mve_vstrh_sg_os_uw, NULL },
+            { NULL, NULL, gen_helper_mve_vstrw_sg_os_uw, NULL },
+            { NULL, NULL, NULL, gen_helper_mve_vstrd_sg_os_ud },
+        },
+    };
+    MVEGenLdStSGFn *fn;
+    unsigned size;
+    unsigned msize;
+    unsigned os;
+    int qd;
+    int qm;
+    int rn;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    rn = extract32(insn, 16, 4);
+    size = extract32(insn, 7, 2);
+    msize = (extract32(insn, 6, 1) << 1) | extract32(insn, 4, 1);
+    os = extract32(insn, 0, 1);
+
+    switch (insn & 0xffb01e00) {
+    case 0xec900e00:
+        if (qd == qm) {
+            return false;
+        }
+        fn = s_load_fns[os][msize][size];
+        break;
+    case 0xfc900e00:
+        if (qd == qm) {
+            return false;
+        }
+        fn = u_load_fns[os][msize][size];
+        break;
+    case 0xec800e00:
+        fn = store_fns[os][msize][size];
+        break;
+    default:
+        return false;
+    }
+
+    return do_mve_ldst_sg(s, fn, qd, qm, rn);
+}
+
+static bool do_mve_ldst_sg_imm(DisasContext *s, MVEGenLdStSGFn *fn,
+                               int qd, int qm, bool add,
+                               unsigned msize, uint32_t imm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 offset_tcg;
+    TCGv_ptr qdptr;
+    TCGv_ptr qmptr;
+    uint32_t offset;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd | qm) || !fn) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    offset = imm << msize;
+    if (!add) {
+        offset = -offset;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    qmptr = mve_qreg_ptr(s, qm);
+    offset_tcg = tcg_const_i32(tcg_ctx, offset);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qmptr, offset_tcg);
+    tcg_temp_free_i32(tcg_ctx, offset_tcg);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_mve_vldst_sg_imm(DisasContext *s, uint32_t insn)
+{
+    MVEGenLdStSGFn * const ldrw_fns[] = {
+        gen_helper_mve_vldrw_sg_uw,
+        gen_helper_mve_vldrw_sg_wb_uw,
+    };
+    MVEGenLdStSGFn * const ldrd_fns[] = {
+        gen_helper_mve_vldrd_sg_ud,
+        gen_helper_mve_vldrd_sg_wb_ud,
+    };
+    MVEGenLdStSGFn * const strw_fns[] = {
+        gen_helper_mve_vstrw_sg_uw,
+        gen_helper_mve_vstrw_sg_wb_uw,
+    };
+    MVEGenLdStSGFn * const strd_fns[] = {
+        gen_helper_mve_vstrd_sg_ud,
+        gen_helper_mve_vstrd_sg_wb_ud,
+    };
+    MVEGenLdStSGFn *fn;
+    unsigned msize;
+    int qd;
+    int qm;
+    bool add;
+    bool writeback;
+    bool is_load;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qm = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    add = extract32(insn, 23, 1);
+    writeback = extract32(insn, 21, 1);
+
+    switch (insn & 0xff111f00) {
+    case 0xfd101e00:
+        fn = ldrw_fns[writeback];
+        msize = 2;
+        is_load = true;
+        break;
+    case 0xfd101f00:
+        fn = ldrd_fns[writeback];
+        msize = 3;
+        is_load = true;
+        break;
+    case 0xfd001e00:
+        fn = strw_fns[writeback];
+        msize = 2;
+        is_load = false;
+        break;
+    case 0xfd001f00:
+        fn = strd_fns[writeback];
+        msize = 3;
+        is_load = false;
+        break;
+    default:
+        return false;
+    }
+
+    if (is_load && qd == qm) {
+        return false;
+    }
+
+    return do_mve_ldst_sg_imm(s, fn, qd, qm, add, msize,
+                              extract32(insn, 0, 7));
+}
+
+static bool do_mve_vldst_il(DisasContext *s, MVEGenLdStIlFn *fn,
+                            int qd, int rnidx, bool writeback,
+                            int addrinc)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 rn;
+    TCGv_i32 qdidx;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd) || !fn ||
+        (rnidx == 13 && writeback) || rnidx == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    rn = load_reg(s, rnidx);
+    qdidx = tcg_const_i32(tcg_ctx, qd);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdidx, rn);
+    tcg_temp_free_i32(tcg_ctx, qdidx);
+
+    if (writeback) {
+        tcg_gen_addi_i32(tcg_ctx, rn, rn, addrinc);
+        store_reg(s, rnidx, rn);
+    } else {
+        tcg_temp_free_i32(tcg_ctx, rn);
+    }
+
+    mve_update_and_store_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_mve_vldst_il(DisasContext *s, uint32_t insn)
+{
+    MVEGenLdStIlFn * const vld2_fns[4][4] = {
+        { gen_helper_mve_vld20b, gen_helper_mve_vld20h,
+          gen_helper_mve_vld20w, NULL },
+        { gen_helper_mve_vld21b, gen_helper_mve_vld21h,
+          gen_helper_mve_vld21w, NULL },
+        { NULL, NULL, NULL, NULL },
+        { NULL, NULL, NULL, NULL },
+    };
+    MVEGenLdStIlFn * const vst2_fns[4][4] = {
+        { gen_helper_mve_vst20b, gen_helper_mve_vst20h,
+          gen_helper_mve_vst20w, NULL },
+        { gen_helper_mve_vst21b, gen_helper_mve_vst21h,
+          gen_helper_mve_vst21w, NULL },
+        { NULL, NULL, NULL, NULL },
+        { NULL, NULL, NULL, NULL },
+    };
+    MVEGenLdStIlFn * const vld4_fns[4][4] = {
+        { gen_helper_mve_vld40b, gen_helper_mve_vld40h,
+          gen_helper_mve_vld40w, NULL },
+        { gen_helper_mve_vld41b, gen_helper_mve_vld41h,
+          gen_helper_mve_vld41w, NULL },
+        { gen_helper_mve_vld42b, gen_helper_mve_vld42h,
+          gen_helper_mve_vld42w, NULL },
+        { gen_helper_mve_vld43b, gen_helper_mve_vld43h,
+          gen_helper_mve_vld43w, NULL },
+    };
+    MVEGenLdStIlFn * const vst4_fns[4][4] = {
+        { gen_helper_mve_vst40b, gen_helper_mve_vst40h,
+          gen_helper_mve_vst40w, NULL },
+        { gen_helper_mve_vst41b, gen_helper_mve_vst41h,
+          gen_helper_mve_vst41w, NULL },
+        { gen_helper_mve_vst42b, gen_helper_mve_vst42h,
+          gen_helper_mve_vst42w, NULL },
+        { gen_helper_mve_vst43b, gen_helper_mve_vst43h,
+          gen_helper_mve_vst43w, NULL },
+    };
+    MVEGenLdStIlFn *fn;
+    unsigned size;
+    unsigned pat;
+    int addrinc;
+    int qd;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    size = extract32(insn, 7, 2);
+    pat = extract32(insn, 5, 2);
+
+    switch (insn & 0xff901e1f) {
+    case 0xfc901e00:
+        if (qd > 6) {
+            return false;
+        }
+        fn = vld2_fns[pat][size];
+        addrinc = 32;
+        break;
+    case 0xfc901e01:
+        if (qd > 4) {
+            return false;
+        }
+        fn = vld4_fns[pat][size];
+        addrinc = 64;
+        break;
+    case 0xfc801e00:
+        if (qd > 6) {
+            return false;
+        }
+        fn = vst2_fns[pat][size];
+        addrinc = 32;
+        break;
+    case 0xfc801e01:
+        if (qd > 4) {
+            return false;
+        }
+        fn = vst4_fns[pat][size];
+        addrinc = 64;
+        break;
+    default:
+        return false;
+    }
+
+    return do_mve_vldst_il(s, fn, qd, extract32(insn, 16, 4),
+                           extract32(insn, 21, 1), addrinc);
+}
+
+static bool trans_mve_vmov_2gp(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 tmp;
+    int qd;
+    int rt;
+    int rt2;
+    int idx;
+    int vd;
+    bool from_gp;
+
+    switch (insn & 0xffb01fe0) {
+    case 0xec000f00:
+        from_gp = false;
+        break;
+    case 0xec100f00:
+        from_gp = true;
+        break;
+    default:
+        return false;
+    }
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    rt = extract32(insn, 0, 4);
+    rt2 = extract32(insn, 16, 4);
+    idx = extract32(insn, 4, 1);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd) ||
+        rt == 13 || rt == 15 || rt2 == 13 || rt2 == 15 ||
+        (!from_gp && rt == rt2)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    vd = qd * 2;
+    if (from_gp) {
+        if (!mve_skip_vmov(s, vd, idx, MO_32)) {
+            tmp = load_reg(s, rt);
+            neon_store_element(tcg_ctx, vd, idx, MO_32, tmp);
+            tcg_temp_free_i32(tcg_ctx, tmp);
+        }
+        if (!mve_skip_vmov(s, vd + 1, idx, MO_32)) {
+            tmp = load_reg(s, rt2);
+            neon_store_element(tcg_ctx, vd + 1, idx, MO_32, tmp);
+            tcg_temp_free_i32(tcg_ctx, tmp);
+        }
+    } else {
+        if (!mve_skip_vmov(s, vd, idx, MO_32)) {
+            tmp = tcg_temp_new_i32(tcg_ctx);
+            neon_load_element(tcg_ctx, tmp, vd, idx, MO_UL);
+            store_reg(s, rt, tmp);
+        }
+        if (!mve_skip_vmov(s, vd + 1, idx, MO_32)) {
+            tmp = tcg_temp_new_i32(tcg_ctx);
+            neon_load_element(tcg_ctx, tmp, vd + 1, idx, MO_UL);
+            store_reg(s, rt2, tmp);
+        }
+    }
+
+    mve_update_and_store_eci(s);
+    return true;
+}
+
+static bool trans_mve_vpsel(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_ptr qnptr;
+    TCGv_ptr qmptr;
+    int qd;
+    int qn;
+    int qm;
+
+    if ((insn & 0xffb11f51) != 0xfe310f01) {
+        return false;
+    }
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qn = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd | qn | qm)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    qnptr = mve_qreg_ptr(s, qn);
+    qmptr = mve_qreg_ptr(s, qm);
+    gen_helper_mve_vpsel(tcg_ctx, tcg_ctx->cpu_env, qdptr, qnptr, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_vcmp(DisasContext *s, int qn, int qm, uint32_t mask,
+                        MVEGenCmpFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qnptr;
+    TCGv_ptr qmptr;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qn | qm)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qnptr = mve_qreg_ptr(s, qn);
+    qmptr = mve_qreg_ptr(s, qm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qnptr, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+    if (mask) {
+        gen_mve_vpst(s, mask);
+    }
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_vcmp_scalar(DisasContext *s, int qn, int rm,
+                               uint32_t mask, MVEGenScalarCmpFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qnptr;
+    TCGv_i32 rmval;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qn) || rm == 13) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qnptr = mve_qreg_ptr(s, qn);
+    if (rm == 15) {
+        rmval = tcg_const_i32(tcg_ctx, 0);
+    } else {
+        rmval = load_reg(s, rm);
+    }
+    fn(tcg_ctx, tcg_ctx->cpu_env, qnptr, rmval);
+    tcg_temp_free_i32(tcg_ctx, rmval);
+    tcg_temp_free_ptr(tcg_ctx, qnptr);
+    if (mask) {
+        gen_mve_vpst(s, mask);
+    }
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static MVEGenCmpFn *mve_vcmp_fn(uint32_t insn, unsigned size)
+{
+    static MVEGenCmpFn * const fns[8][3] = {
+        { gen_helper_mve_vcmpeqb, gen_helper_mve_vcmpeqh,
+          gen_helper_mve_vcmpeqw },
+        { gen_helper_mve_vcmpneb, gen_helper_mve_vcmpneh,
+          gen_helper_mve_vcmpnew },
+        { gen_helper_mve_vcmpcsb, gen_helper_mve_vcmpcsh,
+          gen_helper_mve_vcmpcsw },
+        { gen_helper_mve_vcmphib, gen_helper_mve_vcmphih,
+          gen_helper_mve_vcmphiw },
+        { gen_helper_mve_vcmpgeb, gen_helper_mve_vcmpgeh,
+          gen_helper_mve_vcmpgew },
+        { gen_helper_mve_vcmpltb, gen_helper_mve_vcmplth,
+          gen_helper_mve_vcmpltw },
+        { gen_helper_mve_vcmpgtb, gen_helper_mve_vcmpgth,
+          gen_helper_mve_vcmpgtw },
+        { gen_helper_mve_vcmpleb, gen_helper_mve_vcmpleh,
+          gen_helper_mve_vcmplew },
+    };
+
+    if (size == 3) {
+        return NULL;
+    }
+
+    switch (insn & 0xff811fd1) {
+    case 0xfe010f00:
+        return fns[0][size];
+    case 0xfe010f80:
+        return fns[1][size];
+    case 0xfe010f01:
+        return fns[2][size];
+    case 0xfe010f81:
+        return fns[3][size];
+    case 0xfe011f00:
+        return fns[4][size];
+    case 0xfe011f80:
+        return fns[5][size];
+    case 0xfe011f01:
+        return fns[6][size];
+    case 0xfe011f81:
+        return fns[7][size];
+    default:
+        return NULL;
+    }
+}
+
+static MVEGenScalarCmpFn *mve_vcmp_scalar_fn(uint32_t insn, unsigned size)
+{
+    static MVEGenScalarCmpFn * const fns[8][3] = {
+        { gen_helper_mve_vcmpeq_scalarb, gen_helper_mve_vcmpeq_scalarh,
+          gen_helper_mve_vcmpeq_scalarw },
+        { gen_helper_mve_vcmpne_scalarb, gen_helper_mve_vcmpne_scalarh,
+          gen_helper_mve_vcmpne_scalarw },
+        { gen_helper_mve_vcmpcs_scalarb, gen_helper_mve_vcmpcs_scalarh,
+          gen_helper_mve_vcmpcs_scalarw },
+        { gen_helper_mve_vcmphi_scalarb, gen_helper_mve_vcmphi_scalarh,
+          gen_helper_mve_vcmphi_scalarw },
+        { gen_helper_mve_vcmpge_scalarb, gen_helper_mve_vcmpge_scalarh,
+          gen_helper_mve_vcmpge_scalarw },
+        { gen_helper_mve_vcmplt_scalarb, gen_helper_mve_vcmplt_scalarh,
+          gen_helper_mve_vcmplt_scalarw },
+        { gen_helper_mve_vcmpgt_scalarb, gen_helper_mve_vcmpgt_scalarh,
+          gen_helper_mve_vcmpgt_scalarw },
+        { gen_helper_mve_vcmple_scalarb, gen_helper_mve_vcmple_scalarh,
+          gen_helper_mve_vcmple_scalarw },
+    };
+
+    if (size == 3) {
+        return NULL;
+    }
+
+    switch (insn & 0xff811ff0) {
+    case 0xfe010f40:
+        return fns[0][size];
+    case 0xfe010fc0:
+        return fns[1][size];
+    case 0xfe010f60:
+        return fns[2][size];
+    case 0xfe010fe0:
+        return fns[3][size];
+    case 0xfe011f40:
+        return fns[4][size];
+    case 0xfe011fc0:
+        return fns[5][size];
+    case 0xfe011f60:
+        return fns[6][size];
+    case 0xfe011fe0:
+        return fns[7][size];
+    default:
+        return NULL;
+    }
+}
+
+static MVEGenCmpFn *mve_vcmp_fp_fn(uint32_t insn, unsigned size)
+{
+    static MVEGenCmpFn * const fns[6][4] = {
+        { NULL, gen_helper_mve_vfcmpeqh, gen_helper_mve_vfcmpeqs, NULL },
+        { NULL, gen_helper_mve_vfcmpneh, gen_helper_mve_vfcmpnes, NULL },
+        { NULL, gen_helper_mve_vfcmpgeh, gen_helper_mve_vfcmpges, NULL },
+        { NULL, gen_helper_mve_vfcmplth, gen_helper_mve_vfcmplts, NULL },
+        { NULL, gen_helper_mve_vfcmpgth, gen_helper_mve_vfcmpgts, NULL },
+        { NULL, gen_helper_mve_vfcmpleh, gen_helper_mve_vfcmples, NULL },
+    };
+
+    switch (insn & 0xefb11fd1) {
+    case 0xee310f00:
+        return fns[0][size];
+    case 0xee310f80:
+        return fns[1][size];
+    case 0xee311f00:
+        return fns[2][size];
+    case 0xee311f80:
+        return fns[3][size];
+    case 0xee311f01:
+        return fns[4][size];
+    case 0xee311f81:
+        return fns[5][size];
+    default:
+        return NULL;
+    }
+}
+
+static MVEGenScalarCmpFn *mve_vcmp_fp_scalar_fn(uint32_t insn,
+                                                unsigned size)
+{
+    static MVEGenScalarCmpFn * const fns[6][4] = {
+        { NULL, gen_helper_mve_vfcmpeq_scalarh,
+          gen_helper_mve_vfcmpeq_scalars, NULL },
+        { NULL, gen_helper_mve_vfcmpne_scalarh,
+          gen_helper_mve_vfcmpne_scalars, NULL },
+        { NULL, gen_helper_mve_vfcmpge_scalarh,
+          gen_helper_mve_vfcmpge_scalars, NULL },
+        { NULL, gen_helper_mve_vfcmplt_scalarh,
+          gen_helper_mve_vfcmplt_scalars, NULL },
+        { NULL, gen_helper_mve_vfcmpgt_scalarh,
+          gen_helper_mve_vfcmpgt_scalars, NULL },
+        { NULL, gen_helper_mve_vfcmple_scalarh,
+          gen_helper_mve_vfcmple_scalars, NULL },
+    };
+
+    switch (insn & 0xefb11ff0) {
+    case 0xee310f40:
+        return fns[0][size];
+    case 0xee310fc0:
+        return fns[1][size];
+    case 0xee311f40:
+        return fns[2][size];
+    case 0xee311fc0:
+        return fns[3][size];
+    case 0xee311f60:
+        return fns[4][size];
+    case 0xee311fe0:
+        return fns[5][size];
+    default:
+        return NULL;
+    }
+}
+
+static bool trans_mve_vcmp_fp(DisasContext *s, uint32_t insn)
+{
+    MVEGenCmpFn *fn;
+    MVEGenScalarCmpFn *scalar_fn;
+    unsigned size = extract32(insn, 28, 1) ? 1 : 2;
+    uint32_t mask;
+    int qn;
+    int qm;
+    int rm;
+
+    if (!dc_isar_feature(aa32_mve_fp, s)) {
+        return false;
+    }
+
+    fn = mve_vcmp_fp_fn(insn, size);
+    if (fn) {
+        qn = extract32(insn, 17, 3);
+        qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+        mask = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+        return do_mve_vcmp(s, qn, qm, mask, fn);
+    }
+
+    scalar_fn = mve_vcmp_fp_scalar_fn(insn, size);
+    if (scalar_fn) {
+        qn = extract32(insn, 17, 3);
+        rm = extract32(insn, 0, 4);
+        mask = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+        return do_mve_vcmp_scalar(s, qn, rm, mask, scalar_fn);
+    }
+
+    return false;
+}
+
+static bool trans_mve_vcmp(DisasContext *s, uint32_t insn)
+{
+    MVEGenCmpFn *fn;
+    MVEGenScalarCmpFn *scalar_fn;
+    unsigned size = extract32(insn, 20, 2);
+    uint32_t mask;
+    int qn;
+    int qm;
+    int rm;
+
+    fn = mve_vcmp_fn(insn, size);
+    if (fn) {
+        qn = extract32(insn, 17, 3);
+        qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+        mask = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+        return do_mve_vcmp(s, qn, qm, mask, fn);
+    }
+
+    scalar_fn = mve_vcmp_scalar_fn(insn, size);
+    if (scalar_fn) {
+        qn = extract32(insn, 17, 3);
+        rm = extract32(insn, 0, 4);
+        mask = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+        return do_mve_vcmp_scalar(s, qn, rm, mask, scalar_fn);
+    }
+
+    return false;
+}
+
+static bool trans_mve_vimm_1r(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MVEGenOneOpImmFn *fn;
+    TCGv_ptr qdptr;
+    TCGv_i64 immval;
+    uint64_t imm64;
+    uint32_t imm;
+    unsigned cmode;
+    unsigned op;
+    int qd;
+
+    if ((insn & 0xefb810d0) != 0xef800050) {
+        return false;
+    }
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    imm = (extract32(insn, 28, 1) << 7) |
+          (extract32(insn, 16, 3) << 4) |
+          extract32(insn, 0, 4);
+    cmode = extract32(insn, 8, 4);
+    op = extract32(insn, 5, 1);
+
+    if ((cmode & 1) && cmode < 12) {
+        fn = op ? gen_helper_mve_vandi : gen_helper_mve_vorri;
+    } else {
+        if (cmode == 15 && op) {
+            return false;
+        }
+        fn = gen_helper_mve_vmovi;
+    }
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    imm64 = asimd_imm_const(imm, cmode, op);
+    qdptr = mve_qreg_ptr(s, qd);
+    immval = tcg_const_i64(tcg_ctx, imm64);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, immval);
+    tcg_temp_free_i64(tcg_ctx, immval);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool mve_decode_shl_size_shift(uint32_t insn, unsigned *size,
+                                      uint32_t *shift)
+{
+    if (extract32(insn, 21, 1)) {
+        *size = 2;
+        *shift = extract32(insn, 16, 5);
+        return true;
+    }
+    if (extract32(insn, 20, 1)) {
+        *size = 1;
+        *shift = extract32(insn, 16, 4);
+        return true;
+    }
+    if (extract32(insn, 19, 1)) {
+        *size = 0;
+        *shift = extract32(insn, 16, 3);
+        return true;
+    }
+    return false;
+}
+
+static bool mve_decode_shr_size_shift(uint32_t insn, unsigned *size,
+                                      uint32_t *shift)
+{
+    uint32_t encoded;
+
+    if (extract32(insn, 21, 1)) {
+        *size = 2;
+        encoded = extract32(insn, 16, 5);
+        *shift = 32 - encoded;
+        return true;
+    }
+    if (extract32(insn, 20, 1)) {
+        *size = 1;
+        encoded = extract32(insn, 16, 4);
+        *shift = 16 - encoded;
+        return true;
+    }
+    if (extract32(insn, 19, 1)) {
+        *size = 0;
+        encoded = extract32(insn, 16, 3);
+        *shift = 8 - encoded;
+        return true;
+    }
+    return false;
+}
+
+static bool mve_decode_shll_size_shift(uint32_t insn, unsigned *size,
+                                       uint32_t *shift)
+{
+    if (extract32(insn, 20, 1)) {
+        *size = 1;
+        *shift = extract32(insn, 16, 4);
+        return true;
+    }
+    if (extract32(insn, 19, 1)) {
+        *size = 0;
+        *shift = extract32(insn, 16, 3);
+        return true;
+    }
+    return false;
+}
+
+static bool mve_decode_shll_esize_size_shift(uint32_t insn, unsigned *size,
+                                             uint32_t *shift)
+{
+    switch (extract32(insn, 18, 2)) {
+    case 0:
+        *size = 0;
+        *shift = 8;
+        return true;
+    case 1:
+        *size = 1;
+        *shift = 16;
+        return true;
+    default:
+        return false;
+    }
+}
+
+static bool do_mve_2shift(DisasContext *s, uint32_t insn,
+                          MVEGenTwoOpShiftFn *fn, unsigned size,
+                          uint32_t shift, bool negateshift)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_ptr qmptr;
+    TCGv_i32 shiftval;
+    int qd;
+    int qm;
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    qm = (extract32(insn, 5, 1) << 3) | extract32(insn, 1, 3);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd | qm) || !fn) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    if (negateshift) {
+        shift = -shift;
+    }
+    qdptr = mve_qreg_ptr(s, qd);
+    qmptr = mve_qreg_ptr(s, qm);
+    shiftval = tcg_const_i32(tcg_ctx, shift);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdptr, qmptr, shiftval);
+    tcg_temp_free_i32(tcg_ctx, shiftval);
+    tcg_temp_free_ptr(tcg_ctx, qmptr);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_2shift_scalar(DisasContext *s, uint32_t insn,
+                                 MVEGenTwoOpShiftFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdaptr;
+    TCGv_i32 rmval;
+    int qda;
+    int rm;
+
+    qda = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    rm = extract32(insn, 0, 4);
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qda) || rm == 13 || rm == 15 || !fn) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdaptr = mve_qreg_ptr(s, qda);
+    rmval = load_reg(s, rm);
+    fn(tcg_ctx, tcg_ctx->cpu_env, qdaptr, qdaptr, rmval);
+    tcg_temp_free_i32(tcg_ctx, rmval);
+    tcg_temp_free_ptr(tcg_ctx, qdaptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_mve_scalar_shift(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpShiftFn * const vshls_fns[] = {
+        gen_helper_mve_vshli_sb,
+        gen_helper_mve_vshli_sh,
+        gen_helper_mve_vshli_sw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn * const vshlu_fns[] = {
+        gen_helper_mve_vshli_ub,
+        gen_helper_mve_vshli_uh,
+        gen_helper_mve_vshli_uw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn * const vrshls_fns[] = {
+        gen_helper_mve_vrshli_sb,
+        gen_helper_mve_vrshli_sh,
+        gen_helper_mve_vrshli_sw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn * const vrshlu_fns[] = {
+        gen_helper_mve_vrshli_ub,
+        gen_helper_mve_vrshli_uh,
+        gen_helper_mve_vrshli_uw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn * const vqshls_fns[] = {
+        gen_helper_mve_vqshli_sb,
+        gen_helper_mve_vqshli_sh,
+        gen_helper_mve_vqshli_sw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn * const vqshlu_fns[] = {
+        gen_helper_mve_vqshli_ub,
+        gen_helper_mve_vqshli_uh,
+        gen_helper_mve_vqshli_uw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn * const vqrshls_fns[] = {
+        gen_helper_mve_vqrshli_sb,
+        gen_helper_mve_vqrshli_sh,
+        gen_helper_mve_vqrshli_sw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn * const vqrshlu_fns[] = {
+        gen_helper_mve_vqrshli_ub,
+        gen_helper_mve_vqrshli_uh,
+        gen_helper_mve_vqrshli_uw,
+        NULL,
+    };
+    MVEGenTwoOpShiftFn *fn = NULL;
+    unsigned size = extract32(insn, 18, 2);
+
+    switch (insn & 0xffb31ff0) {
+    case 0xee311e60:
+        fn = vshls_fns[size];
+        break;
+    case 0xfe311e60:
+        fn = vshlu_fns[size];
+        break;
+    case 0xee331e60:
+        fn = vrshls_fns[size];
+        break;
+    case 0xfe331e60:
+        fn = vrshlu_fns[size];
+        break;
+    case 0xee311ee0:
+        fn = vqshls_fns[size];
+        break;
+    case 0xfe311ee0:
+        fn = vqshlu_fns[size];
+        break;
+    case 0xee331ee0:
+        fn = vqrshls_fns[size];
+        break;
+    case 0xfe331ee0:
+        fn = vqrshlu_fns[size];
+        break;
+    default:
+        break;
+    }
+
+    return fn ? do_mve_2shift_scalar(s, insn, fn) : false;
+}
+
+static void gen_mve_gpr_sqshll(TCGContext *tcg_ctx, TCGv_i64 r,
+                               TCGv_i64 n, int64_t shift)
+{
+    TCGv_i32 shiftv = tcg_const_i32(tcg_ctx, shift);
+
+    gen_helper_mve_sqshll(tcg_ctx, r, tcg_ctx->cpu_env, n, shiftv);
+    tcg_temp_free_i32(tcg_ctx, shiftv);
+}
+
+static void gen_mve_gpr_uqshll(TCGContext *tcg_ctx, TCGv_i64 r,
+                               TCGv_i64 n, int64_t shift)
+{
+    TCGv_i32 shiftv = tcg_const_i32(tcg_ctx, shift);
+
+    gen_helper_mve_uqshll(tcg_ctx, r, tcg_ctx->cpu_env, n, shiftv);
+    tcg_temp_free_i32(tcg_ctx, shiftv);
+}
+
+static void gen_mve_gpr_sqshl(TCGContext *tcg_ctx, TCGv_i32 r, TCGv_i32 n,
+                              int32_t shift)
+{
+    TCGv_i32 shiftv = tcg_const_i32(tcg_ctx, shift);
+
+    gen_helper_mve_sqshl(tcg_ctx, r, tcg_ctx->cpu_env, n, shiftv);
+    tcg_temp_free_i32(tcg_ctx, shiftv);
+}
+
+static void gen_mve_gpr_uqshl(TCGContext *tcg_ctx, TCGv_i32 r, TCGv_i32 n,
+                              int32_t shift)
+{
+    TCGv_i32 shiftv = tcg_const_i32(tcg_ctx, shift);
+
+    gen_helper_mve_uqshl(tcg_ctx, r, tcg_ctx->cpu_env, n, shiftv);
+    tcg_temp_free_i32(tcg_ctx, shiftv);
+}
+
+static bool do_mve_gpr_shl_ri(DisasContext *s, uint32_t insn,
+                              MVEGenGPRWideShiftImmFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i64 rda;
+    unsigned rdalo;
+    unsigned rdahi;
+    uint32_t shim;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+        return false;
+    }
+
+    rdalo = extract32(insn, 17, 3) << 1;
+    rdahi = (extract32(insn, 9, 3) << 1) | 1;
+    if (rdahi == 15) {
+        return false;
+    }
+    if (!dc_isar_feature(aa32_mve, s) ||
+        !arm_dc_feature(s, ARM_FEATURE_M_MAIN) ||
+        rdahi == 13) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    shim = (extract32(insn, 12, 3) << 2) | extract32(insn, 6, 2);
+    if (shim == 0) {
+        shim = 32;
+    }
+
+    rda = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_concat_i32_i64(tcg_ctx, rda, tcg_ctx->cpu_R[rdalo],
+                           tcg_ctx->cpu_R[rdahi]);
+    fn(tcg_ctx, rda, rda, shim);
+    tcg_gen_extrl_i64_i32(tcg_ctx, tcg_ctx->cpu_R[rdalo], rda);
+    tcg_gen_extrh_i64_i32(tcg_ctx, tcg_ctx->cpu_R[rdahi], rda);
+    tcg_temp_free_i64(tcg_ctx, rda);
+    return true;
+}
+
+static bool do_mve_gpr_shl_rr(DisasContext *s, uint32_t insn,
+                              MVEGenGPRWideShiftFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i64 rda;
+    unsigned rdalo;
+    unsigned rdahi;
+    unsigned rm;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+        return false;
+    }
+
+    rdalo = extract32(insn, 17, 3) << 1;
+    rdahi = (extract32(insn, 9, 3) << 1) | 1;
+    if (rdahi == 15) {
+        return false;
+    }
+
+    rm = extract32(insn, 12, 4);
+    if (!dc_isar_feature(aa32_mve, s) ||
+        !arm_dc_feature(s, ARM_FEATURE_M_MAIN) ||
+        rdahi == 13 || rm == 13 || rm == 15 ||
+        rm == rdahi || rm == rdalo) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    rda = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_concat_i32_i64(tcg_ctx, rda, tcg_ctx->cpu_R[rdalo],
+                           tcg_ctx->cpu_R[rdahi]);
+    fn(tcg_ctx, rda, tcg_ctx->cpu_env, rda, tcg_ctx->cpu_R[rm]);
+    tcg_gen_extrl_i64_i32(tcg_ctx, tcg_ctx->cpu_R[rdalo], rda);
+    tcg_gen_extrh_i64_i32(tcg_ctx, tcg_ctx->cpu_R[rdahi], rda);
+    tcg_temp_free_i64(tcg_ctx, rda);
+    return true;
+}
+
+static bool do_mve_gpr_sh_ri(DisasContext *s, uint32_t insn,
+                             MVEGenGPRShiftImmFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    unsigned rda;
+    uint32_t shim;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+        return false;
+    }
+
+    rda = extract32(insn, 16, 4);
+    if (!dc_isar_feature(aa32_mve, s) ||
+        !arm_dc_feature(s, ARM_FEATURE_M_MAIN) ||
+        rda == 13 || rda == 15) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    shim = (extract32(insn, 12, 3) << 2) | extract32(insn, 6, 2);
+    if (shim == 0) {
+        shim = 32;
+    }
+    fn(tcg_ctx, tcg_ctx->cpu_R[rda], tcg_ctx->cpu_R[rda], shim);
+    return true;
+}
+
+static bool do_mve_gpr_sh_rr(DisasContext *s, uint32_t insn,
+                             MVEGenGPRShiftFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    unsigned rda;
+    unsigned rm;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_V8_1M)) {
+        return false;
+    }
+
+    rda = extract32(insn, 16, 4);
+    rm = extract32(insn, 12, 4);
+    if (!dc_isar_feature(aa32_mve, s) ||
+        !arm_dc_feature(s, ARM_FEATURE_M_MAIN) ||
+        rda == 13 || rda == 15 || rm == 13 || rm == 15 ||
+        rm == rda) {
+        unallocated_encoding(s);
+        return true;
+    }
+
+    fn(tcg_ctx, tcg_ctx->cpu_R[rda], tcg_ctx->cpu_env,
+       tcg_ctx->cpu_R[rda], tcg_ctx->cpu_R[rm]);
+    return true;
+}
+
+static bool trans_mve_gpr_shift(DisasContext *s, uint32_t insn)
+{
+    uint32_t key;
+
+    key = insn & 0xfff08f3f;
+    switch (key) {
+    case 0xea500f0f:
+        return do_mve_gpr_sh_ri(s, insn, gen_mve_gpr_uqshl);
+    case 0xea500f1f:
+        return do_mve_gpr_sh_ri(s, insn, gen_mve_urshr32_i32);
+    case 0xea500f2f:
+        return do_mve_gpr_sh_ri(s, insn, gen_mve_srshr32_i32);
+    case 0xea500f3f:
+        return do_mve_gpr_sh_ri(s, insn, gen_mve_gpr_sqshl);
+    default:
+        break;
+    }
+
+    key = insn & 0xfff1813f;
+    switch (key) {
+    case 0xea50010f:
+        return do_mve_gpr_shl_ri(s, insn, gen_mve_shl64_i64);
+    case 0xea51010f:
+        return do_mve_gpr_shl_ri(s, insn, gen_mve_gpr_uqshll);
+    case 0xea50011f:
+        return do_mve_gpr_shl_ri(s, insn, gen_mve_shr64_i64);
+    case 0xea51011f:
+        return do_mve_gpr_shl_ri(s, insn, gen_mve_urshr64_i64);
+    case 0xea50012f:
+        return do_mve_gpr_shl_ri(s, insn, gen_mve_sar64_i64);
+    case 0xea51012f:
+        return do_mve_gpr_shl_ri(s, insn, gen_mve_srshr64_i64);
+    case 0xea51013f:
+        return do_mve_gpr_shl_ri(s, insn, gen_mve_gpr_sqshll);
+    default:
+        break;
+    }
+
+    key = insn & 0xfff00fff;
+    switch (key) {
+    case 0xea500f0d:
+        return do_mve_gpr_sh_rr(s, insn, gen_helper_mve_uqrshl);
+    case 0xea500f2d:
+        return do_mve_gpr_sh_rr(s, insn, gen_helper_mve_sqrshr);
+    default:
+        break;
+    }
+
+    key = insn & 0xfff101ff;
+    switch (key) {
+    case 0xea50010d:
+        return do_mve_gpr_shl_rr(s, insn, gen_helper_mve_ushll);
+    case 0xea51010d:
+        return do_mve_gpr_shl_rr(s, insn, gen_helper_mve_uqrshll);
+    case 0xea50012d:
+        return do_mve_gpr_shl_rr(s, insn, gen_helper_mve_sshrl);
+    case 0xea51012d:
+        return do_mve_gpr_shl_rr(s, insn, gen_helper_mve_sqrshrl);
+    case 0xea51018d:
+        return do_mve_gpr_shl_rr(s, insn, gen_helper_mve_uqrshll48);
+    case 0xea5101ad:
+        return do_mve_gpr_shl_rr(s, insn, gen_helper_mve_sqrshrl48);
+    default:
+        return false;
+    }
+}
+
+static bool trans_mve_shift_imm(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpShiftFn *fn = NULL;
+    unsigned size;
+    uint32_t shift;
+    uint32_t key;
+    bool negateshift = false;
+    bool right_shift = false;
+
+    key = insn & 0xff801fd1;
+    switch (key) {
+    case 0xef800550:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshli_ub,
+                gen_helper_mve_vshli_uh,
+                gen_helper_mve_vshli_uw,
+            };
+
+            if (!mve_decode_shl_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xef800050:
+        right_shift = true;
+        negateshift = true;
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshli_sb,
+                gen_helper_mve_vshli_sh,
+                gen_helper_mve_vshli_sw,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xff800050:
+        right_shift = true;
+        negateshift = true;
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshli_ub,
+                gen_helper_mve_vshli_uh,
+                gen_helper_mve_vshli_uw,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xef800250:
+        right_shift = true;
+        negateshift = true;
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vrshli_sb,
+                gen_helper_mve_vrshli_sh,
+                gen_helper_mve_vrshli_sw,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xff800250:
+        right_shift = true;
+        negateshift = true;
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vrshli_ub,
+                gen_helper_mve_vrshli_uh,
+                gen_helper_mve_vrshli_uw,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xff800450:
+        right_shift = true;
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vsrib,
+                gen_helper_mve_vsrih,
+                gen_helper_mve_vsriw,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xff800550:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vslib,
+                gen_helper_mve_vslih,
+                gen_helper_mve_vsliw,
+            };
+
+            if (!mve_decode_shl_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xef800750:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshli_sb,
+                gen_helper_mve_vqshli_sh,
+                gen_helper_mve_vqshli_sw,
+            };
+
+            if (!mve_decode_shl_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xff800750:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshli_ub,
+                gen_helper_mve_vqshli_uh,
+                gen_helper_mve_vqshli_uw,
+            };
+
+            if (!mve_decode_shl_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xff800650:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshlui_sb,
+                gen_helper_mve_vqshlui_sh,
+                gen_helper_mve_vqshlui_sw,
+            };
+
+            if (!mve_decode_shl_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    default:
+        return false;
+    }
+
+    return do_mve_2shift(s, insn, fn, size, shift,
+                         right_shift && negateshift);
+}
+
+static bool trans_mve_vshll(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpShiftFn *fn = NULL;
+    unsigned size;
+    uint32_t shift;
+    uint32_t key;
+    bool esize_shift = false;
+
+    key = insn & 0xffa01fd1;
+    switch (key) {
+    case 0xeea00f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshllbsb,
+                gen_helper_mve_vshllbsh,
+            };
+
+            fn = fns[0];
+            if (!mve_decode_shll_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfea00f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshllbub,
+                gen_helper_mve_vshllbuh,
+            };
+
+            if (!mve_decode_shll_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xeea01f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshlltsb,
+                gen_helper_mve_vshlltsh,
+            };
+
+            if (!mve_decode_shll_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfea01f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshlltub,
+                gen_helper_mve_vshlltuh,
+            };
+
+            if (!mve_decode_shll_size_shift(insn, &size, &shift)) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    default:
+        esize_shift = true;
+        break;
+    }
+
+    if (esize_shift) {
+        key = insn & 0xffb31fd1;
+        switch (key) {
+        case 0xee310e01:
+            {
+                MVEGenTwoOpShiftFn * const fns[] = {
+                    gen_helper_mve_vshllbsb,
+                    gen_helper_mve_vshllbsh,
+                };
+
+                if (!mve_decode_shll_esize_size_shift(insn, &size, &shift)) {
+                    return false;
+                }
+                fn = fns[size];
+            }
+            break;
+        case 0xfe310e01:
+            {
+                MVEGenTwoOpShiftFn * const fns[] = {
+                    gen_helper_mve_vshllbub,
+                    gen_helper_mve_vshllbuh,
+                };
+
+                if (!mve_decode_shll_esize_size_shift(insn, &size, &shift)) {
+                    return false;
+                }
+                fn = fns[size];
+            }
+            break;
+        case 0xee311e01:
+            {
+                MVEGenTwoOpShiftFn * const fns[] = {
+                    gen_helper_mve_vshlltsb,
+                    gen_helper_mve_vshlltsh,
+                };
+
+                if (!mve_decode_shll_esize_size_shift(insn, &size, &shift)) {
+                    return false;
+                }
+                fn = fns[size];
+            }
+            break;
+        case 0xfe311e01:
+            {
+                MVEGenTwoOpShiftFn * const fns[] = {
+                    gen_helper_mve_vshlltub,
+                    gen_helper_mve_vshlltuh,
+                };
+
+                if (!mve_decode_shll_esize_size_shift(insn, &size, &shift)) {
+                    return false;
+                }
+                fn = fns[size];
+            }
+            break;
+        default:
+            return false;
+        }
+    }
+
+    return do_mve_2shift(s, insn, fn, size, shift, false);
+}
+
+static bool trans_mve_shrn(DisasContext *s, uint32_t insn)
+{
+    MVEGenTwoOpShiftFn *fn = NULL;
+    unsigned size;
+    uint32_t shift;
+    uint32_t key;
+
+    key = insn & 0xff801fd1;
+    switch (key) {
+    case 0xee800fc1:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshrnbb,
+                gen_helper_mve_vshrnbh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xee801fc1:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vshrntb,
+                gen_helper_mve_vshrnth,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe800fc1:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vrshrnbb,
+                gen_helper_mve_vrshrnbh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe801fc1:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vrshrntb,
+                gen_helper_mve_vrshrnth,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xee800f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshrnb_sb,
+                gen_helper_mve_vqshrnb_sh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xee801f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshrnt_sb,
+                gen_helper_mve_vqshrnt_sh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe800f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshrnb_ub,
+                gen_helper_mve_vqshrnb_uh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe801f40:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshrnt_ub,
+                gen_helper_mve_vqshrnt_uh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xee800fc0:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshrunbb,
+                gen_helper_mve_vqshrunbh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xee801fc0:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqshruntb,
+                gen_helper_mve_vqshrunth,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xee800f41:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqrshrnb_sb,
+                gen_helper_mve_vqrshrnb_sh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xee801f41:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqrshrnt_sb,
+                gen_helper_mve_vqrshrnt_sh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe800f41:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqrshrnb_ub,
+                gen_helper_mve_vqrshrnb_uh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe801f41:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqrshrnt_ub,
+                gen_helper_mve_vqrshrnt_uh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe800fc0:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqrshrunbb,
+                gen_helper_mve_vqrshrunbh,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    case 0xfe801fc0:
+        {
+            MVEGenTwoOpShiftFn * const fns[] = {
+                gen_helper_mve_vqrshruntb,
+                gen_helper_mve_vqrshrunth,
+            };
+
+            if (!mve_decode_shr_size_shift(insn, &size, &shift) ||
+                size == 2) {
+                return false;
+            }
+            fn = fns[size];
+        }
+        break;
+    default:
+        return false;
+    }
+
+    return do_mve_2shift(s, insn, fn, size, shift, false);
+}
+
+static bool trans_mve_vshlc(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_i32 rdmval;
+    TCGv_i32 shiftval;
+    int qd;
+    int rdm;
+    uint32_t shift;
+
+    if ((insn & 0xffa01ff0) != 0xeea00fc0) {
+        return false;
+    }
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    rdm = extract32(insn, 0, 4);
+    shift = extract32(insn, 16, 5);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd) ||
+        rdm == 13 || rdm == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    rdmval = load_reg(s, rdm);
+    shiftval = tcg_const_i32(tcg_ctx, shift);
+    gen_helper_mve_vshlc(tcg_ctx, rdmval, tcg_ctx->cpu_env, qdptr, rdmval,
+                         shiftval);
+    tcg_temp_free_i32(tcg_ctx, shiftval);
+    store_reg(s, rdm, rdmval);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static void gen_mve_dup_i32(TCGContext *tcg_ctx, TCGv_i32 val,
+                            unsigned size)
+{
+    TCGv_i32 tmp;
+
+    switch (size) {
+    case 0:
+        tmp = tcg_temp_new_i32(tcg_ctx);
+        tcg_gen_ext8u_i32(tcg_ctx, val, val);
+        tcg_gen_shli_i32(tcg_ctx, tmp, val, 8);
+        tcg_gen_or_i32(tcg_ctx, val, val, tmp);
+        tcg_gen_shli_i32(tcg_ctx, tmp, val, 16);
+        tcg_gen_or_i32(tcg_ctx, val, val, tmp);
+        tcg_temp_free_i32(tcg_ctx, tmp);
+        break;
+    case 1:
+        tmp = tcg_temp_new_i32(tcg_ctx);
+        tcg_gen_ext16u_i32(tcg_ctx, val, val);
+        tcg_gen_shli_i32(tcg_ctx, tmp, val, 16);
+        tcg_gen_or_i32(tcg_ctx, val, val, tmp);
+        tcg_temp_free_i32(tcg_ctx, tmp);
+        break;
+    case 2:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static bool trans_mve_vdup(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_i32 rtval;
+    uint32_t key;
+    unsigned size;
+    int qd;
+    int rt;
+
+    key = insn & 0xfff10f7f;
+    switch (key) {
+    case 0xeee00b10:
+        size = 0;
+        break;
+    case 0xeea00b30:
+        size = 1;
+        break;
+    case 0xeea00b10:
+        size = 2;
+        break;
+    default:
+        return false;
+    }
+
+    qd = (extract32(insn, 7, 1) << 3) | extract32(insn, 17, 3);
+    rt = extract32(insn, 12, 4);
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd) ||
+        rt == 13 || rt == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    rtval = load_reg(s, rt);
+    gen_mve_dup_i32(tcg_ctx, rtval, size);
+    gen_helper_mve_vdup(tcg_ctx, tcg_ctx->cpu_env, qdptr, rtval);
+    tcg_temp_free_i32(tcg_ctx, rtval);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_vidup(DisasContext *s, int qd, int rn, uint32_t imm,
+                         MVEGenVIDUPFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_i32 rnval;
+    TCGv_i32 immval;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd)) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    rnval = load_reg(s, rn);
+    immval = tcg_const_i32(tcg_ctx, imm);
+    fn(tcg_ctx, rnval, tcg_ctx->cpu_env, qdptr, rnval, immval);
+    tcg_temp_free_i32(tcg_ctx, immval);
+    store_reg(s, rn, rnval);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool do_mve_viwdup(DisasContext *s, int qd, int rn, int rm,
+                          uint32_t imm, MVEGenVIWDUPFn *fn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr qdptr;
+    TCGv_i32 rnval;
+    TCGv_i32 rmval;
+    TCGv_i32 immval;
+
+    if (!arm_dc_feature(s, ARM_FEATURE_M) ||
+        !dc_isar_feature(aa32_mve, s) ||
+        !mve_check_qreg_bank(qd) ||
+        rm == 13 || rm == 15) {
+        return false;
+    }
+
+    if (!mve_eci_check(s) || !vfp_access_check(s)) {
+        return true;
+    }
+
+    qdptr = mve_qreg_ptr(s, qd);
+    rnval = load_reg(s, rn);
+    rmval = load_reg(s, rm);
+    immval = tcg_const_i32(tcg_ctx, imm);
+    fn(tcg_ctx, rnval, tcg_ctx->cpu_env, qdptr, rnval, rmval, immval);
+    tcg_temp_free_i32(tcg_ctx, immval);
+    tcg_temp_free_i32(tcg_ctx, rmval);
+    store_reg(s, rn, rnval);
+    tcg_temp_free_ptr(tcg_ctx, qdptr);
+
+    mve_update_eci(s);
+    s->mve_no_pred = false;
+    s->base.is_jmp = DISAS_UPDATE;
+    return true;
+}
+
+static bool trans_mve_vidup(DisasContext *s, uint32_t insn)
+{
+    MVEGenVIDUPFn * const vidup_fns[] = {
+        gen_helper_mve_vidupb,
+        gen_helper_mve_viduph,
+        gen_helper_mve_vidupw,
+        NULL,
+    };
+    MVEGenVIWDUPFn * const viwdup_fns[] = {
+        gen_helper_mve_viwdupb,
+        gen_helper_mve_viwduph,
+        gen_helper_mve_viwdupw,
+        NULL,
+    };
+    MVEGenVIWDUPFn * const vdwdup_fns[] = {
+        gen_helper_mve_vdwdupb,
+        gen_helper_mve_vdwduph,
+        gen_helper_mve_vdwdupw,
+        NULL,
+    };
+    unsigned size;
+    uint32_t imm;
+    int qd;
+    int rn;
+    int rm;
+
+    size = extract32(insn, 20, 2);
+    if (size == 3) {
+        return false;
+    }
+
+    qd = (extract32(insn, 22, 1) << 3) | extract32(insn, 13, 3);
+    rn = extract32(insn, 17, 3) * 2;
+    rm = extract32(insn, 1, 3) * 2 + 1;
+    imm = 1U << ((extract32(insn, 7, 1) << 1) | extract32(insn, 0, 1));
+
+    switch (insn & 0xff811f7e) {
+    case 0xee010f6e:
+        return do_mve_vidup(s, qd, rn, imm, vidup_fns[size]);
+    case 0xee011f6e:
+        return do_mve_vidup(s, qd, rn, -imm, vidup_fns[size]);
+    default:
+        break;
+    }
+
+    switch (insn & 0xff811f70) {
+    case 0xee010f60:
+        return do_mve_viwdup(s, qd, rn, rm, imm, viwdup_fns[size]);
+    case 0xee011f60:
+        return do_mve_viwdup(s, qd, rn, rm, imm, vdwdup_fns[size]);
+    default:
+        return false;
+    }
+}
+
+/*
+ * Include the generated decoders.
+ */
+
+#include "decode-a32.inc.c"
+#include "decode-a32-uncond.inc.c"
+#include "decode-t32.inc.c"
+#include "decode-t16.inc.c"
+
+/* Helpers to swap operands for reverse-subtract.  */
+static void gen_rsb(TCGContext *tcg_ctx, TCGv_i32 dst, TCGv_i32 a, TCGv_i32 b)
+{
+    tcg_gen_sub_i32(tcg_ctx, dst, b, a);
+}
+
+static void gen_rsb_CC(TCGContext *tcg_ctx, TCGv_i32 dst, TCGv_i32 a, TCGv_i32 b)
+{
+    gen_sub_CC(tcg_ctx, dst, b, a);
+}
+
+static void gen_rsc(TCGContext *tcg_ctx, TCGv_i32 dest, TCGv_i32 a, TCGv_i32 b)
+{
+    gen_sub_carry(tcg_ctx, dest, b, a);
+}
+
+static void gen_rsc_CC(TCGContext *tcg_ctx, TCGv_i32 dest, TCGv_i32 a, TCGv_i32 b)
+{
+    gen_sbc_CC(tcg_ctx, dest, b, a);
+}
+
+/*
+ * Helpers for the data processing routines.
+ *
+ * After the computation store the results back.
+ * This may be suppressed altogether (STREG_NONE), require a runtime
+ * check against the stack limits (STREG_SP_CHECK), or generate an
+ * exception return.  Oh, or store into a register.
+ *
+ * Always return true, indicating success for a trans_* function.
+ */
+typedef enum {
+   STREG_NONE,
+   STREG_NORMAL,
+   STREG_SP_CHECK,
+   STREG_EXC_RET,
+} StoreRegKind;
+
+static bool store_reg_kind(DisasContext *s, int rd,
+                            TCGv_i32 val, StoreRegKind kind)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    switch (kind) {
+    case STREG_NONE:
+        tcg_temp_free_i32(tcg_ctx, val);
+        return true;
+    case STREG_NORMAL:
+        /* See ALUWritePC: Interworking only from a32 mode. */
+        if (s->thumb) {
+            store_reg(s, rd, val);
+        } else {
+            store_reg_bx(s, rd, val);
+        }
+        return true;
+    case STREG_SP_CHECK:
+        store_sp_checked(s, val);
+        return true;
+    case STREG_EXC_RET:
+        gen_exception_return(s, val);
+        return true;
+    }
+    g_assert_not_reached();
+    // never reach here
+    return true;
+}
+
+/*
+ * Data Processing (register)
+ *
+ * Operate, with set flags, one register source,
+ * one immediate shifted register source, and a destination.
+ */
+static bool op_s_rrr_shi(DisasContext *s, arg_s_rrr_shi *a,
+                         void (*gen)(TCGContext *, TCGv_i32, TCGv_i32, TCGv_i32),
+                         int logic_cc, StoreRegKind kind)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 tmp1, tmp2;
+
+    tmp2 = load_reg(s, a->rm);
+    gen_arm_shift_im(tcg_ctx, tmp2, a->shty, a->shim, logic_cc);
+    tmp1 = load_reg(s, a->rn);
+
+    gen(tcg_ctx, tmp1, tmp1, tmp2);
+    tcg_temp_free_i32(tcg_ctx, tmp2);
+
+    if (logic_cc) {
+        gen_logic_CC(tcg_ctx, tmp1);
+    }
+    return store_reg_kind(s, a->rd, tmp1, kind);
+}
+
+static bool op_s_rxr_shi(DisasContext *s, arg_s_rrr_shi *a,
+                         void (*gen)(TCGContext *, TCGv_i32, TCGv_i32),
+                         int logic_cc, StoreRegKind kind)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 tmp;
+
+    tmp = load_reg(s, a->rm);
+    gen_arm_shift_im(tcg_ctx, tmp, a->shty, a->shim, logic_cc);
+
+    gen(tcg_ctx, tmp, tmp);
+    if (logic_cc) {
+        gen_logic_CC(tcg_ctx, tmp);
+    }
+    return store_reg_kind(s, a->rd, tmp, kind);
+}
+
+/*
+ * Data-processing (register-shifted register)
+ *
+ * Operate, with set flags, one register source,
  * one register shifted register source, and a destination.
  */
 static bool op_s_rrr_shr(DisasContext *s, arg_s_rrr_shr *a,
@@ -11131,7 +16039,49 @@ static void disas_thumb2_insn(DisasContext *s, uint32_t insn)
      * Note disas_vfp is written for a32 with cond field in the
      * top nibble.  The t32 encoding requires 0xe in the top nibble.
      */
-    if (disas_t32(s, insn) ||
+    if (trans_m_profile_vlldm_vlstm(s, insn) ||
+        trans_m_profile_vscclrm(s, insn) ||
+        trans_m_profile_sysreg_mem(s, insn) ||
+        trans_mve_vctp(s, insn) ||
+        trans_mve_vpnot(s, insn) ||
+        trans_mve_vpst(s, insn) ||
+        trans_mve_vpsel(s, insn) ||
+        trans_mve_vcmp_fp(s, insn) ||
+        trans_mve_vcmp(s, insn) ||
+        trans_mve_vimm_1r(s, insn) ||
+        trans_mve_scalar_shift(s, insn) ||
+        trans_mve_gpr_shift(s, insn) ||
+        trans_mve_fp_vcvt_fixed(s, insn) ||
+        trans_mve_shift_imm(s, insn) ||
+        trans_mve_fp_nma(s, insn) ||
+        trans_mve_movn(s, insn) ||
+        trans_mve_vshll(s, insn) ||
+        trans_mve_shrn(s, insn) ||
+        trans_mve_vshlc(s, insn) ||
+        trans_mve_vmlaldav(s, insn) ||
+        trans_mve_vmladav(s, insn) ||
+        trans_mve_vmaxnmv(s, insn) ||
+        trans_mve_vmaxv(s, insn) ||
+        trans_mve_vaddv(s, insn) ||
+        trans_mve_vaddlv(s, insn) ||
+        trans_mve_vabav(s, insn) ||
+        trans_mve_vdup(s, insn) ||
+        trans_mve_vidup(s, insn) ||
+        trans_mve_vmov_2gp(s, insn) ||
+        trans_mve_fp_scalar_2op(s, insn) ||
+        trans_mve_scalar_qdmull(s, insn) ||
+        trans_mve_scalar_2op(s, insn) ||
+        trans_mve_fp_vcmul(s, insn) ||
+        trans_mve_fp_2op(s, insn) ||
+        trans_mve_simple_2op(s, insn) ||
+        trans_mve_fp_convert_round(s, insn) ||
+        trans_mve_simple_1op(s, insn) ||
+        trans_mve_vldst_wn(s, insn) ||
+        trans_mve_vldst_il(s, insn) ||
+        trans_mve_vldst_sg_imm(s, insn) ||
+        trans_mve_vldst_sg(s, insn) ||
+        trans_mve_vldr_vstr(s, insn) ||
+        disas_t32(s, insn) ||
         disas_vfp_uncond(s, insn) ||
         ((insn >> 28) == 0xe && disas_vfp(s, insn))) {
         return;
@@ -11250,8 +16200,16 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs)
     dc->thumb = FIELD_EX32(tb_flags, TBFLAG_AM32, THUMB);
     dc->be_data = FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE;
     condexec = FIELD_EX32(tb_flags, TBFLAG_AM32, CONDEXEC);
-    dc->condexec_mask = (condexec & 0xf) << 1;
-    dc->condexec_cond = condexec >> 4;
+    dc->eci = 0;
+    dc->eci_handled = false;
+    dc->condexec_mask = 0;
+    dc->condexec_cond = 0;
+    if (condexec & 0xf) {
+        dc->condexec_mask = (condexec & 0xf) << 1;
+        dc->condexec_cond = condexec >> 4;
+    } else if (arm_feature(env, ARM_FEATURE_M)) {
+        dc->eci = condexec >> 4;
+    }
 
     core_mmu_idx = FIELD_EX32(tb_flags, TBFLAG_ANY, MMUIDX);
     dc->mmu_idx = core_to_arm_mmu_idx(env, core_mmu_idx);
@@ -11271,7 +16229,9 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs)
         dc->v7m_new_fp_ctxt_needed =
             FIELD_EX32(tb_flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED);
         dc->v7m_lspact = FIELD_EX32(tb_flags, TBFLAG_M32, LSPACT);
+        dc->mve_no_pred = FIELD_EX32(tb_flags, TBFLAG_M32, MVE_NO_PRED);
     } else {
+        dc->mve_no_pred = false;
         dc->be_data =
             FIELD_EX32(tb_flags, TBFLAG_ANY, BE_DATA) ? MO_BE : MO_LE;
         dc->debug_target_el =
@@ -11381,10 +16341,15 @@ static void arm_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu)
 {
     DisasContext *dc = container_of(dcbase, DisasContext, base);
     TCGContext *tcg_ctx = dc->uc->tcg_ctx;
+    uint32_t condexec;
 
-    tcg_gen_insn_start(tcg_ctx, dc->base.pc_next,
-                       (dc->condexec_cond << 4) | (dc->condexec_mask >> 1),
-                       0);
+    if (dc->eci) {
+        condexec = dc->eci << 4;
+    } else {
+        condexec = (dc->condexec_cond << 4) | (dc->condexec_mask >> 1);
+    }
+
+    tcg_gen_insn_start(tcg_ctx, dc->base.pc_next, condexec, 0);
     dc->insn_start = tcg_last_op(tcg_ctx);
 }
 
@@ -11547,6 +16512,8 @@ static void thumb_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
     uint32_t insn;
     bool is_16bit;
     uint32_t insn_size;
+    TCGOp *insn_eci_rewind = NULL;
+    target_ulong insn_eci_pc_curr = 0;
 
     if (arm_pre_translate_insn(dc)) {
         return;
@@ -11571,6 +16538,11 @@ static void thumb_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
     }
     dc->insn = insn;
 
+    if (dc->eci) {
+        insn_eci_rewind = tcg_last_op(tcg_ctx);
+        insn_eci_pc_curr = dc->pc_curr;
+    }
+
     if (dc->condexec_mask && !thumb_insn_is_unconditional(dc, insn)) {
         uint32_t cond = dc->condexec_cond;
 
@@ -11622,6 +16594,14 @@ static void thumb_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
         }
     }
 
+    if (dc->eci && !dc->eci_handled) {
+        tcg_remove_ops_after(tcg_ctx, insn_eci_rewind);
+        dc->pc_curr = insn_eci_pc_curr;
+        dc->condjmp = 0;
+        gen_exception_insn(dc, dc->pc_curr, EXCP_INVSTATE,
+                           syn_uncategorized(), default_exception_el(dc));
+    }
+
     arm_post_translate_insn(dc);
 
     /* Thumb is a variable-length ISA.  Stop translation when the next insn
diff --git a/qemu/target/arm/translate.h b/qemu/target/arm/translate.h
index 62ea7a5277..c4f8b9ce52 100644
--- a/qemu/target/arm/translate.h
+++ b/qemu/target/arm/translate.h
@@ -22,6 +22,8 @@ typedef struct DisasContext {
     /* Thumb-2 conditional execution bits.  */
     int condexec_mask;
     int condexec_cond;
+    int eci;
+    bool eci_handled;
     int thumb;
     int sctlr_b;
     MemOp be_data;
@@ -29,10 +31,17 @@ typedef struct DisasContext {
     ARMMMUIdx mmu_idx; /* MMU index to use for normal loads/stores */
     uint8_t tbii;      /* TBI1|TBI0 for insns */
     uint8_t tbid;      /* TBI1|TBI0 for data */
+    uint8_t tcma;      /* TCMA1|TCMA0 for MTE */
     bool ns;        /* Use non-secure CPREG bank on access */
     int fp_excp_el; /* FP exception EL or 0 if enabled */
     int sve_excp_el; /* SVE exception EL or 0 if enabled */
+    int sme_excp_el; /* SME exception EL or 0 if enabled */
     int sve_len;     /* SVE vector length in bytes */
+    int svl;         /* Streaming SVE vector length in bytes */
+    bool pstate_sm;
+    bool pstate_za;
+    bool sme_trap_nonstreaming;
+    bool is_nonstreaming;
     /* Flag indicating that exceptions from secure mode are routed to EL3. */
     bool secure_routed_to_el3;
     bool vfp_enabled; /* FP enabled via FPSCR.EN */
@@ -44,6 +53,7 @@ typedef struct DisasContext {
     bool v8m_fpccr_s_wrong; /* true if v8M FPCCR.S != v8m_secure */
     bool v7m_new_fp_ctxt_needed; /* ASPEN set but no active FP context */
     bool v7m_lspact; /* FPCCR.LSPACT set */
+    bool mve_no_pred;
     /* Immediate value in AArch32 SVC insn; must be set if is_jmp == DISAS_SWI
      * so that top level loop can generate correct syndrome information.
      */
@@ -76,6 +86,9 @@ typedef struct DisasContext {
     bool unpriv;
     /* True if v8.3-PAuth is active.  */
     bool pauth_active;
+    /* True if v8.5-MemTag allocation tag access is active. */
+    bool ata;
+    bool mte_active[2];
     /* True with v8.5-BTI and SCTLR_ELx.BT* set.  */
     bool bt;
     /* True if any CP15 access is trapped by HSTR_EL2 */
diff --git a/qemu/target/arm/unicorn_arm.c b/qemu/target/arm/unicorn_arm.c
index 0a2a578800..bd2b80d9ed 100644
--- a/qemu/target/arm/unicorn_arm.c
+++ b/qemu/target/arm/unicorn_arm.c
@@ -297,6 +297,18 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
             CHECK_REG_TYPE(int32_t);
             *(int32_t *)value = env->vfp.xregs[ARM_VFP_FPSID];
             break;
+        case UC_ARM_REG_MVFR0:
+            CHECK_REG_TYPE(uint32_t);
+            *(uint32_t *)value = env->vfp.xregs[ARM_VFP_MVFR0];
+            break;
+        case UC_ARM_REG_MVFR1:
+            CHECK_REG_TYPE(uint32_t);
+            *(uint32_t *)value = env->vfp.xregs[ARM_VFP_MVFR1];
+            break;
+        case UC_ARM_REG_MVFR2:
+            CHECK_REG_TYPE(uint32_t);
+            *(uint32_t *)value = env->vfp.xregs[ARM_VFP_MVFR2];
+            break;
         case UC_ARM_REG_IPSR:
             CHECK_REG_TYPE(int32_t);
             *(int32_t *)value = v7m_mrs_xpsr(env, 5);
@@ -349,6 +361,10 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
             CHECK_REG_TYPE(uint32_t);
             *(uint32_t *)value = helper_v7m_mrs(env, 20);
             break;
+        case UC_ARM_REG_VPR:
+            CHECK_REG_TYPE(uint32_t);
+            *(uint32_t *)value = env->v7m.vpr;
+            break;
         case UC_ARM_REG_CP_REG:
             CHECK_REG_TYPE(uc_arm_cp_reg);
             ret = read_cp_reg(env, (uc_arm_cp_reg *)value);
@@ -465,6 +481,18 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             CHECK_REG_TYPE(int32_t);
             env->vfp.xregs[ARM_VFP_FPSID] = *(int32_t *)value;
             break;
+        case UC_ARM_REG_MVFR0:
+            CHECK_REG_TYPE(uint32_t);
+            env->vfp.xregs[ARM_VFP_MVFR0] = *(uint32_t *)value;
+            break;
+        case UC_ARM_REG_MVFR1:
+            CHECK_REG_TYPE(uint32_t);
+            env->vfp.xregs[ARM_VFP_MVFR1] = *(uint32_t *)value;
+            break;
+        case UC_ARM_REG_MVFR2:
+            CHECK_REG_TYPE(uint32_t);
+            env->vfp.xregs[ARM_VFP_MVFR2] = *(uint32_t *)value;
+            break;
         case UC_ARM_REG_IPSR:
             CHECK_REG_TYPE(uint32_t);
             v7m_msr_xpsr(env, 0b1000, 5, *(uint32_t *)value);
@@ -481,6 +509,10 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             CHECK_REG_TYPE(uint32_t);
             helper_v7m_msr(env, 20, *(uint32_t *)value);
             break;
+        case UC_ARM_REG_VPR:
+            CHECK_REG_TYPE(uint32_t);
+            env->v7m.vpr = *(uint32_t *)value;
+            break;
         case UC_ARM_REG_EPSR:
             CHECK_REG_TYPE(uint32_t);
             v7m_msr_xpsr(env, 0b1000, 6, *(uint32_t *)value);
diff --git a/qemu/target/arm/vec_helper.c b/qemu/target/arm/vec_helper.c
index a1839eed81..97c0e274ae 100644
--- a/qemu/target/arm/vec_helper.c
+++ b/qemu/target/arm/vec_helper.c
@@ -35,6 +35,7 @@
 #define H2(x)  (x)
 #define H4(x)  (x)
 #endif
+#define H8(x)  (x)
 
 #define SET_QC() env->vfp.qc[0] = 1
 
@@ -201,188 +202,284 @@ void HELPER(gvec_qrdmlsh_s32)(void *vd, void *vn, void *vm,
  * All elements are treated equally, no matter where they are.
  */
 
-void HELPER(gvec_sdot_b)(void *vd, void *vn, void *vm, uint32_t desc)
+#define DO_DOT(NAME, TYPED, TYPEN, TYPEM)                              \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                       \
+    intptr_t i, oprsz = simd_oprsz(desc);                               \
+    TYPED *d = vd, *a = va;                                             \
+    TYPEN *n = vn;                                                      \
+    TYPEM *m = vm;                                                      \
+                                                                        \
+    for (i = 0; i < oprsz / sizeof(TYPED); i++) {                       \
+        d[i] = a[i] +                                                   \
+               (TYPED)n[i * 4 + 0] * m[i * 4 + 0] +                    \
+               (TYPED)n[i * 4 + 1] * m[i * 4 + 1] +                    \
+               (TYPED)n[i * 4 + 2] * m[i * 4 + 2] +                    \
+               (TYPED)n[i * 4 + 3] * m[i * 4 + 3];                     \
+    }                                                                   \
+    clear_tail(d, oprsz, simd_maxsz(desc));                             \
+}
+
+DO_DOT(gvec_sdot_b, int32_t, int8_t, int8_t)
+DO_DOT(gvec_udot_b, uint32_t, uint8_t, uint8_t)
+DO_DOT(gvec_usdot_b, uint32_t, uint8_t, int8_t)
+DO_DOT(gvec_sdot_h, int64_t, int16_t, int16_t)
+DO_DOT(gvec_udot_h, uint64_t, uint16_t, uint16_t)
+
+#define DO_DOT_IDX(NAME, TYPED, TYPEN, TYPEM, H)                       \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                       \
+    intptr_t i = 0, oprsz = simd_oprsz(desc);                           \
+    intptr_t oprsz_n = oprsz / sizeof(TYPED);                           \
+    intptr_t segend = MIN(16 / sizeof(TYPED), oprsz_n);                 \
+    intptr_t index = simd_data(desc);                                   \
+    TYPED *d = vd, *a = va;                                             \
+    TYPEN *n = vn;                                                      \
+    TYPEM *m_indexed = (TYPEM *)vm + H(index) * 4;                      \
+                                                                        \
+    do {                                                                \
+        TYPED m0 = m_indexed[i * 4 + 0];                                \
+        TYPED m1 = m_indexed[i * 4 + 1];                                \
+        TYPED m2 = m_indexed[i * 4 + 2];                                \
+        TYPED m3 = m_indexed[i * 4 + 3];                                \
+                                                                        \
+        do {                                                            \
+            d[i] = a[i] +                                               \
+                   n[i * 4 + 0] * m0 +                                  \
+                   n[i * 4 + 1] * m1 +                                  \
+                   n[i * 4 + 2] * m2 +                                  \
+                   n[i * 4 + 3] * m3;                                   \
+        } while (++i < segend);                                         \
+        segend = i + (16 / sizeof(TYPED));                              \
+    } while (i < oprsz_n);                                              \
+    clear_tail(d, oprsz, simd_maxsz(desc));                             \
+}
+
+DO_DOT_IDX(gvec_sdot_idx_b, int32_t, int8_t, int8_t, H4)
+DO_DOT_IDX(gvec_udot_idx_b, uint32_t, uint8_t, uint8_t, H4)
+DO_DOT_IDX(gvec_sudot_idx_b, int32_t, int8_t, uint8_t, H4)
+DO_DOT_IDX(gvec_usdot_idx_b, int32_t, uint8_t, int8_t, H4)
+DO_DOT_IDX(gvec_sdot_idx_h, int64_t, int16_t, int16_t, H8)
+DO_DOT_IDX(gvec_udot_idx_h, uint64_t, uint16_t, uint16_t, H8)
+
+#undef DO_DOT
+#undef DO_DOT_IDX
+
+static uint32_t do_smmla_b(uint32_t sum, void *vn, void *vm)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc);
-    uint32_t *d = vd;
     int8_t *n = vn, *m = vm;
+    intptr_t k;
 
-    for (i = 0; i < opr_sz / 4; ++i) {
-        d[i] += n[i * 4 + 0] * m[i * 4 + 0]
-              + n[i * 4 + 1] * m[i * 4 + 1]
-              + n[i * 4 + 2] * m[i * 4 + 2]
-              + n[i * 4 + 3] * m[i * 4 + 3];
+    for (k = 0; k < 8; k++) {
+        sum += n[H1(k)] * m[H1(k)];
     }
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+    return sum;
 }
 
-void HELPER(gvec_udot_b)(void *vd, void *vn, void *vm, uint32_t desc)
+static uint32_t do_ummla_b(uint32_t sum, void *vn, void *vm)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc);
-    uint32_t *d = vd;
     uint8_t *n = vn, *m = vm;
+    intptr_t k;
 
-    for (i = 0; i < opr_sz / 4; ++i) {
-        d[i] += n[i * 4 + 0] * m[i * 4 + 0]
-              + n[i * 4 + 1] * m[i * 4 + 1]
-              + n[i * 4 + 2] * m[i * 4 + 2]
-              + n[i * 4 + 3] * m[i * 4 + 3];
+    for (k = 0; k < 8; k++) {
+        sum += n[H1(k)] * m[H1(k)];
     }
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+    return sum;
 }
 
-void HELPER(gvec_sdot_h)(void *vd, void *vn, void *vm, uint32_t desc)
+static uint32_t do_usmmla_b(uint32_t sum, void *vn, void *vm)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc);
-    uint64_t *d = vd;
-    int16_t *n = vn, *m = vm;
+    uint8_t *n = vn;
+    int8_t *m = vm;
+    intptr_t k;
 
-    for (i = 0; i < opr_sz / 8; ++i) {
-        d[i] += (int64_t)n[i * 4 + 0] * m[i * 4 + 0]
-              + (int64_t)n[i * 4 + 1] * m[i * 4 + 1]
-              + (int64_t)n[i * 4 + 2] * m[i * 4 + 2]
-              + (int64_t)n[i * 4 + 3] * m[i * 4 + 3];
+    for (k = 0; k < 8; k++) {
+        sum += n[H1(k)] * m[H1(k)];
     }
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+    return sum;
 }
 
-void HELPER(gvec_udot_h)(void *vd, void *vn, void *vm, uint32_t desc)
+static void do_mmla_b(void *vd, void *vn, void *vm, void *va, uint32_t desc,
+                      uint32_t (*inner_loop)(uint32_t, void *, void *))
 {
-    intptr_t i, opr_sz = simd_oprsz(desc);
-    uint64_t *d = vd;
-    uint16_t *n = vn, *m = vm;
-
-    for (i = 0; i < opr_sz / 8; ++i) {
-        d[i] += (uint64_t)n[i * 4 + 0] * m[i * 4 + 0]
-              + (uint64_t)n[i * 4 + 1] * m[i * 4 + 1]
-              + (uint64_t)n[i * 4 + 2] * m[i * 4 + 2]
-              + (uint64_t)n[i * 4 + 3] * m[i * 4 + 3];
+    intptr_t seg, oprsz = simd_oprsz(desc);
+    char *n = vn, *m = vm;
+
+    for (seg = 0; seg < oprsz; seg += 16) {
+        uint32_t *d = (uint32_t *)((char *)vd + seg);
+        uint32_t *a = (uint32_t *)((char *)va + seg);
+        uint32_t sum0, sum1, sum2, sum3;
+
+        sum0 = a[H4(0)];
+        sum0 = inner_loop(sum0, n + seg, m + seg);
+        sum1 = a[H4(1)];
+        sum1 = inner_loop(sum1, n + seg, m + seg + 8);
+        sum2 = a[H4(2)];
+        sum2 = inner_loop(sum2, n + seg + 8, m + seg);
+        sum3 = a[H4(3)];
+        sum3 = inner_loop(sum3, n + seg + 8, m + seg + 8);
+
+        d[H4(0)] = sum0;
+        d[H4(1)] = sum1;
+        d[H4(2)] = sum2;
+        d[H4(3)] = sum3;
     }
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+    clear_tail(vd, oprsz, simd_maxsz(desc));
 }
 
-void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
-{
-    intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
-    intptr_t index = simd_data(desc);
-    uint32_t *d = vd;
-    int8_t *n = vn;
-    int8_t *m_indexed = (int8_t *)vm + index * 4;
+#define DO_MMLA_B(NAME, INNER)                                          \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
+{                                                                       \
+    do_mmla_b(vd, vn, vm, va, desc, INNER);                             \
+}
 
-    /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
-     * Otherwise opr_sz is a multiple of 16.
-     */
-    segend = MIN(4, opr_sz_4);
-    i = 0;
-    do {
-        int8_t m0 = m_indexed[i * 4 + 0];
-        int8_t m1 = m_indexed[i * 4 + 1];
-        int8_t m2 = m_indexed[i * 4 + 2];
-        int8_t m3 = m_indexed[i * 4 + 3];
-
-        do {
-            d[i] += n[i * 4 + 0] * m0
-                  + n[i * 4 + 1] * m1
-                  + n[i * 4 + 2] * m2
-                  + n[i * 4 + 3] * m3;
-        } while (++i < segend);
-        segend = i + 4;
-    } while (i < opr_sz_4);
+DO_MMLA_B(gvec_smmla_b, do_smmla_b)
+DO_MMLA_B(gvec_ummla_b, do_ummla_b)
+DO_MMLA_B(gvec_usmmla_b, do_usmmla_b)
 
-    clear_tail(d, opr_sz, simd_maxsz(desc));
-}
+#undef DO_MMLA_B
 
-void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm, uint32_t desc)
+static float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2)
 {
-    intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
-    intptr_t index = simd_data(desc);
-    uint32_t *d = vd;
-    uint8_t *n = vn;
-    uint8_t *m_indexed = (uint8_t *)vm + index * 4;
+    float_status bf_status = {
+        .tininess_before_rounding = float_tininess_before_rounding,
+        .float_rounding_mode = float_round_to_odd_inf,
+        .flush_to_zero = true,
+        .flush_inputs_to_zero = true,
+        .default_nan_mode = true,
+    };
+    float32 t1;
+    float32 t2;
+
+    t1 = float32_mul(e1 << 16, e2 << 16, &bf_status);
+    t2 = float32_mul(e1 & 0xffff0000u, e2 & 0xffff0000u, &bf_status);
+    t1 = float32_add(t1, t2, &bf_status);
+    return float32_add(sum, t1, &bf_status);
+}
 
-    /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
-     * Otherwise opr_sz is a multiple of 16.
-     */
-    segend = MIN(4, opr_sz_4);
-    i = 0;
-    do {
-        uint8_t m0 = m_indexed[i * 4 + 0];
-        uint8_t m1 = m_indexed[i * 4 + 1];
-        uint8_t m2 = m_indexed[i * 4 + 2];
-        uint8_t m3 = m_indexed[i * 4 + 3];
-
-        do {
-            d[i] += n[i * 4 + 0] * m0
-                  + n[i * 4 + 1] * m1
-                  + n[i * 4 + 2] * m2
-                  + n[i * 4 + 3] * m3;
-        } while (++i < segend);
-        segend = i + 4;
-    } while (i < opr_sz_4);
+void HELPER(gvec_bfdot)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
+{
+    intptr_t i;
+    intptr_t oprsz = simd_oprsz(desc);
+    float32 *d = vd;
+    float32 *a = va;
+    uint32_t *n = vn;
+    uint32_t *m = vm;
 
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+    for (i = 0; i < oprsz / 4; ++i) {
+        d[i] = bfdotadd(a[i], n[i], m[i]);
+    }
+    clear_tail(d, oprsz, simd_maxsz(desc));
 }
 
-void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_bfdot_idx)(void *vd, void *vn, void *vm,
+                            void *va, uint32_t desc)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
+    intptr_t i;
+    intptr_t j;
+    intptr_t oprsz = simd_oprsz(desc);
     intptr_t index = simd_data(desc);
-    uint64_t *d = vd;
-    int16_t *n = vn;
-    int16_t *m_indexed = (int16_t *)vm + index * 4;
+    intptr_t elements = oprsz / 4;
+    intptr_t eltspersegment = MIN(16 / 4, elements);
+    float32 *d = vd;
+    float32 *a = va;
+    uint32_t *n = vn;
+    uint32_t *m = vm;
 
-    /* This is supported by SVE only, so opr_sz is always a multiple of 16.
-     * Process the entire segment all at once, writing back the results
-     * only after we've consumed all of the inputs.
-     */
-    for (i = 0; i < opr_sz_8 ; i += 2) {
-        uint64_t d0, d1;
-
-        d0  = n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
-        d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1];
-        d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2];
-        d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3];
-        d1  = n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
-        d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1];
-        d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2];
-        d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3];
-
-        d[i + 0] += d0;
-        d[i + 1] += d1;
+    for (i = 0; i < elements; i += eltspersegment) {
+        uint32_t m_idx = m[i + H4(index)];
+
+        for (j = i; j < i + eltspersegment; j++) {
+            d[j] = bfdotadd(a[j], n[j], m_idx);
+        }
     }
+    clear_tail(d, oprsz, simd_maxsz(desc));
+}
 
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+void HELPER(gvec_bfmmla)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
+{
+    intptr_t s;
+    intptr_t oprsz = simd_oprsz(desc);
+    float32 *d = vd;
+    float32 *a = va;
+    uint32_t *n = vn;
+    uint32_t *m = vm;
+
+    for (s = 0; s < oprsz / 4; s += 4) {
+        float32 sum00;
+        float32 sum01;
+        float32 sum10;
+        float32 sum11;
+
+        sum00 = a[s + H4(0)];
+        sum00 = bfdotadd(sum00, n[s + H4(0)], m[s + H4(0)]);
+        sum00 = bfdotadd(sum00, n[s + H4(1)], m[s + H4(1)]);
+
+        sum01 = a[s + H4(1)];
+        sum01 = bfdotadd(sum01, n[s + H4(0)], m[s + H4(2)]);
+        sum01 = bfdotadd(sum01, n[s + H4(1)], m[s + H4(3)]);
+
+        sum10 = a[s + H4(2)];
+        sum10 = bfdotadd(sum10, n[s + H4(2)], m[s + H4(0)]);
+        sum10 = bfdotadd(sum10, n[s + H4(3)], m[s + H4(1)]);
+
+        sum11 = a[s + H4(3)];
+        sum11 = bfdotadd(sum11, n[s + H4(2)], m[s + H4(2)]);
+        sum11 = bfdotadd(sum11, n[s + H4(3)], m[s + H4(3)]);
+
+        d[s + H4(0)] = sum00;
+        d[s + H4(1)] = sum01;
+        d[s + H4(2)] = sum10;
+        d[s + H4(3)] = sum11;
+    }
+    clear_tail(d, oprsz, simd_maxsz(desc));
 }
 
-void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
+void HELPER(gvec_bfmlal)(void *vd, void *vn, void *vm, void *va,
+                         void *stat, uint32_t desc)
 {
-    intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
-    intptr_t index = simd_data(desc);
-    uint64_t *d = vd;
-    uint16_t *n = vn;
-    uint16_t *m_indexed = (uint16_t *)vm + index * 4;
+    intptr_t i;
+    intptr_t oprsz = simd_oprsz(desc);
+    intptr_t sel = simd_data(desc);
+    float32 *d = vd;
+    float32 *a = va;
+    bfloat16 *n = vn;
+    bfloat16 *m = vm;
 
-    /* This is supported by SVE only, so opr_sz is always a multiple of 16.
-     * Process the entire segment all at once, writing back the results
-     * only after we've consumed all of the inputs.
-     */
-    for (i = 0; i < opr_sz_8 ; i += 2) {
-        uint64_t d0, d1;
-
-        d0  = n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
-        d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1];
-        d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2];
-        d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3];
-        d1  = n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
-        d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1];
-        d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2];
-        d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3];
-
-        d[i + 0] += d0;
-        d[i + 1] += d1;
+    for (i = 0; i < oprsz / 4; ++i) {
+        float32 nn = n[H2(i * 2 + sel)] << 16;
+        float32 mm = m[H2(i * 2 + sel)] << 16;
+
+        d[H4(i)] = float32_muladd(nn, mm, a[H4(i)], 0, stat);
     }
+    clear_tail(d, oprsz, simd_maxsz(desc));
+}
 
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+void HELPER(gvec_bfmlal_idx)(void *vd, void *vn, void *vm,
+                             void *va, void *stat, uint32_t desc)
+{
+    intptr_t i;
+    intptr_t j;
+    intptr_t oprsz = simd_oprsz(desc);
+    intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1);
+    intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 1, 3);
+    intptr_t elements = oprsz / 4;
+    intptr_t eltspersegment = MIN(16 / 4, elements);
+    float32 *d = vd;
+    float32 *a = va;
+    bfloat16 *n = vn;
+    bfloat16 *m = vm;
+
+    for (i = 0; i < elements; i += eltspersegment) {
+        float32 m_idx = m[H2(2 * i + index)] << 16;
+
+        for (j = i; j < i + eltspersegment; j++) {
+            float32 n_j = n[H2(2 * j + sel)] << 16;
+
+            d[H4(j)] = float32_muladd(n_j, m_idx, a[H4(j)], 0, stat);
+        }
+    }
+    clear_tail(d, oprsz, simd_maxsz(desc));
 }
 
 void HELPER(gvec_fcaddh)(void *vd, void *vn, void *vm,
@@ -1005,6 +1102,30 @@ void HELPER(gvec_fmlal_a64)(void *vd, void *vn, void *vm,
              get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
 }
 
+void HELPER(sve2_fmlal_zzzw_s)(void *vd, void *vn, void *vm, void *va,
+                               void *venv, uint32_t desc)
+{
+    intptr_t i, oprsz = simd_oprsz(desc) / sizeof(float32);
+    uint16_t negn = extract32(desc, SIMD_DATA_SHIFT, 1) << 15;
+    intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+    float16 *n = vn, *m = vm;
+    float32 *d = vd, *a = va;
+    CPUARMState *env = venv;
+    float_status *status = &env->vfp.fp_status;
+    bool fz16 = get_flush_inputs_to_zero(&env->vfp.fp_status_f16);
+
+    for (i = 0; i < oprsz; i++) {
+        intptr_t idx = i * 2 + sel;
+        float16 nn_16 = n[H2(idx)] ^ negn;
+        float16 mm_16 = m[H2(idx)];
+        float32 nn = float16_to_float32_by_bits(nn_16, fz16);
+        float32 mm = float16_to_float32_by_bits(mm_16, fz16);
+        float32 aa = a[H4(i)];
+
+        d[H4(i)] = float32_muladd(nn, mm, aa, 0, status);
+    }
+}
+
 static void do_fmlal_idx(float32 *d, void *vn, void *vm, float_status *fpst,
                          uint32_t desc, bool fz16)
 {
@@ -1049,6 +1170,35 @@ void HELPER(gvec_fmlal_idx_a64)(void *vd, void *vn, void *vm,
                  get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
 }
 
+void HELPER(sve2_fmlal_zzxw_s)(void *vd, void *vn, void *vm, void *va,
+                               void *venv, uint32_t desc)
+{
+    intptr_t i, j, oprsz = simd_oprsz(desc);
+    uint16_t negn = extract32(desc, SIMD_DATA_SHIFT, 1) << 15;
+    intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+    intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 2, 3);
+    float16 *n = vn, *m = vm;
+    float32 *d = vd, *a = va;
+    CPUARMState *env = venv;
+    float_status *status = &env->vfp.fp_status;
+    bool fz16 = get_flush_inputs_to_zero(&env->vfp.fp_status_f16);
+
+    for (i = 0; i < oprsz; i += 16) {
+        intptr_t hbase = i / sizeof(float16);
+        intptr_t sbase = i / sizeof(float32);
+        float16 mm_16 = m[H2(hbase + idx)];
+        float32 mm = float16_to_float32_by_bits(mm_16, fz16);
+
+        for (j = 0; j < 4; j++) {
+            float16 nn_16 = n[H2(hbase + j * 2 + sel)] ^ negn;
+            float32 nn = float16_to_float32_by_bits(nn_16, fz16);
+            float32 aa = a[H4(sbase + j)];
+
+            d[H4(sbase + j)] = float32_muladd(nn, mm, aa, 0, status);
+        }
+    }
+}
+
 void HELPER(gvec_sshl_b)(void *vd, void *vn, void *vm, uint32_t desc)
 {
     intptr_t i, opr_sz = simd_oprsz(desc);
@@ -1262,4 +1412,75 @@ void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc)
         d[i] = pmull_h(nn, mm);
     }
 }
+
+static uint64_t pmull_d(uint64_t op1, uint64_t op2)
+{
+    uint64_t result = 0;
+    int i;
+
+    for (i = 0; i < 32; i++) {
+#ifdef _MSC_VER
+        uint64_t mask = 0 - ((op1 >> i) & 1);
+#else
+        uint64_t mask = -((op1 >> i) & 1);
+#endif
+        result ^= (op2 << i) & mask;
+    }
+    return result;
+}
+
+void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc)
+{
+    intptr_t sel = H4(simd_data(desc));
+    intptr_t i, opr_sz = simd_oprsz(desc);
+    uint32_t *n = vn, *m = vm;
+    uint64_t *d = vd;
+
+    for (i = 0; i < opr_sz / 8; i++) {
+        d[i] = pmull_d(n[2 * i + sel], m[2 * i + sel]);
+    }
+}
 #endif
+
+#define DO_SABA(NAME, STYPE, UTYPE)                                     \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, opr_sz = simd_oprsz(desc);                              \
+    STYPE *d = vd, *n = vn, *m = vm;                                     \
+                                                                        \
+    for (i = 0; i < opr_sz / sizeof(STYPE); i++) {                      \
+        UTYPE diff = n[i] < m[i] ? (UTYPE)m[i] - (UTYPE)n[i] :          \
+                     (UTYPE)n[i] - (UTYPE)m[i];                         \
+                                                                        \
+        d[i] = (STYPE)((UTYPE)d[i] + diff);                             \
+    }                                                                   \
+    clear_tail(d, opr_sz, simd_maxsz(desc));                            \
+}
+
+DO_SABA(gvec_saba_b, int8_t, uint8_t)
+DO_SABA(gvec_saba_h, int16_t, uint16_t)
+DO_SABA(gvec_saba_s, int32_t, uint32_t)
+DO_SABA(gvec_saba_d, int64_t, uint64_t)
+
+#undef DO_SABA
+
+#define DO_UABA(NAME, TYPE)                                             \
+void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc)          \
+{                                                                       \
+    intptr_t i, opr_sz = simd_oprsz(desc);                              \
+    TYPE *d = vd, *n = vn, *m = vm;                                      \
+                                                                        \
+    for (i = 0; i < opr_sz / sizeof(TYPE); i++) {                       \
+        TYPE diff = n[i] < m[i] ? m[i] - n[i] : n[i] - m[i];            \
+                                                                        \
+        d[i] += diff;                                                   \
+    }                                                                   \
+    clear_tail(d, opr_sz, simd_maxsz(desc));                            \
+}
+
+DO_UABA(gvec_uaba_b, uint8_t)
+DO_UABA(gvec_uaba_h, uint16_t)
+DO_UABA(gvec_uaba_s, uint32_t)
+DO_UABA(gvec_uaba_d, uint64_t)
+
+#undef DO_UABA
diff --git a/qemu/target/arm/vfp_helper.c b/qemu/target/arm/vfp_helper.c
index 55bce5957a..29f26d7dd4 100644
--- a/qemu/target/arm/vfp_helper.c
+++ b/qemu/target/arm/vfp_helper.c
@@ -89,6 +89,8 @@ static uint32_t vfp_get_fpscr_from_host(CPUARMState *env)
     /* FZ16 does not generate an input denormal exception.  */
     i |= (get_float_exception_flags(&env->vfp.fp_status_f16)
           & ~float_flag_input_denormal);
+    i |= (get_float_exception_flags(&env->vfp.standard_fp_status_f16)
+          & ~float_flag_input_denormal);
     return vfp_exceptbits_from_host(i);
 }
 
@@ -120,7 +122,10 @@ static void vfp_set_fpscr_to_host(CPUARMState *env, uint32_t val)
     if (changed & FPCR_FZ16) {
         bool ftz_enabled = val & FPCR_FZ16;
         set_flush_to_zero(ftz_enabled, &env->vfp.fp_status_f16);
+        set_flush_to_zero(ftz_enabled, &env->vfp.standard_fp_status_f16);
         set_flush_inputs_to_zero(ftz_enabled, &env->vfp.fp_status_f16);
+        set_flush_inputs_to_zero(ftz_enabled,
+                                 &env->vfp.standard_fp_status_f16);
     }
     if (changed & FPCR_FZ) {
         bool ftz_enabled = val & FPCR_FZ;
@@ -142,6 +147,7 @@ static void vfp_set_fpscr_to_host(CPUARMState *env, uint32_t val)
     set_float_exception_flags(i, &env->vfp.fp_status);
     set_float_exception_flags(0, &env->vfp.fp_status_f16);
     set_float_exception_flags(0, &env->vfp.standard_fp_status);
+    set_float_exception_flags(0, &env->vfp.standard_fp_status_f16);
 }
 
 uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env)
@@ -151,6 +157,7 @@ uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env)
     fpscr = env->vfp.xregs[ARM_VFP_FPSCR]
             | (env->vfp.vec_len << 16)
             | (env->vfp.vec_stride << 20);
+    fpscr |= env->v7m.ltpsize << 16;
 
     fpscr |= vfp_get_fpscr_from_host(env);
 
@@ -167,20 +174,31 @@ uint32_t vfp_get_fpscr(CPUARMState *env)
 
 void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val)
 {
+    ARMCPU *cpu = env_archcpu(env);
+
     /* When ARMv8.2-FP16 is not supported, FZ16 is RES0.  */
-    if (!cpu_isar_feature(any_fp16, env_archcpu(env))) {
+    if (!cpu_isar_feature(any_fp16, cpu)) {
         val &= ~FPCR_FZ16;
     }
 
-    if (arm_feature(env, ARM_FEATURE_M)) {
-        /*
-         * M profile FPSCR is RES0 for the QC, STRIDE, FZ16, LEN bits
-         * and also for the trapped-exception-handling bits IxE.
-         */
-        val &= 0xf7c0009f;
+    vfp_set_fpscr_to_host(env, val);
+
+    if (!arm_feature(env, ARM_FEATURE_M)) {
+        env->vfp.vec_len = (val >> 16) & 7;
+        env->vfp.vec_stride = (val >> 20) & 3;
+    } else if (cpu_isar_feature(aa32_mve, cpu)) {
+        env->v7m.ltpsize = extract32(val, FPCR_LTPSIZE_SHIFT,
+                                     FPCR_LTPSIZE_LENGTH);
     }
 
-    vfp_set_fpscr_to_host(env, val);
+    if (arm_feature(env, ARM_FEATURE_NEON) ||
+        (arm_feature(env, ARM_FEATURE_M) &&
+         cpu_isar_feature(aa32_mve, cpu))) {
+        env->vfp.qc[0] = val & FPCR_QC;
+        env->vfp.qc[1] = 0;
+        env->vfp.qc[2] = 0;
+        env->vfp.qc[3] = 0;
+    }
 
     /*
      * We don't implement trapped exception handling, so the
@@ -191,17 +209,6 @@ void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val)
      * in between, then we clear all of the low 16 bits.
      */
     env->vfp.xregs[ARM_VFP_FPSCR] = val & 0xf7c80000;
-    env->vfp.vec_len = (val >> 16) & 7;
-    env->vfp.vec_stride = (val >> 20) & 3;
-
-    /*
-     * The bit we set within fpscr_q is arbitrary; the register as a
-     * whole being zero/non-zero is what counts.
-     */
-    env->vfp.qc[0] = val & FPCR_QC;
-    env->vfp.qc[1] = 0;
-    env->vfp.qc[2] = 0;
-    env->vfp.qc[3] = 0;
 }
 
 void vfp_set_fpscr(CPUARMState *env, uint32_t val)
@@ -421,6 +428,24 @@ uint32_t HELPER(vfp_ultoh)(uint32_t x, uint32_t shift, void *fpst)
 #endif
 }
 
+uint32_t HELPER(vfp_shtoh)(uint32_t x, uint32_t shift, void *fpst)
+{
+#ifdef _MSC_VER
+    return int16_to_float16_scalbn((int16_t)x, 0 - shift, fpst);
+#else
+    return int16_to_float16_scalbn((int16_t)x, -shift, fpst);
+#endif
+}
+
+uint32_t HELPER(vfp_uhtoh)(uint32_t x, uint32_t shift, void *fpst)
+{
+#ifdef _MSC_VER
+    return uint16_to_float16_scalbn((uint16_t)x, 0 - shift, fpst);
+#else
+    return uint16_to_float16_scalbn((uint16_t)x, -shift, fpst);
+#endif
+}
+
 uint32_t HELPER(vfp_sqtoh)(uint64_t x, uint32_t shift, void *fpst)
 {
 #ifdef _MSC_VER
@@ -459,6 +484,26 @@ uint32_t HELPER(vfp_touhh)(uint32_t x, uint32_t shift, void *fpst)
                                     shift, fpst);
 }
 
+uint32_t HELPER(vfp_toshh_round_to_zero)(uint32_t x, uint32_t shift,
+                                         void *fpst)
+{
+    if (unlikely(float16_is_any_nan(x))) {
+        float_raise(float_flag_invalid, fpst);
+        return 0;
+    }
+    return float16_to_int16_scalbn(x, float_round_to_zero, shift, fpst);
+}
+
+uint32_t HELPER(vfp_touhh_round_to_zero)(uint32_t x, uint32_t shift,
+                                         void *fpst)
+{
+    if (unlikely(float16_is_any_nan(x))) {
+        float_raise(float_flag_invalid, fpst);
+        return 0;
+    }
+    return float16_to_uint16_scalbn(x, float_round_to_zero, shift, fpst);
+}
+
 uint32_t HELPER(vfp_toslh)(uint32_t x, uint32_t shift, void *fpst)
 {
     if (unlikely(float16_is_any_nan(x))) {
@@ -582,6 +627,19 @@ uint32_t HELPER(vfp_fcvt_f64_to_f16)(float64 a, void *fpstp, uint32_t ahp_mode)
     return r;
 }
 
+uint32_t HELPER(bfcvt)(float32 x, void *status)
+{
+    return float32_to_bfloat16(x, status);
+}
+
+uint32_t HELPER(bfcvt_pair)(uint64_t pair, void *status)
+{
+    bfloat16 lo = float32_to_bfloat16(extract64(pair, 0, 32), status);
+    bfloat16 hi = float32_to_bfloat16(extract64(pair, 32, 32), status);
+
+    return deposit32(lo, 16, 16, hi);
+}
+
 #define float32_two make_float32(0x40000000)
 #define float32_three make_float32(0x40400000)
 #define float32_one_point_five make_float32(0x3fc00000)
@@ -1072,6 +1130,11 @@ float64 VFP_HELPER(muladd, d)(float64 a, float64 b, float64 c, void *fpstp)
 }
 
 /* ARMv8 round to integral */
+uint32_t HELPER(rinth_exact)(uint32_t x, void *fp_status)
+{
+    return float16_round_to_int(x, fp_status);
+}
+
 float32 HELPER(rints_exact)(float32 x, void *fp_status)
 {
     return float32_round_to_int(x, fp_status);
@@ -1082,6 +1145,22 @@ float64 HELPER(rintd_exact)(float64 x, void *fp_status)
     return float64_round_to_int(x, fp_status);
 }
 
+uint32_t HELPER(rinth)(uint32_t x, void *fp_status)
+{
+    int old_flags = get_float_exception_flags(fp_status), new_flags;
+    float16 ret;
+
+    ret = float16_round_to_int(x, fp_status);
+
+    if (!(old_flags & float_flag_inexact)) {
+        new_flags = get_float_exception_flags(fp_status);
+        set_float_exception_flags(new_flags & ~float_flag_inexact,
+                                  fp_status);
+    }
+
+    return ret;
+}
+
 float32 HELPER(rints)(float32 x, void *fp_status)
 {
     int old_flags = get_float_exception_flags(fp_status), new_flags;
diff --git a/qemu/target/i386/TODO b/qemu/target/i386/TODO
index a8d69cf87f..0842ece151 100644
--- a/qemu/target/i386/TODO
+++ b/qemu/target/i386/TODO
@@ -20,7 +20,7 @@ Optimizations/Features:
 
 - add SVM nested paging support
 - add VMX support
-- add AVX support
+- add remaining AVX-512/EVEX support
 - add SSE5 support
 - fxsave/fxrstor AMD extensions
 - improve monitor/mwait support
diff --git a/qemu/target/i386/cpu.c b/qemu/target/i386/cpu.c
index 86103b09e3..6fed8bc6bd 100644
--- a/qemu/target/i386/cpu.c
+++ b/qemu/target/i386/cpu.c
@@ -578,6 +578,18 @@ static CPUCacheInfo legacy_l3_cache = {
 #define INTEL_PT_CYCLE_BITMAP    0x1fff         /* Support 0,2^(0~11) */
 #define INTEL_PT_PSB_BITMAP      (0x003f << 16) /* Support 2K,4K,8K,16K,32K,64K */
 
+/* CPUID Leaf 0x1D constants: */
+#define INTEL_AMX_TILE_MAX_SUBLEAF     0x1
+#define INTEL_AMX_TOTAL_TILE_BYTES     0x2000
+#define INTEL_AMX_BYTES_PER_TILE       0x400
+#define INTEL_AMX_BYTES_PER_ROW        0x40
+#define INTEL_AMX_TILE_MAX_NAMES       0x8
+#define INTEL_AMX_TILE_MAX_ROWS        0x10
+
+/* CPUID Leaf 0x1E constants: */
+#define INTEL_AMX_TMUL_MAX_K           0x10
+#define INTEL_AMX_TMUL_MAX_N           0x40
+
 #define I486_FEATURES (CPUID_FP87 | CPUID_VME | CPUID_PSE)
 #define PENTIUM_FEATURES (I486_FEATURES | CPUID_DE | CPUID_TSC | \
           CPUID_MSR | CPUID_MCE | CPUID_CX8 | CPUID_MMX | CPUID_APIC)
@@ -604,13 +616,13 @@ static CPUCacheInfo legacy_l3_cache = {
           CPUID_EXT_SSE41 | CPUID_EXT_SSE42 | CPUID_EXT_POPCNT | \
           CPUID_EXT_XSAVE | /* CPUID_EXT_OSXSAVE is dynamic */   \
           CPUID_EXT_MOVBE | CPUID_EXT_AES | CPUID_EXT_HYPERVISOR | \
-          CPUID_EXT_RDRAND)
+          CPUID_EXT_RDRAND | CPUID_EXT_AVX | CPUID_EXT_F16C | \
+          CPUID_EXT_FMA)
           /* missing:
           CPUID_EXT_DTES64, CPUID_EXT_DSCPL, CPUID_EXT_VMX, CPUID_EXT_SMX,
-          CPUID_EXT_EST, CPUID_EXT_TM2, CPUID_EXT_CID, CPUID_EXT_FMA,
+          CPUID_EXT_EST, CPUID_EXT_TM2, CPUID_EXT_CID,
           CPUID_EXT_XTPR, CPUID_EXT_PDCM, CPUID_EXT_PCID, CPUID_EXT_DCA,
-          CPUID_EXT_X2APIC, CPUID_EXT_TSC_DEADLINE_TIMER, CPUID_EXT_AVX,
-          CPUID_EXT_F16C */
+          CPUID_EXT_X2APIC, CPUID_EXT_TSC_DEADLINE_TIMER */
 
 #ifdef TARGET_X86_64
 #define TCG_EXT2_X86_64_FEATURES (CPUID_EXT2_SYSCALL | CPUID_EXT2_LM)
@@ -625,20 +637,21 @@ static CPUCacheInfo legacy_l3_cache = {
 #define TCG_EXT3_FEATURES (CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM | \
           CPUID_EXT3_CR8LEG | CPUID_EXT3_ABM | CPUID_EXT3_SSE4A)
 #define TCG_EXT4_FEATURES 0
-#define TCG_SVM_FEATURES CPUID_SVM_NPT
+#define TCG_SVM_FEATURES (CPUID_SVM_NPT | CPUID_SVM_VGIF | \
+          CPUID_SVM_SVME_ADDR_CHK)
 #define TCG_KVM_FEATURES 0
 #define TCG_7_0_EBX_FEATURES (CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_SMAP | \
           CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ADX | \
           CPUID_7_0_EBX_PCOMMIT | CPUID_7_0_EBX_CLFLUSHOPT |            \
           CPUID_7_0_EBX_CLWB | CPUID_7_0_EBX_MPX | CPUID_7_0_EBX_FSGSBASE | \
-          CPUID_7_0_EBX_ERMS)
+          CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_AVX2)
           /* missing:
-          CPUID_7_0_EBX_HLE, CPUID_7_0_EBX_AVX2,
+          CPUID_7_0_EBX_HLE
           CPUID_7_0_EBX_INVPCID, CPUID_7_0_EBX_RTM,
           CPUID_7_0_EBX_RDSEED */
-#define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | \
+#define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_UMIP | CPUID_7_0_ECX_PKU | \
           /* CPUID_7_0_ECX_OSPKE is dynamic */ \
-          CPUID_7_0_ECX_LA57)
+          CPUID_7_0_ECX_LA57 | CPUID_7_0_ECX_PKS | CPUID_7_0_ECX_VAES)
 #define TCG_7_0_EDX_FEATURES 0
 #define TCG_7_1_EAX_FEATURES 0
 #define TCG_APM_FEATURES 0
@@ -852,12 +865,12 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         .type = CPUID_FEATURE_WORD,
         .feat_names = {
             NULL, NULL, "avx512-4vnniw", "avx512-4fmaps",
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, "md-clear", NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, NULL /* pconfig */, NULL,
-            NULL, NULL, NULL, NULL,
-            NULL, NULL, "spec-ctrl", "stibp",
+            "fsrm", NULL, NULL, NULL,
+            "avx512-vp2intersect", NULL, "md-clear", NULL,
+            NULL, NULL, "serialize", NULL,
+            "tsx-ldtrk", NULL, NULL /* pconfig */, "arch-lbr",
+            NULL, NULL, "amx-bf16", "avx512-fp16",
+            "amx-tile", "amx-int8", "spec-ctrl", "stibp",
             NULL, "arch-capabilities", "core-capability", "ssbd",
         },
         .cpuid = {
@@ -920,7 +933,7 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         .type = CPUID_FEATURE_WORD,
         .feat_names = {
             "xsaveopt", "xsavec", "xgetbv1", "xsaves",
-            NULL, NULL, NULL, NULL,
+            "xfd", NULL, NULL, NULL,
             NULL, NULL, NULL, NULL,
             NULL, NULL, NULL, NULL,
             NULL, NULL, NULL, NULL,
@@ -935,6 +948,32 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         },
         .tcg_features = TCG_XSAVE_FEATURES,
     },
+    [FEAT_XSAVE_XSS_LO] = {
+        .type = CPUID_FEATURE_WORD,
+        .feat_names = {
+            NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL, NULL,
+            NULL, NULL, NULL, NULL,
+        },
+        .cpuid = {
+            .eax = 0xd,
+            .needs_ecx = true, .ecx = 1,
+            .reg = R_ECX,
+        },
+    },
+    [FEAT_XSAVE_XSS_HI] = {
+        .type = CPUID_FEATURE_WORD,
+        .cpuid = {
+            .eax = 0xd,
+            .needs_ecx = true, .ecx = 1,
+            .reg = R_EDX,
+        },
+    },
     [FEAT_6_EAX] = {
         .type = CPUID_FEATURE_WORD,
         .feat_names = {
@@ -950,7 +989,7 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         .cpuid = { .eax = 6, .reg = R_EAX, },
         .tcg_features = TCG_6_EAX_FEATURES,
     },
-    [FEAT_XSAVE_COMP_LO] = {
+    [FEAT_XSAVE_XCR0_LO] = {
         .type = CPUID_FEATURE_WORD,
         .cpuid = {
             .eax = 0xD,
@@ -959,7 +998,7 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         },
         .tcg_features = ~0U,
     },
-    [FEAT_XSAVE_COMP_HI] = {
+    [FEAT_XSAVE_XCR0_HI] = {
         .type = CPUID_FEATURE_WORD,
         .cpuid = {
             .eax = 0xD,
@@ -1194,12 +1233,7 @@ static const X86RegisterInfo32 x86_reg_info_32[CPU_NB_REGS32] = {
 };
 #undef REGISTER
 
-typedef struct ExtSaveArea {
-    uint32_t feature, bits;
-    uint32_t offset, size;
-} ExtSaveArea;
-
-static const ExtSaveArea x86_ext_save_areas[] = {
+ExtSaveArea x86_ext_save_areas[XSAVE_STATE_AREA_COUNT] = {
     [XSTATE_FP_BIT] = {
         /* x87 FP state component is always enabled if XSAVE is supported */
         .feature = FEAT_1_ECX, .bits = CPUID_EXT_XSAVE,
@@ -1242,17 +1276,33 @@ static const ExtSaveArea x86_ext_save_areas[] = {
           { .feature = FEAT_7_0_ECX, .bits = CPUID_7_0_ECX_PKU,
             .offset = offsetof(X86XSaveArea, pkru_state),
             .size = sizeof(XSavePKRU) },
+    [XSTATE_ARCH_LBR_BIT] = {
+        .feature = FEAT_7_0_EDX, .bits = CPUID_7_0_EDX_ARCH_LBR,
+        .offset = 0,
+        .size = sizeof(XSavesArchLBR),
+    },
+    [XSTATE_XTILE_CFG_BIT] = {
+        .feature = FEAT_7_0_EDX, .bits = CPUID_7_0_EDX_AMX_TILE,
+        .offset = 0xac0,
+        .size = sizeof(XSaveXTILECFG),
+    },
+    [XSTATE_XTILE_DATA_BIT] = {
+        .feature = FEAT_7_0_EDX, .bits = CPUID_7_0_EDX_AMX_TILE,
+        .offset = 0xb00,
+        .size = sizeof(XSaveXTILEDATA),
+    },
 };
 
-static uint32_t xsave_area_size(uint64_t mask)
+uint32_t xsave_area_size(uint64_t mask, bool compacted)
 {
     int i;
-    uint64_t ret = 0;
+    uint64_t ret = x86_ext_save_areas[0].size;
 
-    for (i = 0; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
+    for (i = 2; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
         const ExtSaveArea *esa = &x86_ext_save_areas[i];
         if ((mask >> i) & 1) {
-            ret = MAX(ret, esa->offset + esa->size);
+            uint32_t offset = compacted ? ret : esa->offset;
+            ret = MAX(ret, offset + esa->size);
         }
     }
     return ret;
@@ -1263,10 +1313,16 @@ static inline bool accel_uses_host_cpuid(void)
     return false;
 }
 
-static inline uint64_t x86_cpu_xsave_components(X86CPU *cpu)
+static inline uint64_t x86_cpu_xsave_xcr0_components(X86CPU *cpu)
 {
-    return ((uint64_t)cpu->env.features[FEAT_XSAVE_COMP_HI]) << 32 |
-           cpu->env.features[FEAT_XSAVE_COMP_LO];
+    return ((uint64_t)cpu->env.features[FEAT_XSAVE_XCR0_HI]) << 32 |
+           cpu->env.features[FEAT_XSAVE_XCR0_LO];
+}
+
+static inline uint64_t x86_cpu_xsave_xss_components(X86CPU *cpu)
+{
+    return ((uint64_t)cpu->env.features[FEAT_XSAVE_XSS_HI]) << 32 |
+           cpu->env.features[FEAT_XSAVE_XSS_LO];
 }
 
 const char *get_register_name_32(unsigned int reg)
@@ -4227,23 +4283,37 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         }
 
         if (count == 0) {
-            *ecx = xsave_area_size(x86_cpu_xsave_components(cpu));
-            *eax = env->features[FEAT_XSAVE_COMP_LO];
-            *edx = env->features[FEAT_XSAVE_COMP_HI];
+            *ecx = xsave_area_size(x86_cpu_xsave_xcr0_components(cpu), false);
+            *eax = env->features[FEAT_XSAVE_XCR0_LO];
+            *edx = env->features[FEAT_XSAVE_XCR0_HI];
             /*
              * The initial value of xcr0 and ebx == 0, On host without kvm
              * commit 412a3c41(e.g., CentOS 6), the ebx's value always == 0
              * even through guest update xcr0, this will crash some legacy guest
              * (e.g., CentOS 6), So set ebx == ecx to workaroud it.
              */
-            *ebx = xsave_area_size(env->xcr0);
+            *ebx = xsave_area_size(env->xcr0, false);
         } else if (count == 1) {
+            uint64_t xstate = x86_cpu_xsave_xcr0_components(cpu) |
+                              x86_cpu_xsave_xss_components(cpu);
+
             *eax = env->features[FEAT_XSAVE];
+            *ebx = xsave_area_size(xstate, true);
+            *ecx = env->features[FEAT_XSAVE_XSS_LO];
+            *edx = env->features[FEAT_XSAVE_XSS_HI];
+            *ecx &= ~XSTATE_ARCH_LBR_MASK;
         } else if (count < ARRAY_SIZE(x86_ext_save_areas)) {
-            if ((x86_cpu_xsave_components(cpu) >> count) & 1) {
-                const ExtSaveArea *esa = &x86_ext_save_areas[count];
+            const ExtSaveArea *esa = &x86_ext_save_areas[count];
+
+            if ((x86_cpu_xsave_xcr0_components(cpu) >> count) & 1) {
                 *eax = esa->size;
                 *ebx = esa->offset;
+                *ecx = esa->ecx &
+                       (ESA_FEATURE_ALIGN64_MASK | ESA_FEATURE_XFD_MASK);
+            } else if ((x86_cpu_xsave_xss_components(cpu) >> count) & 1) {
+                *eax = esa->size;
+                *ebx = 0;
+                *ecx = 1;
             }
         }
         break;
@@ -4268,6 +4338,42 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         }
         break;
     }
+    case 0x1D: {
+        /* AMX TILE */
+        *eax = 0;
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        if (!(env->features[FEAT_7_0_EDX] & CPUID_7_0_EDX_AMX_TILE)) {
+            break;
+        }
+
+        if (count == 0) {
+            *eax = INTEL_AMX_TILE_MAX_SUBLEAF;
+        } else if (count == 1) {
+            *eax = INTEL_AMX_TOTAL_TILE_BYTES |
+                   (INTEL_AMX_BYTES_PER_TILE << 16);
+            *ebx = INTEL_AMX_BYTES_PER_ROW |
+                   (INTEL_AMX_TILE_MAX_NAMES << 16);
+            *ecx = INTEL_AMX_TILE_MAX_ROWS;
+        }
+        break;
+    }
+    case 0x1E: {
+        /* AMX TMUL */
+        *eax = 0;
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        if (!(env->features[FEAT_7_0_EDX] & CPUID_7_0_EDX_AMX_TILE)) {
+            break;
+        }
+
+        if (count == 0) {
+            *ebx = INTEL_AMX_TMUL_MAX_K | (INTEL_AMX_TMUL_MAX_N << 8);
+        }
+        break;
+    }
     case 0x40000000:
         /*
          * CPUID code in kvm_arch_init_vcpu() ignores stuff
@@ -4546,6 +4652,9 @@ static void x86_cpu_reset(CPUState *dev)
     }
     for (i = 2; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
         const ExtSaveArea *esa = &x86_ext_save_areas[i];
+        if (!((1ULL << i) & CPUID_XSTATE_XCR0_MASK)) {
+            continue;
+        }
         if (env->features[esa->feature] & esa->bits) {
             xcr0 |= 1ull << i;
         }
@@ -4635,6 +4744,10 @@ static void x86_cpu_enable_xsave_components(X86CPU *cpu)
     uint64_t mask;
 
     if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) {
+        env->features[FEAT_XSAVE_XCR0_LO] = 0;
+        env->features[FEAT_XSAVE_XCR0_HI] = 0;
+        env->features[FEAT_XSAVE_XSS_LO] = 0;
+        env->features[FEAT_XSAVE_XSS_HI] = 0;
         return;
     }
 
@@ -4646,8 +4759,10 @@ static void x86_cpu_enable_xsave_components(X86CPU *cpu)
         }
     }
 
-    env->features[FEAT_XSAVE_COMP_LO] = mask;
-    env->features[FEAT_XSAVE_COMP_HI] = mask >> 32;
+    env->features[FEAT_XSAVE_XCR0_LO] = mask & CPUID_XSTATE_XCR0_MASK;
+    env->features[FEAT_XSAVE_XCR0_HI] = (mask & CPUID_XSTATE_XCR0_MASK) >> 32;
+    env->features[FEAT_XSAVE_XSS_LO] = mask & CPUID_XSTATE_XSS_MASK;
+    env->features[FEAT_XSAVE_XSS_HI] = (mask & CPUID_XSTATE_XSS_MASK) >> 32;
 }
 
 /***** Steps involved on loading and filtering CPUID data
diff --git a/qemu/target/i386/cpu.h b/qemu/target/i386/cpu.h
index fbb2706d20..bd98aaf3d7 100644
--- a/qemu/target/i386/cpu.h
+++ b/qemu/target/i386/cpu.h
@@ -166,6 +166,8 @@ typedef enum X86Seg {
 #define HF_IOBPT_SHIFT      24 /* an io breakpoint enabled */
 #define HF_MPX_EN_SHIFT     25 /* MPX Enabled (CR4+XCR0+BNDCFGx) */
 #define HF_MPX_IU_SHIFT     26 /* BND registers in-use */
+#define HF_UMIP_SHIFT       27 /* CR4.UMIP */
+#define HF_AVX_EN_SHIFT     28 /* AVX Enabled (CR4+XCR0) */
 
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
 #define HF_INHIBIT_IRQ_MASK  (1 << HF_INHIBIT_IRQ_SHIFT)
@@ -191,6 +193,8 @@ typedef enum X86Seg {
 #define HF_IOBPT_MASK        (1 << HF_IOBPT_SHIFT)
 #define HF_MPX_EN_MASK       (1 << HF_MPX_EN_SHIFT)
 #define HF_MPX_IU_MASK       (1 << HF_MPX_IU_SHIFT)
+#define HF_UMIP_MASK         (1 << HF_UMIP_SHIFT)
+#define HF_AVX_EN_MASK       (1 << HF_AVX_EN_SHIFT)
 
 /* hflags2 */
 
@@ -237,6 +241,7 @@ typedef enum X86Seg {
 #define CR4_OSFXSR_SHIFT 9
 #define CR4_OSFXSR_MASK (1U << CR4_OSFXSR_SHIFT)
 #define CR4_OSXMMEXCPT_MASK  (1U << 10)
+#define CR4_UMIP_MASK   (1U << 11)
 #define CR4_LA57_MASK   (1U << 12)
 #define CR4_VMXE_MASK   (1U << 13)
 #define CR4_SMXE_MASK   (1U << 14)
@@ -246,6 +251,17 @@ typedef enum X86Seg {
 #define CR4_SMEP_MASK   (1U << 20)
 #define CR4_SMAP_MASK   (1U << 21)
 #define CR4_PKE_MASK   (1U << 22)
+#define CR4_PKS_MASK   (1U << 24)
+
+#define CR4_RESERVED_MASK \
+    (~(target_ulong)(CR4_VME_MASK | CR4_PVI_MASK | CR4_TSD_MASK | \
+                     CR4_DE_MASK | CR4_PSE_MASK | CR4_PAE_MASK | \
+                     CR4_MCE_MASK | CR4_PGE_MASK | CR4_PCE_MASK | \
+                     CR4_OSFXSR_MASK | CR4_OSXMMEXCPT_MASK | \
+                     CR4_UMIP_MASK | CR4_LA57_MASK | \
+                     CR4_FSGSBASE_MASK | CR4_PCIDE_MASK | \
+                     CR4_OSXSAVE_MASK | CR4_SMEP_MASK | CR4_SMAP_MASK | \
+                     CR4_PKE_MASK | CR4_PKS_MASK))
 
 #define DR6_BD          (1 << 13)
 #define DR6_BS          (1 << 14)
@@ -354,6 +370,12 @@ typedef enum X86Seg {
 
 #define MSR_IA32_TSX_CTRL		0x122
 #define MSR_IA32_TSCDEADLINE            0x6e0
+#define MSR_IA32_PKRS                   0x6e1
+#define MSR_ARCH_LBR_CTL                0x000014ce
+#define MSR_ARCH_LBR_DEPTH              0x000014cf
+#define MSR_ARCH_LBR_FROM_0             0x00001500
+#define MSR_ARCH_LBR_TO_0               0x00001600
+#define MSR_ARCH_LBR_INFO_0             0x00001200
 
 #define FEATURE_CONTROL_LOCKED                    (1<<0)
 #define FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX (1<<2)
@@ -455,6 +477,9 @@ typedef enum X86Seg {
 
 #define MSR_VM_HSAVE_PA                 0xc0010117
 
+#define MSR_IA32_XFD                    0x000001c4
+#define MSR_IA32_XFD_ERR                0x000001c5
+
 #define MSR_IA32_BNDCFGS                0x00000d90
 #define MSR_IA32_XSS                    0x00000da0
 #define MSR_IA32_UMWAIT_CONTROL         0xe1
@@ -487,6 +512,9 @@ typedef enum X86Seg {
 #define XSTATE_ZMM_Hi256_BIT            6
 #define XSTATE_Hi16_ZMM_BIT             7
 #define XSTATE_PKRU_BIT                 9
+#define XSTATE_ARCH_LBR_BIT             15
+#define XSTATE_XTILE_CFG_BIT            17
+#define XSTATE_XTILE_DATA_BIT           18
 
 #define XSTATE_FP_MASK                  (1ULL << XSTATE_FP_BIT)
 #define XSTATE_SSE_MASK                 (1ULL << XSTATE_SSE_BIT)
@@ -497,6 +525,26 @@ typedef enum X86Seg {
 #define XSTATE_ZMM_Hi256_MASK           (1ULL << XSTATE_ZMM_Hi256_BIT)
 #define XSTATE_Hi16_ZMM_MASK            (1ULL << XSTATE_Hi16_ZMM_BIT)
 #define XSTATE_PKRU_MASK                (1ULL << XSTATE_PKRU_BIT)
+#define XSTATE_ARCH_LBR_MASK            (1ULL << XSTATE_ARCH_LBR_BIT)
+#define XSTATE_XTILE_CFG_MASK           (1ULL << XSTATE_XTILE_CFG_BIT)
+#define XSTATE_XTILE_DATA_MASK          (1ULL << XSTATE_XTILE_DATA_BIT)
+
+#define XSTATE_DYNAMIC_MASK             (XSTATE_XTILE_DATA_MASK)
+
+#define ESA_FEATURE_ALIGN64_BIT         1
+#define ESA_FEATURE_XFD_BIT             2
+
+#define ESA_FEATURE_ALIGN64_MASK        (1U << ESA_FEATURE_ALIGN64_BIT)
+#define ESA_FEATURE_XFD_MASK            (1U << ESA_FEATURE_XFD_BIT)
+
+#define CPUID_XSTATE_XCR0_MASK  (XSTATE_FP_MASK | XSTATE_SSE_MASK | \
+                                 XSTATE_YMM_MASK | XSTATE_BNDREGS_MASK | \
+                                 XSTATE_BNDCSR_MASK | XSTATE_OPMASK_MASK | \
+                                 XSTATE_ZMM_Hi256_MASK | \
+                                 XSTATE_Hi16_ZMM_MASK | XSTATE_PKRU_MASK | \
+                                 XSTATE_XTILE_CFG_MASK | XSTATE_XTILE_DATA_MASK)
+
+#define CPUID_XSTATE_XSS_MASK   (XSTATE_ARCH_LBR_MASK)
 
 /* CPUID feature words */
 typedef enum FeatureWord {
@@ -521,8 +569,10 @@ typedef enum FeatureWord {
     FEAT_SVM,           /* CPUID[8000_000A].EDX */
     FEAT_XSAVE,         /* CPUID[EAX=0xd,ECX=1].EAX */
     FEAT_6_EAX,         /* CPUID[6].EAX */
-    FEAT_XSAVE_COMP_LO, /* CPUID[EAX=0xd,ECX=0].EAX */
-    FEAT_XSAVE_COMP_HI, /* CPUID[EAX=0xd,ECX=0].EDX */
+    FEAT_XSAVE_XCR0_LO, /* CPUID[EAX=0xd,ECX=0].EAX */
+    FEAT_XSAVE_XCR0_HI, /* CPUID[EAX=0xd,ECX=0].EDX */
+    FEAT_XSAVE_XSS_LO,  /* CPUID[EAX=0xd,ECX=1].ECX */
+    FEAT_XSAVE_XSS_HI,  /* CPUID[EAX=0xd,ECX=1].EDX */
     FEAT_ARCH_CAPABILITIES,
     FEAT_CORE_CAPABILITY,
     FEAT_VMX_PROCBASED_CTLS,
@@ -675,6 +725,8 @@ typedef uint64_t FeatureWordArray[FEATURE_WORDS];
 #define CPUID_SVM_DECODEASSIST (1U << 7)
 #define CPUID_SVM_PAUSEFILTER  (1U << 10)
 #define CPUID_SVM_PFTHRESHOLD  (1U << 12)
+#define CPUID_SVM_VGIF         (1U << 16)
+#define CPUID_SVM_SVME_ADDR_CHK (1U << 28)
 
 /* Support RDFSBASE/RDGSBASE/WRFSBASE/WRGSBASE */
 #define CPUID_7_0_EBX_FSGSBASE          (1U << 0)
@@ -763,11 +815,27 @@ typedef uint64_t FeatureWordArray[FEATURE_WORDS];
 #define CPUID_7_0_ECX_MOVDIRI           (1U << 27)
 /* Move 64 Bytes as Direct Store Instruction */
 #define CPUID_7_0_ECX_MOVDIR64B         (1U << 28)
+/* Protection Keys for Supervisor-mode Pages */
+#define CPUID_7_0_ECX_PKS               (1U << 31)
 
 /* AVX512 Neural Network Instructions */
 #define CPUID_7_0_EDX_AVX512_4VNNIW     (1U << 2)
 /* AVX512 Multiply Accumulation Single Precision */
 #define CPUID_7_0_EDX_AVX512_4FMAPS     (1U << 3)
+/* Fast Short REP MOV */
+#define CPUID_7_0_EDX_FSRM              (1U << 4)
+/* AVX512 VP2INTERSECT instructions */
+#define CPUID_7_0_EDX_AVX512_VP2INTERSECT (1U << 8)
+/* SERIALIZE instruction */
+#define CPUID_7_0_EDX_SERIALIZE         (1U << 14)
+/* TSX suspend load address tracking instruction */
+#define CPUID_7_0_EDX_TSX_LDTRK         (1U << 16)
+/* Architectural Last Branch Records */
+#define CPUID_7_0_EDX_ARCH_LBR          (1U << 19)
+/* AVX512 FP16 instruction */
+#define CPUID_7_0_EDX_AVX512_FP16       (1U << 23)
+/* AMX tile state */
+#define CPUID_7_0_EDX_AMX_TILE          (1U << 24)
 /* Speculation Control */
 #define CPUID_7_0_EDX_SPEC_CTRL         (1U << 26)
 /* Single Thread Indirect Branch Predictors */
@@ -797,6 +865,7 @@ typedef uint64_t FeatureWordArray[FEATURE_WORDS];
 #define CPUID_XSAVE_XSAVEC     (1U << 1)
 #define CPUID_XSAVE_XGETBV1    (1U << 2)
 #define CPUID_XSAVE_XSAVES     (1U << 3)
+#define CPUID_D_1_EAX_XFD      (1U << 4)
 
 #define CPUID_6_EAX_ARAT       (1U << 2)
 
@@ -1109,26 +1178,18 @@ typedef struct SegmentCache {
         uint16_t _w_##n[(bits)/16]; \
         uint32_t _l_##n[(bits)/32]; \
         uint64_t _q_##n[(bits)/64]; \
+        float16  _h_##n[(bits)/16]; \
         float32  _s_##n[(bits)/32]; \
         float64  _d_##n[(bits)/64]; \
     }
 
-typedef union {
-    uint8_t _b[16];
-    uint16_t _w[8];
-    uint32_t _l[4];
-    uint64_t _q[2];
-    float32 _s[4];
-    float64 _d[2];
+typedef union XMMReg {
+    uint64_t _q_XMMReg[128 / 64];
 } XMMReg;
 
-typedef union {
-    uint8_t _b[32];
-    uint16_t _w[16];
-    uint32_t _l[8];
-    uint64_t _q[4];
-    float32 _s[8];
-    float64 _d[4];
+typedef union YMMReg {
+    uint64_t _q_YMMReg[256 / 64];
+    XMMReg _x_YMMReg[256 / 128];
 } YMMReg;
 
 #if 0
@@ -1142,7 +1203,17 @@ typedef union {
 } ZMMReg;
 #endif
 
-typedef MMREG_UNION(ZMMReg, 512) ZMMReg;
+typedef union ZMMReg {
+    uint8_t _b_ZMMReg[512 / 8];
+    uint16_t _w_ZMMReg[512 / 16];
+    uint32_t _l_ZMMReg[512 / 32];
+    uint64_t _q_ZMMReg[512 / 64];
+    float16 _h_ZMMReg[512 / 16];
+    float32 _s_ZMMReg[512 / 32];
+    float64 _d_ZMMReg[512 / 64];
+    XMMReg _x_ZMMReg[512 / 128];
+    YMMReg _y_ZMMReg[512 / 256];
+} ZMMReg;
 typedef MMREG_UNION(MMXReg, 64)  MMXReg;
 
 typedef struct BNDReg {
@@ -1163,9 +1234,17 @@ typedef struct BNDCSReg {
 #define ZMM_B(n) _b_ZMMReg[63 - (n)]
 #define ZMM_W(n) _w_ZMMReg[31 - (n)]
 #define ZMM_L(n) _l_ZMMReg[15 - (n)]
+#define ZMM_H(n) _h_ZMMReg[31 - (n)]
 #define ZMM_S(n) _s_ZMMReg[15 - (n)]
 #define ZMM_Q(n) _q_ZMMReg[7 - (n)]
 #define ZMM_D(n) _d_ZMMReg[7 - (n)]
+#define ZMM_X(n) _x_ZMMReg[3 - (n)]
+#define ZMM_Y(n) _y_ZMMReg[1 - (n)]
+
+#define XMM_Q(n) _q_XMMReg[1 - (n)]
+
+#define YMM_Q(n) _q_YMMReg[3 - (n)]
+#define YMM_X(n) _x_YMMReg[1 - (n)]
 
 #define MMX_B(n) _b_MMXReg[7 - (n)]
 #define MMX_W(n) _w_MMXReg[3 - (n)]
@@ -1175,9 +1254,17 @@ typedef struct BNDCSReg {
 #define ZMM_B(n) _b_ZMMReg[n]
 #define ZMM_W(n) _w_ZMMReg[n]
 #define ZMM_L(n) _l_ZMMReg[n]
+#define ZMM_H(n) _h_ZMMReg[n]
 #define ZMM_S(n) _s_ZMMReg[n]
 #define ZMM_Q(n) _q_ZMMReg[n]
 #define ZMM_D(n) _d_ZMMReg[n]
+#define ZMM_X(n) _x_ZMMReg[n]
+#define ZMM_Y(n) _y_ZMMReg[n]
+
+#define XMM_Q(n) _q_XMMReg[n]
+
+#define YMM_Q(n) _q_YMMReg[n]
+#define YMM_X(n) _x_YMMReg[n]
 
 #define MMX_B(n) _b_MMXReg[n]
 #define MMX_W(n) _w_MMXReg[n]
@@ -1278,6 +1365,34 @@ typedef struct XSavePKRU {
     uint32_t padding;
 } XSavePKRU;
 
+/* Ext. save area 17: AMX XTILECFG state */
+typedef struct XSaveXTILECFG {
+    uint8_t xtilecfg[64];
+} XSaveXTILECFG;
+
+/* Ext. save area 18: AMX XTILEDATA state */
+typedef struct XSaveXTILEDATA {
+    uint8_t xtiledata[8][1024];
+} XSaveXTILEDATA;
+
+typedef struct LBREntry {
+    uint64_t from;
+    uint64_t to;
+    uint64_t info;
+} LBREntry;
+
+#define ARCH_LBR_NR_ENTRIES 32
+
+/* Ext. save area 15: supervisor mode Arch LBR state */
+typedef struct XSavesArchLBR {
+    uint64_t lbr_ctl;
+    uint64_t lbr_depth;
+    uint64_t ler_from;
+    uint64_t ler_to;
+    uint64_t ler_info;
+    LBREntry lbr_records[ARCH_LBR_NR_ENTRIES];
+} XSavesArchLBR;
+
 typedef struct X86XSaveArea {
     X86LegacyXSaveArea legacy;
     X86XSaveHeader header;
@@ -1312,6 +1427,19 @@ QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, hi16_zmm_state) != 0x680);
 QEMU_BUILD_BUG_ON(sizeof(XSaveHi16_ZMM) != 0x400);
 QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, pkru_state) != 0xA80);
 QEMU_BUILD_BUG_ON(sizeof(XSavePKRU) != 0x8);
+QEMU_BUILD_BUG_ON(sizeof(XSaveXTILECFG) != 0x40);
+QEMU_BUILD_BUG_ON(sizeof(XSaveXTILEDATA) != 0x2000);
+QEMU_BUILD_BUG_ON(sizeof(XSavesArchLBR) != 0x328);
+
+typedef struct ExtSaveArea {
+    uint32_t feature, bits;
+    uint32_t offset, size;
+    uint32_t ecx;
+} ExtSaveArea;
+
+#define XSAVE_STATE_AREA_COUNT (XSTATE_XTILE_DATA_BIT + 1)
+
+extern ExtSaveArea x86_ext_save_areas[XSAVE_STATE_AREA_COUNT];
 
 typedef enum TPRAccess {
     TPR_ACCESS_READ,
@@ -1433,8 +1561,8 @@ typedef struct CPUX86State {
     float_status mmx_status; /* for 3DNow! float ops */
     float_status sse_status;
     uint32_t mxcsr;
-    ZMMReg xmm_regs[CPU_NB_REGS == 8 ? 8 : 32];
-    ZMMReg xmm_t0;
+    QEMU_ALIGN(16, ZMMReg xmm_regs[CPU_NB_REGS == 8 ? 8 : 32]);
+    QEMU_ALIGN(16, ZMMReg xmm_t0);
     MMXReg mmx_t0;
 
     /*
@@ -1446,6 +1574,8 @@ typedef struct CPUX86State {
     uint64_t opmask_regs[NB_OPMASK_REGS];
     YMMReg zmmh_regs[CPU_NB_REGS];          /* currently not in use */
     ZMMReg hi16_zmm_regs[CPU_NB_REGS];      /* currently not in use */
+    uint8_t xtilecfg[64];
+    uint8_t xtiledata[8192];
 
     /* sysenter registers */
     uint32_t sysenter_cs;
@@ -1486,6 +1616,7 @@ typedef struct CPUX86State {
     uint64_t msr_smi_count;
 
     uint32_t pkru;
+    uint32_t pkrs;
     uint32_t tsx_ctrl;
 
     uint64_t spec_ctrl;
@@ -1501,6 +1632,13 @@ typedef struct CPUX86State {
     uint64_t pv_eoi_en_msr;
     uint64_t poll_control_msr;
 
+    uint64_t msr_xfd;
+    uint64_t msr_xfd_err;
+
+    uint64_t msr_lbr_ctl;
+    uint64_t msr_lbr_depth;
+    LBREntry lbr_records[ARCH_LBR_NR_ENTRIES];
+
     /* exception/interrupt handling */
     int error_code;
     int exception_is_int;
@@ -1745,6 +1883,7 @@ void x86_register_ferr_irq(qemu_irq irq);
 void cpu_set_ignne(CPUX86State *env);
 
 /* mpx_helper.c */
+void cpu_sync_avx_hflag(CPUX86State *env);
 void cpu_sync_bndcs_hflags(CPUX86State *env);
 
 /* this function must always be used to load data in the segment
@@ -2073,6 +2212,7 @@ static inline bool cpu_vmx_maybe_enabled(CPUX86State *env)
 /* fpu_helper.c */
 void update_fp_status(CPUX86State *env);
 void update_mxcsr_status(CPUX86State *env);
+void update_mxcsr_from_sse_status(CPUX86State *env);
 
 static inline void cpu_set_mxcsr(CPUX86State *env, uint32_t mxcsr)
 {
@@ -2133,10 +2273,42 @@ const char *get_register_name_32(unsigned int reg);
 /* cpu.c */
 bool cpu_is_bsp(X86CPU *cpu);
 
-void x86_cpu_xrstor_all_areas(X86CPU *cpu, const X86XSaveArea *buf);
-void x86_cpu_xsave_all_areas(X86CPU *cpu, X86XSaveArea *buf);
+void x86_cpu_xrstor_all_areas(X86CPU *cpu, const void *buf, uint32_t buflen);
+void x86_cpu_xsave_all_areas(X86CPU *cpu, void *buf, uint32_t buflen);
+uint32_t xsave_area_size(uint64_t mask, bool compacted);
 void x86_update_hflags(CPUX86State* env);
 
+static inline uint64_t cr4_reserved_bits(CPUX86State *env)
+{
+    uint64_t reserved_bits = CR4_RESERVED_MASK;
+
+    if (!env->features[FEAT_XSAVE]) {
+        reserved_bits |= CR4_OSXSAVE_MASK;
+    }
+    if (!(env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_SMEP)) {
+        reserved_bits |= CR4_SMEP_MASK;
+    }
+    if (!(env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_SMAP)) {
+        reserved_bits |= CR4_SMAP_MASK;
+    }
+    if (!(env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_FSGSBASE)) {
+        reserved_bits |= CR4_FSGSBASE_MASK;
+    }
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_PKU)) {
+        reserved_bits |= CR4_PKE_MASK;
+    }
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_LA57)) {
+        reserved_bits |= CR4_LA57_MASK;
+    }
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_UMIP)) {
+        reserved_bits |= CR4_UMIP_MASK;
+    }
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_PKS)) {
+        reserved_bits |= CR4_PKS_MASK;
+    }
+    return reserved_bits;
+}
+
 int uc_check_cpu_x86_load_seg(CPUX86State *env, int seg_reg, int sel);
 X86CPU *cpu_x86_init(struct uc_struct *uc);
 
diff --git a/qemu/target/i386/decode-new.c.inc b/qemu/target/i386/decode-new.c.inc
new file mode 100644
index 0000000000..37a8b1533a
--- /dev/null
+++ b/qemu/target/i386/decode-new.c.inc
@@ -0,0 +1,1874 @@
+/*
+ * New-style decoder for i386 instructions
+ *
+ *  Copyright (c) 2022 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * The decoder is mostly based on tables copied from the Intel SDM.  As
+ * a result, most operand load and writeback is done entirely in common
+ * table-driven code using the same operand type (X86_TYPE_*) and
+ * size (X86_SIZE_*) codes used in the manual.
+ *
+ * The main difference is that the V, U and W types are extended to
+ * cover MMX as well; if an instruction is like
+ *
+ *      por   Pq, Qq
+ *  66  por   Vx, Hx, Wx
+ *
+ * only the second row is included and the instruction is marked as a
+ * valid MMX instruction.  The MMX flag directs the decoder to rewrite
+ * the V/U/H/W types to P/N/P/Q if there is no prefix, as well as changing
+ * "x" to "q" if there is no prefix.
+ *
+ * In addition, the ss/ps/sd/pd types are sometimes mushed together as "x"
+ * if the difference is expressed via prefixes.  Individual instructions
+ * are separated by prefix in the generator functions.
+ *
+ * There are a couple cases in which instructions (e.g. MOVD) write the
+ * whole XMM or MM register but are established incorrectly in the manual
+ * as "d" or "q".  These have to be fixed for the decoder to work correctly.
+ */
+
+#define X86_OP_NONE { 0 },
+
+#define X86_OP_GROUP3(op, op0_, s0_, op1_, s1_, op2_, s2_, ...) { \
+    .decode = glue(decode_, op),                                  \
+    .op0 = glue(X86_TYPE_, op0_),                                 \
+    .s0 = glue(X86_SIZE_, s0_),                                   \
+    .op1 = glue(X86_TYPE_, op1_),                                 \
+    .s1 = glue(X86_SIZE_, s1_),                                   \
+    .op2 = glue(X86_TYPE_, op2_),                                 \
+    .s2 = glue(X86_SIZE_, s2_),                                   \
+    .is_decode = true,                                            \
+    __VA_ARGS__                                                   \
+}
+
+#define X86_OP_GROUP2(op, op0, s0, op1, s1, ...)                  \
+    X86_OP_GROUP3(op, op0, s0, 2op, s0, op1, s1, __VA_ARGS__)
+#define X86_OP_GROUP0(op, ...)                                    \
+    X86_OP_GROUP3(op, None, None, None, None, None, None, __VA_ARGS__)
+
+#define X86_OP_ENTRY3(op, op0_, s0_, op1_, s1_, op2_, s2_, ...) { \
+    .gen = glue(gen_, op),                                        \
+    .op0 = glue(X86_TYPE_, op0_),                                 \
+    .s0 = glue(X86_SIZE_, s0_),                                   \
+    .op1 = glue(X86_TYPE_, op1_),                                 \
+    .s1 = glue(X86_SIZE_, s1_),                                   \
+    .op2 = glue(X86_TYPE_, op2_),                                 \
+    .s2 = glue(X86_SIZE_, s2_),                                   \
+    __VA_ARGS__                                                   \
+}
+
+#define X86_OP_ENTRY4(op, op0_, s0_, op1_, s1_, op2_, s2_, ...)   \
+    X86_OP_ENTRY3(op, op0_, s0_, op1_, s1_, op2_, s2_,            \
+        .op3 = X86_TYPE_I, .s3 = X86_SIZE_b,                      \
+        __VA_ARGS__)
+
+#define X86_OP_ENTRY2(op, op0, s0, op1, s1, ...)                  \
+    X86_OP_ENTRY3(op, op0, s0, 2op, s0, op1, s1, __VA_ARGS__)
+#define X86_OP_ENTRYw(op, op0, s0, ...)                           \
+    X86_OP_ENTRY3(op, op0, s0, None, None, None, None, __VA_ARGS__)
+#define X86_OP_ENTRYr(op, op0, s0, ...)                           \
+    X86_OP_ENTRY3(op, None, None, None, None, op0, s0, __VA_ARGS__)
+#define X86_OP_ENTRY0(op, ...)                                    \
+    X86_OP_ENTRY3(op, None, None, None, None, None, None, __VA_ARGS__)
+
+#define cpuid(feat) .cpuid = X86_FEAT_##feat,
+#define i64 .special = X86_SPECIAL_i64,
+#define o64 .special = X86_SPECIAL_o64,
+#define xchg .special = X86_SPECIAL_Locked,
+#define mmx .special = X86_SPECIAL_MMX,
+#define zext0 .special = X86_SPECIAL_ZExtOp0,
+#define zext2 .special = X86_SPECIAL_ZExtOp2,
+#define avx_movx .special = X86_SPECIAL_AVXExtMov,
+
+#define vex1 .vex_class = 1,
+#define vex1_rep3 .vex_class = 1, .vex_special = X86_VEX_REPScalar,
+#define vex2 .vex_class = 2,
+#define vex2_rep3 .vex_class = 2, .vex_special = X86_VEX_REPScalar,
+#define vex3 .vex_class = 3,
+#define vex4 .vex_class = 4,
+#define vex4_unal .vex_class = 4, .vex_special = X86_VEX_SSEUnaligned,
+#define vex4_rep5 .vex_class = 4, .vex_special = X86_VEX_REPScalar,
+#define vex5 .vex_class = 5,
+#define vex6 .vex_class = 6,
+#define vex7 .vex_class = 7,
+#define vex8 .vex_class = 8,
+#define vex11 .vex_class = 11,
+#define vex12 .vex_class = 12,
+#define vex13 .vex_class = 13,
+
+#define avx2_256 .vex_special = X86_VEX_AVX2_256,
+
+#define P_00          1
+#define P_66          (1 << PREFIX_DATA)
+#define P_F3          (1 << PREFIX_REPZ)
+#define P_F2          (1 << PREFIX_REPNZ)
+
+#define p_00          .valid_prefix = P_00,
+#define p_66          .valid_prefix = P_66,
+#define p_f3          .valid_prefix = P_F3,
+#define p_f2          .valid_prefix = P_F2,
+#define p_00_66       .valid_prefix = P_00 | P_66,
+#define p_00_f3       .valid_prefix = P_00 | P_F3,
+#define p_66_f2       .valid_prefix = P_66 | P_F2,
+#define p_00_66_f3    .valid_prefix = P_00 | P_66 | P_F3,
+#define p_66_f3_f2    .valid_prefix = P_66 | P_F3 | P_F2,
+#define p_00_66_f3_f2 .valid_prefix = P_00 | P_66 | P_F3 | P_F2,
+
+static uint8_t get_modrm(DisasContext *s, CPUX86State *env)
+{
+    if (!s->has_modrm) {
+        s->modrm = x86_ldub_code(env, s);
+        s->has_modrm = true;
+    }
+    return s->modrm;
+}
+
+static inline const X86OpEntry *decode_by_prefix(DisasContext *s, const X86OpEntry entries[4])
+{
+    if (s->prefix & PREFIX_REPNZ) {
+        return &entries[3];
+    } else if (s->prefix & PREFIX_REPZ) {
+        return &entries[2];
+    } else if (s->prefix & PREFIX_DATA) {
+        return &entries[1];
+    } else {
+        return &entries[0];
+    }
+}
+
+static void decode_group15(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    /* only includes ldmxcsr and stmxcsr, because they have AVX variants.  */
+    static const X86OpEntry group15_reg[8] = {
+    };
+
+    static const X86OpEntry group15_mem[8] = {
+        [2] = X86_OP_ENTRYr(LDMXCSR,    E,d, vex5),
+        [3] = X86_OP_ENTRYw(STMXCSR,    E,d, vex5),
+    };
+
+    uint8_t modrm = get_modrm(s, env);
+    if ((modrm >> 6) == 3) {
+        *entry = group15_reg[(modrm >> 3) & 7];
+    } else {
+        *entry = group15_mem[(modrm >> 3) & 7];
+    }
+}
+
+static void decode_group17(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86GenFunc group17_gen[8] = {
+        NULL, gen_BLSR, gen_BLSMSK, gen_BLSI,
+    };
+    int op = (get_modrm(s, env) >> 3) & 7;
+    entry->gen = group17_gen[op];
+}
+
+static void decode_group12(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_group12[8] = {
+        {},
+        {},
+        X86_OP_ENTRY3(PSRLW_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        {},
+        X86_OP_ENTRY3(PSRAW_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        {},
+        X86_OP_ENTRY3(PSLLW_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        {},
+    };
+
+    int op = (get_modrm(s, env) >> 3) & 7;
+    *entry = opcodes_group12[op];
+}
+
+static void decode_group13(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_group13[8] = {
+        {},
+        {},
+        X86_OP_ENTRY3(PSRLD_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        {},
+        X86_OP_ENTRY3(PSRAD_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        {},
+        X86_OP_ENTRY3(PSLLD_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        {},
+    };
+
+    int op = (get_modrm(s, env) >> 3) & 7;
+    *entry = opcodes_group13[op];
+}
+
+static void decode_group14(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_group14[8] = {
+        /* grp14 */
+        {},
+        {},
+        X86_OP_ENTRY3(PSRLQ_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        X86_OP_ENTRY3(PSRLDQ_i, H,x, U,x, I,b, vex7 avx2_256 p_66),
+        {},
+        {},
+        X86_OP_ENTRY3(PSLLQ_i,  H,x, U,x, I,b, vex7 mmx avx2_256 p_00_66),
+        X86_OP_ENTRY3(PSLLDQ_i, H,x, U,x, I,b, vex7 avx2_256 p_66),
+    };
+
+    int op = (get_modrm(s, env) >> 3) & 7;
+    *entry = opcodes_group14[op];
+}
+
+static void decode_0F6F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F6F[4] = {
+        X86_OP_ENTRY3(MOVDQ,       P,q, None,None, Q,q, vex5 mmx),  /* movq */
+        X86_OP_ENTRY3(MOVDQ,       V,x, None,None, W,x, vex1),      /* movdqa */
+        X86_OP_ENTRY3(MOVDQ,       V,x, None,None, W,x, vex4_unal), /* movdqu */
+        {},
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F6F);
+}
+
+static void decode_0F70(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry pshufw[4] = {
+        X86_OP_ENTRY3(PSHUFW,  P,q, Q,q, I,b, vex4 mmx),
+        X86_OP_ENTRY3(PSHUFD,  V,x, W,x, I,b, vex4 avx2_256),
+        X86_OP_ENTRY3(PSHUFHW, V,x, W,x, I,b, vex4 avx2_256),
+        X86_OP_ENTRY3(PSHUFLW, V,x, W,x, I,b, vex4 avx2_256),
+    };
+
+    *entry = *decode_by_prefix(s, pshufw);
+}
+
+static void decode_0F77(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    if (!(s->prefix & PREFIX_VEX)) {
+        entry->gen = gen_EMMS;
+    } else if (!s->vex_l) {
+        entry->gen = gen_VZEROUPPER;
+        entry->vex_class = 8;
+    } else {
+        entry->gen = gen_VZEROALL;
+        entry->vex_class = 8;
+    }
+}
+
+static void decode_0F78(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F78[4] = {
+        {},
+        X86_OP_ENTRY3(EXTRQ_i,       V,x, None,None, I,w,  cpuid(SSE4A)), /* AMD extension */
+        {},
+        X86_OP_ENTRY3(INSERTQ_i,     V,x, U,x, I,w,        cpuid(SSE4A)), /* AMD extension */
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F78);
+}
+
+static void decode_0F79(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    if (s->prefix & PREFIX_REPNZ) {
+        entry->gen = gen_INSERTQ_r; /* AMD extension */
+    } else if (s->prefix & PREFIX_DATA) {
+        entry->gen = gen_EXTRQ_r; /* AMD extension */
+    } else {
+        entry->gen = NULL;
+    };
+}
+
+static void decode_0F7E(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F7E[4] = {
+        X86_OP_ENTRY3(MOVD_from,  E,y, None,None, P,y, vex5 mmx),
+        X86_OP_ENTRY3(MOVD_from,  E,y, None,None, V,y, vex5),
+        X86_OP_ENTRY3(MOVQ,       V,x, None,None, W,q, vex5),  /* wrong dest Vy on SDM! */
+        {},
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F7E);
+}
+
+static void decode_0F7F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F7F[4] = {
+        X86_OP_ENTRY3(MOVDQ,       W,x, None,None, V,x, vex5 mmx), /* movq */
+        X86_OP_ENTRY3(MOVDQ,       W,x, None,None, V,x, vex1), /* movdqa */
+        X86_OP_ENTRY3(MOVDQ,       W,x, None,None, V,x, vex4_unal), /* movdqu */
+        {},
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F7F);
+}
+
+static void decode_0FD6(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry movq[4] = {
+        {},
+        X86_OP_ENTRY3(MOVQ,    W,x,  None, None, V,q, vex5),
+        X86_OP_ENTRY3(MOVq_dq, V,dq, None, None, N,q),
+        X86_OP_ENTRY3(MOVq_dq, P,q,  None, None, U,q),
+    };
+
+    *entry = *decode_by_prefix(s, movq);
+}
+
+static const X86OpEntry opcodes_0F38_00toEF[240] = {
+    [0x00] = X86_OP_ENTRY3(PSHUFB,    V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x01] = X86_OP_ENTRY3(PHADDW,    V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x02] = X86_OP_ENTRY3(PHADDD,    V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x03] = X86_OP_ENTRY3(PHADDSW,   V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x04] = X86_OP_ENTRY3(PMADDUBSW, V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x05] = X86_OP_ENTRY3(PHSUBW,    V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x06] = X86_OP_ENTRY3(PHSUBD,    V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x07] = X86_OP_ENTRY3(PHSUBSW,   V,x,  H,x,   W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+
+    [0x10] = X86_OP_ENTRY2(PBLENDVB,  V,x,         W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x13] = X86_OP_ENTRY2(VCVTPH2PS, V,x,         W,xh, vex11 cpuid(F16C) p_66),
+    [0x14] = X86_OP_ENTRY2(BLENDVPS,  V,x,         W,x,  vex4 cpuid(SSE41) p_66),
+    [0x15] = X86_OP_ENTRY2(BLENDVPD,  V,x,         W,x,  vex4 cpuid(SSE41) p_66),
+    /* Listed incorrectly as type 4 */
+    [0x16] = X86_OP_ENTRY3(VPERMD,    V,qq, H,qq,      W,qq,  vex6 cpuid(AVX2) p_66),
+    [0x17] = X86_OP_ENTRY3(VPTEST,    None,None, V,x,  W,x,   vex4 cpuid(SSE41) p_66),
+
+    /*
+     * Source operand listed as Mq/Ux and similar in the manual; incorrectly listed
+     * as 128-bit only in 2-17.
+     */
+    [0x20] = X86_OP_ENTRY3(VPMOVSXBW, V,x,  None,None, W,q,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x21] = X86_OP_ENTRY3(VPMOVSXBD, V,x,  None,None, W,d,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x22] = X86_OP_ENTRY3(VPMOVSXBQ, V,x,  None,None, W,w,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x23] = X86_OP_ENTRY3(VPMOVSXWD, V,x,  None,None, W,q,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x24] = X86_OP_ENTRY3(VPMOVSXWQ, V,x,  None,None, W,d,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x25] = X86_OP_ENTRY3(VPMOVSXDQ, V,x,  None,None, W,q,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+
+    /* Same as PMOVSX.  */
+    [0x30] = X86_OP_ENTRY3(VPMOVZXBW, V,x,  None,None, W,q,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x31] = X86_OP_ENTRY3(VPMOVZXBD, V,x,  None,None, W,d,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x32] = X86_OP_ENTRY3(VPMOVZXBQ, V,x,  None,None, W,w,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x33] = X86_OP_ENTRY3(VPMOVZXWD, V,x,  None,None, W,q,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x34] = X86_OP_ENTRY3(VPMOVZXWQ, V,x,  None,None, W,d,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x35] = X86_OP_ENTRY3(VPMOVZXDQ, V,x,  None,None, W,q,   vex5 cpuid(SSE41) avx_movx avx2_256 p_66),
+    [0x36] = X86_OP_ENTRY3(VPERMD,    V,qq, H,qq,      W,qq,  vex6 cpuid(AVX2) p_66),
+    [0x37] = X86_OP_ENTRY3(PCMPGTQ,   V,x,  H,x,       W,x,   vex4 cpuid(SSE42) avx2_256 p_66),
+
+    [0x40] = X86_OP_ENTRY3(PMULLD,      V,x,  H,x,       W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x41] = X86_OP_ENTRY3(VPHMINPOSUW, V,dq, None,None, W,dq, vex4 cpuid(SSE41) p_66),
+    /* Listed incorrectly as type 4 */
+    [0x45] = X86_OP_ENTRY3(VPSRLV,      V,x,  H,x,       W,x,  vex6 cpuid(AVX2) p_66),
+    [0x46] = X86_OP_ENTRY3(VPSRAV,      V,x,  H,x,       W,x,  vex6 cpuid(AVX2) p_66),
+    [0x47] = X86_OP_ENTRY3(VPSLLV,      V,x,  H,x,       W,x,  vex6 cpuid(AVX2) p_66),
+
+    [0x90] = X86_OP_ENTRY3(VPGATHERD, V,x,  H,x,  M,d,  vex12 cpuid(AVX2) p_66), /* vpgatherdd/q */
+    [0x91] = X86_OP_ENTRY3(VPGATHERQ, V,x,  H,x,  M,q,  vex12 cpuid(AVX2) p_66), /* vpgatherqd/q */
+    [0x92] = X86_OP_ENTRY3(VPGATHERD, V,x,  H,x,  M,d,  vex12 cpuid(AVX2) p_66), /* vgatherdps/d */
+    [0x93] = X86_OP_ENTRY3(VPGATHERQ, V,x,  H,x,  M,q,  vex12 cpuid(AVX2) p_66), /* vgatherqps/d */
+
+    /* Should be exception type 2 but they do not have legacy SSE equivalents? */
+    [0x96] = X86_OP_ENTRY3(VFMADDSUB132Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x97] = X86_OP_ENTRY3(VFMSUBADD132Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+
+    [0xa6] = X86_OP_ENTRY3(VFMADDSUB213Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xa7] = X86_OP_ENTRY3(VFMSUBADD213Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+
+    [0xb6] = X86_OP_ENTRY3(VFMADDSUB231Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xb7] = X86_OP_ENTRY3(VFMSUBADD231Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+
+    [0x08] = X86_OP_ENTRY3(PSIGNB,    V,x,        H,x,  W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x09] = X86_OP_ENTRY3(PSIGNW,    V,x,        H,x,  W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x0a] = X86_OP_ENTRY3(PSIGND,    V,x,        H,x,  W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x0b] = X86_OP_ENTRY3(PMULHRSW,  V,x,        H,x,  W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x0c] = X86_OP_ENTRY3(VPERMILPS, V,x,        H,x,  W,x,  vex4 cpuid(AVX) p_00_66),
+    [0x0d] = X86_OP_ENTRY3(VPERMILPD, V,x,        H,x,  W,x,  vex4 cpuid(AVX) p_66),
+    [0x0e] = X86_OP_ENTRY3(VTESTPS,   None,None,  V,x,  W,x,  vex4 cpuid(AVX) p_66),
+    [0x0f] = X86_OP_ENTRY3(VTESTPD,   None,None,  V,x,  W,x,  vex4 cpuid(AVX) p_66),
+
+    [0x18] = X86_OP_ENTRY3(VPBROADCASTD,   V,x,  None,None, W,d,  vex6 cpuid(AVX) p_66), /* vbroadcastss */
+    [0x19] = X86_OP_ENTRY3(VPBROADCASTQ,   V,qq, None,None, W,q,  vex6 cpuid(AVX) p_66), /* vbroadcastsd */
+    [0x1a] = X86_OP_ENTRY3(VBROADCASTx128, V,qq, None,None, WM,dq,vex6 cpuid(AVX) p_66),
+    [0x1c] = X86_OP_ENTRY3(PABSB,          V,x,  None,None, W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x1d] = X86_OP_ENTRY3(PABSW,          V,x,  None,None, W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+    [0x1e] = X86_OP_ENTRY3(PABSD,          V,x,  None,None, W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+
+    [0x28] = X86_OP_ENTRY3(PMULDQ,        V,x, H,x,       W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x29] = X86_OP_ENTRY3(PCMPEQQ,       V,x, H,x,       W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x2a] = X86_OP_ENTRY3(MOVDQ,         V,x, None,None, WM,x, vex1 cpuid(SSE41) avx2_256 p_66), /* movntdqa */
+    [0x2b] = X86_OP_ENTRY3(VPACKUSDW,     V,x, H,x,       W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x2c] = X86_OP_ENTRY3(VMASKMOVPS,    V,x, H,x,       WM,x, vex6 cpuid(AVX) p_66),
+    [0x2d] = X86_OP_ENTRY3(VMASKMOVPD,    V,x, H,x,       WM,x, vex6 cpuid(AVX) p_66),
+    /* Incorrectly listed as Mx,Hx,Vx in the manual */
+    [0x2e] = X86_OP_ENTRY3(VMASKMOVPS_st, M,x, V,x,       H,x,  vex6 cpuid(AVX) p_66),
+    [0x2f] = X86_OP_ENTRY3(VMASKMOVPD_st, M,x, V,x,       H,x,  vex6 cpuid(AVX) p_66),
+
+    [0x38] = X86_OP_ENTRY3(PMINSB,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x39] = X86_OP_ENTRY3(PMINSD,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x3a] = X86_OP_ENTRY3(PMINUW,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x3b] = X86_OP_ENTRY3(PMINUD,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x3c] = X86_OP_ENTRY3(PMAXSB,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x3d] = X86_OP_ENTRY3(PMAXSD,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x3e] = X86_OP_ENTRY3(PMAXUW,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x3f] = X86_OP_ENTRY3(PMAXUD,        V,x,  H,x, W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+
+    [0x58] = X86_OP_ENTRY3(VPBROADCASTD,   V,x,  None,None, W,d,  vex6 cpuid(AVX2) p_66),
+    [0x59] = X86_OP_ENTRY3(VPBROADCASTQ,   V,x,  None,None, W,q,  vex6 cpuid(AVX2) p_66),
+    [0x5a] = X86_OP_ENTRY3(VBROADCASTx128, V,qq, None,None, WM,dq,vex6 cpuid(AVX2) p_66),
+
+    [0x78] = X86_OP_ENTRY3(VPBROADCASTB,   V,x,  None,None, W,b,  vex6 cpuid(AVX2) p_66),
+    [0x79] = X86_OP_ENTRY3(VPBROADCASTW,   V,x,  None,None, W,w,  vex6 cpuid(AVX2) p_66),
+
+    [0x8c] = X86_OP_ENTRY3(VPMASKMOV,    V,x,  H,x, WM,x, vex6 cpuid(AVX2) p_66),
+    [0x8e] = X86_OP_ENTRY3(VPMASKMOV_st, M,x,  V,x, H,x,  vex6 cpuid(AVX2) p_66),
+
+    /* Should be exception type 2 or 3 but they do not have legacy SSE equivalents? */
+    [0x98] = X86_OP_ENTRY3(VFMADD132Px,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x99] = X86_OP_ENTRY3(VFMADD132Sx,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x9a] = X86_OP_ENTRY3(VFMSUB132Px,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x9b] = X86_OP_ENTRY3(VFMSUB132Sx,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x9c] = X86_OP_ENTRY3(VFNMADD132Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x9d] = X86_OP_ENTRY3(VFNMADD132Sx, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x9e] = X86_OP_ENTRY3(VFNMSUB132Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0x9f] = X86_OP_ENTRY3(VFNMSUB132Sx, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+
+    [0xa8] = X86_OP_ENTRY3(VFMADD213Px,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xa9] = X86_OP_ENTRY3(VFMADD213Sx,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xaa] = X86_OP_ENTRY3(VFMSUB213Px,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xab] = X86_OP_ENTRY3(VFMSUB213Sx,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xac] = X86_OP_ENTRY3(VFNMADD213Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xad] = X86_OP_ENTRY3(VFNMADD213Sx, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xae] = X86_OP_ENTRY3(VFNMSUB213Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xaf] = X86_OP_ENTRY3(VFNMSUB213Sx, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+
+    [0xb8] = X86_OP_ENTRY3(VFMADD231Px,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xb9] = X86_OP_ENTRY3(VFMADD231Sx,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xba] = X86_OP_ENTRY3(VFMSUB231Px,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xbb] = X86_OP_ENTRY3(VFMSUB231Sx,  V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xbc] = X86_OP_ENTRY3(VFNMADD231Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xbd] = X86_OP_ENTRY3(VFNMADD231Sx, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xbe] = X86_OP_ENTRY3(VFNMSUB231Px, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+    [0xbf] = X86_OP_ENTRY3(VFNMSUB231Sx, V,x,  H,x, W,x,  vex6 cpuid(FMA) p_66),
+
+    [0xdb] = X86_OP_ENTRY3(VAESIMC,     V,dq, None,None, W,dq, vex4 cpuid(AES) p_66),
+    [0xdc] = X86_OP_ENTRY3(VAESENC,     V,x,  H,x,       W,x,  vex4 cpuid(AES) p_66),
+    [0xdd] = X86_OP_ENTRY3(VAESENCLAST, V,x,  H,x,       W,x,  vex4 cpuid(AES) p_66),
+    [0xde] = X86_OP_ENTRY3(VAESDEC,     V,x,  H,x,       W,x,  vex4 cpuid(AES) p_66),
+    [0xdf] = X86_OP_ENTRY3(VAESDECLAST, V,x,  H,x,       W,x,  vex4 cpuid(AES) p_66),
+};
+
+/* five rows for no prefix, 66, F3, F2, 66+F2  */
+static const X86OpEntry opcodes_0F38_F0toFF[16][5] = {
+    [0] = {
+        X86_OP_ENTRY3(MOVBE, G,y, M,y, None,None, cpuid(MOVBE)),
+        X86_OP_ENTRY3(MOVBE, G,w, M,w, None,None, cpuid(MOVBE)),
+        {},
+        X86_OP_ENTRY2(CRC32, G,d, E,b, cpuid(SSE42)),
+        X86_OP_ENTRY2(CRC32, G,d, E,b, cpuid(SSE42)),
+    },
+    [1] = {
+        X86_OP_ENTRY3(MOVBE, M,y, G,y, None,None, cpuid(MOVBE)),
+        X86_OP_ENTRY3(MOVBE, M,w, G,w, None,None, cpuid(MOVBE)),
+        {},
+        X86_OP_ENTRY2(CRC32, G,d, E,y, cpuid(SSE42)),
+        X86_OP_ENTRY2(CRC32, G,d, E,w, cpuid(SSE42)),
+    },
+    [2] = {
+        X86_OP_ENTRY3(ANDN, G,y, B,y, E,y, vex13 cpuid(BMI1)),
+        {},
+        {},
+        {},
+        {},
+    },
+    [3] = {
+        X86_OP_GROUP3(group17, B,y, E,y, None,None, vex13 cpuid(BMI1)),
+        {},
+        {},
+        {},
+        {},
+    },
+    [5] = {
+        X86_OP_ENTRY3(BZHI, G,y, E,y, B,y, vex13 cpuid(BMI1)),
+        {},
+        X86_OP_ENTRY3(PEXT, G,y, B,y, E,y, vex13 cpuid(BMI2)),
+        X86_OP_ENTRY3(PDEP, G,y, B,y, E,y, vex13 cpuid(BMI2)),
+        {},
+    },
+    [6] = {
+        {},
+        X86_OP_ENTRY2(ADCX, G,y, E,y, cpuid(ADX)),
+        X86_OP_ENTRY2(ADOX, G,y, E,y, cpuid(ADX)),
+        X86_OP_ENTRY3(MULX, /* B,y, */ G,y, E,y, 2,y, vex13 cpuid(BMI2)),
+        {},
+    },
+    [7] = {
+        X86_OP_ENTRY3(BEXTR, G,y, E,y, B,y, vex13 cpuid(BMI1)),
+        X86_OP_ENTRY3(SHLX, G,y, E,y, B,y, vex13 cpuid(BMI1)),
+        X86_OP_ENTRY3(SARX, G,y, E,y, B,y, vex13 cpuid(BMI1)),
+        X86_OP_ENTRY3(SHRX, G,y, E,y, B,y, vex13 cpuid(BMI1)),
+        {},
+    },
+};
+
+static void decode_0F38(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    *b = x86_ldub_code(env, s);
+    if (*b < 0xf0) {
+        *entry = opcodes_0F38_00toEF[*b];
+    } else {
+        int row = 0;
+        if (s->prefix & PREFIX_REPZ) {
+            /* The REPZ (F3) prefix has priority over 66 */
+            row = 2;
+        } else {
+            row += s->prefix & PREFIX_REPNZ ? 3 : 0;
+            row += s->prefix & PREFIX_DATA ? 1 : 0;
+        }
+        *entry = opcodes_0F38_F0toFF[*b & 15][row];
+    }
+}
+
+static void decode_VINSERTPS(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry
+        vinsertps_reg = X86_OP_ENTRY4(VINSERTPS_r, V,dq, H,dq, U,dq, vex5 cpuid(SSE41) p_66),
+        vinsertps_mem = X86_OP_ENTRY4(VINSERTPS_m, V,dq, H,dq, M,d,  vex5 cpuid(SSE41) p_66);
+
+    int modrm = get_modrm(s, env);
+    *entry = (modrm >> 6) == 3 ? vinsertps_reg : vinsertps_mem;
+}
+
+static const X86OpEntry opcodes_0F3A[256] = {
+    /*
+     * These are VEX-only, but incorrectly listed in the manual as exception type 4.
+     * Also the "qq" instructions are sometimes omitted by Table 2-17, but are VEX256
+     * only.
+     */
+    [0x00] = X86_OP_ENTRY3(VPERMQ,      V,qq, W,qq, I,b,  vex6 cpuid(AVX2) p_66),
+    [0x01] = X86_OP_ENTRY3(VPERMQ,      V,qq, W,qq, I,b,  vex6 cpuid(AVX2) p_66), /* VPERMPD */
+    [0x02] = X86_OP_ENTRY4(VBLENDPS,    V,x,  H,x,  W,x,  vex6 cpuid(AVX2) p_66), /* VPBLENDD */
+    [0x04] = X86_OP_ENTRY3(VPERMILPS_i, V,x,  W,x,  I,b,  vex6 cpuid(AVX) p_66),
+    [0x05] = X86_OP_ENTRY3(VPERMILPD_i, V,x,  W,x,  I,b,  vex6 cpuid(AVX) p_66),
+    [0x06] = X86_OP_ENTRY4(VPERM2x128,  V,qq, H,qq, W,qq, vex6 cpuid(AVX) p_66),
+
+    [0x14] = X86_OP_ENTRY3(PEXTRB,     E,b,  V,dq, I,b,  vex5 cpuid(SSE41) zext0 p_66),
+    [0x15] = X86_OP_ENTRY3(PEXTRW,     E,w,  V,dq, I,b,  vex5 cpuid(SSE41) zext0 p_66),
+    [0x16] = X86_OP_ENTRY3(PEXTR,      E,y,  V,dq, I,b,  vex5 cpuid(SSE41) p_66),
+    [0x17] = X86_OP_ENTRY3(VEXTRACTPS, E,d,  V,dq, I,b,  vex5 cpuid(SSE41) p_66),
+    [0x1d] = X86_OP_ENTRY3(VCVTPS2PH,  W,xh, V,x,  I,b,  vex11 cpuid(F16C) p_66),
+
+    [0x20] = X86_OP_ENTRY4(PINSRB,     V,dq, H,dq, E,b,  vex5 cpuid(SSE41) zext2 p_66),
+    [0x21] = X86_OP_GROUP0(VINSERTPS),
+    [0x22] = X86_OP_ENTRY4(PINSR,      V,dq, H,dq, E,y,  vex5 cpuid(SSE41) p_66),
+
+    [0x40] = X86_OP_ENTRY4(VDDPS,      V,x,  H,x,  W,x,  vex2 cpuid(SSE41) p_66),
+    [0x41] = X86_OP_ENTRY4(VDDPD,      V,dq, H,dq, W,dq, vex2 cpuid(SSE41) p_66),
+    [0x42] = X86_OP_ENTRY4(VMPSADBW,   V,x,  H,x,  W,x,  vex2 cpuid(SSE41) avx2_256 p_66),
+    [0x44] = X86_OP_ENTRY4(PCLMULQDQ,  V,dq, H,dq, W,dq, vex4 cpuid(PCLMULQDQ) p_66),
+    [0x46] = X86_OP_ENTRY4(VPERM2x128, V,qq, H,qq, W,qq, vex6 cpuid(AVX2) p_66),
+
+    [0x60] = X86_OP_ENTRY4(PCMPESTRM,  None,None, V,dq, W,dq, vex4_unal cpuid(SSE42) p_66),
+    [0x61] = X86_OP_ENTRY4(PCMPESTRI,  None,None, V,dq, W,dq, vex4_unal cpuid(SSE42) p_66),
+    [0x62] = X86_OP_ENTRY4(PCMPISTRM,  None,None, V,dq, W,dq, vex4_unal cpuid(SSE42) p_66),
+    [0x63] = X86_OP_ENTRY4(PCMPISTRI,  None,None, V,dq, W,dq, vex4_unal cpuid(SSE42) p_66),
+
+    [0x08] = X86_OP_ENTRY3(VROUNDPS,   V,x,  W,x,  I,b,  vex2 cpuid(SSE41) p_66),
+    [0x09] = X86_OP_ENTRY3(VROUNDPD,   V,x,  W,x,  I,b,  vex2 cpuid(SSE41) p_66),
+    /*
+     * Not listed as four operand in the manual.  Also writes and reads 128-bits
+     * from the first two operands due to the V operand picking higher entries of
+     * the H operand; the "Vss,Hss,Wss" description from the manual is incorrect.
+     * For other unary operations such as VSQRTSx this is hidden by the "REPScalar"
+     * value of vex_special, because the table lists the operand types of VSQRTPx.
+     */
+    [0x0a] = X86_OP_ENTRY4(VROUNDSS,   V,x,  H,x, W,ss, vex3 cpuid(SSE41) p_66),
+    [0x0b] = X86_OP_ENTRY4(VROUNDSD,   V,x,  H,x, W,sd, vex3 cpuid(SSE41) p_66),
+    [0x0c] = X86_OP_ENTRY4(VBLENDPS,   V,x,  H,x,  W,x,  vex4 cpuid(SSE41) p_66),
+    [0x0d] = X86_OP_ENTRY4(VBLENDPD,   V,x,  H,x,  W,x,  vex4 cpuid(SSE41) p_66),
+    [0x0e] = X86_OP_ENTRY4(VPBLENDW,   V,x,  H,x,  W,x,  vex4 cpuid(SSE41) avx2_256 p_66),
+    [0x0f] = X86_OP_ENTRY4(PALIGNR,    V,x,  H,x,  W,x,  vex4 cpuid(SSSE3) mmx avx2_256 p_00_66),
+
+    [0x18] = X86_OP_ENTRY4(VINSERTx128,  V,qq, H,qq, W,dq, vex6 cpuid(AVX) p_66),
+    [0x19] = X86_OP_ENTRY3(VEXTRACTx128, W,dq, V,qq, I,b,  vex6 cpuid(AVX) p_66),
+
+    [0x38] = X86_OP_ENTRY4(VINSERTx128,  V,qq, H,qq, W,dq, vex6 cpuid(AVX2) p_66),
+    [0x39] = X86_OP_ENTRY3(VEXTRACTx128, W,dq, V,qq, I,b,  vex6 cpuid(AVX2) p_66),
+
+    /* Listed incorrectly as type 4 */
+    [0x4a] = X86_OP_ENTRY4(VBLENDVPS, V,x,  H,x,  W,x,   vex6 cpuid(AVX) p_66),
+    [0x4b] = X86_OP_ENTRY4(VBLENDVPD, V,x,  H,x,  W,x,   vex6 cpuid(AVX) p_66),
+    [0x4c] = X86_OP_ENTRY4(VPBLENDVB, V,x,  H,x,  W,x,   vex6 cpuid(AVX) p_66 avx2_256),
+
+    [0xdf] = X86_OP_ENTRY3(VAESKEYGEN, V,dq, W,dq, I,b,  vex4 cpuid(AES) p_66),
+
+    [0xF0] = X86_OP_ENTRY3(RORX, G,y, E,y, I,b, vex13 cpuid(BMI2) p_f2),
+};
+
+static void decode_0F3A(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    *b = x86_ldub_code(env, s);
+    *entry = opcodes_0F3A[*b];
+}
+
+/*
+ * There are some mistakes in the operands in the manual, and the load/store/register
+ * cases are easiest to keep separate, so the entries for 10-17 follow simplicity and
+ * efficiency of implementation rather than copying what the manual says.
+ *
+ * In particular:
+ *
+ * 1) "VMOVSS m32, xmm1" and "VMOVSD m64, xmm1" do not support VEX.vvvv != 1111b,
+ * but this is not mentioned in the tables.
+ *
+ * 2) MOVHLPS, MOVHPS, MOVHPD, MOVLPD, MOVLPS read the high quadword of one of their
+ * operands, which must therefore be dq; MOVLPD and MOVLPS also write the high
+ * quadword of the V operand.
+ */
+static void decode_0F10(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F10_reg[4] = {
+        X86_OP_ENTRY3(MOVDQ,   V,x,  None,None, W,x, vex4_unal), /* MOVUPS */
+        X86_OP_ENTRY3(MOVDQ,   V,x,  None,None, W,x, vex4_unal), /* MOVUPD */
+        X86_OP_ENTRY3(VMOVSS,  V,x,  H,x,       W,x, vex5),
+        X86_OP_ENTRY3(VMOVLPx, V,x,  H,x,       W,x, vex5), /* MOVSD */
+    };
+
+    static const X86OpEntry opcodes_0F10_mem[4] = {
+        X86_OP_ENTRY3(MOVDQ,      V,x,  None,None, W,x,  vex4_unal), /* MOVUPS */
+        X86_OP_ENTRY3(MOVDQ,      V,x,  None,None, W,x,  vex4_unal), /* MOVUPD */
+        X86_OP_ENTRY3(VMOVSS_ld,  V,x,  H,x,       M,ss, vex5),
+        X86_OP_ENTRY3(VMOVSD_ld,  V,x,  H,x,       M,sd, vex5),
+    };
+
+    if ((get_modrm(s, env) >> 6) == 3) {
+        *entry = *decode_by_prefix(s, opcodes_0F10_reg);
+    } else {
+        *entry = *decode_by_prefix(s, opcodes_0F10_mem);
+    }
+}
+
+static void decode_0F11(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F11_reg[4] = {
+        X86_OP_ENTRY3(MOVDQ,   W,x,  None,None, V,x, vex4), /* MOVUPS */
+        X86_OP_ENTRY3(MOVDQ,   W,x,  None,None, V,x, vex4), /* MOVUPD */
+        X86_OP_ENTRY3(VMOVSS,  W,x,  H,x,       V,x, vex5),
+        X86_OP_ENTRY3(VMOVLPx, W,x,  H,x,       V,q, vex5), /* MOVSD */
+    };
+
+    static const X86OpEntry opcodes_0F11_mem[4] = {
+        X86_OP_ENTRY3(MOVDQ,      W,x,  None,None, V,x, vex4), /* MOVUPS */
+        X86_OP_ENTRY3(MOVDQ,      W,x,  None,None, V,x, vex4), /* MOVUPD */
+        X86_OP_ENTRY3(VMOVSS_st,  M,ss, None,None, V,x, vex5),
+        X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex5), /* MOVSD */
+    };
+
+    if ((get_modrm(s, env) >> 6) == 3) {
+        *entry = *decode_by_prefix(s, opcodes_0F11_reg);
+    } else {
+        *entry = *decode_by_prefix(s, opcodes_0F11_mem);
+    }
+}
+
+static void decode_0F12(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F12_mem[4] = {
+        /*
+         * Use dq for operand for compatibility with gen_MOVSD and
+         * to allow VEX128 only.
+         */
+        X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq,      M,q, vex5), /* MOVLPS */
+        X86_OP_ENTRY3(VMOVLPx_ld, V,dq, H,dq,      M,q, vex5), /* MOVLPD */
+        X86_OP_ENTRY3(VMOVSLDUP,  V,x,  None,None, W,x, vex4 cpuid(SSE3)),
+        X86_OP_ENTRY3(VMOVDDUP,   V,x,  None,None, WM,q, vex5 cpuid(SSE3)), /* qq if VEX.256 */
+    };
+    static const X86OpEntry opcodes_0F12_reg[4] = {
+        X86_OP_ENTRY3(VMOVHLPS,  V,dq, H,dq,       U,dq, vex7),
+        X86_OP_ENTRY3(VMOVLPx,   W,x,  H,x,        U,q,  vex5), /* MOVLPD */
+        X86_OP_ENTRY3(VMOVSLDUP, V,x,  None,None,  U,x,  vex4 cpuid(SSE3)),
+        X86_OP_ENTRY3(VMOVDDUP,  V,x,  None,None,  U,x,  vex5 cpuid(SSE3)),
+    };
+
+    if ((get_modrm(s, env) >> 6) == 3) {
+        *entry = *decode_by_prefix(s, opcodes_0F12_reg);
+    } else {
+        *entry = *decode_by_prefix(s, opcodes_0F12_mem);
+        if ((s->prefix & PREFIX_REPNZ) && s->vex_l) {
+            entry->s2 = X86_SIZE_qq;
+        }
+    }
+}
+
+static void decode_0F16(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F16_mem[4] = {
+        /*
+         * Operand 1 technically only reads the low 64 bits, but uses dq so that
+         * it is easier to check for op0 == op1 in an endianness-neutral manner.
+         */
+        X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq,      M,q, vex5), /* MOVHPS */
+        X86_OP_ENTRY3(VMOVHPx_ld, V,dq, H,dq,      M,q, vex5), /* MOVHPD */
+        X86_OP_ENTRY3(VMOVSHDUP,  V,x,  None,None, W,x, vex4 cpuid(SSE3)),
+        {},
+    };
+    static const X86OpEntry opcodes_0F16_reg[4] = {
+        /* Same as above, operand 1 could be Hq if it wasn't for big-endian.  */
+        X86_OP_ENTRY3(VMOVLHPS,  V,dq, H,dq,      U,q, vex7),
+        X86_OP_ENTRY3(VMOVHPx,   V,x,  H,x,       U,x, vex5), /* MOVHPD */
+        X86_OP_ENTRY3(VMOVSHDUP, V,x,  None,None, U,x, vex4 cpuid(SSE3)),
+        {},
+    };
+
+    if ((get_modrm(s, env) >> 6) == 3) {
+        *entry = *decode_by_prefix(s, opcodes_0F16_reg);
+    } else {
+        *entry = *decode_by_prefix(s, opcodes_0F16_mem);
+    }
+}
+
+static void decode_0F2A(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F2A[4] = {
+        X86_OP_ENTRY3(CVTPI2Px,  V,x,  None,None, Q,q),
+        X86_OP_ENTRY3(CVTPI2Px,  V,x,  None,None, Q,q),
+        X86_OP_ENTRY3(VCVTSI2Sx, V,x,  H,x, E,y,        vex3),
+        X86_OP_ENTRY3(VCVTSI2Sx, V,x,  H,x, E,y,        vex3),
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F2A);
+}
+
+static void decode_0F2B(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F2B[4] = {
+        X86_OP_ENTRY3(MOVDQ,      M,x,  None,None, V,x, vex1), /* MOVNTPS */
+        X86_OP_ENTRY3(MOVDQ,      M,x,  None,None, V,x, vex1), /* MOVNTPD */
+        /* AMD extensions */
+        X86_OP_ENTRY3(VMOVSS_st,  M,ss, None,None, V,x, vex4 cpuid(SSE4A)), /* MOVNTSS */
+        X86_OP_ENTRY3(VMOVLPx_st, M,sd, None,None, V,x, vex4 cpuid(SSE4A)), /* MOVNTSD */
+    };
+
+    *entry = *decode_by_prefix(s, opcodes_0F2B);
+}
+
+static void decode_0F2C(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F2C[4] = {
+        /* Listed as ps/pd in the manual, but CVTTPS2PI only reads 64-bit.  */
+        X86_OP_ENTRY3(CVTTPx2PI,  P,q,  None,None, W,q),
+        X86_OP_ENTRY3(CVTTPx2PI,  P,q,  None,None, W,dq),
+        X86_OP_ENTRY3(VCVTTSx2SI, G,y,  None,None, W,ss, vex3),
+        X86_OP_ENTRY3(VCVTTSx2SI, G,y,  None,None, W,sd, vex3),
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F2C);
+}
+
+static void decode_0F2D(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F2D[4] = {
+        /* Listed as ps/pd in the manual, but CVTPS2PI only reads 64-bit.  */
+        X86_OP_ENTRY3(CVTPx2PI,  P,q,  None,None, W,q),
+        X86_OP_ENTRY3(CVTPx2PI,  P,q,  None,None, W,dq),
+        X86_OP_ENTRY3(VCVTSx2SI, G,y,  None,None, W,ss, vex3),
+        X86_OP_ENTRY3(VCVTSx2SI, G,y,  None,None, W,sd, vex3),
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F2D);
+}
+
+static void decode_VxCOMISx(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    /*
+     * VUCOMISx and VCOMISx are different and use no-prefix and 0x66 for SS and SD
+     * respectively.  Scalar values usually are associated with 0xF2 and 0xF3, for
+     * which X86_VEX_REPScalar exists, but here it has to be decoded by hand.
+     */
+    entry->s1 = entry->s2 = (s->prefix & PREFIX_DATA ? X86_SIZE_sd : X86_SIZE_ss);
+    entry->gen = (*b == 0x2E ? gen_VUCOMI : gen_VCOMI);
+}
+
+static void decode_sse_unary(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    if (!(s->prefix & (PREFIX_REPZ | PREFIX_REPNZ))) {
+        entry->op1 = X86_TYPE_None;
+        entry->s1 = X86_SIZE_None;
+    }
+    switch (*b) {
+    case 0x51: entry->gen = gen_VSQRT; break;
+    case 0x52: entry->gen = gen_VRSQRT; break;
+    case 0x53: entry->gen = gen_VRCP; break;
+    }
+}
+
+static void decode_0F5A(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F5A[4] = {
+        X86_OP_ENTRY2(VCVTPS2PD,  V,x,       W,xh, vex2),      /* VCVTPS2PD */
+        X86_OP_ENTRY2(VCVTPD2PS,  V,x,       W,x,  vex2),      /* VCVTPD2PS */
+        X86_OP_ENTRY3(VCVTSS2SD,  V,x,  H,x, W,x,  vex2_rep3), /* VCVTSS2SD */
+        X86_OP_ENTRY3(VCVTSD2SS,  V,x,  H,x, W,x,  vex2_rep3), /* VCVTSD2SS */
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F5A);
+}
+
+static void decode_0F5B(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0F5B[4] = {
+        X86_OP_ENTRY2(VCVTDQ2PS,   V,x, W,x,      vex2),
+        X86_OP_ENTRY2(VCVTPS2DQ,   V,x, W,x,      vex2),
+        X86_OP_ENTRY2(VCVTTPS2DQ,  V,x, W,x,      vex2),
+        {},
+    };
+    *entry = *decode_by_prefix(s, opcodes_0F5B);
+}
+
+static void decode_0FE6(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    static const X86OpEntry opcodes_0FE6[4] = {
+        {},
+        X86_OP_ENTRY2(VCVTTPD2DQ,  V,x, W,x,      vex2),
+        X86_OP_ENTRY2(VCVTDQ2PD,   V,x, W,x,      vex5),
+        X86_OP_ENTRY2(VCVTPD2DQ,   V,x, W,x,      vex2),
+    };
+    *entry = *decode_by_prefix(s, opcodes_0FE6);
+}
+
+static const X86OpEntry opcodes_0F[256] = {
+    [0x0E] = X86_OP_ENTRY0(EMMS,                              cpuid(3DNOW)), /* femms */
+    /*
+     * 3DNow!'s opcode byte comes *after* modrm and displacements, making it
+     * more like an Ib operand.  Dispatch to the right helper in a single gen_*
+     * function.
+     */
+    [0x0F] = X86_OP_ENTRY3(3dnow,       P,q, Q,q, I,b,        cpuid(3DNOW)),
+
+    [0x10] = X86_OP_GROUP0(0F10),
+    [0x11] = X86_OP_GROUP0(0F11),
+    [0x12] = X86_OP_GROUP0(0F12),
+    [0x13] = X86_OP_ENTRY3(VMOVLPx_st,  M,q, None,None, V,q,  vex5 p_00_66),
+    [0x14] = X86_OP_ENTRY3(VUNPCKLPx,   V,x, H,x, W,x,        vex4 p_00_66),
+    [0x15] = X86_OP_ENTRY3(VUNPCKHPx,   V,x, H,x, W,x,        vex4 p_00_66),
+    [0x16] = X86_OP_GROUP0(0F16),
+    /* Incorrectly listed as Mq,Vq in the manual */
+    [0x17] = X86_OP_ENTRY3(VMOVHPx_st,  M,q, None,None, V,dq, vex5 p_00_66),
+
+    [0x50] = X86_OP_ENTRY3(MOVMSK,     G,y, None,None, U,x, vex7 p_00_66),
+    [0x51] = X86_OP_GROUP3(sse_unary,  V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2), /* sqrtps */
+    [0x52] = X86_OP_GROUP3(sse_unary,  V,x, H,x, W,x, vex4_rep5 p_00_f3), /* rsqrtps */
+    [0x53] = X86_OP_GROUP3(sse_unary,  V,x, H,x, W,x, vex4_rep5 p_00_f3), /* rcpps */
+    [0x54] = X86_OP_ENTRY3(PAND,       V,x, H,x, W,x,  vex4 p_00_66), /* vand */
+    [0x55] = X86_OP_ENTRY3(PANDN,      V,x, H,x, W,x,  vex4 p_00_66), /* vandn */
+    [0x56] = X86_OP_ENTRY3(POR,        V,x, H,x, W,x,  vex4 p_00_66), /* vor */
+    [0x57] = X86_OP_ENTRY3(PXOR,       V,x, H,x, W,x,  vex4 p_00_66), /* vxor */
+
+    [0x60] = X86_OP_ENTRY3(PUNPCKLBW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x61] = X86_OP_ENTRY3(PUNPCKLWD,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x62] = X86_OP_ENTRY3(PUNPCKLDQ,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x63] = X86_OP_ENTRY3(PACKSSWB,   V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x64] = X86_OP_ENTRY3(PCMPGTB,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x65] = X86_OP_ENTRY3(PCMPGTW,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x66] = X86_OP_ENTRY3(PCMPGTD,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x67] = X86_OP_ENTRY3(PACKUSWB,   V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+
+    [0x70] = X86_OP_GROUP0(0F70),
+    [0x71] = X86_OP_GROUP0(group12),
+    [0x72] = X86_OP_GROUP0(group13),
+    [0x73] = X86_OP_GROUP0(group14),
+    [0x74] = X86_OP_ENTRY3(PCMPEQB,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x75] = X86_OP_ENTRY3(PCMPEQW,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x76] = X86_OP_ENTRY3(PCMPEQD,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x77] = X86_OP_GROUP0(0F77),
+
+    [0x28] = X86_OP_ENTRY3(MOVDQ,      V,x,  None,None, W,x, vex1 p_00_66), /* MOVAPS */
+    [0x29] = X86_OP_ENTRY3(MOVDQ,      W,x,  None,None, V,x, vex1 p_00_66), /* MOVAPS */
+    [0x2A] = X86_OP_GROUP0(0F2A),
+    [0x2B] = X86_OP_GROUP0(0F2B),
+    [0x2C] = X86_OP_GROUP0(0F2C),
+    [0x2D] = X86_OP_GROUP0(0F2D),
+    [0x2E] = X86_OP_GROUP3(VxCOMISx,   None,None, V,x, W,x,  vex3 p_00_66), /* VUCOMISS/SD */
+    [0x2F] = X86_OP_GROUP3(VxCOMISx,   None,None, V,x, W,x,  vex3 p_00_66), /* VCOMISS/SD */
+
+    [0x38] = X86_OP_GROUP0(0F38),
+    [0x3a] = X86_OP_GROUP0(0F3A),
+
+    [0x58] = X86_OP_ENTRY3(VADD,       V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2),
+    [0x59] = X86_OP_ENTRY3(VMUL,       V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2),
+    [0x5a] = X86_OP_GROUP0(0F5A),
+    [0x5b] = X86_OP_GROUP0(0F5B),
+    [0x5c] = X86_OP_ENTRY3(VSUB,       V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2),
+    [0x5d] = X86_OP_ENTRY3(VMIN,       V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2),
+    [0x5e] = X86_OP_ENTRY3(VDIV,       V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2),
+    [0x5f] = X86_OP_ENTRY3(VMAX,       V,x, H,x, W,x, vex2_rep3 p_00_66_f3_f2),
+
+    [0x68] = X86_OP_ENTRY3(PUNPCKHBW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x69] = X86_OP_ENTRY3(PUNPCKHWD,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x6a] = X86_OP_ENTRY3(PUNPCKHDQ,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x6b] = X86_OP_ENTRY3(PACKSSDW,   V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0x6c] = X86_OP_ENTRY3(PUNPCKLQDQ, V,x, H,x, W,x,  vex4 p_66 avx2_256),
+    [0x6d] = X86_OP_ENTRY3(PUNPCKHQDQ, V,x, H,x, W,x,  vex4 p_66 avx2_256),
+    [0x6e] = X86_OP_ENTRY3(MOVD_to,    V,x, None,None, E,y, vex5 mmx p_00_66),  /* wrong dest Vy on SDM! */
+    [0x6f] = X86_OP_GROUP0(0F6F),
+
+    [0x78] = X86_OP_GROUP0(0F78),
+    [0x79] = X86_OP_GROUP2(0F79,       V,x, U,x,       cpuid(SSE4A)),
+    [0x7c] = X86_OP_ENTRY3(VHADD,      V,x, H,x, W,x,  vex2 cpuid(SSE3) p_66_f2),
+    [0x7d] = X86_OP_ENTRY3(VHSUB,      V,x, H,x, W,x,  vex2 cpuid(SSE3) p_66_f2),
+    [0x7e] = X86_OP_GROUP0(0F7E),
+    [0x7f] = X86_OP_GROUP0(0F7F),
+
+    [0xae] = X86_OP_GROUP0(group15),
+
+    [0xc2] = X86_OP_ENTRY4(VCMP,       V,x, H,x, W,x,       vex2_rep3 p_00_66_f3_f2),
+    [0xc4] = X86_OP_ENTRY4(PINSRW,     V,dq,H,dq,E,w,       vex5 mmx p_00_66),
+    [0xc5] = X86_OP_ENTRY3(PEXTRW,     G,d, U,dq,I,b,       vex5 mmx p_00_66),
+    [0xc6] = X86_OP_ENTRY4(VSHUF,      V,x, H,x, W,x,       vex4 p_00_66),
+
+    [0xd0] = X86_OP_ENTRY3(VADDSUB,   V,x, H,x, W,x,        vex2 cpuid(SSE3) p_66_f2),
+    [0xd1] = X86_OP_ENTRY3(PSRLW_r,   V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xd2] = X86_OP_ENTRY3(PSRLD_r,   V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xd3] = X86_OP_ENTRY3(PSRLQ_r,   V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xd4] = X86_OP_ENTRY3(PADDQ,     V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xd5] = X86_OP_ENTRY3(PMULLW,    V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xd6] = X86_OP_GROUP0(0FD6),
+    [0xd7] = X86_OP_ENTRY3(PMOVMSKB,  G,d, None,None, U,x,  vex7 mmx avx2_256 p_00_66),
+
+    [0xe0] = X86_OP_ENTRY3(PAVGB,     V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xe1] = X86_OP_ENTRY3(PSRAW_r,   V,x, H,x, W,x,        vex7 mmx avx2_256 p_00_66),
+    [0xe2] = X86_OP_ENTRY3(PSRAD_r,   V,x, H,x, W,x,        vex7 mmx avx2_256 p_00_66),
+    [0xe3] = X86_OP_ENTRY3(PAVGW,     V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xe4] = X86_OP_ENTRY3(PMULHUW,   V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xe5] = X86_OP_ENTRY3(PMULHW,    V,x, H,x, W,x,        vex4 mmx avx2_256 p_00_66),
+    [0xe6] = X86_OP_GROUP0(0FE6),
+    [0xe7] = X86_OP_ENTRY3(MOVDQ,     W,x, None,None, V,x,  vex1 mmx p_00_66), /* MOVNTQ/MOVNTDQ */
+
+    [0xf0] = X86_OP_ENTRY3(MOVDQ,    V,x, None,None, WM,x,  vex4_unal cpuid(SSE3) p_f2), /* LDDQU */
+    [0xf1] = X86_OP_ENTRY3(PSLLW_r,  V,x, H,x, W,x,         vex7 mmx avx2_256 p_00_66),
+    [0xf2] = X86_OP_ENTRY3(PSLLD_r,  V,x, H,x, W,x,         vex7 mmx avx2_256 p_00_66),
+    [0xf3] = X86_OP_ENTRY3(PSLLQ_r,  V,x, H,x, W,x,         vex7 mmx avx2_256 p_00_66),
+    [0xf4] = X86_OP_ENTRY3(PMULUDQ,  V,x, H,x, W,x,         vex4 mmx avx2_256 p_00_66),
+    [0xf5] = X86_OP_ENTRY3(PMADDWD,  V,x, H,x, W,x,         vex4 mmx avx2_256 p_00_66),
+    [0xf6] = X86_OP_ENTRY3(PSADBW,   V,x, H,x, W,x,         vex4 mmx avx2_256 p_00_66),
+    [0xf7] = X86_OP_ENTRY3(MASKMOV,  None,None, V,dq, U,dq, vex4_unal avx2_256 mmx p_00_66),
+
+    /* Incorrectly missing from 2-17 */
+    [0xd8] = X86_OP_ENTRY3(PSUBUSB,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xd9] = X86_OP_ENTRY3(PSUBUSW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xda] = X86_OP_ENTRY3(PMINUB,   V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xdb] = X86_OP_ENTRY3(PAND,     V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xdc] = X86_OP_ENTRY3(PADDUSB,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xdd] = X86_OP_ENTRY3(PADDUSW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xde] = X86_OP_ENTRY3(PMAXUB,   V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xdf] = X86_OP_ENTRY3(PANDN,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+
+    [0xe8] = X86_OP_ENTRY3(PSUBSB,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xe9] = X86_OP_ENTRY3(PSUBSW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xea] = X86_OP_ENTRY3(PMINSW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xeb] = X86_OP_ENTRY3(POR,     V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xec] = X86_OP_ENTRY3(PADDSB,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xed] = X86_OP_ENTRY3(PADDSW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xee] = X86_OP_ENTRY3(PMAXSW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xef] = X86_OP_ENTRY3(PXOR,    V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+
+    [0xf8] = X86_OP_ENTRY3(PSUBB,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xf9] = X86_OP_ENTRY3(PSUBW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xfa] = X86_OP_ENTRY3(PSUBD,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xfb] = X86_OP_ENTRY3(PSUBQ,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xfc] = X86_OP_ENTRY3(PADDB,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xfd] = X86_OP_ENTRY3(PADDW,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    [0xfe] = X86_OP_ENTRY3(PADDD,  V,x, H,x, W,x,  vex4 mmx avx2_256 p_00_66),
+    /* 0xff = UD0 */
+};
+
+static void do_decode_0F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    *entry = opcodes_0F[*b];
+}
+
+static void decode_0F(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    *b = x86_ldub_code(env, s);
+    do_decode_0F(s, env, entry, b);
+}
+
+static const X86OpEntry opcodes_root[256] = {
+    [0x0F] = X86_OP_GROUP0(0F),
+};
+
+#undef mmx
+#undef vex1
+#undef vex2
+#undef vex3
+#undef vex4
+#undef vex4_unal
+#undef vex5
+#undef vex6
+#undef vex7
+#undef vex8
+#undef vex11
+#undef vex12
+#undef vex13
+
+/*
+ * Decode the fixed part of the opcode and place the last
+ * in b.
+ */
+static void decode_root(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b)
+{
+    *entry = opcodes_root[*b];
+}
+
+
+static int decode_modrm(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                        X86DecodedOp *op, X86OpType type)
+{
+    int modrm = get_modrm(s, env);
+    if ((modrm >> 6) == 3) {
+        if (s->prefix & PREFIX_LOCK) {
+            decode->e.gen = gen_illegal;
+            return 0xff;
+        }
+        op->n = (modrm & 7);
+        if (type != X86_TYPE_Q && type != X86_TYPE_N) {
+            op->n |= REX_B(s);
+        }
+    } else {
+        op->has_ea = true;
+        op->n = -1;
+        decode->mem = gen_lea_modrm_0(env, s, get_modrm(s, env));
+    }
+    return modrm;
+}
+
+static bool decode_op_size(DisasContext *s, X86OpEntry *e, X86OpSize size, MemOp *ot)
+{
+    switch (size) {
+    case X86_SIZE_b:  /* byte */
+        *ot = MO_8;
+        return true;
+
+    case X86_SIZE_d:  /* 32-bit */
+    case X86_SIZE_ss: /* SSE/AVX scalar single precision */
+        *ot = MO_32;
+        return true;
+
+    case X86_SIZE_p:  /* Far pointer, return offset size */
+    case X86_SIZE_s:  /* Descriptor, return offset size */
+    case X86_SIZE_v:  /* 16/32/64-bit, based on operand size */
+        *ot = s->dflag;
+        return true;
+
+    case X86_SIZE_pi: /* MMX */
+    case X86_SIZE_q:  /* 64-bit */
+    case X86_SIZE_sd: /* SSE/AVX scalar double precision */
+        *ot = MO_64;
+        return true;
+
+    case X86_SIZE_w:  /* 16-bit */
+        *ot = MO_16;
+        return true;
+
+    case X86_SIZE_y:  /* 32/64-bit, based on operand size */
+        *ot = s->dflag == MO_16 ? MO_32 : s->dflag;
+        return true;
+
+    case X86_SIZE_z:  /* 16-bit for 16-bit operand size, else 32-bit */
+        *ot = s->dflag == MO_16 ? MO_16 : MO_32;
+        return true;
+
+    case X86_SIZE_dq: /* SSE/AVX 128-bit */
+        if (e->special == X86_SPECIAL_MMX &&
+            !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) {
+            *ot = MO_64;
+            return true;
+        }
+        if (s->vex_l && e->s0 != X86_SIZE_qq && e->s1 != X86_SIZE_qq) {
+            return false;
+        }
+        *ot = MO_128;
+        return true;
+
+    case X86_SIZE_qq: /* AVX 256-bit */
+        if (!s->vex_l) {
+            return false;
+        }
+        *ot = MO_256;
+        return true;
+
+    case X86_SIZE_x:  /* 128/256-bit, based on operand size */
+        if (e->special == X86_SPECIAL_MMX &&
+            !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) {
+            *ot = MO_64;
+            return true;
+        }
+        /* fall through */
+    case X86_SIZE_ps: /* SSE/AVX packed single precision */
+    case X86_SIZE_pd: /* SSE/AVX packed double precision */
+        *ot = s->vex_l ? MO_256 : MO_128;
+        return true;
+
+    case X86_SIZE_xh: /* SSE/AVX packed half register */
+        *ot = s->vex_l ? MO_128 : MO_64;
+        return true;
+
+    case X86_SIZE_d64:  /* Default to 64-bit in 64-bit mode */
+        *ot = CODE64(s) && s->dflag == MO_32 ? MO_64 : s->dflag;
+        return true;
+
+    case X86_SIZE_f64:  /* Ignore size override prefix in 64-bit mode */
+        *ot = CODE64(s) ? MO_64 : s->dflag;
+        return true;
+
+    default:
+        *ot = -1;
+        return true;
+    }
+}
+
+static bool decode_op(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                      X86DecodedOp *op, X86OpType type, int b)
+{
+    int modrm;
+
+    switch (type) {
+    case X86_TYPE_None:  /* Implicit or absent */
+    case X86_TYPE_A:  /* Implicit */
+    case X86_TYPE_F:  /* EFLAGS/RFLAGS */
+        break;
+
+    case X86_TYPE_B:  /* VEX.vvvv selects a GPR */
+        op->unit = X86_OP_INT;
+        op->n = s->vex_v;
+        break;
+
+    case X86_TYPE_C:  /* REG in the modrm byte selects a control register */
+        op->unit = X86_OP_CR;
+        goto get_reg;
+
+    case X86_TYPE_D:  /* REG in the modrm byte selects a debug register */
+        op->unit = X86_OP_DR;
+        goto get_reg;
+
+    case X86_TYPE_G:  /* REG in the modrm byte selects a GPR */
+        op->unit = X86_OP_INT;
+        goto get_reg;
+
+    case X86_TYPE_S:  /* reg selects a segment register */
+        op->unit = X86_OP_SEG;
+        goto get_reg;
+
+    case X86_TYPE_P:
+        op->unit = X86_OP_MMX;
+        goto get_reg;
+
+    case X86_TYPE_V:  /* reg in the modrm byte selects an XMM/YMM register */
+        if (decode->e.special == X86_SPECIAL_MMX &&
+            !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) {
+            op->unit = X86_OP_MMX;
+        } else {
+            op->unit = X86_OP_SSE;
+        }
+    get_reg:
+        op->n = ((get_modrm(s, env) >> 3) & 7);
+        if (op->unit != X86_OP_MMX) {
+            op->n |= REX_R(s);
+        }
+        break;
+
+    case X86_TYPE_E:  /* ALU modrm operand */
+        op->unit = X86_OP_INT;
+        goto get_modrm;
+
+    case X86_TYPE_Q:  /* MMX modrm operand */
+        op->unit = X86_OP_MMX;
+        goto get_modrm;
+
+    case X86_TYPE_W:  /* XMM/YMM modrm operand */
+        if (decode->e.special == X86_SPECIAL_MMX &&
+            !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) {
+            op->unit = X86_OP_MMX;
+        } else {
+            op->unit = X86_OP_SSE;
+        }
+        goto get_modrm;
+
+    case X86_TYPE_N:  /* R/M in the modrm byte selects an MMX register */
+        op->unit = X86_OP_MMX;
+        goto get_modrm_reg;
+
+    case X86_TYPE_U:  /* R/M in the modrm byte selects an XMM/YMM register */
+        if (decode->e.special == X86_SPECIAL_MMX &&
+            !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) {
+            op->unit = X86_OP_MMX;
+        } else {
+            op->unit = X86_OP_SSE;
+        }
+        goto get_modrm_reg;
+
+    case X86_TYPE_R:  /* R/M in the modrm byte selects a register */
+        op->unit = X86_OP_INT;
+    get_modrm_reg:
+        modrm = get_modrm(s, env);
+        if ((modrm >> 6) != 3) {
+            return false;
+        }
+        goto get_modrm;
+
+    case X86_TYPE_WM:  /* modrm byte selects an XMM/YMM memory operand */
+        op->unit = X86_OP_SSE;
+        /* fall through */
+    case X86_TYPE_M:  /* modrm byte selects a memory operand */
+        modrm = get_modrm(s, env);
+        if ((modrm >> 6) == 3) {
+            return false;
+        }
+    get_modrm:
+        decode_modrm(s, env, decode, op, type);
+        break;
+
+    case X86_TYPE_O:  /* Absolute address encoded in the instruction */
+        op->unit = X86_OP_INT;
+        op->has_ea = true;
+        op->n = -1;
+        decode->mem = (AddressParts) {
+            .def_seg = R_DS,
+            .base = -1,
+            .index = -1,
+            .disp = insn_get_addr(env, s, s->aflag)
+        };
+        break;
+
+    case X86_TYPE_H:  /* For AVX, VEX.vvvv selects an XMM/YMM register */
+        if ((s->prefix & PREFIX_VEX)) {
+            op->unit = X86_OP_SSE;
+            op->n = s->vex_v;
+            break;
+        }
+        if (op == &decode->op[0]) {
+            /* shifts place the destination in VEX.vvvv, use modrm */
+            return decode_op(s, env, decode, op, decode->e.op1, b);
+        } else {
+            return decode_op(s, env, decode, op, decode->e.op0, b);
+        }
+
+    case X86_TYPE_I:  /* Immediate */
+        op->unit = X86_OP_IMM;
+        decode->immediate = insn_get_signed(env, s, op->ot);
+        break;
+
+    case X86_TYPE_J:  /* Relative offset for a jump */
+        op->unit = X86_OP_IMM;
+        decode->immediate = insn_get_signed(env, s, op->ot);
+        decode->immediate += s->pc - s->cs_base;
+        if (s->dflag == MO_16) {
+            decode->immediate &= 0xffff;
+        } else if (!CODE64(s)) {
+            decode->immediate &= 0xffffffffu;
+        }
+        break;
+
+    case X86_TYPE_L:  /* The upper 4 bits of the immediate select a 128-bit register */
+        op->n = insn_get(env, s, op->ot) >> 4;
+        break;
+
+    case X86_TYPE_X:  /* string source */
+        op->n = -1;
+        decode->mem = (AddressParts) {
+            .def_seg = R_DS,
+            .base = R_ESI,
+            .index = -1,
+        };
+        break;
+
+    case X86_TYPE_Y:  /* string destination */
+        op->n = -1;
+        decode->mem = (AddressParts) {
+            .def_seg = R_ES,
+            .base = R_EDI,
+            .index = -1,
+        };
+        break;
+
+    case X86_TYPE_2op:
+        *op = decode->op[0];
+        break;
+
+    case X86_TYPE_LoBits:
+        op->n = (b & 7) | REX_B(s);
+        op->unit = X86_OP_INT;
+        break;
+
+    case X86_TYPE_0: case X86_TYPE_1: case X86_TYPE_2: case X86_TYPE_3: case X86_TYPE_4: case X86_TYPE_5: case X86_TYPE_6: case X86_TYPE_7:
+        op->n = type - X86_TYPE_0;
+        op->unit = X86_OP_INT;
+        break;
+
+    case X86_TYPE_ES: case X86_TYPE_CS: case X86_TYPE_SS: case X86_TYPE_DS: case X86_TYPE_FS: case X86_TYPE_GS:
+        op->n = type - X86_TYPE_ES;
+        op->unit = X86_OP_SEG;
+        break;
+    }
+
+    return true;
+}
+
+static bool validate_sse_prefix(DisasContext *s, X86OpEntry *e)
+{
+    uint16_t sse_prefixes;
+
+    if (!e->valid_prefix) {
+        return true;
+    }
+    if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) {
+        /* In SSE instructions, 0xF3 and 0xF2 cancel 0x66.  */
+        s->prefix &= ~PREFIX_DATA;
+    }
+
+    /* Now, either zero or one bit is set in sse_prefixes.  */
+    sse_prefixes = s->prefix & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA);
+    return e->valid_prefix & (1 << sse_prefixes);
+}
+
+static bool decode_insn(DisasContext *s, CPUX86State *env, X86DecodeFunc decode_func,
+                        X86DecodedInsn *decode)
+{
+    X86OpEntry *e = &decode->e;
+
+    decode_func(s, env, e, &decode->b);
+    while (e->is_decode) {
+        e->is_decode = false;
+        e->decode(s, env, e, &decode->b);
+    }
+
+    if (!validate_sse_prefix(s, e)) {
+        return false;
+    }
+
+    /* First compute size of operands in order to initialize s->rip_offset.  */
+    if (e->op0 != X86_TYPE_None) {
+        if (!decode_op_size(s, e, e->s0, &decode->op[0].ot)) {
+            return false;
+        }
+        if (e->op0 == X86_TYPE_I) {
+            s->rip_offset += 1 << decode->op[0].ot;
+        }
+    }
+    if (e->op1 != X86_TYPE_None) {
+        if (!decode_op_size(s, e, e->s1, &decode->op[1].ot)) {
+            return false;
+        }
+        if (e->op1 == X86_TYPE_I) {
+            s->rip_offset += 1 << decode->op[1].ot;
+        }
+    }
+    if (e->op2 != X86_TYPE_None) {
+        if (!decode_op_size(s, e, e->s2, &decode->op[2].ot)) {
+            return false;
+        }
+        if (e->op2 == X86_TYPE_I) {
+            s->rip_offset += 1 << decode->op[2].ot;
+        }
+    }
+    if (e->op3 != X86_TYPE_None) {
+        /*
+         * A couple instructions actually use the extra immediate byte for an Lx
+         * register operand; those are handled in the gen_* functions as one off.
+         */
+        assert(e->op3 == X86_TYPE_I && e->s3 == X86_SIZE_b);
+        s->rip_offset += 1;
+    }
+
+    if (e->op0 != X86_TYPE_None &&
+        !decode_op(s, env, decode, &decode->op[0], e->op0, decode->b)) {
+        return false;
+    }
+
+    if (e->op1 != X86_TYPE_None &&
+        !decode_op(s, env, decode, &decode->op[1], e->op1, decode->b)) {
+        return false;
+    }
+
+    if (e->op2 != X86_TYPE_None &&
+        !decode_op(s, env, decode, &decode->op[2], e->op2, decode->b)) {
+        return false;
+    }
+
+    if (e->op3 != X86_TYPE_None) {
+        decode->immediate = insn_get_signed(env, s, MO_8);
+    }
+
+    return true;
+}
+
+static bool has_cpuid_feature(DisasContext *s, X86CPUIDFeature cpuid)
+{
+    switch (cpuid) {
+    case X86_FEAT_None:
+        return true;
+    case X86_FEAT_F16C:
+        return (s->cpuid_ext_features & CPUID_EXT_F16C);
+    case X86_FEAT_FMA:
+        return (s->cpuid_ext_features & CPUID_EXT_FMA);
+    case X86_FEAT_MOVBE:
+        return (s->cpuid_ext_features & CPUID_EXT_MOVBE);
+    case X86_FEAT_PCLMULQDQ:
+        return (s->cpuid_ext_features & CPUID_EXT_PCLMULQDQ);
+    case X86_FEAT_SSE:
+        return (s->cpuid_features & CPUID_SSE);
+    case X86_FEAT_SSE2:
+        return (s->cpuid_features & CPUID_SSE2);
+    case X86_FEAT_SSE3:
+        return (s->cpuid_ext_features & CPUID_EXT_SSE3);
+    case X86_FEAT_SSSE3:
+        return (s->cpuid_ext_features & CPUID_EXT_SSSE3);
+    case X86_FEAT_SSE41:
+        return (s->cpuid_ext_features & CPUID_EXT_SSE41);
+    case X86_FEAT_SSE42:
+        return (s->cpuid_ext_features & CPUID_EXT_SSE42);
+    case X86_FEAT_AES:
+        if (!(s->cpuid_ext_features & CPUID_EXT_AES)) {
+            return false;
+        } else if (!(s->prefix & PREFIX_VEX)) {
+            return true;
+        } else if (!(s->cpuid_ext_features & CPUID_EXT_AVX)) {
+            return false;
+        } else {
+            return !s->vex_l || (s->cpuid_7_0_ecx_features & CPUID_7_0_ECX_VAES);
+        }
+
+    case X86_FEAT_AVX:
+        return (s->cpuid_ext_features & CPUID_EXT_AVX);
+
+    case X86_FEAT_3DNOW:
+        return (s->cpuid_ext2_features & CPUID_EXT2_3DNOW);
+    case X86_FEAT_SSE4A:
+        return (s->cpuid_ext3_features & CPUID_EXT3_SSE4A);
+
+    case X86_FEAT_ADX:
+        return (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_ADX);
+    case X86_FEAT_BMI1:
+        return (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI1);
+    case X86_FEAT_BMI2:
+        return (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI2);
+    case X86_FEAT_AVX2:
+        return (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_AVX2);
+    }
+    g_assert_not_reached();
+    return false;
+}
+
+static bool validate_vex(DisasContext *s, X86DecodedInsn *decode)
+{
+    X86OpEntry *e = &decode->e;
+
+    switch (e->vex_special) {
+    case X86_VEX_REPScalar:
+        /*
+         * Instructions which differ between 00/66 and F2/F3 in the
+         * exception classification and the size of the memory operand.
+         */
+        assert(e->vex_class == 1 || e->vex_class == 2 || e->vex_class == 4);
+        if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) {
+            e->vex_class = e->vex_class < 4 ? 3 : 5;
+            if (s->vex_l) {
+                goto illegal;
+            }
+            assert(decode->e.s2 == X86_SIZE_x);
+            if (decode->op[2].has_ea) {
+                decode->op[2].ot = s->prefix & PREFIX_REPZ ? MO_32 : MO_64;
+            }
+        }
+        break;
+
+    case X86_VEX_SSEUnaligned:
+        /* handled in sse_needs_alignment.  */
+        break;
+
+    case X86_VEX_AVX2_256:
+        if ((s->prefix & PREFIX_VEX) && s->vex_l && !has_cpuid_feature(s, X86_FEAT_AVX2)) {
+            goto illegal;
+        }
+    }
+
+    /* TODO: instructions that require VEX.W=0 (Table 2-16) */
+
+    switch (e->vex_class) {
+    case 0:
+        if (s->prefix & PREFIX_VEX) {
+            goto illegal;
+        }
+        return true;
+    case 1:
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+    case 7:
+        if (s->prefix & PREFIX_VEX) {
+            if (!(s->flags & HF_AVX_EN_MASK)) {
+                goto illegal;
+            }
+        } else if (e->special != X86_SPECIAL_MMX ||
+                   (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))) {
+            if (!(s->flags & HF_OSFXSR_MASK)) {
+                goto illegal;
+            }
+        }
+        break;
+    case 12:
+        /* Must have a VSIB byte and no address prefix.  */
+        assert(s->has_modrm);
+        if ((s->modrm & 7) != 4 || s->aflag == MO_16) {
+            goto illegal;
+        }
+
+        /* Check no overlap between registers.  */
+        if (!decode->op[0].has_ea &&
+            (decode->op[0].n == decode->mem.index || decode->op[0].n == decode->op[1].n)) {
+            goto illegal;
+        }
+        assert(!decode->op[1].has_ea);
+        if (decode->op[1].n == decode->mem.index) {
+            goto illegal;
+        }
+        if (!decode->op[2].has_ea &&
+            (decode->op[2].n == decode->mem.index || decode->op[2].n == decode->op[1].n)) {
+            goto illegal;
+        }
+        /* fall through */
+    case 6:
+    case 11:
+        if (!(s->prefix & PREFIX_VEX)) {
+            goto illegal;
+        }
+        if (!(s->flags & HF_AVX_EN_MASK)) {
+            goto illegal;
+        }
+        break;
+    case 8:
+        /* Non-VEX case handled in decode_0F77.  */
+        assert(s->prefix & PREFIX_VEX);
+        if (!(s->flags & HF_AVX_EN_MASK)) {
+            goto illegal;
+        }
+        break;
+    case 13:
+        if (!(s->prefix & PREFIX_VEX)) {
+            goto illegal;
+        }
+        if (s->vex_l) {
+            goto illegal;
+        }
+        /* All integer instructions use VEX.vvvv, so exit.  */
+        return true;
+    }
+
+    if (s->vex_v != 0 &&
+        e->op0 != X86_TYPE_H && e->op0 != X86_TYPE_B &&
+        e->op1 != X86_TYPE_H && e->op1 != X86_TYPE_B &&
+        e->op2 != X86_TYPE_H && e->op2 != X86_TYPE_B) {
+        goto illegal;
+    }
+
+    if (s->flags & HF_TS_MASK) {
+        goto nm_exception;
+    }
+    if (s->flags & HF_EM_MASK) {
+        goto illegal;
+    }
+    return true;
+
+nm_exception:
+    gen_NM_exception(s);
+    return false;
+illegal:
+    gen_illegal_opcode(s);
+    return false;
+}
+
+static void decode_temp_free(TCGContext *tcg_ctx, X86DecodedOp *op)
+{
+    if (op->v_ptr) {
+        tcg_temp_free_ptr(tcg_ctx, op->v_ptr);
+    }
+}
+
+static void decode_temps_free(TCGContext *tcg_ctx, X86DecodedInsn *decode)
+{
+    decode_temp_free(tcg_ctx, &decode->op[0]);
+    decode_temp_free(tcg_ctx, &decode->op[1]);
+    decode_temp_free(tcg_ctx, &decode->op[2]);
+}
+
+/*
+ * Convert one instruction. s->base.is_jmp is set if the translation must
+ * be stopped.
+ */
+static void disas_insn_new(DisasContext *s, CPUState *cpu, int b)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    CPUX86State *env = cpu->env_ptr;
+    bool first = true;
+    X86DecodedInsn decode;
+    X86DecodeFunc decode_func = decode_root;
+
+    s->has_modrm = false;
+
+ next_byte:
+    if (first) {
+        first = false;
+    } else {
+        b = x86_ldub_code(env, s);
+    }
+    /* Collect prefixes.  */
+    switch (b) {
+    case 0xf3:
+        s->prefix |= PREFIX_REPZ;
+        s->prefix &= ~PREFIX_REPNZ;
+        goto next_byte;
+    case 0xf2:
+        s->prefix |= PREFIX_REPNZ;
+        s->prefix &= ~PREFIX_REPZ;
+        goto next_byte;
+    case 0xf0:
+        s->prefix |= PREFIX_LOCK;
+        goto next_byte;
+    case 0x2e:
+        s->override = R_CS;
+        goto next_byte;
+    case 0x36:
+        s->override = R_SS;
+        goto next_byte;
+    case 0x3e:
+        s->override = R_DS;
+        goto next_byte;
+    case 0x26:
+        s->override = R_ES;
+        goto next_byte;
+    case 0x64:
+        s->override = R_FS;
+        goto next_byte;
+    case 0x65:
+        s->override = R_GS;
+        goto next_byte;
+    case 0x66:
+        s->prefix |= PREFIX_DATA;
+        goto next_byte;
+    case 0x67:
+        s->prefix |= PREFIX_ADR;
+        goto next_byte;
+#ifdef TARGET_X86_64
+    case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47: case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
+        if (CODE64(s)) {
+            /* REX prefix */
+            s->prefix |= PREFIX_REX;
+            s->vex_w = (b >> 3) & 1;
+            s->rex_r = (b & 0x4) << 1;
+            s->rex_x = (b & 0x2) << 2;
+            s->rex_b = (b & 0x1) << 3;
+            goto next_byte;
+        }
+        break;
+#endif
+    case 0xc5: /* 2-byte VEX */
+    case 0xc4: /* 3-byte VEX */
+        /*
+         * VEX prefixes cannot be used except in 32-bit mode.
+         * Otherwise the instruction is LES or LDS.
+         */
+        if (CODE32(s) && !VM86(s)) {
+            static const int pp_prefix[4] = {
+                0, PREFIX_DATA, PREFIX_REPZ, PREFIX_REPNZ
+            };
+            int vex3, vex2 = x86_ldub_code(env, s);
+
+            if (!CODE64(s) && (vex2 & 0xc0) != 0xc0) {
+                /*
+                 * 4.1.4.6: In 32-bit mode, bits [7:6] must be 11b,
+                 * otherwise the instruction is LES or LDS.
+                 */
+                s->pc--; /* rewind the advance_pc() x86_ldub_code() did */
+                break;
+            }
+
+            /* 4.1.1-4.1.3: No preceding lock, 66, f2, f3, or rex prefixes. */
+            if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ
+                             | PREFIX_LOCK | PREFIX_DATA | PREFIX_REX)) {
+                goto illegal_op;
+            }
+#ifdef TARGET_X86_64
+            s->rex_r = (~vex2 >> 4) & 8;
+#endif
+            if (b == 0xc5) {
+                /* 2-byte VEX prefix: RVVVVlpp, implied 0f leading opcode byte */
+                vex3 = vex2;
+                decode_func = decode_0F;
+            } else {
+                /* 3-byte VEX prefix: RXBmmmmm wVVVVlpp */
+                vex3 = x86_ldub_code(env, s);
+#ifdef TARGET_X86_64
+                s->rex_x = (~vex2 >> 3) & 8;
+                s->rex_b = (~vex2 >> 2) & 8;
+#endif
+                s->vex_w = (vex3 >> 7) & 1;
+                switch (vex2 & 0x1f) {
+                case 0x01: /* Implied 0f leading opcode bytes.  */
+                    decode_func = decode_0F;
+                    break;
+                case 0x02: /* Implied 0f 38 leading opcode bytes.  */
+                    decode_func = decode_0F38;
+                    break;
+                case 0x03: /* Implied 0f 3a leading opcode bytes.  */
+                    decode_func = decode_0F3A;
+                    break;
+                default:   /* Reserved for future use.  */
+                    goto unknown_op;
+                }
+            }
+            s->vex_v = (~vex3 >> 3) & 0xf;
+            s->vex_l = (vex3 >> 2) & 1;
+            s->prefix |= pp_prefix[vex3 & 3] | PREFIX_VEX;
+        }
+        break;
+    default:
+        if (b >= 0x100) {
+            b -= 0x100;
+            decode_func = do_decode_0F;
+        }
+        break;
+    }
+
+    /* Post-process prefixes.  */
+    if (CODE64(s)) {
+        /*
+         * In 64-bit mode, the default data size is 32-bit.  Select 64-bit
+         * data with rex_w, and 16-bit data with 0x66; rex_w takes precedence
+         * over 0x66 if both are present.
+         */
+        s->dflag = (REX_W(s) ? MO_64 : s->prefix & PREFIX_DATA ? MO_16 : MO_32);
+        /* In 64-bit mode, 0x67 selects 32-bit addressing.  */
+        s->aflag = (s->prefix & PREFIX_ADR ? MO_32 : MO_64);
+    } else {
+        /* In 16/32-bit mode, 0x66 selects the opposite data size.  */
+        if (CODE32(s) ^ ((s->prefix & PREFIX_DATA) != 0)) {
+            s->dflag = MO_32;
+        } else {
+            s->dflag = MO_16;
+        }
+        /* In 16/32-bit mode, 0x67 selects the opposite addressing.  */
+        if (CODE32(s) ^ ((s->prefix & PREFIX_ADR) != 0)) {
+            s->aflag = MO_32;
+        }  else {
+            s->aflag = MO_16;
+        }
+    }
+
+    memset(&decode, 0, sizeof(decode));
+    decode.b = b;
+    if (!decode_insn(s, env, decode_func, &decode)) {
+        goto illegal_op;
+    }
+    if (!decode.e.gen) {
+        goto unknown_op;
+    }
+
+    if (!has_cpuid_feature(s, decode.e.cpuid)) {
+        goto illegal_op;
+    }
+
+    switch (decode.e.special) {
+    case X86_SPECIAL_None:
+        break;
+
+    case X86_SPECIAL_Locked:
+        if (decode.op[0].has_ea) {
+            s->prefix |= PREFIX_LOCK;
+        }
+        break;
+
+    case X86_SPECIAL_ProtMode:
+        if (!PE(s) || VM86(s)) {
+            goto illegal_op;
+        }
+        break;
+
+    case X86_SPECIAL_i64:
+        if (CODE64(s)) {
+            goto illegal_op;
+        }
+        break;
+    case X86_SPECIAL_o64:
+        if (!CODE64(s)) {
+            goto illegal_op;
+        }
+        break;
+
+    case X86_SPECIAL_ZExtOp0:
+        assert(decode.op[0].unit == X86_OP_INT);
+        if (!decode.op[0].has_ea) {
+            decode.op[0].ot = MO_32;
+        }
+        break;
+
+    case X86_SPECIAL_ZExtOp2:
+        assert(decode.op[2].unit == X86_OP_INT);
+        if (!decode.op[2].has_ea) {
+            decode.op[2].ot = MO_32;
+        }
+        break;
+
+    case X86_SPECIAL_AVXExtMov:
+        if (!decode.op[2].has_ea) {
+            decode.op[2].ot = s->vex_l ? MO_256 : MO_128;
+        } else if (s->vex_l) {
+            decode.op[2].ot++;
+        }
+        break;
+
+    default:
+        break;
+    }
+
+    if (!validate_vex(s, &decode)) {
+        return;
+    }
+    if (decode.e.special == X86_SPECIAL_MMX &&
+        !(s->prefix & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))) {
+        gen_helper_enter_mmx(tcg_ctx, cpu_env);
+    }
+
+    if (decode.op[0].has_ea || decode.op[1].has_ea || decode.op[2].has_ea) {
+        gen_load_ea(s, &decode.mem, decode.e.vex_class == 12);
+    }
+    if (s->prefix & PREFIX_LOCK) {
+        if (decode.op[0].unit != X86_OP_INT || !decode.op[0].has_ea) {
+            goto illegal_op;
+        }
+        gen_load(s, &decode, 2, s->T1);
+        decode.e.gen(s, env, &decode);
+    } else {
+        if (decode.op[0].unit == X86_OP_MMX) {
+            compute_mmx_offset(&decode.op[0]);
+        } else if (decode.op[0].unit == X86_OP_SSE) {
+            compute_xmm_offset(&decode.op[0]);
+        }
+        gen_load(s, &decode, 1, s->T0);
+        gen_load(s, &decode, 2, s->T1);
+        decode.e.gen(s, env, &decode);
+        gen_writeback(s, &decode, 0, s->T0);
+    }
+    decode_temps_free(tcg_ctx, &decode);
+    return;
+ illegal_op:
+    gen_illegal_opcode(s);
+    return;
+ unknown_op:
+    gen_unknown_opcode(env, s);
+}
diff --git a/qemu/target/i386/decode-new.h b/qemu/target/i386/decode-new.h
new file mode 100644
index 0000000000..afaf5e85d1
--- /dev/null
+++ b/qemu/target/i386/decode-new.h
@@ -0,0 +1,250 @@
+/*
+ * Decode table flags, mostly based on Intel SDM.
+ *
+ *  Copyright (c) 2022 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+typedef enum X86OpType {
+    X86_TYPE_None,
+
+    X86_TYPE_A, /* Implicit */
+    X86_TYPE_B, /* VEX.vvvv selects a GPR */
+    X86_TYPE_C, /* REG in the modrm byte selects a control register */
+    X86_TYPE_D, /* REG in the modrm byte selects a debug register */
+    X86_TYPE_E, /* ALU modrm operand */
+    X86_TYPE_F, /* EFLAGS/RFLAGS */
+    X86_TYPE_G, /* REG in the modrm byte selects a GPR */
+    X86_TYPE_H, /* For AVX, VEX.vvvv selects an XMM/YMM register */
+    X86_TYPE_I, /* Immediate */
+    X86_TYPE_J, /* Relative offset for a jump */
+    X86_TYPE_L, /* The upper 4 bits of the immediate select a 128-bit register */
+    X86_TYPE_M, /* modrm byte selects a memory operand */
+    X86_TYPE_N, /* R/M in the modrm byte selects an MMX register */
+    X86_TYPE_O, /* Absolute address encoded in the instruction */
+    X86_TYPE_P, /* reg in the modrm byte selects an MMX register */
+    X86_TYPE_Q, /* MMX modrm operand */
+    X86_TYPE_R, /* R/M in the modrm byte selects a register */
+    X86_TYPE_S, /* reg selects a segment register */
+    X86_TYPE_U, /* R/M in the modrm byte selects an XMM/YMM register */
+    X86_TYPE_V, /* reg in the modrm byte selects an XMM/YMM register */
+    X86_TYPE_W, /* XMM/YMM modrm operand */
+    X86_TYPE_X, /* string source */
+    X86_TYPE_Y, /* string destination */
+
+    /* Custom */
+    X86_TYPE_WM, /* modrm byte selects an XMM/YMM memory operand */
+    X86_TYPE_2op, /* 2-operand RMW instruction */
+    X86_TYPE_LoBits, /* encoded in bits 0-2 of the operand + REX.B */
+    X86_TYPE_0, /* Hard-coded GPRs (RAX..RDI) */
+    X86_TYPE_1,
+    X86_TYPE_2,
+    X86_TYPE_3,
+    X86_TYPE_4,
+    X86_TYPE_5,
+    X86_TYPE_6,
+    X86_TYPE_7,
+    X86_TYPE_ES, /* Hard-coded segment registers */
+    X86_TYPE_CS,
+    X86_TYPE_SS,
+    X86_TYPE_DS,
+    X86_TYPE_FS,
+    X86_TYPE_GS,
+} X86OpType;
+
+typedef enum X86OpSize {
+    X86_SIZE_None,
+
+    X86_SIZE_a,  /* BOUND operand */
+    X86_SIZE_b,  /* byte */
+    X86_SIZE_d,  /* 32-bit */
+    X86_SIZE_dq, /* SSE/AVX 128-bit */
+    X86_SIZE_p,  /* Far pointer */
+    X86_SIZE_pd, /* SSE/AVX packed double precision */
+    X86_SIZE_pi, /* MMX */
+    X86_SIZE_ps, /* SSE/AVX packed single precision */
+    X86_SIZE_q,  /* 64-bit */
+    X86_SIZE_qq, /* AVX 256-bit */
+    X86_SIZE_s,  /* Descriptor */
+    X86_SIZE_sd, /* SSE/AVX scalar double precision */
+    X86_SIZE_ss, /* SSE/AVX scalar single precision */
+    X86_SIZE_si, /* 32-bit GPR */
+    X86_SIZE_v,  /* 16/32/64-bit, based on operand size */
+    X86_SIZE_w,  /* 16-bit */
+    X86_SIZE_x,  /* 128/256-bit, based on operand size */
+    X86_SIZE_y,  /* 32/64-bit, based on operand size */
+    X86_SIZE_z,  /* 16-bit for 16-bit operand size, else 32-bit */
+
+    /* Custom */
+    X86_SIZE_d64,
+    X86_SIZE_f64,
+    X86_SIZE_xh, /* SSE/AVX packed half register */
+} X86OpSize;
+
+typedef enum X86CPUIDFeature {
+    X86_FEAT_None,
+    X86_FEAT_3DNOW,
+    X86_FEAT_ADX,
+    X86_FEAT_AES,
+    X86_FEAT_AVX,
+    X86_FEAT_AVX2,
+    X86_FEAT_BMI1,
+    X86_FEAT_BMI2,
+    X86_FEAT_F16C,
+    X86_FEAT_FMA,
+    X86_FEAT_MOVBE,
+    X86_FEAT_PCLMULQDQ,
+    X86_FEAT_SSE,
+    X86_FEAT_SSE2,
+    X86_FEAT_SSE3,
+    X86_FEAT_SSSE3,
+    X86_FEAT_SSE41,
+    X86_FEAT_SSE42,
+    X86_FEAT_SSE4A,
+} X86CPUIDFeature;
+
+/* Execution flags */
+
+typedef enum X86OpUnit {
+    X86_OP_SKIP,    /* not valid or managed by emission function */
+    X86_OP_SEG,     /* segment selector */
+    X86_OP_CR,      /* control register */
+    X86_OP_DR,      /* debug register */
+    X86_OP_INT,     /* loaded into/stored from s->T0/T1 */
+    X86_OP_IMM,     /* immediate */
+    X86_OP_SSE,     /* address in either s->ptrX or s->A0 depending on has_ea */
+    X86_OP_MMX,     /* address in either s->ptrX or s->A0 depending on has_ea */
+} X86OpUnit;
+
+typedef enum X86InsnSpecial {
+    X86_SPECIAL_None,
+
+    /* Always locked if it has a memory operand (XCHG) */
+    X86_SPECIAL_Locked,
+
+    /* Fault outside protected mode */
+    X86_SPECIAL_ProtMode,
+
+    /*
+     * Register operand 0/2 is zero extended to 32 bits.  Rd/Mb or Rd/Mw
+     * in the manual.
+     */
+    X86_SPECIAL_ZExtOp0,
+    X86_SPECIAL_ZExtOp2,
+
+    /*
+     * Register operand 2 is extended to full width, while a memory operand
+     * is doubled in size if VEX.L=1.
+     */
+    X86_SPECIAL_AVXExtMov,
+
+    /*
+     * MMX instruction exists with no prefix; if there is no prefix, V/H/W/U operands
+     * become P/P/Q/N, and size "x" becomes "q".
+     */
+    X86_SPECIAL_MMX,
+
+    /* Illegal or exclusive to 64-bit mode */
+    X86_SPECIAL_i64,
+    X86_SPECIAL_o64,
+} X86InsnSpecial;
+
+/*
+ * Special cases for instructions that operate on XMM/YMM registers.  Intel
+ * retconned all of them to have VEX exception classes other than 0 and 13, so
+ * all these only matter for instructions that have a VEX exception class.
+ * Based on tables in the "AVX and SSE Instruction Exception Specification"
+ * section of the manual.
+ */
+typedef enum X86VEXSpecial {
+    /* Legacy SSE instructions that allow unaligned operands */
+    X86_VEX_SSEUnaligned,
+
+    /*
+     * Used for instructions that distinguish the XMM operand type with an
+     * instruction prefix; legacy SSE encodings will allow unaligned operands
+     * for scalar operands only (identified by a REP prefix).  In this case,
+     * the decoding table uses "x" for the vector operands instead of specifying
+     * pd/ps/sd/ss individually.
+     */
+    X86_VEX_REPScalar,
+
+    /*
+     * VEX instructions that only support 256-bit operands with AVX2 (Table 2-17
+     * column 3).  Columns 2 and 4 (instructions limited to 256- and 127-bit
+     * operands respectively) are implicit in the presence of dq and qq
+     * operands, and thus handled by decode_op_size.
+     */
+    X86_VEX_AVX2_256,
+} X86VEXSpecial;
+
+
+typedef struct X86OpEntry  X86OpEntry;
+typedef struct X86DecodedInsn X86DecodedInsn;
+
+/* Decode function for multibyte opcodes.  */
+typedef void (*X86DecodeFunc)(DisasContext *s, CPUX86State *env, X86OpEntry *entry, uint8_t *b);
+
+/* Code generation function.  */
+typedef void (*X86GenFunc)(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode);
+
+struct X86OpEntry {
+    /* Based on the is_decode flags.  */
+    union {
+        X86GenFunc gen;
+        X86DecodeFunc decode;
+    };
+    /* op0 is always written, op1 and op2 are always read.  */
+    X86OpType    op0:8;
+    X86OpSize    s0:8;
+    X86OpType    op1:8;
+    X86OpSize    s1:8;
+    X86OpType    op2:8;
+    X86OpSize    s2:8;
+    /* Must be I and b respectively if present.  */
+    X86OpType    op3:8;
+    X86OpSize    s3:8;
+
+    X86InsnSpecial special:8;
+    X86CPUIDFeature cpuid:8;
+    unsigned     vex_class:8;
+    X86VEXSpecial vex_special:8;
+    uint16_t     valid_prefix:16;
+    bool         is_decode:1;
+};
+typedef struct X86DecodedOp {
+    int8_t n;
+    MemOp ot;     /* For b/c/d/p/s/q/v/w/y/z */
+    X86OpUnit unit;
+    bool has_ea;
+    int offset;   /* For MMX and SSE */
+
+    /*
+     * This field is used internally by macros OP0_PTR/OP1_PTR/OP2_PTR,
+     * do not access directly!
+     */
+    TCGv_ptr v_ptr;
+} X86DecodedOp;
+
+struct X86DecodedInsn {
+    X86OpEntry e;
+    X86DecodedOp op[3];
+    target_ulong immediate;
+    AddressParts mem;
+
+    uint8_t b;
+};
diff --git a/qemu/target/i386/emit.c.inc b/qemu/target/i386/emit.c.inc
new file mode 100644
index 0000000000..9ba74bf069
--- /dev/null
+++ b/qemu/target/i386/emit.c.inc
@@ -0,0 +1,2456 @@
+/*
+ * New-style TCG opcode generator for i386 instructions
+ *
+ *  Copyright (c) 2022 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define ZMM_OFFSET(reg) offsetof(CPUX86State, xmm_regs[reg])
+
+typedef void (*SSEFunc_i_ep)(TCGContext *tcg_ctx, TCGv_i32 val, TCGv_ptr env,
+                             TCGv_ptr reg);
+typedef void (*SSEFunc_l_ep)(TCGContext *tcg_ctx, TCGv_i64 val, TCGv_ptr env,
+                             TCGv_ptr reg);
+typedef void (*SSEFunc_0_epp)(TCGContext *tcg_ctx, TCGv_ptr env,
+                              TCGv_ptr reg_a, TCGv_ptr reg_b);
+typedef void (*SSEFunc_0_eppp)(TCGContext *tcg_ctx, TCGv_ptr env,
+                               TCGv_ptr reg_a, TCGv_ptr reg_b,
+                               TCGv_ptr reg_c);
+typedef void (*SSEFunc_0_epppp)(TCGContext *tcg_ctx, TCGv_ptr env,
+                                TCGv_ptr reg_a, TCGv_ptr reg_b,
+                                TCGv_ptr reg_c, TCGv_ptr reg_d);
+typedef void (*SSEFunc_0_eppi)(TCGContext *tcg_ctx, TCGv_ptr env,
+                               TCGv_ptr reg_a, TCGv_ptr reg_b,
+                               TCGv_i32 val);
+typedef void (*SSEFunc_0_epppi)(TCGContext *tcg_ctx, TCGv_ptr env,
+                                TCGv_ptr reg_a, TCGv_ptr reg_b,
+                                TCGv_ptr reg_c, TCGv_i32 val);
+typedef void (*SSEFunc_0_ppi)(TCGContext *tcg_ctx, TCGv_ptr reg_a,
+                              TCGv_ptr reg_b, TCGv_i32 val);
+typedef void (*SSEFunc_0_pppi)(TCGContext *tcg_ctx, TCGv_ptr reg_a,
+                               TCGv_ptr reg_b, TCGv_ptr reg_c,
+                               TCGv_i32 val);
+typedef void (*SSEFunc_0_eppt)(TCGContext *tcg_ctx, TCGv_ptr env,
+                               TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv val);
+typedef void (*SSEFunc_0_epppti)(TCGContext *tcg_ctx, TCGv_ptr env,
+                                 TCGv_ptr reg_a, TCGv_ptr reg_b,
+                                 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale);
+typedef void (*SSEFunc_0_eppppi)(TCGContext *tcg_ctx, TCGv_ptr env,
+                                 TCGv_ptr reg_a, TCGv_ptr reg_b,
+                                 TCGv_ptr reg_c, TCGv_ptr reg_d,
+                                 TCGv_i32 flags);
+typedef void (*SSEFunc_0_eppppii)(TCGContext *tcg_ctx, TCGv_ptr env,
+                                  TCGv_ptr reg_a, TCGv_ptr reg_b,
+                                  TCGv_ptr reg_c, TCGv_ptr reg_d,
+                                  TCGv_i32 even, TCGv_i32 odd);
+
+static inline TCGv_i32 tcg_constant8u_i32(TCGContext *tcg_ctx, uint8_t val)
+{
+    return tcg_constant_i32(tcg_ctx, val);
+}
+
+static void gen_NM_exception(DisasContext *s)
+{
+    gen_exception(s, EXCP07_PREX);
+}
+
+static void gen_illegal(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_illegal_opcode(s);
+}
+
+static void gen_load_ea(DisasContext *s, AddressParts *mem, bool is_vsib)
+{
+    TCGv ea = gen_lea_modrm_1(s, *mem, is_vsib);
+    gen_lea_v_seg(s, s->aflag, ea, mem->def_seg, s->override);
+}
+
+static inline int mmx_offset(MemOp ot)
+{
+    switch (ot) {
+    case MO_8:
+        return offsetof(MMXReg, MMX_B(0));
+    case MO_16:
+        return offsetof(MMXReg, MMX_W(0));
+    case MO_32:
+        return offsetof(MMXReg, MMX_L(0));
+    case MO_64:
+        return offsetof(MMXReg, MMX_Q(0));
+    default:
+        g_assert_not_reached();
+    }
+    return 0;
+}
+
+static inline int xmm_offset(MemOp ot)
+{
+    switch (ot) {
+    case MO_8:
+        return offsetof(ZMMReg, ZMM_B(0));
+    case MO_16:
+        return offsetof(ZMMReg, ZMM_W(0));
+    case MO_32:
+        return offsetof(ZMMReg, ZMM_L(0));
+    case MO_64:
+        return offsetof(ZMMReg, ZMM_Q(0));
+    case MO_128:
+        return offsetof(ZMMReg, ZMM_X(0));
+    case MO_256:
+        return offsetof(ZMMReg, ZMM_Y(0));
+    default:
+        g_assert_not_reached();
+    }
+    return 0;
+}
+
+static int vector_reg_offset(X86DecodedOp *op)
+{
+    assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE);
+
+    if (op->unit == X86_OP_MMX) {
+        return op->offset - mmx_offset(op->ot);
+    } else {
+        return op->offset - xmm_offset(op->ot);
+    }
+}
+
+static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n)
+{
+    int base_ofs = vector_reg_offset(op);
+    switch(ot) {
+    case MO_8:
+        if (op->unit == X86_OP_MMX) {
+            return base_ofs + offsetof(MMXReg, MMX_B(n));
+        } else {
+            return base_ofs + offsetof(ZMMReg, ZMM_B(n));
+        }
+    case MO_16:
+        if (op->unit == X86_OP_MMX) {
+            return base_ofs + offsetof(MMXReg, MMX_W(n));
+        } else {
+            return base_ofs + offsetof(ZMMReg, ZMM_W(n));
+        }
+    case MO_32:
+        if (op->unit == X86_OP_MMX) {
+            return base_ofs + offsetof(MMXReg, MMX_L(n));
+        } else {
+            return base_ofs + offsetof(ZMMReg, ZMM_L(n));
+        }
+    case MO_64:
+        if (op->unit == X86_OP_MMX) {
+            return base_ofs;
+        } else {
+            return base_ofs + offsetof(ZMMReg, ZMM_Q(n));
+        }
+    case MO_128:
+        assert(op->unit == X86_OP_SSE);
+        return base_ofs + offsetof(ZMMReg, ZMM_X(n));
+    case MO_256:
+        assert(op->unit == X86_OP_SSE);
+        return base_ofs + offsetof(ZMMReg, ZMM_Y(n));
+    default:
+        g_assert_not_reached();
+    }
+    return 0;
+}
+
+static void compute_mmx_offset(X86DecodedOp *op)
+{
+    if (!op->has_ea) {
+        op->offset = offsetof(CPUX86State, fpregs[op->n].mmx) + mmx_offset(op->ot);
+    } else {
+        op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot);
+    }
+}
+
+static void compute_xmm_offset(X86DecodedOp *op)
+{
+    if (!op->has_ea) {
+        op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot);
+    } else {
+        op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot);
+    }
+}
+
+static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    switch(ot) {
+    case MO_8:
+        gen_op_ld_v(s, MO_8, temp, s->A0);
+        tcg_gen_st8_tl(tcg_ctx, temp, cpu_env, dest_ofs);
+        break;
+    case MO_16:
+        gen_op_ld_v(s, MO_16, temp, s->A0);
+        tcg_gen_st16_tl(tcg_ctx, temp, cpu_env, dest_ofs);
+        break;
+    case MO_32:
+        gen_op_ld_v(s, MO_32, temp, s->A0);
+        tcg_gen_st32_tl(tcg_ctx, temp, cpu_env, dest_ofs);
+        break;
+    case MO_64:
+        gen_ldq_env_A0(s, dest_ofs);
+        break;
+    case MO_128:
+        gen_ldo_env_A0(s, dest_ofs, aligned);
+        break;
+    case MO_256:
+        gen_ldy_env_A0(s, dest_ofs, aligned);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot)
+{
+    switch (decode->e.vex_class) {
+    case 2:
+    case 4:
+        if ((s->prefix & PREFIX_VEX) ||
+            decode->e.vex_special == X86_VEX_SSEUnaligned) {
+            /* MOST legacy SSE instructions require aligned memory operands, but not all.  */
+            return false;
+        }
+        /* fall through */
+    case 1:
+        return ot >= MO_128;
+
+    default:
+        return false;
+    }
+}
+
+static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    X86DecodedOp *op = &decode->op[opn];
+
+    switch (op->unit) {
+    case X86_OP_SKIP:
+        return;
+    case X86_OP_SEG:
+        tcg_gen_ld32u_tl(tcg_ctx, v, cpu_env,
+                         offsetof(CPUX86State,segs[op->n].selector));
+        break;
+    case X86_OP_CR:
+        tcg_gen_ld_tl(tcg_ctx, v, cpu_env, offsetof(CPUX86State, cr[op->n]));
+        break;
+    case X86_OP_DR:
+        tcg_gen_ld_tl(tcg_ctx, v, cpu_env, offsetof(CPUX86State, dr[op->n]));
+        break;
+    case X86_OP_INT:
+        if (op->has_ea) {
+            gen_op_ld_v(s, op->ot, v, s->A0);
+        } else {
+            gen_op_mov_v_reg(s, op->ot, v, op->n);
+        }
+        break;
+    case X86_OP_IMM:
+        tcg_gen_movi_tl(tcg_ctx, v, decode->immediate);
+        break;
+
+    case X86_OP_MMX:
+        compute_mmx_offset(op);
+        goto load_vector;
+
+    case X86_OP_SSE:
+        compute_xmm_offset(op);
+    load_vector:
+        if (op->has_ea) {
+            bool aligned = sse_needs_alignment(s, decode, op->ot);
+            gen_load_sse(s, v, op->ot, op->offset, aligned);
+        }
+        break;
+
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static TCGv_ptr op_ptr(TCGContext *tcg_ctx, X86DecodedInsn *decode, int opn)
+{
+    X86DecodedOp *op = &decode->op[opn];
+    if (op->v_ptr) {
+        return op->v_ptr;
+    }
+    op->v_ptr = tcg_temp_new_ptr(tcg_ctx);
+
+    /* The temporary points to the MMXReg or ZMMReg.  */
+    tcg_gen_addi_ptr(tcg_ctx, op->v_ptr, cpu_env, vector_reg_offset(op));
+    return op->v_ptr;
+}
+
+#define OP_PTR0 op_ptr(tcg_ctx, decode, 0)
+#define OP_PTR1 op_ptr(tcg_ctx, decode, 1)
+#define OP_PTR2 op_ptr(tcg_ctx, decode, 2)
+
+static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    X86DecodedOp *op = &decode->op[opn];
+    switch (op->unit) {
+    case X86_OP_SKIP:
+        break;
+    case X86_OP_SEG:
+        /* Note that gen_movl_seg_T0 takes care of interrupt shadow and TF.  */
+        gen_movl_seg_T0(s, op->n);
+        break;
+    case X86_OP_INT:
+        if (op->has_ea) {
+            gen_op_st_v(s, op->ot, v, s->A0);
+        } else {
+            gen_op_mov_reg_v(s, op->ot, op->n, v);
+        }
+        break;
+    case X86_OP_MMX:
+        break;
+    case X86_OP_SSE:
+        if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) {
+            tcg_gen_gvec_dup_imm(tcg_ctx, MO_64,
+                                 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)),
+                                 16, 16, 0);
+        }
+        break;
+    case X86_OP_CR:
+    case X86_OP_DR:
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static inline int vector_len(DisasContext *s, X86DecodedInsn *decode)
+{
+    if (decode->e.special == X86_SPECIAL_MMX &&
+        !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) {
+        return 8;
+    }
+    return s->vex_l ? 32 : 16;
+}
+
+static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+    int vec_len = vector_len(s, decode);
+    bool aligned = sse_needs_alignment(s, decode, ot);
+
+    if (!decode->op[0].has_ea) {
+        tcg_gen_gvec_mov(tcg_ctx, MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len);
+        return;
+    }
+
+    switch (ot) {
+    case MO_64:
+        gen_stq_env_A0(s, src_ofs);
+        break;
+    case MO_128:
+        gen_sto_env_A0(s, src_ofs, aligned);
+        break;
+    case MO_256:
+        gen_sty_env_A0(s, src_ofs, aligned);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static void gen_helper_pavgusb(TCGContext *tcg_ctx, TCGv_ptr env,
+                               TCGv_ptr reg_a, TCGv_ptr reg_b)
+{
+    gen_helper_pavgb_mmx(tcg_ctx, env, reg_a, reg_a, reg_b);
+}
+
+#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1)
+static const SSEFunc_0_epp fns_3dnow[] = {
+    [0x0c] = gen_helper_pi2fw,
+    [0x0d] = gen_helper_pi2fd,
+    [0x1c] = gen_helper_pf2iw,
+    [0x1d] = gen_helper_pf2id,
+    [0x8a] = gen_helper_pfnacc,
+    [0x8e] = gen_helper_pfpnacc,
+    [0x90] = gen_helper_pfcmpge,
+    [0x94] = gen_helper_pfmin,
+    [0x96] = gen_helper_pfrcp,
+    [0x97] = gen_helper_pfrsqrt,
+    [0x9a] = gen_helper_pfsub,
+    [0x9e] = gen_helper_pfadd,
+    [0xa0] = gen_helper_pfcmpgt,
+    [0xa4] = gen_helper_pfmax,
+    [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */
+    [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */
+    [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */
+    [0xaa] = gen_helper_pfsubr,
+    [0xae] = gen_helper_pfacc,
+    [0xb0] = gen_helper_pfcmpeq,
+    [0xb4] = gen_helper_pfmul,
+    [0xb7] = gen_helper_pmulhrw_mmx,
+    [0xbb] = gen_helper_pswapd,
+    [0xbf] = gen_helper_pavgusb,
+};
+
+static void gen_3dnow(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    uint8_t b = decode->immediate;
+    SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL;
+
+    if (!fn) {
+        gen_illegal_opcode(s);
+        return;
+    }
+    if (s->flags & HF_TS_MASK) {
+        gen_NM_exception(s);
+        return;
+    }
+    if (s->flags & HF_EM_MASK) {
+        gen_illegal_opcode(s);
+        return;
+    }
+
+    gen_helper_enter_mmx(tcg_ctx, cpu_env);
+    if (fn == FN_3DNOW_MOVE) {
+       tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[1].offset);
+       tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset);
+    } else {
+       fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1);
+    }
+}
+
+/*
+ * 00 = v*ps Vps, Hps, Wpd
+ * 66 = v*pd Vpd, Hpd, Wps
+ * f3 = v*ss Vss, Hss, Wps
+ * f2 = v*sd Vsd, Hsd, Wps
+ */
+static inline void gen_unary_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                              SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm,
+                              SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm,
+                              SSEFunc_0_eppp sd, SSEFunc_0_eppp ss)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) {
+        SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd;
+        if (!fn) {
+            gen_illegal_opcode(s);
+            return;
+        }
+        fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+    } else {
+        SSEFunc_0_epp ps, pd, fn;
+        ps = s->vex_l ? ps_ymm : ps_xmm;
+        pd = s->vex_l ? pd_ymm : pd_xmm;
+        fn = s->prefix & PREFIX_DATA ? pd : ps;
+        if (!fn) {
+            gen_illegal_opcode(s);
+            return;
+        }
+        fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+}
+#define UNARY_FP_SSE(uname, lname)                                                 \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_unary_fp_sse(s, env, decode,                                               \
+                     gen_helper_##lname##pd_xmm,                                   \
+                     gen_helper_##lname##ps_xmm,                                   \
+                     gen_helper_##lname##pd_ymm,                                   \
+                     gen_helper_##lname##ps_ymm,                                   \
+                     gen_helper_##lname##sd,                                       \
+                     gen_helper_##lname##ss);                                      \
+}
+UNARY_FP_SSE(VSQRT, sqrt)
+
+/*
+ * 00 = v*ps Vps, Hps, Wpd
+ * 66 = v*pd Vpd, Hpd, Wps
+ * f3 = v*ss Vss, Hss, Wps
+ * f2 = v*sd Vsd, Hsd, Wps
+ */
+static inline void gen_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                              SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm,
+                              SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm,
+                              SSEFunc_0_eppp sd, SSEFunc_0_eppp ss)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_0_eppp ps, pd, fn;
+    if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) {
+        fn = s->prefix & PREFIX_REPZ ? ss : sd;
+    } else {
+        ps = s->vex_l ? ps_ymm : ps_xmm;
+        pd = s->vex_l ? pd_ymm : pd_xmm;
+        fn = s->prefix & PREFIX_DATA ? pd : ps;
+    }
+    if (fn) {
+        fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+    } else {
+        gen_illegal_opcode(s);
+    }
+}
+
+#define FP_SSE(uname, lname)                                                       \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_fp_sse(s, env, decode,                                                     \
+               gen_helper_##lname##pd_xmm,                                         \
+               gen_helper_##lname##ps_xmm,                                         \
+               gen_helper_##lname##pd_ymm,                                         \
+               gen_helper_##lname##ps_ymm,                                         \
+               gen_helper_##lname##sd,                                             \
+               gen_helper_##lname##ss);                                            \
+}
+FP_SSE(VADD, add)
+FP_SSE(VMUL, mul)
+FP_SSE(VSUB, sub)
+FP_SSE(VMIN, min)
+FP_SSE(VDIV, div)
+FP_SSE(VMAX, max)
+
+#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd)                         \
+static void gen_##uname##Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;                                          \
+    SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \
+    SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \
+    SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm;                                   \
+                                                                                   \
+    fn(tcg_ctx, cpu_env, OP_PTR0, ptr0, ptr1, ptr2,                                         \
+       tcg_constant_i32(tcg_ctx, even),                                                     \
+       tcg_constant_i32(tcg_ctx, (even) ^ (odd)));                                          \
+}
+
+#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags)                                    \
+FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags)                              \
+static void gen_##uname##Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;                                          \
+    SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss;        \
+                                                                                   \
+    fn(tcg_ctx, cpu_env, OP_PTR0, ptr0, ptr1, ptr2,                                         \
+       tcg_constant_i32(tcg_ctx, flags));                                                   \
+}                                                                                  \
+
+FMA_SSE(VFMADD231,  OP_PTR1, OP_PTR2, OP_PTR0, 0)
+FMA_SSE(VFMADD213,  OP_PTR1, OP_PTR0, OP_PTR2, 0)
+FMA_SSE(VFMADD132,  OP_PTR0, OP_PTR2, OP_PTR1, 0)
+
+FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product)
+FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product)
+FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product)
+
+FMA_SSE(VFMSUB231,  OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c)
+FMA_SSE(VFMSUB213,  OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c)
+FMA_SSE(VFMSUB132,  OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c)
+
+FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product)
+FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product)
+FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product)
+
+FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0)
+FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0)
+FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0)
+
+FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c)
+FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c)
+FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c)
+
+#define FP_UNPACK_SSE(uname, lname)                                                \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    /* PS maps to the DQ integer instruction, PD maps to QDQ.  */                  \
+    gen_fp_sse(s, env, decode,                                                     \
+               gen_helper_##lname##qdq_xmm,                                        \
+               gen_helper_##lname##dq_xmm,                                         \
+               gen_helper_##lname##qdq_ymm,                                        \
+               gen_helper_##lname##dq_ymm,                                         \
+               NULL, NULL);                                                        \
+}
+FP_UNPACK_SSE(VUNPCKLPx, punpckl)
+FP_UNPACK_SSE(VUNPCKHPx, punpckh)
+
+/*
+ * 00 = v*ps Vps, Wpd
+ * f3 = v*ss Vss, Wps
+ */
+static inline void gen_unary_fp32_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                      SSEFunc_0_epp ps_xmm,
+                                      SSEFunc_0_epp ps_ymm,
+                                      SSEFunc_0_eppp ss)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) {
+        goto illegal_op;
+    } else if (s->prefix & PREFIX_REPZ) {
+        if (!ss) {
+            goto illegal_op;
+        }
+        ss(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+    } else {
+        SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm;
+        if (!fn) {
+            goto illegal_op;
+        }
+        fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+    return;
+
+illegal_op:
+    gen_illegal_opcode(s);
+}
+#define UNARY_FP32_SSE(uname, lname)                                               \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_unary_fp32_sse(s, env, decode,                                             \
+                       gen_helper_##lname##ps_xmm,                                 \
+                       gen_helper_##lname##ps_ymm,                                 \
+                       gen_helper_##lname##ss);                                    \
+}
+UNARY_FP32_SSE(VRSQRT, rsqrt)
+UNARY_FP32_SSE(VRCP, rcp)
+
+/*
+ * 66 = v*pd Vpd, Hpd, Wpd
+ * f2 = v*ps Vps, Hps, Wps
+ */
+static inline void gen_horizontal_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                         SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm,
+                                         SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_0_eppp ps, pd, fn;
+    ps = s->vex_l ? ps_ymm : ps_xmm;
+    pd = s->vex_l ? pd_ymm : pd_xmm;
+    fn = s->prefix & PREFIX_DATA ? pd : ps;
+    fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+}
+#define HORIZONTAL_FP_SSE(uname, lname)                                            \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_horizontal_fp_sse(s, env, decode,                                          \
+                          gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm,  \
+                          gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \
+}
+HORIZONTAL_FP_SSE(VHADD, hadd)
+HORIZONTAL_FP_SSE(VHSUB, hsub)
+HORIZONTAL_FP_SSE(VADDSUB, addsub)
+
+static inline void gen_ternary_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                   int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm;
+    TCGv_ptr ptr3 = tcg_temp_new_ptr(tcg_ctx);
+
+    /* The format of the fourth input is Lx */
+    tcg_gen_addi_ptr(tcg_ctx, ptr3, cpu_env, ZMM_OFFSET(op3));
+    fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3);
+    tcg_temp_free_ptr(tcg_ctx, ptr3);
+}
+#define TERNARY_SSE(uname, uvname, lname)                                          \
+static void gen_##uvname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_ternary_sse(s, env, decode, (uint8_t)decode->immediate >> 4,               \
+                    gen_helper_##lname##_xmm, gen_helper_##lname##_ymm);           \
+}                                                                                  \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_ternary_sse(s, env, decode, 0,                                             \
+                  gen_helper_##lname##_xmm, gen_helper_##lname##_ymm);             \
+}
+TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps)
+TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd)
+TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb)
+
+static inline void gen_binary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                      SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    if (!s->vex_l) {
+        xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+    } else {
+        ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+    }
+}
+
+#define BINARY_IMM_SSE(uname, lname)                                               \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_binary_imm_sse(s, env, decode,                                             \
+                       gen_helper_##lname##_xmm,                                   \
+                       gen_helper_##lname##_ymm);                                  \
+}
+
+BINARY_IMM_SSE(VBLENDPD,   blendpd)
+BINARY_IMM_SSE(VBLENDPS,   blendps)
+BINARY_IMM_SSE(VPBLENDW,   pblendw)
+BINARY_IMM_SSE(VDDPS,      dpps)
+#define gen_helper_dppd_ymm NULL
+BINARY_IMM_SSE(VDDPD,      dppd)
+BINARY_IMM_SSE(VMPSADBW,   mpsadbw)
+BINARY_IMM_SSE(PCLMULQDQ,  pclmulqdq)
+
+
+#define UNARY_INT_GVEC(uname, func, ...)                                           \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;                                          \
+    int vec_len = vector_len(s, decode);                                          \
+                                                                                   \
+    func(tcg_ctx, __VA_ARGS__, decode->op[0].offset,                               \
+         decode->op[2].offset, vec_len, vec_len);                                  \
+}
+UNARY_INT_GVEC(PABSB,          tcg_gen_gvec_abs, MO_8)
+UNARY_INT_GVEC(PABSW,          tcg_gen_gvec_abs, MO_16)
+UNARY_INT_GVEC(PABSD,          tcg_gen_gvec_abs, MO_32)
+UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128)
+UNARY_INT_GVEC(VPBROADCASTB,   tcg_gen_gvec_dup_mem, MO_8)
+UNARY_INT_GVEC(VPBROADCASTW,   tcg_gen_gvec_dup_mem, MO_16)
+UNARY_INT_GVEC(VPBROADCASTD,   tcg_gen_gvec_dup_mem, MO_32)
+UNARY_INT_GVEC(VPBROADCASTQ,   tcg_gen_gvec_dup_mem, MO_64)
+
+
+#define BINARY_INT_GVEC(uname, func, ...)                                          \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;                                          \
+    int vec_len = vector_len(s, decode);                                          \
+                                                                                   \
+    func(tcg_ctx, __VA_ARGS__,                                                     \
+         decode->op[0].offset, decode->op[1].offset,                               \
+         decode->op[2].offset, vec_len, vec_len);                                  \
+}
+
+BINARY_INT_GVEC(PADDB,   tcg_gen_gvec_add, MO_8)
+BINARY_INT_GVEC(PADDW,   tcg_gen_gvec_add, MO_16)
+BINARY_INT_GVEC(PADDD,   tcg_gen_gvec_add, MO_32)
+BINARY_INT_GVEC(PADDQ,   tcg_gen_gvec_add, MO_64)
+BINARY_INT_GVEC(PADDSB,  tcg_gen_gvec_ssadd, MO_8)
+BINARY_INT_GVEC(PADDSW,  tcg_gen_gvec_ssadd, MO_16)
+BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8)
+BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16)
+BINARY_INT_GVEC(PAND,    tcg_gen_gvec_and, MO_64)
+BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8)
+BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32)
+BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16)
+BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64)
+BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8)
+BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16)
+BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32)
+BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64)
+BINARY_INT_GVEC(PMAXSB,  tcg_gen_gvec_smax, MO_8)
+BINARY_INT_GVEC(PMAXSW,  tcg_gen_gvec_smax, MO_16)
+BINARY_INT_GVEC(PMAXSD,  tcg_gen_gvec_smax, MO_32)
+BINARY_INT_GVEC(PMAXUB,  tcg_gen_gvec_umax, MO_8)
+BINARY_INT_GVEC(PMAXUW,  tcg_gen_gvec_umax, MO_16)
+BINARY_INT_GVEC(PMAXUD,  tcg_gen_gvec_umax, MO_32)
+BINARY_INT_GVEC(PMINSB,  tcg_gen_gvec_smin, MO_8)
+BINARY_INT_GVEC(PMINSW,  tcg_gen_gvec_smin, MO_16)
+BINARY_INT_GVEC(PMINSD,  tcg_gen_gvec_smin, MO_32)
+BINARY_INT_GVEC(PMINUB,  tcg_gen_gvec_umin, MO_8)
+BINARY_INT_GVEC(PMINUW,  tcg_gen_gvec_umin, MO_16)
+BINARY_INT_GVEC(PMINUD,  tcg_gen_gvec_umin, MO_32)
+BINARY_INT_GVEC(PMULLW,  tcg_gen_gvec_mul, MO_16)
+BINARY_INT_GVEC(PMULLD,  tcg_gen_gvec_mul, MO_32)
+BINARY_INT_GVEC(POR,     tcg_gen_gvec_or, MO_64)
+BINARY_INT_GVEC(PSUBB,   tcg_gen_gvec_sub, MO_8)
+BINARY_INT_GVEC(PSUBW,   tcg_gen_gvec_sub, MO_16)
+BINARY_INT_GVEC(PSUBD,   tcg_gen_gvec_sub, MO_32)
+BINARY_INT_GVEC(PSUBQ,   tcg_gen_gvec_sub, MO_64)
+BINARY_INT_GVEC(PSUBSB,  tcg_gen_gvec_sssub, MO_8)
+BINARY_INT_GVEC(PSUBSW,  tcg_gen_gvec_sssub, MO_16)
+BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8)
+BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16)
+BINARY_INT_GVEC(PXOR,    tcg_gen_gvec_xor, MO_64)
+
+
+/*
+ * 00 = p*  Pq, Qq (if mmx not NULL; no VEX)
+ * 66 = vp* Vx, Hx, Wx
+ *
+ * These are really the same encoding, because 1) V is the same as P when VEX.V
+ * is not present 2) P and Q are the same as H and W apart from MM/XMM
+ */
+static inline void gen_binary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                      SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX));
+
+    if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) {
+        /* VEX encoding is not applicable to MMX instructions.  */
+        gen_illegal_opcode(s);
+        return;
+    }
+    if (!(s->prefix & PREFIX_DATA)) {
+        mmx(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+    } else if (!s->vex_l) {
+        xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+    } else {
+        ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+    }
+}
+
+
+#define BINARY_INT_MMX(uname, lname)                                               \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_binary_int_sse(s, env, decode,                                             \
+                          gen_helper_##lname##_mmx,                                \
+                          gen_helper_##lname##_xmm,                                \
+                          gen_helper_##lname##_ymm);                               \
+}
+BINARY_INT_MMX(PUNPCKLBW,  punpcklbw)
+BINARY_INT_MMX(PUNPCKLWD,  punpcklwd)
+BINARY_INT_MMX(PUNPCKLDQ,  punpckldq)
+BINARY_INT_MMX(PACKSSWB,   packsswb)
+BINARY_INT_MMX(PACKUSWB,   packuswb)
+BINARY_INT_MMX(PUNPCKHBW,  punpckhbw)
+BINARY_INT_MMX(PUNPCKHWD,  punpckhwd)
+BINARY_INT_MMX(PUNPCKHDQ,  punpckhdq)
+BINARY_INT_MMX(PACKSSDW,   packssdw)
+
+BINARY_INT_MMX(PAVGB,   pavgb)
+BINARY_INT_MMX(PAVGW,   pavgw)
+BINARY_INT_MMX(PMADDWD, pmaddwd)
+BINARY_INT_MMX(PMULHUW, pmulhuw)
+BINARY_INT_MMX(PMULHW,  pmulhw)
+BINARY_INT_MMX(PMULUDQ, pmuludq)
+BINARY_INT_MMX(PSADBW,  psadbw)
+
+BINARY_INT_MMX(PSLLW_r, psllw)
+BINARY_INT_MMX(PSLLD_r, pslld)
+BINARY_INT_MMX(PSLLQ_r, psllq)
+BINARY_INT_MMX(PSRLW_r, psrlw)
+BINARY_INT_MMX(PSRLD_r, psrld)
+BINARY_INT_MMX(PSRLQ_r, psrlq)
+BINARY_INT_MMX(PSRAW_r, psraw)
+BINARY_INT_MMX(PSRAD_r, psrad)
+
+BINARY_INT_MMX(PHADDW,    phaddw)
+BINARY_INT_MMX(PHADDSW,   phaddsw)
+BINARY_INT_MMX(PHADDD,    phaddd)
+BINARY_INT_MMX(PHSUBW,    phsubw)
+BINARY_INT_MMX(PHSUBSW,   phsubsw)
+BINARY_INT_MMX(PHSUBD,    phsubd)
+BINARY_INT_MMX(PMADDUBSW, pmaddubsw)
+BINARY_INT_MMX(PSHUFB,    pshufb)
+BINARY_INT_MMX(PSIGNB,    psignb)
+BINARY_INT_MMX(PSIGNW,    psignw)
+BINARY_INT_MMX(PSIGND,    psignd)
+BINARY_INT_MMX(PMULHRSW,  pmulhrsw)
+
+/* Instructions with no MMX equivalent.  */
+#define BINARY_INT_SSE(uname, lname)                                               \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_binary_int_sse(s, env, decode,                                             \
+                          NULL,                                                    \
+                          gen_helper_##lname##_xmm,                                \
+                          gen_helper_##lname##_ymm);                               \
+}
+
+/* Instructions with no MMX equivalent.  */
+BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq)
+BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq)
+BINARY_INT_SSE(VPACKUSDW,  packusdw)
+BINARY_INT_SSE(VPERMILPS,  vpermilps)
+BINARY_INT_SSE(VPERMILPD,  vpermilpd)
+BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd)
+BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq)
+
+BINARY_INT_SSE(PMULDQ,    pmuldq)
+
+BINARY_INT_SSE(VAESDEC, aesdec)
+BINARY_INT_SSE(VAESDECLAST, aesdeclast)
+BINARY_INT_SSE(VAESENC, aesenc)
+BINARY_INT_SSE(VAESENCLAST, aesenclast)
+
+#define UNARY_CMP_SSE(uname, lname)                                                \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;                                          \
+    if (!s->vex_l) {                                                               \
+        gen_helper_##lname##_xmm(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2);              \
+    } else {                                                                       \
+        gen_helper_##lname##_ymm(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2);              \
+    }                                                                              \
+    set_cc_op(s, CC_OP_EFLAGS);                                                    \
+}
+UNARY_CMP_SSE(VPTEST,     ptest)
+UNARY_CMP_SSE(VTESTPS,    vtestps)
+UNARY_CMP_SSE(VTESTPD,    vtestpd)
+
+static inline void gen_unary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                     SSEFunc_0_epp xmm, SSEFunc_0_epp ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (!s->vex_l) {
+        xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    } else {
+        ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+}
+
+#define UNARY_INT_SSE(uname, lname)                                                \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_unary_int_sse(s, env, decode,                                              \
+                      gen_helper_##lname##_xmm,                                    \
+                      gen_helper_##lname##_ymm);                                   \
+}
+
+UNARY_INT_SSE(VPMOVSXBW,    pmovsxbw)
+UNARY_INT_SSE(VPMOVSXBD,    pmovsxbd)
+UNARY_INT_SSE(VPMOVSXBQ,    pmovsxbq)
+UNARY_INT_SSE(VPMOVSXWD,    pmovsxwd)
+UNARY_INT_SSE(VPMOVSXWQ,    pmovsxwq)
+UNARY_INT_SSE(VPMOVSXDQ,    pmovsxdq)
+
+UNARY_INT_SSE(VPMOVZXBW,    pmovzxbw)
+UNARY_INT_SSE(VPMOVZXBD,    pmovzxbd)
+UNARY_INT_SSE(VPMOVZXBQ,    pmovzxbq)
+UNARY_INT_SSE(VPMOVZXWD,    pmovzxwd)
+UNARY_INT_SSE(VPMOVZXWQ,    pmovzxwq)
+UNARY_INT_SSE(VPMOVZXDQ,    pmovzxdq)
+
+UNARY_INT_SSE(VMOVSLDUP,    pmovsldup)
+UNARY_INT_SSE(VMOVSHDUP,    pmovshdup)
+UNARY_INT_SSE(VMOVDDUP,     pmovdldup)
+
+UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd)
+UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq)
+UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq)
+UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps)
+UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq)
+UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq)
+UNARY_INT_SSE(VCVTPH2PS, cvtph2ps)
+
+
+static inline void gen_unary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                     SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    if (!s->vex_l) {
+        xmm(tcg_ctx, OP_PTR0, OP_PTR1, imm);
+    } else {
+        ymm(tcg_ctx, OP_PTR0, OP_PTR1, imm);
+    }
+}
+
+#define UNARY_IMM_SSE(uname, lname)                                                \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_unary_imm_sse(s, env, decode,                                              \
+                      gen_helper_##lname##_xmm,                                    \
+                      gen_helper_##lname##_ymm);                                   \
+}
+
+UNARY_IMM_SSE(PSHUFD,     pshufd)
+UNARY_IMM_SSE(PSHUFHW,    pshufhw)
+UNARY_IMM_SSE(PSHUFLW,    pshuflw)
+#define gen_helper_vpermq_xmm NULL
+UNARY_IMM_SSE(VPERMQ,      vpermq)
+UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm)
+UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm)
+
+static inline void gen_unary_imm_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                        SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    if (!s->vex_l) {
+        xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, imm);
+    } else {
+        ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, imm);
+    }
+}
+
+#define UNARY_IMM_FP_SSE(uname, lname)                                             \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_unary_imm_fp_sse(s, env, decode,                                           \
+                      gen_helper_##lname##_xmm,                                    \
+                      gen_helper_##lname##_ymm);                                   \
+}
+
+UNARY_IMM_FP_SSE(VROUNDPS,    roundps)
+UNARY_IMM_FP_SSE(VROUNDPD,    roundpd)
+
+static inline void gen_vexw_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm,
+                                SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm;
+    SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm;
+    SSEFunc_0_eppp fn = s->vex_w ? q : d;
+    fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+}
+
+/* VEX.W affects whether to operate on 32- or 64-bit elements.  */
+#define VEXW_AVX(uname, lname)                                                     \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_vexw_avx(s, env, decode,                                                   \
+                 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm,             \
+                 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm);            \
+}
+VEXW_AVX(VPSLLV,    vpsllv)
+VEXW_AVX(VPSRLV,    vpsrlv)
+VEXW_AVX(VPSRAV,    vpsrav)
+VEXW_AVX(VPMASKMOV, vpmaskmov)
+
+/* Same as above, but with extra arguments to the helper.  */
+static inline void gen_vsib_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm,
+                                SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm;
+    SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm;
+    SSEFunc_0_epppti fn = s->vex_w ? q : d;
+    TCGv_i32 scale = tcg_constant_i32(tcg_ctx, decode->mem.scale);
+    TCGv_ptr index = tcg_temp_new_ptr(tcg_ctx);
+
+    /* Pass third input as (index, base, scale) */
+    tcg_gen_addi_ptr(tcg_ctx, index, cpu_env, ZMM_OFFSET(decode->mem.index));
+    fn(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, index, s->A0, scale);
+
+    /*
+     * There are two output operands, so zero OP1's high 128 bits
+     * in the VEX.128 case.
+     */
+    if (!s->vex_l) {
+        int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1);
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, ymmh_ofs, 16, 16, 0);
+    }
+    tcg_temp_free_ptr(tcg_ctx, index);
+}
+#define VSIB_AVX(uname, lname)                                                     \
+static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \
+{                                                                                  \
+    gen_vsib_avx(s, env, decode,                                                   \
+                 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm,             \
+                 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm);            \
+}
+VSIB_AVX(VPGATHERD, vpgatherd)
+VSIB_AVX(VPGATHERQ, vpgatherq)
+
+static void gen_ADCOX(DisasContext *s, CPUX86State *env, MemOp ot, int cc_op)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int opposite_cc_op;
+    TCGv carry_in = NULL;
+    TCGv carry_out = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2);
+    TCGv zero;
+
+    if (cc_op == s->cc_op || s->cc_op == CC_OP_ADCOX) {
+        /* Re-use the carry-out from a previous round.  */
+        carry_in = carry_out;
+    } else {
+        /* We don't have a carry-in, get it out of EFLAGS.  */
+        if (s->cc_op != CC_OP_ADCX && s->cc_op != CC_OP_ADOX) {
+            gen_compute_eflags(s);
+        }
+        carry_in = s->tmp0;
+        tcg_gen_extract_tl(tcg_ctx, carry_in, cpu_cc_src,
+            ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1);
+    }
+
+    switch (ot) {
+#ifdef TARGET_X86_64
+    case MO_32:
+        /* If TL is 64-bit just do everything in 64-bit arithmetic.  */
+        tcg_gen_add_i64(tcg_ctx, s->T0, s->T0, s->T1);
+        tcg_gen_add_i64(tcg_ctx, s->T0, s->T0, carry_in);
+        tcg_gen_shri_i64(tcg_ctx, carry_out, s->T0, 32);
+        break;
+#endif
+    default:
+        zero = tcg_constant_tl(tcg_ctx, 0);
+        tcg_gen_add2_tl(tcg_ctx, s->T0, carry_out, s->T0, zero, carry_in, zero);
+        tcg_gen_add2_tl(tcg_ctx, s->T0, carry_out, s->T0, carry_out, s->T1, zero);
+        break;
+    }
+
+    opposite_cc_op = cc_op == CC_OP_ADCX ? CC_OP_ADOX : CC_OP_ADCX;
+    if (s->cc_op == CC_OP_ADCOX || s->cc_op == opposite_cc_op) {
+        /* Merge with the carry-out from the opposite instruction.  */
+        set_cc_op(s, CC_OP_ADCOX);
+    } else {
+        set_cc_op(s, cc_op);
+    }
+}
+
+static void gen_ADCX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADCX);
+}
+
+static void gen_ADOX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADOX);
+}
+
+static void gen_ANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+
+    tcg_gen_andc_tl(tcg_ctx, s->T0, s->T1, s->T0);
+    gen_op_update1_cc(s);
+    set_cc_op(s, CC_OP_LOGICB + ot);
+}
+
+static void gen_BEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+    TCGv bound = tcg_constant_tl(tcg_ctx, ot == MO_64 ? 63 : 31);
+    TCGv zero = tcg_constant_tl(tcg_ctx, 0);
+    TCGv mone = tcg_constant_tl(tcg_ctx, -1);
+
+    /*
+     * Extract START, and shift the operand.
+     * Shifts larger than operand size get zeros.
+     */
+    tcg_gen_ext8u_tl(tcg_ctx, s->A0, s->T1);
+    if (TARGET_LONG_BITS == 64 && ot == MO_32) {
+        tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
+    }
+    tcg_gen_shr_tl(tcg_ctx, s->T0, s->T0, s->A0);
+
+    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero);
+
+    /*
+     * Extract the LEN into an inverse mask.  Lengths larger than
+     * operand size get all zeros, length 0 gets all ones.
+     */
+    tcg_gen_extract_tl(tcg_ctx, s->A0, s->T1, 8, 8);
+    tcg_gen_shl_tl(tcg_ctx, s->T1, mone, s->A0);
+    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero);
+    tcg_gen_andc_tl(tcg_ctx, s->T0, s->T0, s->T1);
+
+    gen_op_update1_cc(s);
+    set_cc_op(s, CC_OP_LOGICB + ot);
+}
+
+static void gen_BLSI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T0);
+    tcg_gen_neg_tl(tcg_ctx, s->T1, s->T0);
+    tcg_gen_and_tl(tcg_ctx, s->T0, s->T0, s->T1);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
+    set_cc_op(s, CC_OP_BLSIB + ot);
+}
+
+static void gen_BLSMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T0);
+    tcg_gen_subi_tl(tcg_ctx, s->T1, s->T0, 1);
+    tcg_gen_xor_tl(tcg_ctx, s->T0, s->T0, s->T1);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
+    set_cc_op(s, CC_OP_BMILGB + ot);
+}
+
+static void gen_BLSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T0);
+    tcg_gen_subi_tl(tcg_ctx, s->T1, s->T0, 1);
+    tcg_gen_and_tl(tcg_ctx, s->T0, s->T0, s->T1);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
+    set_cc_op(s, CC_OP_BMILGB + ot);
+}
+
+static void gen_BZHI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+    TCGv bound = tcg_constant_tl(tcg_ctx, ot == MO_64 ? 63 : 31);
+    TCGv zero = tcg_constant_tl(tcg_ctx, 0);
+    TCGv mone = tcg_constant_tl(tcg_ctx, -1);
+
+    tcg_gen_ext8u_tl(tcg_ctx, s->T1, s->T1);
+
+    /*
+     * Note that since we're using BMILG (in order to get O
+     * cleared) we need to store the inverse into C.
+     */
+    tcg_gen_setcond_tl(tcg_ctx, TCG_COND_LEU, cpu_cc_src, s->T1, bound);
+
+    tcg_gen_shl_tl(tcg_ctx, s->A0, mone, s->T1);
+    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero);
+    tcg_gen_andc_tl(tcg_ctx, s->T0, s->T0, s->A0);
+
+    gen_op_update1_cc(s);
+    set_cc_op(s, CC_OP_BMILGB + ot);
+}
+
+static void gen_CRC32(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[2].ot;
+
+    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
+    gen_helper_crc32(tcg_ctx, s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(tcg_ctx, 8 << ot));
+}
+
+static void gen_CVTPI2Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_enter_mmx(tcg_ctx, cpu_env);
+    if (s->prefix & PREFIX_DATA) {
+        gen_helper_cvtpi2pd(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    } else {
+        gen_helper_cvtpi2ps(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+}
+
+static void gen_CVTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_enter_mmx(tcg_ctx, cpu_env);
+    if (s->prefix & PREFIX_DATA) {
+        gen_helper_cvtpd2pi(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    } else {
+        gen_helper_cvtps2pi(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+}
+
+static void gen_CVTTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_enter_mmx(tcg_ctx, cpu_env);
+    if (s->prefix & PREFIX_DATA) {
+        gen_helper_cvttpd2pi(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    } else {
+        gen_helper_cvttps2pi(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+}
+
+static void gen_EMMS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_emms(tcg_ctx, cpu_env);
+}
+
+static void gen_EXTRQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 length = tcg_constant_i32(tcg_ctx, decode->immediate & 63);
+    TCGv_i32 index = tcg_constant_i32(tcg_ctx, (decode->immediate >> 8) & 63);
+
+    gen_helper_extrq_i(tcg_ctx, cpu_env, OP_PTR0, index, length);
+}
+
+static void gen_EXTRQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_extrq_r(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+}
+
+static void gen_INSERTQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 length = tcg_constant_i32(tcg_ctx, decode->immediate & 63);
+    TCGv_i32 index = tcg_constant_i32(tcg_ctx, (decode->immediate >> 8) & 63);
+
+    gen_helper_insertq_i(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, index, length);
+}
+
+static void gen_INSERTQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_insertq_r(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+}
+
+static void gen_LDMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (s->vex_l) {
+        gen_illegal_opcode(s);
+        return;
+    }
+    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T1);
+    gen_helper_ldmxcsr(tcg_ctx, cpu_env, s->tmp2_i32);
+}
+
+static void gen_MASKMOV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_mov_tl(tcg_ctx, s->A0, cpu_regs[R_EDI]);
+    gen_extu(tcg_ctx, s->aflag, s->A0);
+    gen_add_A0_ds_seg(s);
+
+    if (s->prefix & PREFIX_DATA) {
+        gen_helper_maskmov_xmm(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2, s->A0);
+    } else {
+        gen_helper_maskmov_mmx(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2, s->A0);
+    }
+}
+
+static void gen_MOVBE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+
+    /* M operand type does not load/store */
+    if (decode->e.op0 == X86_TYPE_M) {
+        tcg_gen_qemu_st_tl(tcg_ctx, s->T0, s->A0, s->mem_index, ot | MO_BE);
+    } else {
+        tcg_gen_qemu_ld_tl(tcg_ctx, s->T0, s->A0, s->mem_index, ot | MO_BE);
+    }
+}
+
+static void gen_MOVD_from(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[2].ot;
+
+    switch (ot) {
+    case MO_32:
+#ifdef TARGET_X86_64
+        tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env, decode->op[2].offset);
+        break;
+    case MO_64:
+#endif
+        tcg_gen_ld_tl(tcg_ctx, s->T0, cpu_env, decode->op[2].offset);
+        break;
+    default:
+        abort();
+    }
+}
+
+static void gen_MOVD_to(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[2].ot;
+    int vec_len = vector_len(s, decode);
+    int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0);
+
+    tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+
+    switch (ot) {
+    case MO_32:
+#ifdef TARGET_X86_64
+        tcg_gen_st32_tl(tcg_ctx, s->T1, cpu_env, lo_ofs);
+        break;
+    case MO_64:
+#endif
+        tcg_gen_st_tl(tcg_ctx, s->T1, cpu_env, lo_ofs);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static void gen_MOVDQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_store_sse(s, decode, decode->op[2].offset);
+}
+
+static void gen_MOVMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_i_ep ps, pd, fn;
+    ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm;
+    pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm;
+    fn = s->prefix & PREFIX_DATA ? pd : ps;
+    fn(tcg_ctx, s->tmp2_i32, cpu_env, OP_PTR2);
+    tcg_gen_extu_i32_tl(tcg_ctx, s->T0, s->tmp2_i32);
+}
+
+static void gen_MOVQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+    int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0);
+
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[2].offset);
+    if (decode->op[0].has_ea) {
+        tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
+    } else {
+        /*
+         * tcg_gen_gvec_dup_i64(tcg_ctx, MO_64, op0.offset, 8, vec_len, s->tmp1_64) would
+         * seem to work, but it does not on big-endian platforms; the cleared parts
+         * are always at higher addresses, but cross-endian emulation inverts the
+         * byte order so that the cleared parts need to be at *lower* addresses.
+         * Because oprsz is 8, we see this here even for SSE; but more in general,
+         * it disqualifies using oprsz < maxsz to emulate VEX128.
+         */
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+        tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, lo_ofs);
+    }
+}
+
+static void gen_MOVq_dq(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_enter_mmx(tcg_ctx, cpu_env);
+    /* Otherwise the same as any other movq.  */
+    return gen_MOVQ(s, env, decode);
+}
+
+static void gen_MULX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+
+    /* low part of result in VEX.vvvv, high in MODRM */
+    switch (ot) {
+    default:
+        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
+        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, s->T1);
+        tcg_gen_mulu2_i32(tcg_ctx, s->tmp2_i32, s->tmp3_i32,
+                          s->tmp2_i32, s->tmp3_i32);
+        tcg_gen_extu_i32_tl(tcg_ctx, cpu_regs[s->vex_v], s->tmp2_i32);
+        tcg_gen_extu_i32_tl(tcg_ctx, s->T0, s->tmp3_i32);
+        break;
+#ifdef TARGET_X86_64
+    case MO_64:
+        tcg_gen_mulu2_i64(tcg_ctx, cpu_regs[s->vex_v], s->T0, s->T0, s->T1);
+        break;
+#endif
+    }
+
+}
+
+static void gen_PALIGNR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    if (!(s->prefix & PREFIX_DATA)) {
+        gen_helper_palignr_mmx(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+    } else if (!s->vex_l) {
+        gen_helper_palignr_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+    } else {
+        gen_helper_palignr_ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+    }
+}
+
+static void gen_PANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    /* Careful, operand order is reversed!  */
+    tcg_gen_gvec_andc(tcg_ctx, MO_64,
+                      decode->op[0].offset, decode->op[2].offset,
+                      decode->op[1].offset, vec_len, vec_len);
+}
+
+static void gen_PCMPESTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    gen_helper_pcmpestri_xmm(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2, imm);
+    set_cc_op(s, CC_OP_EFLAGS);
+}
+
+static void gen_PCMPESTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    gen_helper_pcmpestrm_xmm(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2, imm);
+    set_cc_op(s, CC_OP_EFLAGS);
+    if ((s->prefix & PREFIX_VEX) && !s->vex_l) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)),
+                             16, 16, 0);
+    }
+}
+
+static void gen_PCMPISTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    gen_helper_pcmpistri_xmm(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2, imm);
+    set_cc_op(s, CC_OP_EFLAGS);
+}
+
+static void gen_PCMPISTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    gen_helper_pcmpistrm_xmm(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2, imm);
+    set_cc_op(s, CC_OP_EFLAGS);
+    if ((s->prefix & PREFIX_VEX) && !s->vex_l) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)),
+                             16, 16, 0);
+    }
+}
+
+static void gen_PDEP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[1].ot;
+    if (ot < MO_64) {
+        tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
+    }
+    gen_helper_pdep(tcg_ctx, s->T0, s->T0, s->T1);
+}
+
+static void gen_PEXT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[1].ot;
+    if (ot < MO_64) {
+        tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
+    }
+    gen_helper_pext(tcg_ctx, s->T0, s->T0, s->T1);
+}
+
+static inline void gen_pextr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+    int mask = (vec_len >> ot) - 1;
+    int val = decode->immediate & mask;
+
+    switch (ot) {
+    case MO_8:
+        tcg_gen_ld8u_tl(tcg_ctx, s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
+        break;
+    case MO_16:
+        tcg_gen_ld16u_tl(tcg_ctx, s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
+        break;
+    case MO_32:
+#ifdef TARGET_X86_64
+        tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
+        break;
+    case MO_64:
+#endif
+        tcg_gen_ld_tl(tcg_ctx, s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val));
+        break;
+    default:
+        abort();
+    }
+}
+
+static void gen_PEXTRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_pextr(s, env, decode, MO_8);
+}
+
+static void gen_PEXTRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_pextr(s, env, decode, MO_16);
+}
+
+static void gen_PEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    MemOp ot = decode->op[0].ot;
+    gen_pextr(s, env, decode, ot);
+}
+
+static inline void gen_pinsr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+    int mask = (vec_len >> ot) - 1;
+    int val = decode->immediate & mask;
+
+    if (decode->op[1].offset != decode->op[0].offset) {
+        assert(vec_len == 16);
+        gen_store_sse(s, decode, decode->op[1].offset);
+    }
+
+    switch (ot) {
+    case MO_8:
+        tcg_gen_st8_tl(tcg_ctx, s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
+        break;
+    case MO_16:
+        tcg_gen_st16_tl(tcg_ctx, s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
+        break;
+    case MO_32:
+#ifdef TARGET_X86_64
+        tcg_gen_st32_tl(tcg_ctx, s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
+        break;
+    case MO_64:
+#endif
+        tcg_gen_st_tl(tcg_ctx, s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val));
+        break;
+    default:
+        abort();
+    }
+}
+
+static void gen_PINSRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_pinsr(s, env, decode, MO_8);
+}
+
+static void gen_PINSRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_pinsr(s, env, decode, MO_16);
+}
+
+static void gen_PINSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_pinsr(s, env, decode, decode->op[2].ot);
+}
+
+static void gen_pmovmskb_i64(TCGContext *tcg_ctx, TCGv_i64 d, TCGv_i64 s)
+{
+    TCGv_i64 t = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_andi_i64(tcg_ctx, d, s, 0x8080808080808080ull);
+
+    /*
+     * After each shift+or pair:
+     * 0:  a.......b.......c.......d.......e.......f.......g.......h.......
+     * 7:  ab......bc......cd......de......ef......fg......gh......h.......
+     * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h.......
+     * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h.......
+     * The result is left in the high bits of the word.
+     */
+    tcg_gen_shli_i64(tcg_ctx, t, d, 7);
+    tcg_gen_or_i64(tcg_ctx, d, d, t);
+    tcg_gen_shli_i64(tcg_ctx, t, d, 14);
+    tcg_gen_or_i64(tcg_ctx, d, d, t);
+    tcg_gen_shli_i64(tcg_ctx, t, d, 28);
+    tcg_gen_or_i64(tcg_ctx, d, d, t);
+}
+
+static void gen_pmovmskb_vec(TCGContext *tcg_ctx, unsigned vece,
+                             TCGv_vec d, TCGv_vec s)
+{
+    TCGv_vec t = tcg_temp_new_vec_matching(tcg_ctx, d);
+    TCGv_vec m = tcg_constant_vec_matching(tcg_ctx, d, MO_8, 0x80);
+
+    /* See above */
+    tcg_gen_and_vec(tcg_ctx, vece, d, s, m);
+    tcg_gen_shli_vec(tcg_ctx, vece, t, d, 7);
+    tcg_gen_or_vec(tcg_ctx, vece, d, d, t);
+    tcg_gen_shli_vec(tcg_ctx, vece, t, d, 14);
+    tcg_gen_or_vec(tcg_ctx, vece, d, d, t);
+    tcg_gen_shli_vec(tcg_ctx, vece, t, d, 28);
+    tcg_gen_or_vec(tcg_ctx, vece, d, d, t);
+}
+
+#ifdef TARGET_X86_64
+#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64
+#else
+#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32
+#endif
+
+static void gen_PMOVMSKB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
+    static const GVecGen2 g = {
+        .fni8 = gen_pmovmskb_i64,
+        .fniv = gen_pmovmskb_vec,
+        .opt_opc = vecop_list,
+        .vece = MO_64,
+        .prefer_i64 = TCG_TARGET_REG_BITS == 64
+    };
+    MemOp ot = decode->op[2].ot;
+    int vec_len = vector_len(s, decode);
+    TCGv t = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_gvec_2(tcg_ctx, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset,
+                   vec_len, vec_len, &g);
+    tcg_gen_ld8u_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1)));
+    while (vec_len > 8) {
+        vec_len -= 8;
+        if (TCG_TARGET_HAS_extract2_tl) {
+            /*
+             * Load the next byte of the result into the high byte of T.
+             * TCG does a similar expansion of deposit to shl+extract2; by
+             * loading the whole word, the shift left is avoided.
+             */
+#ifdef TARGET_X86_64
+            tcg_gen_ld_tl(tcg_ctx, t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8)));
+#else
+            tcg_gen_ld_tl(tcg_ctx, t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4)));
+#endif
+
+            tcg_gen_extract2_tl(tcg_ctx, s->T0, t, s->T0, TARGET_LONG_BITS - 8);
+        } else {
+            /*
+             * The _previous_ value is deposited into bits 8 and higher of t.  Because
+             * those bits are known to be zero after ld8u, this becomes a shift+or
+             * if deposit is not available.
+             */
+            tcg_gen_ld8u_tl(tcg_ctx, t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1)));
+            tcg_gen_deposit_tl(tcg_ctx, s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8);
+        }
+    }
+    tcg_temp_free(tcg_ctx, t);
+}
+
+static void gen_PSHUFW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    gen_helper_pshufw_mmx(tcg_ctx, OP_PTR0, OP_PTR1, imm);
+}
+
+static void gen_PSRLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 16) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    } else {
+        tcg_gen_gvec_shri(tcg_ctx, MO_16,
+                          decode->op[0].offset, decode->op[1].offset,
+                          decode->immediate, vec_len, vec_len);
+    }
+}
+
+static void gen_PSLLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 16) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    } else {
+        tcg_gen_gvec_shli(tcg_ctx, MO_16,
+                          decode->op[0].offset, decode->op[1].offset,
+                          decode->immediate, vec_len, vec_len);
+    }
+}
+
+static void gen_PSRAW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 16) {
+        decode->immediate = 15;
+    }
+    tcg_gen_gvec_sari(tcg_ctx, MO_16,
+                      decode->op[0].offset, decode->op[1].offset,
+                      decode->immediate, vec_len, vec_len);
+}
+
+static void gen_PSRLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 32) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    } else {
+        tcg_gen_gvec_shri(tcg_ctx, MO_32,
+                          decode->op[0].offset, decode->op[1].offset,
+                          decode->immediate, vec_len, vec_len);
+    }
+}
+
+static void gen_PSLLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 32) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    } else {
+        tcg_gen_gvec_shli(tcg_ctx, MO_32,
+                          decode->op[0].offset, decode->op[1].offset,
+                          decode->immediate, vec_len, vec_len);
+    }
+}
+
+static void gen_PSRAD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 32) {
+        decode->immediate = 31;
+    }
+    tcg_gen_gvec_sari(tcg_ctx, MO_32,
+                      decode->op[0].offset, decode->op[1].offset,
+                      decode->immediate, vec_len, vec_len);
+}
+
+static void gen_PSRLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 64) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    } else {
+        tcg_gen_gvec_shri(tcg_ctx, MO_64,
+                          decode->op[0].offset, decode->op[1].offset,
+                          decode->immediate, vec_len, vec_len);
+    }
+}
+
+static void gen_PSLLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    if (decode->immediate >= 64) {
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    } else {
+        tcg_gen_gvec_shli(tcg_ctx, MO_64,
+                          decode->op[0].offset, decode->op[1].offset,
+                          decode->immediate, vec_len, vec_len);
+    }
+}
+
+static TCGv_ptr make_imm8u_xmm_vec(TCGContext *tcg_ctx, uint8_t imm,
+                                   int vec_len)
+{
+    MemOp ot = vec_len == 16 ? MO_128 : MO_256;
+    TCGv_i32 imm_v = tcg_constant8u_i32(tcg_ctx, imm);
+    TCGv_ptr ptr = tcg_temp_new_ptr(tcg_ctx);
+
+    tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot),
+                         vec_len, vec_len, 0);
+
+    tcg_gen_addi_ptr(tcg_ctx, ptr, cpu_env, offsetof(CPUX86State, xmm_t0));
+    tcg_gen_st_i32(tcg_ctx, imm_v, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0)));
+    return ptr;
+}
+
+static void gen_PSRLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+    TCGv_ptr imm_vec = make_imm8u_xmm_vec(tcg_ctx, decode->immediate,
+                                          vec_len);
+
+    if (s->vex_l) {
+        gen_helper_psrldq_ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, imm_vec);
+    } else {
+        gen_helper_psrldq_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, imm_vec);
+    }
+    tcg_temp_free_ptr(tcg_ctx, imm_vec);
+}
+
+static void gen_PSLLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+    TCGv_ptr imm_vec = make_imm8u_xmm_vec(tcg_ctx, decode->immediate,
+                                          vec_len);
+
+    if (s->vex_l) {
+        gen_helper_pslldq_ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, imm_vec);
+    } else {
+        gen_helper_pslldq_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, imm_vec);
+    }
+    tcg_temp_free_ptr(tcg_ctx, imm_vec);
+}
+
+static void gen_RORX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+    int b = decode->immediate;
+
+    if (ot == MO_64) {
+        tcg_gen_rotri_tl(tcg_ctx, s->T0, s->T0, b & 63);
+    } else {
+        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
+        tcg_gen_rotri_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, b & 31);
+        tcg_gen_extu_i32_tl(tcg_ctx, s->T0, s->tmp2_i32);
+    }
+}
+
+static void gen_SARX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+    int mask;
+
+    mask = ot == MO_64 ? 63 : 31;
+    tcg_gen_andi_tl(tcg_ctx, s->T1, s->T1, mask);
+    if (ot != MO_64) {
+        tcg_gen_ext32s_tl(tcg_ctx, s->T0, s->T0);
+    }
+    tcg_gen_sar_tl(tcg_ctx, s->T0, s->T0, s->T1);
+}
+
+static void gen_SHLX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+    int mask;
+
+    mask = ot == MO_64 ? 63 : 31;
+    tcg_gen_andi_tl(tcg_ctx, s->T1, s->T1, mask);
+    tcg_gen_shl_tl(tcg_ctx, s->T0, s->T0, s->T1);
+}
+
+static void gen_SHRX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    MemOp ot = decode->op[0].ot;
+    int mask;
+
+    mask = ot == MO_64 ? 63 : 31;
+    tcg_gen_andi_tl(tcg_ctx, s->T1, s->T1, mask);
+    if (ot != MO_64) {
+        tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
+    }
+    tcg_gen_shr_tl(tcg_ctx, s->T0, s->T0, s->T1);
+}
+
+static void gen_VAESKEYGEN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    assert(!s->vex_l);
+    gen_helper_aeskeygenassist_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, imm);
+}
+
+static void gen_STMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (s->vex_l) {
+        gen_illegal_opcode(s);
+        return;
+    }
+    gen_helper_update_mxcsr(tcg_ctx, cpu_env);
+    tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, mxcsr));
+}
+
+static void gen_VAESIMC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(!s->vex_l);
+    gen_helper_aesimc_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+}
+
+/*
+ * 00 = v*ps Vps, Hps, Wpd
+ * 66 = v*pd Vpd, Hpd, Wps
+ * f3 = v*ss Vss, Hss, Wps
+ * f2 = v*sd Vsd, Hsd, Wps
+ */
+#define SSE_CMP(x) { \
+    gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \
+    gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \
+    gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm}
+static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = {
+    SSE_CMP(cmpeq),
+    SSE_CMP(cmplt),
+    SSE_CMP(cmple),
+    SSE_CMP(cmpunord),
+    SSE_CMP(cmpneq),
+    SSE_CMP(cmpnlt),
+    SSE_CMP(cmpnle),
+    SSE_CMP(cmpord),
+
+    SSE_CMP(cmpequ),
+    SSE_CMP(cmpnge),
+    SSE_CMP(cmpngt),
+    SSE_CMP(cmpfalse),
+    SSE_CMP(cmpnequ),
+    SSE_CMP(cmpge),
+    SSE_CMP(cmpgt),
+    SSE_CMP(cmptrue),
+
+    SSE_CMP(cmpeqs),
+    SSE_CMP(cmpltq),
+    SSE_CMP(cmpleq),
+    SSE_CMP(cmpunords),
+    SSE_CMP(cmpneqq),
+    SSE_CMP(cmpnltq),
+    SSE_CMP(cmpnleq),
+    SSE_CMP(cmpords),
+
+    SSE_CMP(cmpequs),
+    SSE_CMP(cmpngeq),
+    SSE_CMP(cmpngtq),
+    SSE_CMP(cmpfalses),
+    SSE_CMP(cmpnequs),
+    SSE_CMP(cmpgeq),
+    SSE_CMP(cmpgtq),
+    SSE_CMP(cmptrues),
+};
+#undef SSE_CMP
+
+static void gen_VCMP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7);
+    int b =
+        s->prefix & PREFIX_REPZ  ? 2 /* ss */ :
+        s->prefix & PREFIX_REPNZ ? 3 /* sd */ :
+        !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2);
+
+    gen_helper_cmp_funcs[index][b](tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+}
+
+static void gen_VCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_0_epp fn;
+    fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss;
+    fn(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2);
+    set_cc_op(s, CC_OP_EFLAGS);
+}
+
+static void gen_VCVTPD2PS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (s->vex_l) {
+        gen_helper_cvtpd2ps_ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    } else {
+        gen_helper_cvtpd2ps_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+}
+
+static void gen_VCVTPS2PD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (s->vex_l) {
+        gen_helper_cvtps2pd_ymm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    } else {
+        gen_helper_cvtps2pd_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+    }
+}
+
+static void gen_VCVTPS2PH(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_unary_imm_fp_sse(s, env, decode,
+                      gen_helper_cvtps2ph_xmm,
+                      gen_helper_cvtps2ph_ymm);
+    /*
+     * VCVTPS2PH is the only instruction that performs an operation on a
+     * register source and then *stores* into memory.
+     */
+    if (decode->op[0].has_ea) {
+        gen_store_sse(s, decode, decode->op[0].offset);
+    }
+}
+
+static void gen_VCVTSD2SS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_cvtsd2ss(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+}
+
+static void gen_VCVTSS2SD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_helper_cvtss2sd(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2);
+}
+
+static void gen_VCVTSI2Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+    TCGv_i32 in;
+
+    tcg_gen_gvec_mov(tcg_ctx, MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
+
+#ifdef TARGET_X86_64
+    MemOp ot = decode->op[2].ot;
+    if (ot == MO_64) {
+        if (s->prefix & PREFIX_REPNZ) {
+            gen_helper_cvtsq2sd(tcg_ctx, cpu_env, OP_PTR0, s->T1);
+        } else {
+            gen_helper_cvtsq2ss(tcg_ctx, cpu_env, OP_PTR0, s->T1);
+        }
+        return;
+    }
+    in = s->tmp2_i32;
+    tcg_gen_trunc_tl_i32(tcg_ctx, in, s->T1);
+#else
+    in = s->T1;
+#endif
+
+    if (s->prefix & PREFIX_REPNZ) {
+        gen_helper_cvtsi2sd(tcg_ctx, cpu_env, OP_PTR0, in);
+    } else {
+        gen_helper_cvtsi2ss(tcg_ctx, cpu_env, OP_PTR0, in);
+    }
+}
+
+static inline void gen_VCVTtSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                                  SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq,
+                                  SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 out;
+
+#ifdef TARGET_X86_64
+    MemOp ot = decode->op[0].ot;
+    if (ot == MO_64) {
+        if (s->prefix & PREFIX_REPNZ) {
+            sd2sq(tcg_ctx, s->T0, cpu_env, OP_PTR2);
+        } else {
+            ss2sq(tcg_ctx, s->T0, cpu_env, OP_PTR2);
+        }
+        return;
+    }
+
+    out = s->tmp2_i32;
+#else
+    out = s->T0;
+#endif
+    if (s->prefix & PREFIX_REPNZ) {
+        sd2si(tcg_ctx, out, cpu_env, OP_PTR2);
+    } else {
+        ss2si(tcg_ctx, out, cpu_env, OP_PTR2);
+    }
+#ifdef TARGET_X86_64
+    tcg_gen_extu_i32_tl(tcg_ctx, s->T0, out);
+#endif
+}
+
+#ifndef TARGET_X86_64
+#define gen_helper_cvtss2sq NULL
+#define gen_helper_cvtsd2sq NULL
+#define gen_helper_cvttss2sq NULL
+#define gen_helper_cvttsd2sq NULL
+#endif
+
+static void gen_VCVTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_VCVTtSx2SI(s, env, decode,
+                   gen_helper_cvtss2si, gen_helper_cvtss2sq,
+                   gen_helper_cvtsd2si, gen_helper_cvtsd2sq);
+}
+
+static void gen_VCVTTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_VCVTtSx2SI(s, env, decode,
+                   gen_helper_cvttss2si, gen_helper_cvttss2sq,
+                   gen_helper_cvttsd2si, gen_helper_cvttsd2sq);
+}
+
+static void gen_VEXTRACTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int mask = decode->immediate & 1;
+    int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask);
+    if (decode->op[0].has_ea) {
+        /* VEX-only instruction, no alignment requirements.  */
+        gen_sto_env_A0(s, src_ofs, false);
+    } else {
+        tcg_gen_gvec_mov(tcg_ctx, MO_64, decode->op[0].offset, src_ofs, 16, 16);
+    }
+}
+
+static void gen_VEXTRACTPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_pextr(s, env, decode, MO_32);
+}
+
+static void gen_vinsertps(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int val = decode->immediate;
+    int dest_word = (val >> 4) & 3;
+    int new_mask = (val & 15) | (1 << dest_word);
+    int vec_len = 16;
+
+    assert(!s->vex_l);
+
+    if (new_mask == 15) {
+        /* All zeroes except possibly for the inserted element */
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    } else if (decode->op[1].offset != decode->op[0].offset) {
+        gen_store_sse(s, decode, decode->op[1].offset);
+    }
+
+    if (new_mask != (val & 15)) {
+        tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, cpu_env,
+                       vector_elem_offset(&decode->op[0], MO_32, dest_word));
+    }
+
+    if (new_mask != 15) {
+        TCGv_i32 zero = tcg_constant_i32(tcg_ctx, 0); /* float32_zero */
+        int i;
+        for (i = 0; i < 4; i++) {
+            if ((val >> i) & 1) {
+                tcg_gen_st_i32(tcg_ctx, zero, cpu_env,
+                               vector_elem_offset(&decode->op[0], MO_32, i));
+            }
+        }
+    }
+}
+
+static void gen_VINSERTPS_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int val = decode->immediate;
+    tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, cpu_env,
+                   vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3));
+    gen_vinsertps(s, env, decode);
+}
+
+static void gen_VINSERTPS_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0, s->mem_index, MO_LEUL);
+    gen_vinsertps(s, env, decode);
+}
+
+static void gen_VINSERTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int mask = decode->immediate & 1;
+    tcg_gen_gvec_mov(tcg_ctx, MO_64,
+                     decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)),
+                     decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16);
+    tcg_gen_gvec_mov(tcg_ctx, MO_64,
+                     decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)),
+                     decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16);
+}
+
+static inline void gen_maskmov(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode,
+                               SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (!s->vex_l) {
+        xmm(tcg_ctx, cpu_env, OP_PTR2, OP_PTR1, s->A0);
+    } else {
+        ymm(tcg_ctx, cpu_env, OP_PTR2, OP_PTR1, s->A0);
+    }
+}
+
+static void gen_VMASKMOVPD_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_maskmov(s, env, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm);
+}
+
+static void gen_VMASKMOVPS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_maskmov(s, env, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm);
+}
+
+static void gen_VMOVHPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
+    if (decode->op[0].offset != decode->op[1].offset) {
+        tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
+        tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
+    }
+}
+
+static void gen_VMOVHPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
+}
+
+static void gen_VMOVHPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    if (decode->op[0].offset != decode->op[2].offset) {
+        tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
+        tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
+    }
+    if (decode->op[0].offset != decode->op[1].offset) {
+        tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
+        tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
+    }
+}
+
+static void gen_VMOVHLPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1)));
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
+    if (decode->op[0].offset != decode->op[1].offset) {
+        tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1)));
+        tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
+    }
+}
+
+static void gen_VMOVLHPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[2].offset);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1)));
+    if (decode->op[0].offset != decode->op[1].offset) {
+        tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0)));
+        tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
+    }
+}
+
+/*
+ * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx.
+ * Use a gvec move to move everything above the bottom 64 bits.
+ */
+
+static void gen_VMOVLPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0)));
+    tcg_gen_gvec_mov(tcg_ctx, MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0)));
+}
+
+static void gen_VMOVLPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
+    tcg_gen_gvec_mov(tcg_ctx, MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0)));
+}
+
+static void gen_VMOVLPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0)));
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
+}
+
+static void gen_VMOVSD_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i64 zero = tcg_constant_i64(tcg_ctx, 0);
+
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
+    tcg_gen_st_i64(tcg_ctx, zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1)));
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0)));
+}
+
+static void gen_VMOVSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0)));
+    tcg_gen_gvec_mov(tcg_ctx, MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len);
+    tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0)));
+}
+
+static void gen_VMOVSS_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int vec_len = vector_len(s, decode);
+
+    tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0, s->mem_index, MO_LEUL);
+    tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, decode->op[0].offset, vec_len, vec_len, 0);
+    tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0)));
+}
+
+static void gen_VMOVSS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0)));
+    tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0, s->mem_index, MO_LEUL);
+}
+
+static void gen_VPMASKMOV_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    if (s->vex_w) {
+        gen_VMASKMOVPD_st(s, env, decode);
+    } else {
+        gen_VMASKMOVPS_st(s, env, decode);
+    }
+}
+
+static void gen_VPERMD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(s->vex_l);
+    gen_helper_vpermd_ymm(tcg_ctx, OP_PTR0, OP_PTR1, OP_PTR2);
+}
+
+static void gen_VPERM2x128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    assert(s->vex_l);
+    gen_helper_vpermdq_ymm(tcg_ctx, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+}
+
+static void gen_VPHMINPOSUW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(!s->vex_l);
+    gen_helper_phminposuw_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR2);
+}
+
+static void gen_VROUNDSD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    assert(!s->vex_l);
+    gen_helper_roundsd_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+}
+
+static void gen_VROUNDSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant8u_i32(tcg_ctx, decode->immediate);
+    assert(!s->vex_l);
+    gen_helper_roundss_xmm(tcg_ctx, cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+}
+
+static void gen_VSHUF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 imm = tcg_constant_i32(tcg_ctx, decode->immediate);
+    SSEFunc_0_pppi ps, pd, fn;
+    ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm;
+    pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm;
+    fn = s->prefix & PREFIX_DATA ? pd : ps;
+    fn(tcg_ctx, OP_PTR0, OP_PTR1, OP_PTR2, imm);
+}
+
+static void gen_VUCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    SSEFunc_0_epp fn;
+    fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss;
+    fn(tcg_ctx, cpu_env, OP_PTR1, OP_PTR2);
+    set_cc_op(s, CC_OP_EFLAGS);
+}
+
+static void gen_VZEROALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_ptr ptr = tcg_temp_new_ptr(tcg_ctx);
+
+    tcg_gen_addi_ptr(tcg_ctx, ptr, cpu_env, offsetof(CPUX86State, xmm_regs));
+    gen_helper_memset(tcg_ctx, ptr, ptr, tcg_constant_i32(tcg_ctx, 0),
+                      tcg_constant_ptr(tcg_ctx, CPU_NB_REGS * sizeof(ZMMReg)));
+    tcg_temp_free_ptr(tcg_ctx, ptr);
+}
+
+static void gen_VZEROUPPER(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int i;
+
+    for (i = 0; i < CPU_NB_REGS; i++) {
+        int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1));
+        tcg_gen_gvec_dup_imm(tcg_ctx, MO_64, offset, 16, 16, 0);
+    }
+}
diff --git a/qemu/target/i386/fpu_helper.c b/qemu/target/i386/fpu_helper.c
index b3f537000f..5b7dd8ecff 100644
--- a/qemu/target/i386/fpu_helper.c
+++ b/qemu/target/i386/fpu_helper.c
@@ -6,7 +6,7 @@
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
+ * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -20,13 +20,19 @@
 #include "qemu/osdep.h"
 #include <math.h>
 #include "cpu.h"
-#include "exec/helper-proto.h"
-#include "qemu/host-utils.h"
 #include "exec/exec-all.h"
-#include "exec/cpu_ldst.h"
+#include "exec/helper-proto.h"
 #include "fpu/softfloat.h"
+#include "fpu/softfloat-macros.h"
+
+/* float macros */
+#define FT0    (env->ft0)
+#define ST0    (env->fpregs[env->fpstt].d)
+#define ST(n)  (env->fpregs[(env->fpstt + (n)) & 7].d)
+#define ST1    ST(1)
 
-#define FPU_RC_MASK         0xc00
+#define FPU_RC_SHIFT        10
+#define FPU_RC_MASK         (3 << FPU_RC_SHIFT)
 #define FPU_RC_NEAR         0x000
 #define FPU_RC_DOWN         0x400
 #define FPU_RC_UP           0x800
@@ -55,8 +61,23 @@
 #define FPUC_EM 0x3f
 
 #define floatx80_lg2 make_floatx80(0x3ffd, 0x9a209a84fbcff799LL)
+#define floatx80_lg2_d make_floatx80(0x3ffd, 0x9a209a84fbcff798LL)
 #define floatx80_l2e make_floatx80(0x3fff, 0xb8aa3b295c17f0bcLL)
+#define floatx80_l2e_d make_floatx80(0x3fff, 0xb8aa3b295c17f0bbLL)
 #define floatx80_l2t make_floatx80(0x4000, 0xd49a784bcd1b8afeLL)
+#define floatx80_l2t_u make_floatx80(0x4000, 0xd49a784bcd1b8affLL)
+#define floatx80_ln2_d make_floatx80(0x3ffe, 0xb17217f7d1cf79abLL)
+#define floatx80_pi_d make_floatx80(0x4000, 0xc90fdaa22168c234LL)
+
+void x86_register_ferr_irq(qemu_irq irq)
+{
+    (void)irq;
+}
+
+void fpu_check_raise_ferr_irq(CPUX86State *env)
+{
+    (void)env;
+}
 
 static void cpu_clear_ignne(CPUX86State *env)
 {
@@ -66,17 +87,8 @@ static void cpu_clear_ignne(CPUX86State *env)
 void cpu_set_ignne(CPUX86State *env)
 {
     env->hflags2 |= HF2_IGNNE_MASK;
-    /*
-     * We get here in response to a write to port F0h.  The chipset should
-     * deassert FP_IRQ and FERR# instead should stay signaled until FPSW_SE is
-     * cleared, because FERR# and FP_IRQ are two separate pins on real
-     * hardware.  However, we don't model FERR# as a qemu_irq, so we just
-     * do directly what the chipset would do, i.e. deassert FP_IRQ.
-     */
-    // qemu_irq_lower(ferr_irq);
 }
 
-
 static inline void fpush(CPUX86State *env)
 {
     env->fpstt = (env->fpstt - 1) & 7;
@@ -89,8 +101,7 @@ static inline void fpop(CPUX86State *env)
     env->fpstt = (env->fpstt + 1) & 7;
 }
 
-static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr,
-                                   uintptr_t retaddr)
+static floatx80 do_fldt(CPUX86State *env, target_ulong ptr, uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
@@ -99,8 +110,8 @@ static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr,
     return temp.d;
 }
 
-static inline void helper_fstt(CPUX86State *env, floatx80 f, target_ulong ptr,
-                               uintptr_t retaddr)
+static void do_fstt(CPUX86State *env, floatx80 f, target_ulong ptr,
+                    uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
@@ -141,12 +152,32 @@ static void fpu_set_exception(CPUX86State *env, int mask)
     }
 }
 
+static inline uint8_t save_exception_flags(CPUX86State *env)
+{
+    uint8_t old_flags = get_float_exception_flags(&env->fp_status);
+    set_float_exception_flags(0, &env->fp_status);
+    return old_flags;
+}
+
+static void merge_exception_flags(CPUX86State *env, uint8_t old_flags)
+{
+    uint8_t new_flags = get_float_exception_flags(&env->fp_status);
+    float_raise(old_flags, &env->fp_status);
+    fpu_set_exception(env,
+                      ((new_flags & float_flag_invalid ? FPUS_IE : 0) |
+                       (new_flags & float_flag_divbyzero ? FPUS_ZE : 0) |
+                       (new_flags & float_flag_overflow ? FPUS_OE : 0) |
+                       (new_flags & float_flag_underflow ? FPUS_UE : 0) |
+                       (new_flags & float_flag_inexact ? FPUS_PE : 0) |
+                       (new_flags & float_flag_input_denormal ? FPUS_DE : 0)));
+}
+
 static inline floatx80 helper_fdiv(CPUX86State *env, floatx80 a, floatx80 b)
 {
-    if (floatx80_is_zero(b)) {
-        fpu_set_exception(env, FPUS_ZE);
-    }
-    return floatx80_div(a, b, &env->fp_status);
+    uint8_t old_flags = save_exception_flags(env);
+    floatx80 ret = floatx80_div(a, b, &env->fp_status);
+    merge_exception_flags(env, old_flags);
+    return ret;
 }
 
 static void fpu_raise_exception(CPUX86State *env, uintptr_t retaddr)
@@ -154,10 +185,16 @@ static void fpu_raise_exception(CPUX86State *env, uintptr_t retaddr)
     if (env->cr[0] & CR0_NE_MASK) {
         raise_exception_ra(env, EXCP10_COPR, retaddr);
     }
+#if !defined(CONFIG_USER_ONLY)
+    else {
+        fpu_check_raise_ferr_irq(env);
+    }
+#endif
 }
 
 void helper_flds_FT0(CPUX86State *env, uint32_t val)
 {
+    uint8_t old_flags = save_exception_flags(env);
     union {
         float32 f;
         uint32_t i;
@@ -165,10 +202,12 @@ void helper_flds_FT0(CPUX86State *env, uint32_t val)
 
     u.i = val;
     FT0 = float32_to_floatx80(u.f, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fldl_FT0(CPUX86State *env, uint64_t val)
 {
+    uint8_t old_flags = save_exception_flags(env);
     union {
         float64 f;
         uint64_t i;
@@ -176,6 +215,7 @@ void helper_fldl_FT0(CPUX86State *env, uint64_t val)
 
     u.i = val;
     FT0 = float64_to_floatx80(u.f, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fildl_FT0(CPUX86State *env, int32_t val)
@@ -185,6 +225,7 @@ void helper_fildl_FT0(CPUX86State *env, int32_t val)
 
 void helper_flds_ST0(CPUX86State *env, uint32_t val)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int new_fpstt;
     union {
         float32 f;
@@ -196,10 +237,12 @@ void helper_flds_ST0(CPUX86State *env, uint32_t val)
     env->fpregs[new_fpstt].d = float32_to_floatx80(u.f, &env->fp_status);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fldl_ST0(CPUX86State *env, uint64_t val)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int new_fpstt;
     union {
         float64 f;
@@ -211,114 +254,146 @@ void helper_fldl_ST0(CPUX86State *env, uint64_t val)
     env->fpregs[new_fpstt].d = float64_to_floatx80(u.f, &env->fp_status);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
+    merge_exception_flags(env, old_flags);
+}
+
+static FloatX80RoundPrec tmp_maximise_precision(float_status *st)
+{
+    FloatX80RoundPrec old = get_floatx80_rounding_precision(st);
+    set_floatx80_rounding_precision(floatx80_precision_x, st);
+    return old;
 }
 
 void helper_fildl_ST0(CPUX86State *env, int32_t val)
 {
     int new_fpstt;
+    FloatX80RoundPrec old = tmp_maximise_precision(&env->fp_status);
 
     new_fpstt = (env->fpstt - 1) & 7;
     env->fpregs[new_fpstt].d = int32_to_floatx80(val, &env->fp_status);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
+
+    set_floatx80_rounding_precision(old, &env->fp_status);
 }
 
 void helper_fildll_ST0(CPUX86State *env, int64_t val)
 {
     int new_fpstt;
+    FloatX80RoundPrec old = tmp_maximise_precision(&env->fp_status);
 
     new_fpstt = (env->fpstt - 1) & 7;
     env->fpregs[new_fpstt].d = int64_to_floatx80(val, &env->fp_status);
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
+
+    set_floatx80_rounding_precision(old, &env->fp_status);
 }
 
 uint32_t helper_fsts_ST0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     union {
         float32 f;
         uint32_t i;
     } u;
 
     u.f = floatx80_to_float32(ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
     return u.i;
 }
 
 uint64_t helper_fstl_ST0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     union {
         float64 f;
         uint64_t i;
     } u;
 
     u.f = floatx80_to_float64(ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
     return u.i;
 }
 
 int32_t helper_fist_ST0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int32_t val;
 
     val = floatx80_to_int32(ST0, &env->fp_status);
     if (val != (int16_t)val) {
+        set_float_exception_flags(float_flag_invalid, &env->fp_status);
         val = -32768;
     }
+    merge_exception_flags(env, old_flags);
     return val;
 }
 
 int32_t helper_fistl_ST0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int32_t val;
-    signed char old_exp_flags;
-
-    old_exp_flags = get_float_exception_flags(&env->fp_status);
-    set_float_exception_flags(0, &env->fp_status);
 
     val = floatx80_to_int32(ST0, &env->fp_status);
     if (get_float_exception_flags(&env->fp_status) & float_flag_invalid) {
         val = 0x80000000;
     }
-    set_float_exception_flags(get_float_exception_flags(&env->fp_status)
-                                | old_exp_flags, &env->fp_status);
+    merge_exception_flags(env, old_flags);
     return val;
 }
 
 int64_t helper_fistll_ST0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int64_t val;
-    signed char old_exp_flags;
-
-    old_exp_flags = get_float_exception_flags(&env->fp_status);
-    set_float_exception_flags(0, &env->fp_status);
 
     val = floatx80_to_int64(ST0, &env->fp_status);
     if (get_float_exception_flags(&env->fp_status) & float_flag_invalid) {
         val = 0x8000000000000000ULL;
     }
-    set_float_exception_flags(get_float_exception_flags(&env->fp_status)
-                                | old_exp_flags, &env->fp_status);
+    merge_exception_flags(env, old_flags);
     return val;
 }
 
 int32_t helper_fistt_ST0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int32_t val;
 
     val = floatx80_to_int32_round_to_zero(ST0, &env->fp_status);
     if (val != (int16_t)val) {
+        set_float_exception_flags(float_flag_invalid, &env->fp_status);
         val = -32768;
     }
+    merge_exception_flags(env, old_flags);
     return val;
 }
 
 int32_t helper_fisttl_ST0(CPUX86State *env)
 {
-    return floatx80_to_int32_round_to_zero(ST0, &env->fp_status);
+    uint8_t old_flags = save_exception_flags(env);
+    int32_t val;
+
+    val = floatx80_to_int32_round_to_zero(ST0, &env->fp_status);
+    if (get_float_exception_flags(&env->fp_status) & float_flag_invalid) {
+        val = 0x80000000;
+    }
+    merge_exception_flags(env, old_flags);
+    return val;
 }
 
 int64_t helper_fisttll_ST0(CPUX86State *env)
 {
-    return floatx80_to_int64_round_to_zero(ST0, &env->fp_status);
+    uint8_t old_flags = save_exception_flags(env);
+    int64_t val;
+
+    val = floatx80_to_int64_round_to_zero(ST0, &env->fp_status);
+    if (get_float_exception_flags(&env->fp_status) & float_flag_invalid) {
+        val = 0x8000000000000000ULL;
+    }
+    merge_exception_flags(env, old_flags);
+    return val;
 }
 
 void helper_fldt_ST0(CPUX86State *env, target_ulong ptr)
@@ -326,14 +401,14 @@ void helper_fldt_ST0(CPUX86State *env, target_ulong ptr)
     int new_fpstt;
 
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt].d = helper_fldt(env, ptr, GETPC());
+    env->fpregs[new_fpstt].d = do_fldt(env, ptr, GETPC());
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
 
 void helper_fstt_ST0(CPUX86State *env, target_ulong ptr)
 {
-    helper_fstt(env, ST0, ptr, GETPC());
+    do_fstt(env, ST0, ptr, GETPC());
 }
 
 void helper_fpush(CPUX86State *env)
@@ -400,62 +475,78 @@ static const int fcom_ccval[4] = {0x0100, 0x4000, 0x0000, 0x4500};
 
 void helper_fcom_ST0_FT0(CPUX86State *env)
 {
-    int ret;
+    uint8_t old_flags = save_exception_flags(env);
+    FloatRelation ret;
 
     ret = floatx80_compare(ST0, FT0, &env->fp_status);
     env->fpus = (env->fpus & ~0x4500) | fcom_ccval[ret + 1];
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fucom_ST0_FT0(CPUX86State *env)
 {
-    int ret;
+    uint8_t old_flags = save_exception_flags(env);
+    FloatRelation ret;
 
     ret = floatx80_compare_quiet(ST0, FT0, &env->fp_status);
     env->fpus = (env->fpus & ~0x4500) | fcom_ccval[ret + 1];
+    merge_exception_flags(env, old_flags);
 }
 
 static const int fcomi_ccval[4] = {CC_C, CC_Z, 0, CC_Z | CC_P | CC_C};
 
 void helper_fcomi_ST0_FT0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int eflags;
-    int ret;
+    FloatRelation ret;
 
     ret = floatx80_compare(ST0, FT0, &env->fp_status);
     eflags = cpu_cc_compute_all(env, CC_OP);
     eflags = (eflags & ~(CC_Z | CC_P | CC_C)) | fcomi_ccval[ret + 1];
     CC_SRC = eflags;
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fucomi_ST0_FT0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int eflags;
-    int ret;
+    FloatRelation ret;
 
     ret = floatx80_compare_quiet(ST0, FT0, &env->fp_status);
     eflags = cpu_cc_compute_all(env, CC_OP);
     eflags = (eflags & ~(CC_Z | CC_P | CC_C)) | fcomi_ccval[ret + 1];
     CC_SRC = eflags;
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fadd_ST0_FT0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST0 = floatx80_add(ST0, FT0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fmul_ST0_FT0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST0 = floatx80_mul(ST0, FT0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fsub_ST0_FT0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST0 = floatx80_sub(ST0, FT0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fsubr_ST0_FT0(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST0 = floatx80_sub(FT0, ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fdiv_ST0_FT0(CPUX86State *env)
@@ -472,22 +563,30 @@ void helper_fdivr_ST0_FT0(CPUX86State *env)
 
 void helper_fadd_STN_ST0(CPUX86State *env, int st_index)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST(st_index) = floatx80_add(ST(st_index), ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fmul_STN_ST0(CPUX86State *env, int st_index)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST(st_index) = floatx80_mul(ST(st_index), ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fsub_STN_ST0(CPUX86State *env, int st_index)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST(st_index) = floatx80_sub(ST(st_index), ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fsubr_STN_ST0(CPUX86State *env, int st_index)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST(st_index) = floatx80_sub(ST0, ST(st_index), &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fdiv_STN_ST0(CPUX86State *env, int st_index)
@@ -519,58 +618,81 @@ void helper_fabs_ST0(CPUX86State *env)
 
 void helper_fld1_ST0(CPUX86State *env)
 {
-    //ST0 = floatx80_one;
-    floatx80 one = { 0x8000000000000000LL, 0x3fff };
-    ST0 = one;
+    ST0 = floatx80_one;
 }
 
 void helper_fldl2t_ST0(CPUX86State *env)
 {
-    //ST0 = floatx80_l2t;
-    floatx80 l2t = { 0xd49a784bcd1b8afeLL, 0x4000 };
-    ST0 = l2t;
+    switch (env->fpuc & FPU_RC_MASK) {
+    case FPU_RC_UP:
+        ST0 = floatx80_l2t_u;
+        break;
+    default:
+        ST0 = floatx80_l2t;
+        break;
+    }
 }
 
 void helper_fldl2e_ST0(CPUX86State *env)
 {
-    //ST0 = floatx80_l2e;
-    floatx80 l2e = { 0xb8aa3b295c17f0bcLL, 0x3fff };
-    ST0 = l2e;
+    switch (env->fpuc & FPU_RC_MASK) {
+    case FPU_RC_DOWN:
+    case FPU_RC_CHOP:
+        ST0 = floatx80_l2e_d;
+        break;
+    default:
+        ST0 = floatx80_l2e;
+        break;
+    }
 }
 
 void helper_fldpi_ST0(CPUX86State *env)
 {
-    //ST0 = floatx80_pi;
-    floatx80 pi = { 0xc90fdaa22168c235LL, 0x4000 };
-    ST0 = pi;
+    switch (env->fpuc & FPU_RC_MASK) {
+    case FPU_RC_DOWN:
+    case FPU_RC_CHOP:
+        ST0 = floatx80_pi_d;
+        break;
+    default:
+        ST0 = floatx80_pi;
+        break;
+    }
 }
 
 void helper_fldlg2_ST0(CPUX86State *env)
 {
-    //ST0 = floatx80_lg2;
-    floatx80 lg2 = { 0x9a209a84fbcff799LL, 0x3ffd };
-    ST0 = lg2;
+    switch (env->fpuc & FPU_RC_MASK) {
+    case FPU_RC_DOWN:
+    case FPU_RC_CHOP:
+        ST0 = floatx80_lg2_d;
+        break;
+    default:
+        ST0 = floatx80_lg2;
+        break;
+    }
 }
 
 void helper_fldln2_ST0(CPUX86State *env)
 {
-    //ST0 = floatx80_ln2;
-    floatx80 ln2 = { 0xb17217f7d1cf79acLL, 0x3ffe };
-    ST0 = ln2;
+    switch (env->fpuc & FPU_RC_MASK) {
+    case FPU_RC_DOWN:
+    case FPU_RC_CHOP:
+        ST0 = floatx80_ln2_d;
+        break;
+    default:
+        ST0 = floatx80_ln2;
+        break;
+    }
 }
 
 void helper_fldz_ST0(CPUX86State *env)
 {
-    //ST0 = floatx80_zero;
-    floatx80 zero = { 0x0000000000000000LL, 0x0000 };
-    ST0 = zero;
+    ST0 = floatx80_zero;
 }
 
 void helper_fldz_FT0(CPUX86State *env)
 {
-    //FT0 = floatx80_zero;
-    floatx80 zero = { 0x0000000000000000LL, 0x0000 };
-    FT0 = zero;
+    FT0 = floatx80_zero;
 }
 
 uint32_t helper_fnstsw(CPUX86State *env)
@@ -583,40 +705,40 @@ uint32_t helper_fnstcw(CPUX86State *env)
     return env->fpuc;
 }
 
+static void set_x86_rounding_mode(unsigned mode, float_status *status)
+{
+    static FloatRoundMode x86_round_mode[4] = {
+        float_round_nearest_even,
+        float_round_down,
+        float_round_up,
+        float_round_to_zero
+    };
+    assert(mode < ARRAY_SIZE(x86_round_mode));
+    set_float_rounding_mode(x86_round_mode[mode], status);
+}
+
 void update_fp_status(CPUX86State *env)
 {
-    int rnd_type;
+    int rnd_mode;
+    FloatX80RoundPrec rnd_prec;
 
     /* set rounding mode */
-    switch (env->fpuc & FPU_RC_MASK) {
-    default:
-    case FPU_RC_NEAR:
-        rnd_type = float_round_nearest_even;
-        break;
-    case FPU_RC_DOWN:
-        rnd_type = float_round_down;
-        break;
-    case FPU_RC_UP:
-        rnd_type = float_round_up;
-        break;
-    case FPU_RC_CHOP:
-        rnd_type = float_round_to_zero;
-        break;
-    }
-    set_float_rounding_mode(rnd_type, &env->fp_status);
+    rnd_mode = (env->fpuc & FPU_RC_MASK) >> FPU_RC_SHIFT;
+    set_x86_rounding_mode(rnd_mode, &env->fp_status);
+
     switch ((env->fpuc >> 8) & 3) {
     case 0:
-        rnd_type = 32;
+        rnd_prec = floatx80_precision_s;
         break;
     case 2:
-        rnd_type = 64;
+        rnd_prec = floatx80_precision_d;
         break;
     case 3:
     default:
-        rnd_type = 80;
+        rnd_prec = floatx80_precision_x;
         break;
     }
-    set_floatx80_rounding_precision(rnd_type, &env->fp_status);
+    set_floatx80_rounding_precision(rnd_prec, &env->fp_status);
 }
 
 void helper_fldcw(CPUX86State *env, uint32_t val)
@@ -684,18 +806,31 @@ void helper_fbld_ST0(CPUX86State *env, target_ulong ptr)
 
 void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
 {
+    uint8_t old_flags = save_exception_flags(env);
     int v;
     target_ulong mem_ref, mem_end;
     int64_t val;
+    CPU_LDoubleU temp;
+
+    temp.d = ST0;
 
     val = floatx80_to_int64(ST0, &env->fp_status);
     mem_ref = ptr;
+    if (val >= 1000000000000000000LL || val <= -1000000000000000000LL) {
+        set_float_exception_flags(float_flag_invalid, &env->fp_status);
+        while (mem_ref < ptr + 7) {
+            cpu_stb_data_ra(env, mem_ref++, 0, GETPC());
+        }
+        cpu_stb_data_ra(env, mem_ref++, 0xc0, GETPC());
+        cpu_stb_data_ra(env, mem_ref++, 0xff, GETPC());
+        cpu_stb_data_ra(env, mem_ref++, 0xff, GETPC());
+        merge_exception_flags(env, old_flags);
+        return;
+    }
     mem_end = mem_ref + 9;
-    if (val < 0) {
+    if (SIGND(temp)) {
         cpu_stb_data_ra(env, mem_end, 0x80, GETPC());
-        if (val != 0x8000000000000000LL) {
-            val = -val;
-        }
+        val = -val;
     } else {
         cpu_stb_data_ra(env, mem_end, 0x00, GETPC());
     }
@@ -705,35 +840,402 @@ void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
         }
         v = val % 100;
         val = val / 100;
-        v = (int)((unsigned int)(v / 10) << 4) | (v % 10);
+        v = ((v / 10) << 4) | (v % 10);
         cpu_stb_data_ra(env, mem_ref++, v, GETPC());
     }
     while (mem_ref < mem_end) {
         cpu_stb_data_ra(env, mem_ref++, 0, GETPC());
     }
+    merge_exception_flags(env, old_flags);
 }
 
-void helper_f2xm1(CPUX86State *env)
-{
-    double val = floatx80_to_double(env, ST0);
+/* 128-bit significand of log(2).  */
+#define ln2_sig_high 0xb17217f7d1cf79abULL
+#define ln2_sig_low 0xc9e3b39803f2f6afULL
 
-    val = pow(2.0, val) - 1.0;
-    ST0 = double_to_floatx80(env, val);
-}
+/*
+ * Polynomial coefficients for an approximation to (2^x - 1) / x, on
+ * the interval [-1/64, 1/64].
+ */
+#define f2xm1_coeff_0 make_floatx80(0x3ffe, 0xb17217f7d1cf79acULL)
+#define f2xm1_coeff_0_low make_floatx80(0xbfbc, 0xd87edabf495b3762ULL)
+#define f2xm1_coeff_1 make_floatx80(0x3ffc, 0xf5fdeffc162c7543ULL)
+#define f2xm1_coeff_2 make_floatx80(0x3ffa, 0xe35846b82505fcc7ULL)
+#define f2xm1_coeff_3 make_floatx80(0x3ff8, 0x9d955b7dd273b899ULL)
+#define f2xm1_coeff_4 make_floatx80(0x3ff5, 0xaec3ff3c4ef4ac0cULL)
+#define f2xm1_coeff_5 make_floatx80(0x3ff2, 0xa184897c3a7f0de9ULL)
+#define f2xm1_coeff_6 make_floatx80(0x3fee, 0xffe634d0ec30d504ULL)
+#define f2xm1_coeff_7 make_floatx80(0x3feb, 0xb160111d2db515e4ULL)
+
+struct f2xm1_data {
+    /*
+     * A value very close to a multiple of 1/32, such that 2^t and 2^t - 1
+     * are very close to exact floatx80 values.
+     */
+    floatx80 t;
+    /* The value of 2^t.  */
+    floatx80 exp2;
+    /* The value of 2^t - 1.  */
+    floatx80 exp2m1;
+};
+
+static const struct f2xm1_data f2xm1_table[65] = {
+    { make_floatx80_init(0xbfff, 0x8000000000000000ULL),
+      make_floatx80_init(0x3ffe, 0x8000000000000000ULL),
+      make_floatx80_init(0xbffe, 0x8000000000000000ULL) },
+    { make_floatx80_init(0xbffe, 0xf800000000002e7eULL),
+      make_floatx80_init(0x3ffe, 0x82cd8698ac2b9160ULL),
+      make_floatx80_init(0xbffd, 0xfa64f2cea7a8dd40ULL) },
+    { make_floatx80_init(0xbffe, 0xefffffffffffe960ULL),
+      make_floatx80_init(0x3ffe, 0x85aac367cc488345ULL),
+      make_floatx80_init(0xbffd, 0xf4aa7930676ef976ULL) },
+    { make_floatx80_init(0xbffe, 0xe800000000006f10ULL),
+      make_floatx80_init(0x3ffe, 0x88980e8092da5c14ULL),
+      make_floatx80_init(0xbffd, 0xeecfe2feda4b47d8ULL) },
+    { make_floatx80_init(0xbffe, 0xe000000000008a45ULL),
+      make_floatx80_init(0x3ffe, 0x8b95c1e3ea8ba2a5ULL),
+      make_floatx80_init(0xbffd, 0xe8d47c382ae8bab6ULL) },
+    { make_floatx80_init(0xbffe, 0xd7ffffffffff8a9eULL),
+      make_floatx80_init(0x3ffe, 0x8ea4398b45cd8116ULL),
+      make_floatx80_init(0xbffd, 0xe2b78ce97464fdd4ULL) },
+    { make_floatx80_init(0xbffe, 0xd0000000000019a0ULL),
+      make_floatx80_init(0x3ffe, 0x91c3d373ab11b919ULL),
+      make_floatx80_init(0xbffd, 0xdc785918a9dc8dceULL) },
+    { make_floatx80_init(0xbffe, 0xc7ffffffffff14dfULL),
+      make_floatx80_init(0x3ffe, 0x94f4efa8fef76836ULL),
+      make_floatx80_init(0xbffd, 0xd61620ae02112f94ULL) },
+    { make_floatx80_init(0xbffe, 0xc000000000006530ULL),
+      make_floatx80_init(0x3ffe, 0x9837f0518db87fbbULL),
+      make_floatx80_init(0xbffd, 0xcf901f5ce48f008aULL) },
+    { make_floatx80_init(0xbffe, 0xb7ffffffffff1723ULL),
+      make_floatx80_init(0x3ffe, 0x9b8d39b9d54eb74cULL),
+      make_floatx80_init(0xbffd, 0xc8e58c8c55629168ULL) },
+    { make_floatx80_init(0xbffe, 0xb00000000000b5e1ULL),
+      make_floatx80_init(0x3ffe, 0x9ef5326091a0c366ULL),
+      make_floatx80_init(0xbffd, 0xc2159b3edcbe7934ULL) },
+    { make_floatx80_init(0xbffe, 0xa800000000006f8aULL),
+      make_floatx80_init(0x3ffe, 0xa27043030c49370aULL),
+      make_floatx80_init(0xbffd, 0xbb1f79f9e76d91ecULL) },
+    { make_floatx80_init(0xbffe, 0x9fffffffffff816aULL),
+      make_floatx80_init(0x3ffe, 0xa5fed6a9b15171cfULL),
+      make_floatx80_init(0xbffd, 0xb40252ac9d5d1c62ULL) },
+    { make_floatx80_init(0xbffe, 0x97ffffffffffb621ULL),
+      make_floatx80_init(0x3ffe, 0xa9a15ab4ea7c30e6ULL),
+      make_floatx80_init(0xbffd, 0xacbd4a962b079e34ULL) },
+    { make_floatx80_init(0xbffe, 0x8fffffffffff162bULL),
+      make_floatx80_init(0x3ffe, 0xad583eea42a1b886ULL),
+      make_floatx80_init(0xbffd, 0xa54f822b7abc8ef4ULL) },
+    { make_floatx80_init(0xbffe, 0x87ffffffffff4d34ULL),
+      make_floatx80_init(0x3ffe, 0xb123f581d2ac7b51ULL),
+      make_floatx80_init(0xbffd, 0x9db814fc5aa7095eULL) },
+    { make_floatx80_init(0xbffe, 0x800000000000227dULL),
+      make_floatx80_init(0x3ffe, 0xb504f333f9de539dULL),
+      make_floatx80_init(0xbffd, 0x95f619980c4358c6ULL) },
+    { make_floatx80_init(0xbffd, 0xefffffffffff3978ULL),
+      make_floatx80_init(0x3ffe, 0xb8fbaf4762fbd0a1ULL),
+      make_floatx80_init(0xbffd, 0x8e08a1713a085ebeULL) },
+    { make_floatx80_init(0xbffd, 0xe00000000000df81ULL),
+      make_floatx80_init(0x3ffe, 0xbd08a39f580bfd8cULL),
+      make_floatx80_init(0xbffd, 0x85eeb8c14fe804e8ULL) },
+    { make_floatx80_init(0xbffd, 0xd00000000000bccfULL),
+      make_floatx80_init(0x3ffe, 0xc12c4cca667062f6ULL),
+      make_floatx80_init(0xbffc, 0xfb4eccd6663e7428ULL) },
+    { make_floatx80_init(0xbffd, 0xc00000000000eff0ULL),
+      make_floatx80_init(0x3ffe, 0xc5672a1155069abeULL),
+      make_floatx80_init(0xbffc, 0xea6357baabe59508ULL) },
+    { make_floatx80_init(0xbffd, 0xb000000000000fe6ULL),
+      make_floatx80_init(0x3ffe, 0xc9b9bd866e2f234bULL),
+      make_floatx80_init(0xbffc, 0xd91909e6474372d4ULL) },
+    { make_floatx80_init(0xbffd, 0x9fffffffffff2172ULL),
+      make_floatx80_init(0x3ffe, 0xce248c151f84bf00ULL),
+      make_floatx80_init(0xbffc, 0xc76dcfab81ed0400ULL) },
+    { make_floatx80_init(0xbffd, 0x8fffffffffffafffULL),
+      make_floatx80_init(0x3ffe, 0xd2a81d91f12afb2bULL),
+      make_floatx80_init(0xbffc, 0xb55f89b83b541354ULL) },
+    { make_floatx80_init(0xbffc, 0xffffffffffff81a3ULL),
+      make_floatx80_init(0x3ffe, 0xd744fccad69d7d5eULL),
+      make_floatx80_init(0xbffc, 0xa2ec0cd4a58a0a88ULL) },
+    { make_floatx80_init(0xbffc, 0xdfffffffffff1568ULL),
+      make_floatx80_init(0x3ffe, 0xdbfbb797daf25a44ULL),
+      make_floatx80_init(0xbffc, 0x901121a0943696f0ULL) },
+    { make_floatx80_init(0xbffc, 0xbfffffffffff68daULL),
+      make_floatx80_init(0x3ffe, 0xe0ccdeec2a94f811ULL),
+      make_floatx80_init(0xbffb, 0xf999089eab583f78ULL) },
+    { make_floatx80_init(0xbffc, 0x9fffffffffff4690ULL),
+      make_floatx80_init(0x3ffe, 0xe5b906e77c83657eULL),
+      make_floatx80_init(0xbffb, 0xd237c8c41be4d410ULL) },
+    { make_floatx80_init(0xbffb, 0xffffffffffff8aeeULL),
+      make_floatx80_init(0x3ffe, 0xeac0c6e7dd24427cULL),
+      make_floatx80_init(0xbffb, 0xa9f9c8c116ddec20ULL) },
+    { make_floatx80_init(0xbffb, 0xbfffffffffff2d18ULL),
+      make_floatx80_init(0x3ffe, 0xefe4b99bdcdb06ebULL),
+      make_floatx80_init(0xbffb, 0x80da33211927c8a8ULL) },
+    { make_floatx80_init(0xbffa, 0xffffffffffff8ccbULL),
+      make_floatx80_init(0x3ffe, 0xf5257d152486d0f4ULL),
+      make_floatx80_init(0xbffa, 0xada82eadb792f0c0ULL) },
+    { make_floatx80_init(0xbff9, 0xffffffffffff11feULL),
+      make_floatx80_init(0x3ffe, 0xfa83b2db722a0846ULL),
+      make_floatx80_init(0xbff9, 0xaf89a491babef740ULL) },
+    { floatx80_zero_init,
+      make_floatx80_init(0x3fff, 0x8000000000000000ULL),
+      floatx80_zero_init },
+    { make_floatx80_init(0x3ff9, 0xffffffffffff2680ULL),
+      make_floatx80_init(0x3fff, 0x82cd8698ac2b9f6fULL),
+      make_floatx80_init(0x3ff9, 0xb361a62b0ae7dbc0ULL) },
+    { make_floatx80_init(0x3ffb, 0x800000000000b500ULL),
+      make_floatx80_init(0x3fff, 0x85aac367cc488345ULL),
+      make_floatx80_init(0x3ffa, 0xb5586cf9891068a0ULL) },
+    { make_floatx80_init(0x3ffb, 0xbfffffffffff4b67ULL),
+      make_floatx80_init(0x3fff, 0x88980e8092da7cceULL),
+      make_floatx80_init(0x3ffb, 0x8980e8092da7cce0ULL) },
+    { make_floatx80_init(0x3ffb, 0xffffffffffffff57ULL),
+      make_floatx80_init(0x3fff, 0x8b95c1e3ea8bd6dfULL),
+      make_floatx80_init(0x3ffb, 0xb95c1e3ea8bd6df0ULL) },
+    { make_floatx80_init(0x3ffc, 0x9fffffffffff811fULL),
+      make_floatx80_init(0x3fff, 0x8ea4398b45cd4780ULL),
+      make_floatx80_init(0x3ffb, 0xea4398b45cd47800ULL) },
+    { make_floatx80_init(0x3ffc, 0xbfffffffffff9980ULL),
+      make_floatx80_init(0x3fff, 0x91c3d373ab11b919ULL),
+      make_floatx80_init(0x3ffc, 0x8e1e9b9d588dc8c8ULL) },
+    { make_floatx80_init(0x3ffc, 0xdffffffffffff631ULL),
+      make_floatx80_init(0x3fff, 0x94f4efa8fef70864ULL),
+      make_floatx80_init(0x3ffc, 0xa7a77d47f7b84320ULL) },
+    { make_floatx80_init(0x3ffc, 0xffffffffffff2499ULL),
+      make_floatx80_init(0x3fff, 0x9837f0518db892d4ULL),
+      make_floatx80_init(0x3ffc, 0xc1bf828c6dc496a0ULL) },
+    { make_floatx80_init(0x3ffd, 0x8fffffffffff80fbULL),
+      make_floatx80_init(0x3fff, 0x9b8d39b9d54e3a79ULL),
+      make_floatx80_init(0x3ffc, 0xdc69cdceaa71d3c8ULL) },
+    { make_floatx80_init(0x3ffd, 0x9fffffffffffbc23ULL),
+      make_floatx80_init(0x3fff, 0x9ef5326091a10313ULL),
+      make_floatx80_init(0x3ffc, 0xf7a993048d081898ULL) },
+    { make_floatx80_init(0x3ffd, 0xafffffffffff20ecULL),
+      make_floatx80_init(0x3fff, 0xa27043030c49370aULL),
+      make_floatx80_init(0x3ffd, 0x89c10c0c3124dc28ULL) },
+    { make_floatx80_init(0x3ffd, 0xc00000000000fd2cULL),
+      make_floatx80_init(0x3fff, 0xa5fed6a9b15171cfULL),
+      make_floatx80_init(0x3ffd, 0x97fb5aa6c545c73cULL) },
+    { make_floatx80_init(0x3ffd, 0xd0000000000093beULL),
+      make_floatx80_init(0x3fff, 0xa9a15ab4ea7c30e6ULL),
+      make_floatx80_init(0x3ffd, 0xa6856ad3a9f0c398ULL) },
+    { make_floatx80_init(0x3ffd, 0xe00000000000c2aeULL),
+      make_floatx80_init(0x3fff, 0xad583eea42a17876ULL),
+      make_floatx80_init(0x3ffd, 0xb560fba90a85e1d8ULL) },
+    { make_floatx80_init(0x3ffd, 0xefffffffffff1e3fULL),
+      make_floatx80_init(0x3fff, 0xb123f581d2abef6cULL),
+      make_floatx80_init(0x3ffd, 0xc48fd6074aafbdb0ULL) },
+    { make_floatx80_init(0x3ffd, 0xffffffffffff1c23ULL),
+      make_floatx80_init(0x3fff, 0xb504f333f9de2cadULL),
+      make_floatx80_init(0x3ffd, 0xd413cccfe778b2b4ULL) },
+    { make_floatx80_init(0x3ffe, 0x8800000000006344ULL),
+      make_floatx80_init(0x3fff, 0xb8fbaf4762fbd0a1ULL),
+      make_floatx80_init(0x3ffd, 0xe3eebd1d8bef4284ULL) },
+    { make_floatx80_init(0x3ffe, 0x9000000000005d67ULL),
+      make_floatx80_init(0x3fff, 0xbd08a39f580c668dULL),
+      make_floatx80_init(0x3ffd, 0xf4228e7d60319a34ULL) },
+    { make_floatx80_init(0x3ffe, 0x9800000000009127ULL),
+      make_floatx80_init(0x3fff, 0xc12c4cca6670e042ULL),
+      make_floatx80_init(0x3ffe, 0x82589994cce1c084ULL) },
+    { make_floatx80_init(0x3ffe, 0x9fffffffffff06f9ULL),
+      make_floatx80_init(0x3fff, 0xc5672a11550655c3ULL),
+      make_floatx80_init(0x3ffe, 0x8ace5422aa0cab86ULL) },
+    { make_floatx80_init(0x3ffe, 0xa7fffffffffff80dULL),
+      make_floatx80_init(0x3fff, 0xc9b9bd866e2f234bULL),
+      make_floatx80_init(0x3ffe, 0x93737b0cdc5e4696ULL) },
+    { make_floatx80_init(0x3ffe, 0xafffffffffff1470ULL),
+      make_floatx80_init(0x3fff, 0xce248c151f83fd69ULL),
+      make_floatx80_init(0x3ffe, 0x9c49182a3f07fad2ULL) },
+    { make_floatx80_init(0x3ffe, 0xb800000000000e0aULL),
+      make_floatx80_init(0x3fff, 0xd2a81d91f12aec5cULL),
+      make_floatx80_init(0x3ffe, 0xa5503b23e255d8b8ULL) },
+    { make_floatx80_init(0x3ffe, 0xc00000000000b7faULL),
+      make_floatx80_init(0x3fff, 0xd744fccad69dd630ULL),
+      make_floatx80_init(0x3ffe, 0xae89f995ad3bac60ULL) },
+    { make_floatx80_init(0x3ffe, 0xc800000000003aa6ULL),
+      make_floatx80_init(0x3fff, 0xdbfbb797daf25a44ULL),
+      make_floatx80_init(0x3ffe, 0xb7f76f2fb5e4b488ULL) },
+    { make_floatx80_init(0x3ffe, 0xd00000000000a6aeULL),
+      make_floatx80_init(0x3fff, 0xe0ccdeec2a954685ULL),
+      make_floatx80_init(0x3ffe, 0xc199bdd8552a8d0aULL) },
+    { make_floatx80_init(0x3ffe, 0xd800000000004165ULL),
+      make_floatx80_init(0x3fff, 0xe5b906e77c837155ULL),
+      make_floatx80_init(0x3ffe, 0xcb720dcef906e2aaULL) },
+    { make_floatx80_init(0x3ffe, 0xe00000000000582cULL),
+      make_floatx80_init(0x3fff, 0xeac0c6e7dd24713aULL),
+      make_floatx80_init(0x3ffe, 0xd5818dcfba48e274ULL) },
+    { make_floatx80_init(0x3ffe, 0xe800000000001a5dULL),
+      make_floatx80_init(0x3fff, 0xefe4b99bdcdb06ebULL),
+      make_floatx80_init(0x3ffe, 0xdfc97337b9b60dd6ULL) },
+    { make_floatx80_init(0x3ffe, 0xefffffffffffc1efULL),
+      make_floatx80_init(0x3fff, 0xf5257d152486a2faULL),
+      make_floatx80_init(0x3ffe, 0xea4afa2a490d45f4ULL) },
+    { make_floatx80_init(0x3ffe, 0xf800000000001069ULL),
+      make_floatx80_init(0x3fff, 0xfa83b2db722a0e5cULL),
+      make_floatx80_init(0x3ffe, 0xf50765b6e4541cb8ULL) },
+    { make_floatx80_init(0x3fff, 0x8000000000000000ULL),
+      make_floatx80_init(0x4000, 0x8000000000000000ULL),
+      make_floatx80_init(0x3fff, 0x8000000000000000ULL) },
+};
 
-void helper_fyl2x(CPUX86State *env)
+void helper_f2xm1(CPUX86State *env)
 {
-    double fptemp = floatx80_to_double(env, ST0);
-
-    if (fptemp > 0.0) {
-        fptemp = log(fptemp) / log(2.0); /* log2(ST) */
-        fptemp *= floatx80_to_double(env, ST1);
-        ST1 = double_to_floatx80(env, fptemp);
-        fpop(env);
+    uint8_t old_flags = save_exception_flags(env);
+    uint64_t sig = extractFloatx80Frac(ST0);
+    int32_t exp = extractFloatx80Exp(ST0);
+    bool sign = extractFloatx80Sign(ST0);
+
+    if (floatx80_invalid_encoding(ST0)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST0 = floatx80_default_nan(&env->fp_status);
+    } else if (floatx80_is_any_nan(ST0)) {
+        if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
+            float_raise(float_flag_invalid, &env->fp_status);
+            ST0 = floatx80_silence_nan(ST0, &env->fp_status);
+        }
+    } else if (exp > 0x3fff ||
+               (exp == 0x3fff && sig != (0x8000000000000000ULL))) {
+        /* Out of range for the instruction, treat as invalid.  */
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST0 = floatx80_default_nan(&env->fp_status);
+    } else if (exp == 0x3fff) {
+        /* Argument 1 or -1, exact result 1 or -0.5.  */
+        if (sign) {
+            ST0 = make_floatx80(0xbffe, 0x8000000000000000ULL);
+        }
+    } else if (exp < 0x3fb0) {
+        if (!floatx80_is_zero(ST0)) {
+            /*
+             * Multiplying the argument by an extra-precision version
+             * of log(2) is sufficiently precise.  Zero arguments are
+             * returned unchanged.
+             */
+            uint64_t sig0, sig1, sig2;
+            if (exp == 0) {
+                normalizeFloatx80Subnormal(sig, &exp, &sig);
+            }
+            mul128By64To192(ln2_sig_high, ln2_sig_low, sig, &sig0, &sig1,
+                            &sig2);
+            /* This result is inexact.  */
+            sig1 |= 1;
+            ST0 = normalizeRoundAndPackFloatx80(floatx80_precision_x,
+                                                sign, exp, sig0, sig1,
+                                                &env->fp_status);
+        }
     } else {
-        env->fpus &= ~0x4700;
-        env->fpus |= 0x400;
+        floatx80 tmp, y, accum;
+        bool asign, bsign;
+        int32_t n, aexp, bexp;
+        uint64_t asig0, asig1, asig2, bsig0, bsig1;
+        FloatRoundMode save_mode = env->fp_status.float_rounding_mode;
+        FloatX80RoundPrec save_prec =
+            env->fp_status.floatx80_rounding_precision;
+        env->fp_status.float_rounding_mode = float_round_nearest_even;
+        env->fp_status.floatx80_rounding_precision = floatx80_precision_x;
+
+        /* Find the nearest multiple of 1/32 to the argument.  */
+        tmp = floatx80_scalbn(ST0, 5, &env->fp_status);
+        n = 32 + floatx80_to_int32(tmp, &env->fp_status);
+        y = floatx80_sub(ST0, f2xm1_table[n].t, &env->fp_status);
+
+        if (floatx80_is_zero(y)) {
+            /*
+             * Use the value of 2^t - 1 from the table, to avoid
+             * needing to special-case zero as a result of
+             * multiplication below.
+             */
+            ST0 = f2xm1_table[n].t;
+            set_float_exception_flags(float_flag_inexact, &env->fp_status);
+            env->fp_status.float_rounding_mode = save_mode;
+        } else {
+            /*
+             * Compute the lower parts of a polynomial expansion for
+             * (2^y - 1) / y.
+             */
+            accum = floatx80_mul(f2xm1_coeff_7, y, &env->fp_status);
+            accum = floatx80_add(f2xm1_coeff_6, accum, &env->fp_status);
+            accum = floatx80_mul(accum, y, &env->fp_status);
+            accum = floatx80_add(f2xm1_coeff_5, accum, &env->fp_status);
+            accum = floatx80_mul(accum, y, &env->fp_status);
+            accum = floatx80_add(f2xm1_coeff_4, accum, &env->fp_status);
+            accum = floatx80_mul(accum, y, &env->fp_status);
+            accum = floatx80_add(f2xm1_coeff_3, accum, &env->fp_status);
+            accum = floatx80_mul(accum, y, &env->fp_status);
+            accum = floatx80_add(f2xm1_coeff_2, accum, &env->fp_status);
+            accum = floatx80_mul(accum, y, &env->fp_status);
+            accum = floatx80_add(f2xm1_coeff_1, accum, &env->fp_status);
+            accum = floatx80_mul(accum, y, &env->fp_status);
+            accum = floatx80_add(f2xm1_coeff_0_low, accum, &env->fp_status);
+
+            /*
+             * The full polynomial expansion is f2xm1_coeff_0 + accum
+             * (where accum has much lower magnitude, and so, in
+             * particular, carry out of the addition is not possible).
+             * (This expansion is only accurate to about 70 bits, not
+             * 128 bits.)
+             */
+            aexp = extractFloatx80Exp(f2xm1_coeff_0);
+            asign = extractFloatx80Sign(f2xm1_coeff_0);
+            shift128RightJamming(extractFloatx80Frac(accum), 0,
+                                 aexp - extractFloatx80Exp(accum),
+                                 &asig0, &asig1);
+            bsig0 = extractFloatx80Frac(f2xm1_coeff_0);
+            bsig1 = 0;
+            if (asign == extractFloatx80Sign(accum)) {
+                add128(bsig0, bsig1, asig0, asig1, &asig0, &asig1);
+            } else {
+                sub128(bsig0, bsig1, asig0, asig1, &asig0, &asig1);
+            }
+            /* And thus compute an approximation to 2^y - 1.  */
+            mul128By64To192(asig0, asig1, extractFloatx80Frac(y),
+                            &asig0, &asig1, &asig2);
+            aexp += extractFloatx80Exp(y) - 0x3ffe;
+            asign ^= extractFloatx80Sign(y);
+            if (n != 32) {
+                /*
+                 * Multiply this by the precomputed value of 2^t and
+                 * add that of 2^t - 1.
+                 */
+                mul128By64To192(asig0, asig1,
+                                extractFloatx80Frac(f2xm1_table[n].exp2),
+                                &asig0, &asig1, &asig2);
+                aexp += extractFloatx80Exp(f2xm1_table[n].exp2) - 0x3ffe;
+                bexp = extractFloatx80Exp(f2xm1_table[n].exp2m1);
+                bsig0 = extractFloatx80Frac(f2xm1_table[n].exp2m1);
+                bsig1 = 0;
+                if (bexp < aexp) {
+                    shift128RightJamming(bsig0, bsig1, aexp - bexp,
+                                         &bsig0, &bsig1);
+                } else if (aexp < bexp) {
+                    shift128RightJamming(asig0, asig1, bexp - aexp,
+                                         &asig0, &asig1);
+                    aexp = bexp;
+                }
+                /* The sign of 2^t - 1 is always that of the result.  */
+                bsign = extractFloatx80Sign(f2xm1_table[n].exp2m1);
+                if (asign == bsign) {
+                    /* Avoid possible carry out of the addition.  */
+                    shift128RightJamming(asig0, asig1, 1,
+                                         &asig0, &asig1);
+                    shift128RightJamming(bsig0, bsig1, 1,
+                                         &bsig0, &bsig1);
+                    ++aexp;
+                    add128(asig0, asig1, bsig0, bsig1, &asig0, &asig1);
+                } else {
+                    sub128(bsig0, bsig1, asig0, asig1, &asig0, &asig1);
+                    asign = bsign;
+                }
+            }
+            env->fp_status.float_rounding_mode = save_mode;
+            /* This result is inexact.  */
+            asig1 |= 1;
+            ST0 = normalizeRoundAndPackFloatx80(floatx80_precision_x,
+                                                asign, aexp, asig0, asig1,
+                                                &env->fp_status);
+        }
+
+        env->fp_status.floatx80_rounding_precision = save_prec;
     }
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fptan(CPUX86State *env)
@@ -743,194 +1245,1006 @@ void helper_fptan(CPUX86State *env)
     if ((fptemp > MAXTAN) || (fptemp < -MAXTAN)) {
         env->fpus |= 0x400;
     } else {
-        floatx80 one = { 0x8000000000000000LL, 0x3fff };
         fptemp = tan(fptemp);
         ST0 = double_to_floatx80(env, fptemp);
         fpush(env);
-        ST0 = one;
+        ST0 = floatx80_one;
         env->fpus &= ~0x400; /* C2 <-- 0 */
         /* the above code is for |arg| < 2**52 only */
     }
 }
 
+/* Values of pi/4, pi/2, 3pi/4 and pi, with 128-bit precision.  */
+#define pi_4_exp 0x3ffe
+#define pi_4_sig_high 0xc90fdaa22168c234ULL
+#define pi_4_sig_low 0xc4c6628b80dc1cd1ULL
+#define pi_2_exp 0x3fff
+#define pi_2_sig_high 0xc90fdaa22168c234ULL
+#define pi_2_sig_low 0xc4c6628b80dc1cd1ULL
+#define pi_34_exp 0x4000
+#define pi_34_sig_high 0x96cbe3f9990e91a7ULL
+#define pi_34_sig_low 0x9394c9e8a0a5159dULL
+#define pi_exp 0x4000
+#define pi_sig_high 0xc90fdaa22168c234ULL
+#define pi_sig_low 0xc4c6628b80dc1cd1ULL
+
+/*
+ * Polynomial coefficients for an approximation to atan(x), with only
+ * odd powers of x used, for x in the interval [-1/16, 1/16].  (Unlike
+ * for some other approximations, no low part is needed for the first
+ * coefficient here to achieve a sufficiently accurate result, because
+ * the coefficient in this minimax approximation is very close to
+ * exactly 1.)
+ */
+#define fpatan_coeff_0 make_floatx80(0x3fff, 0x8000000000000000ULL)
+#define fpatan_coeff_1 make_floatx80(0xbffd, 0xaaaaaaaaaaaaaa43ULL)
+#define fpatan_coeff_2 make_floatx80(0x3ffc, 0xccccccccccbfe4f8ULL)
+#define fpatan_coeff_3 make_floatx80(0xbffc, 0x92492491fbab2e66ULL)
+#define fpatan_coeff_4 make_floatx80(0x3ffb, 0xe38e372881ea1e0bULL)
+#define fpatan_coeff_5 make_floatx80(0xbffb, 0xba2c0104bbdd0615ULL)
+#define fpatan_coeff_6 make_floatx80(0x3ffb, 0x9baf7ebf898b42efULL)
+
+struct fpatan_data {
+    /* High and low parts of atan(x).  */
+    floatx80 atan_high, atan_low;
+};
+
+static const struct fpatan_data fpatan_table[9] = {
+    { floatx80_zero_init,
+      floatx80_zero_init },
+    { make_floatx80_init(0x3ffb, 0xfeadd4d5617b6e33ULL),
+      make_floatx80_init(0xbfb9, 0xdda19d8305ddc420ULL) },
+    { make_floatx80_init(0x3ffc, 0xfadbafc96406eb15ULL),
+      make_floatx80_init(0x3fbb, 0xdb8f3debef442fccULL) },
+    { make_floatx80_init(0x3ffd, 0xb7b0ca0f26f78474ULL),
+      make_floatx80_init(0xbfbc, 0xeab9bdba460376faULL) },
+    { make_floatx80_init(0x3ffd, 0xed63382b0dda7b45ULL),
+      make_floatx80_init(0x3fbc, 0xdfc88bd978751a06ULL) },
+    { make_floatx80_init(0x3ffe, 0x8f005d5ef7f59f9bULL),
+      make_floatx80_init(0x3fbd, 0xb906bc2ccb886e90ULL) },
+    { make_floatx80_init(0x3ffe, 0xa4bc7d1934f70924ULL),
+      make_floatx80_init(0x3fbb, 0xcd43f9522bed64f8ULL) },
+    { make_floatx80_init(0x3ffe, 0xb8053e2bc2319e74ULL),
+      make_floatx80_init(0xbfbc, 0xd3496ab7bd6eef0cULL) },
+    { make_floatx80_init(0x3ffe, 0xc90fdaa22168c235ULL),
+      make_floatx80_init(0xbfbc, 0xece675d1fc8f8cbcULL) },
+};
+
 void helper_fpatan(CPUX86State *env)
 {
-    double fptemp, fpsrcop;
+    uint8_t old_flags = save_exception_flags(env);
+    uint64_t arg0_sig = extractFloatx80Frac(ST0);
+    int32_t arg0_exp = extractFloatx80Exp(ST0);
+    bool arg0_sign = extractFloatx80Sign(ST0);
+    uint64_t arg1_sig = extractFloatx80Frac(ST1);
+    int32_t arg1_exp = extractFloatx80Exp(ST1);
+    bool arg1_sign = extractFloatx80Sign(ST1);
+
+    if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_silence_nan(ST0, &env->fp_status);
+    } else if (floatx80_is_signaling_nan(ST1, &env->fp_status)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_silence_nan(ST1, &env->fp_status);
+    } else if (floatx80_invalid_encoding(ST0) ||
+               floatx80_invalid_encoding(ST1)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_default_nan(&env->fp_status);
+    } else if (floatx80_is_any_nan(ST0)) {
+        ST1 = ST0;
+    } else if (floatx80_is_any_nan(ST1)) {
+        /* Pass this NaN through.  */
+    } else if (floatx80_is_zero(ST1) && !arg0_sign) {
+        /* Pass this zero through.  */
+    } else if (((floatx80_is_infinity(ST0) && !floatx80_is_infinity(ST1)) ||
+                 arg0_exp - arg1_exp >= 80) &&
+               !arg0_sign) {
+        /*
+         * Dividing ST1 by ST0 gives the correct result up to
+         * rounding, and avoids spurious underflow exceptions that
+         * might result from passing some small values through the
+         * polynomial approximation, but if a finite nonzero result of
+         * division is exact, the result of fpatan is still inexact
+         * (and underflowing where appropriate).
+         */
+        FloatX80RoundPrec save_prec =
+            env->fp_status.floatx80_rounding_precision;
+        env->fp_status.floatx80_rounding_precision = floatx80_precision_x;
+        ST1 = floatx80_div(ST1, ST0, &env->fp_status);
+        env->fp_status.floatx80_rounding_precision = save_prec;
+        if (!floatx80_is_zero(ST1) &&
+            !(get_float_exception_flags(&env->fp_status) &
+              float_flag_inexact)) {
+            /*
+             * The mathematical result is very slightly closer to zero
+             * than this exact result.  Round a value with the
+             * significand adjusted accordingly to get the correct
+             * exceptions, and possibly an adjusted result depending
+             * on the rounding mode.
+             */
+            uint64_t sig = extractFloatx80Frac(ST1);
+            int32_t exp = extractFloatx80Exp(ST1);
+            bool sign = extractFloatx80Sign(ST1);
+            if (exp == 0) {
+                normalizeFloatx80Subnormal(sig, &exp, &sig);
+            }
+            ST1 = normalizeRoundAndPackFloatx80(floatx80_precision_x,
+                                                sign, exp, sig - 1,
+                                                -1, &env->fp_status);
+        }
+    } else {
+        /* The result is inexact.  */
+        bool rsign = arg1_sign;
+        int32_t rexp;
+        uint64_t rsig0, rsig1;
+        if (floatx80_is_zero(ST1)) {
+            /*
+             * ST0 is negative.  The result is pi with the sign of
+             * ST1.
+             */
+            rexp = pi_exp;
+            rsig0 = pi_sig_high;
+            rsig1 = pi_sig_low;
+        } else if (floatx80_is_infinity(ST1)) {
+            if (floatx80_is_infinity(ST0)) {
+                if (arg0_sign) {
+                    rexp = pi_34_exp;
+                    rsig0 = pi_34_sig_high;
+                    rsig1 = pi_34_sig_low;
+                } else {
+                    rexp = pi_4_exp;
+                    rsig0 = pi_4_sig_high;
+                    rsig1 = pi_4_sig_low;
+                }
+            } else {
+                rexp = pi_2_exp;
+                rsig0 = pi_2_sig_high;
+                rsig1 = pi_2_sig_low;
+            }
+        } else if (floatx80_is_zero(ST0) || arg1_exp - arg0_exp >= 80) {
+            rexp = pi_2_exp;
+            rsig0 = pi_2_sig_high;
+            rsig1 = pi_2_sig_low;
+        } else if (floatx80_is_infinity(ST0) || arg0_exp - arg1_exp >= 80) {
+            /* ST0 is negative.  */
+            rexp = pi_exp;
+            rsig0 = pi_sig_high;
+            rsig1 = pi_sig_low;
+        } else {
+            /*
+             * ST0 and ST1 are finite, nonzero and with exponents not
+             * too far apart.
+             */
+            int32_t adj_exp, num_exp, den_exp, xexp, yexp, n, texp, zexp, aexp;
+            int32_t azexp, axexp;
+            bool adj_sub, ysign, zsign;
+            uint64_t adj_sig0, adj_sig1, num_sig, den_sig, xsig0, xsig1;
+            uint64_t msig0, msig1, msig2, remsig0, remsig1, remsig2;
+            uint64_t ysig0, ysig1, tsig, zsig0, zsig1, asig0, asig1;
+            uint64_t azsig0, azsig1;
+            uint64_t azsig2, azsig3, axsig0, axsig1;
+            floatx80 x8;
+            FloatRoundMode save_mode = env->fp_status.float_rounding_mode;
+            FloatX80RoundPrec save_prec =
+                env->fp_status.floatx80_rounding_precision;
+            env->fp_status.float_rounding_mode = float_round_nearest_even;
+            env->fp_status.floatx80_rounding_precision = floatx80_precision_x;
+
+            if (arg0_exp == 0) {
+                normalizeFloatx80Subnormal(arg0_sig, &arg0_exp, &arg0_sig);
+            }
+            if (arg1_exp == 0) {
+                normalizeFloatx80Subnormal(arg1_sig, &arg1_exp, &arg1_sig);
+            }
+            if (arg0_exp > arg1_exp ||
+                (arg0_exp == arg1_exp && arg0_sig >= arg1_sig)) {
+                /* Work with abs(ST1) / abs(ST0).  */
+                num_exp = arg1_exp;
+                num_sig = arg1_sig;
+                den_exp = arg0_exp;
+                den_sig = arg0_sig;
+                if (arg0_sign) {
+                    /* The result is subtracted from pi.  */
+                    adj_exp = pi_exp;
+                    adj_sig0 = pi_sig_high;
+                    adj_sig1 = pi_sig_low;
+                    adj_sub = true;
+                } else {
+                    /* The result is used as-is.  */
+                    adj_exp = 0;
+                    adj_sig0 = 0;
+                    adj_sig1 = 0;
+                    adj_sub = false;
+                }
+            } else {
+                /* Work with abs(ST0) / abs(ST1).  */
+                num_exp = arg0_exp;
+                num_sig = arg0_sig;
+                den_exp = arg1_exp;
+                den_sig = arg1_sig;
+                /* The result is added to or subtracted from pi/2.  */
+                adj_exp = pi_2_exp;
+                adj_sig0 = pi_2_sig_high;
+                adj_sig1 = pi_2_sig_low;
+                adj_sub = !arg0_sign;
+            }
+
+            /*
+             * Compute x = num/den, where 0 < x <= 1 and x is not too
+             * small.
+             */
+            xexp = num_exp - den_exp + 0x3ffe;
+            remsig0 = num_sig;
+            remsig1 = 0;
+            if (den_sig <= remsig0) {
+                shift128Right(remsig0, remsig1, 1, &remsig0, &remsig1);
+                ++xexp;
+            }
+            xsig0 = estimateDiv128To64(remsig0, remsig1, den_sig);
+            mul64To128(den_sig, xsig0, &msig0, &msig1);
+            sub128(remsig0, remsig1, msig0, msig1, &remsig0, &remsig1);
+            while ((int64_t) remsig0 < 0) {
+                --xsig0;
+                add128(remsig0, remsig1, 0, den_sig, &remsig0, &remsig1);
+            }
+            xsig1 = estimateDiv128To64(remsig1, 0, den_sig);
+            /*
+             * No need to correct any estimation error in xsig1; even
+             * with such error, it is accurate enough.
+             */
+
+            /*
+             * Split x as x = t + y, where t = n/8 is the nearest
+             * multiple of 1/8 to x.
+             */
+            x8 = normalizeRoundAndPackFloatx80(floatx80_precision_x,
+                                               false, xexp + 3, xsig0,
+                                               xsig1, &env->fp_status);
+            n = floatx80_to_int32(x8, &env->fp_status);
+            if (n == 0) {
+                ysign = false;
+                yexp = xexp;
+                ysig0 = xsig0;
+                ysig1 = xsig1;
+                texp = 0;
+                tsig = 0;
+            } else {
+                int shift = clz32(n) + 32;
+                texp = 0x403b - shift;
+                tsig = n;
+                tsig <<= shift;
+                if (texp == xexp) {
+                    sub128(xsig0, xsig1, tsig, 0, &ysig0, &ysig1);
+                    if ((int64_t) ysig0 >= 0) {
+                        ysign = false;
+                        if (ysig0 == 0) {
+                            if (ysig1 == 0) {
+                                yexp = 0;
+                            } else {
+                                shift = clz64(ysig1) + 64;
+                                yexp = xexp - shift;
+                                shift128Left(ysig0, ysig1, shift,
+                                             &ysig0, &ysig1);
+                            }
+                        } else {
+                            shift = clz64(ysig0);
+                            yexp = xexp - shift;
+                            shift128Left(ysig0, ysig1, shift, &ysig0, &ysig1);
+                        }
+                    } else {
+                        ysign = true;
+                        sub128(0, 0, ysig0, ysig1, &ysig0, &ysig1);
+                        if (ysig0 == 0) {
+                            shift = clz64(ysig1) + 64;
+                        } else {
+                            shift = clz64(ysig0);
+                        }
+                        yexp = xexp - shift;
+                        shift128Left(ysig0, ysig1, shift, &ysig0, &ysig1);
+                    }
+                } else {
+                    /*
+                     * t's exponent must be greater than x's because t
+                     * is positive and the nearest multiple of 1/8 to
+                     * x, and if x has a greater exponent, the power
+                     * of 2 with that exponent is also a multiple of
+                     * 1/8.
+                     */
+                    uint64_t usig0, usig1;
+                    shift128RightJamming(xsig0, xsig1, texp - xexp,
+                                         &usig0, &usig1);
+                    ysign = true;
+                    sub128(tsig, 0, usig0, usig1, &ysig0, &ysig1);
+                    if (ysig0 == 0) {
+                        shift = clz64(ysig1) + 64;
+                    } else {
+                        shift = clz64(ysig0);
+                    }
+                    yexp = texp - shift;
+                    shift128Left(ysig0, ysig1, shift, &ysig0, &ysig1);
+                }
+            }
+
+            /*
+             * Compute z = y/(1+tx), so arctan(x) = arctan(t) +
+             * arctan(z).
+             */
+            zsign = ysign;
+            if (texp == 0 || yexp == 0) {
+                zexp = yexp;
+                zsig0 = ysig0;
+                zsig1 = ysig1;
+            } else {
+                /*
+                 * t <= 1, x <= 1 and if both are 1 then y is 0, so tx < 1.
+                 */
+                int32_t dexp = texp + xexp - 0x3ffe;
+                uint64_t dsig0, dsig1, dsig2;
+                mul128By64To192(xsig0, xsig1, tsig, &dsig0, &dsig1, &dsig2);
+                /*
+                 * dexp <= 0x3fff (and if equal, dsig0 has a leading 0
+                 * bit).  Add 1 to produce the denominator 1+tx.
+                 */
+                shift128RightJamming(dsig0, dsig1, 0x3fff - dexp,
+                                     &dsig0, &dsig1);
+                dsig0 |= 0x8000000000000000ULL;
+                zexp = yexp - 1;
+                remsig0 = ysig0;
+                remsig1 = ysig1;
+                remsig2 = 0;
+                if (dsig0 <= remsig0) {
+                    shift128Right(remsig0, remsig1, 1, &remsig0, &remsig1);
+                    ++zexp;
+                }
+                zsig0 = estimateDiv128To64(remsig0, remsig1, dsig0);
+                mul128By64To192(dsig0, dsig1, zsig0, &msig0, &msig1, &msig2);
+                sub192(remsig0, remsig1, remsig2, msig0, msig1, msig2,
+                       &remsig0, &remsig1, &remsig2);
+                while ((int64_t) remsig0 < 0) {
+                    --zsig0;
+                    add192(remsig0, remsig1, remsig2, 0, dsig0, dsig1,
+                           &remsig0, &remsig1, &remsig2);
+                }
+                zsig1 = estimateDiv128To64(remsig1, remsig2, dsig0);
+                /* No need to correct any estimation error in zsig1.  */
+            }
+
+            if (zexp == 0) {
+                azexp = 0;
+                azsig0 = 0;
+                azsig1 = 0;
+            } else {
+                floatx80 z2, accum;
+                uint64_t z2sig0, z2sig1, z2sig2, z2sig3;
+                /* Compute z^2.  */
+                mul128To256(zsig0, zsig1, zsig0, zsig1,
+                            &z2sig0, &z2sig1, &z2sig2, &z2sig3);
+                z2 = normalizeRoundAndPackFloatx80(floatx80_precision_x, false,
+                                                   zexp + zexp - 0x3ffe,
+                                                   z2sig0, z2sig1,
+                                                   &env->fp_status);
+
+                /* Compute the lower parts of the polynomial expansion.  */
+                accum = floatx80_mul(fpatan_coeff_6, z2, &env->fp_status);
+                accum = floatx80_add(fpatan_coeff_5, accum, &env->fp_status);
+                accum = floatx80_mul(accum, z2, &env->fp_status);
+                accum = floatx80_add(fpatan_coeff_4, accum, &env->fp_status);
+                accum = floatx80_mul(accum, z2, &env->fp_status);
+                accum = floatx80_add(fpatan_coeff_3, accum, &env->fp_status);
+                accum = floatx80_mul(accum, z2, &env->fp_status);
+                accum = floatx80_add(fpatan_coeff_2, accum, &env->fp_status);
+                accum = floatx80_mul(accum, z2, &env->fp_status);
+                accum = floatx80_add(fpatan_coeff_1, accum, &env->fp_status);
+                accum = floatx80_mul(accum, z2, &env->fp_status);
+
+                /*
+                 * The full polynomial expansion is z*(fpatan_coeff_0 + accum).
+                 * fpatan_coeff_0 is 1, and accum is negative and much smaller.
+                 */
+                aexp = extractFloatx80Exp(fpatan_coeff_0);
+                shift128RightJamming(extractFloatx80Frac(accum), 0,
+                                     aexp - extractFloatx80Exp(accum),
+                                     &asig0, &asig1);
+                sub128(extractFloatx80Frac(fpatan_coeff_0), 0, asig0, asig1,
+                       &asig0, &asig1);
+                /* Multiply by z to compute arctan(z).  */
+                azexp = aexp + zexp - 0x3ffe;
+                mul128To256(asig0, asig1, zsig0, zsig1, &azsig0, &azsig1,
+                            &azsig2, &azsig3);
+            }
+
+            /* Add arctan(t) (positive or zero) and arctan(z) (sign zsign).  */
+            if (texp == 0) {
+                /* z is positive.  */
+                axexp = azexp;
+                axsig0 = azsig0;
+                axsig1 = azsig1;
+            } else {
+                bool low_sign = extractFloatx80Sign(fpatan_table[n].atan_low);
+                int32_t low_exp = extractFloatx80Exp(fpatan_table[n].atan_low);
+                uint64_t low_sig0 =
+                    extractFloatx80Frac(fpatan_table[n].atan_low);
+                uint64_t low_sig1 = 0;
+                axexp = extractFloatx80Exp(fpatan_table[n].atan_high);
+                axsig0 = extractFloatx80Frac(fpatan_table[n].atan_high);
+                axsig1 = 0;
+                shift128RightJamming(low_sig0, low_sig1, axexp - low_exp,
+                                     &low_sig0, &low_sig1);
+                if (low_sign) {
+                    sub128(axsig0, axsig1, low_sig0, low_sig1,
+                           &axsig0, &axsig1);
+                } else {
+                    add128(axsig0, axsig1, low_sig0, low_sig1,
+                           &axsig0, &axsig1);
+                }
+                if (azexp >= axexp) {
+                    shift128RightJamming(axsig0, axsig1, azexp - axexp + 1,
+                                         &axsig0, &axsig1);
+                    axexp = azexp + 1;
+                    shift128RightJamming(azsig0, azsig1, 1,
+                                         &azsig0, &azsig1);
+                } else {
+                    shift128RightJamming(axsig0, axsig1, 1,
+                                         &axsig0, &axsig1);
+                    shift128RightJamming(azsig0, azsig1, axexp - azexp + 1,
+                                         &azsig0, &azsig1);
+                    ++axexp;
+                }
+                if (zsign) {
+                    sub128(axsig0, axsig1, azsig0, azsig1,
+                           &axsig0, &axsig1);
+                } else {
+                    add128(axsig0, axsig1, azsig0, azsig1,
+                           &axsig0, &axsig1);
+                }
+            }
+
+            if (adj_exp == 0) {
+                rexp = axexp;
+                rsig0 = axsig0;
+                rsig1 = axsig1;
+            } else {
+                /*
+                 * Add or subtract arctan(x) (exponent axexp,
+                 * significand axsig0 and axsig1, positive, not
+                 * necessarily normalized) to the number given by
+                 * adj_exp, adj_sig0 and adj_sig1, according to
+                 * adj_sub.
+                 */
+                if (adj_exp >= axexp) {
+                    shift128RightJamming(axsig0, axsig1, adj_exp - axexp + 1,
+                                         &axsig0, &axsig1);
+                    rexp = adj_exp + 1;
+                    shift128RightJamming(adj_sig0, adj_sig1, 1,
+                                         &adj_sig0, &adj_sig1);
+                } else {
+                    shift128RightJamming(axsig0, axsig1, 1,
+                                         &axsig0, &axsig1);
+                    shift128RightJamming(adj_sig0, adj_sig1,
+                                         axexp - adj_exp + 1,
+                                         &adj_sig0, &adj_sig1);
+                    rexp = axexp + 1;
+                }
+                if (adj_sub) {
+                    sub128(adj_sig0, adj_sig1, axsig0, axsig1,
+                           &rsig0, &rsig1);
+                } else {
+                    add128(adj_sig0, adj_sig1, axsig0, axsig1,
+                           &rsig0, &rsig1);
+                }
+            }
+
+            env->fp_status.float_rounding_mode = save_mode;
+            env->fp_status.floatx80_rounding_precision = save_prec;
+        }
+        /* This result is inexact.  */
+        rsig1 |= 1;
+        ST1 = normalizeRoundAndPackFloatx80(floatx80_precision_x, rsign, rexp,
+                                            rsig0, rsig1, &env->fp_status);
+    }
 
-    fpsrcop = floatx80_to_double(env, ST1);
-    fptemp = floatx80_to_double(env, ST0);
-    ST1 = double_to_floatx80(env, atan2(fpsrcop, fptemp));
     fpop(env);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fxtract(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     CPU_LDoubleU temp;
 
     temp.d = ST0;
 
     if (floatx80_is_zero(ST0)) {
         /* Easy way to generate -inf and raising division by 0 exception */
-        floatx80 zero = { 0x0000000000000000LL, 0x0000 };
-        floatx80 one  = { 0x8000000000000000LL, 0x3fff };
-        ST0 = floatx80_div(floatx80_chs(one), zero,
+        ST0 = floatx80_div(floatx80_chs(floatx80_one), floatx80_zero,
                            &env->fp_status);
         fpush(env);
         ST0 = temp.d;
+    } else if (floatx80_invalid_encoding(ST0)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST0 = floatx80_default_nan(&env->fp_status);
+        fpush(env);
+        ST0 = ST1;
+    } else if (floatx80_is_any_nan(ST0)) {
+        if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
+            float_raise(float_flag_invalid, &env->fp_status);
+            ST0 = floatx80_silence_nan(ST0, &env->fp_status);
+        }
+        fpush(env);
+        ST0 = ST1;
+    } else if (floatx80_is_infinity(ST0)) {
+        fpush(env);
+        ST0 = ST1;
+        ST1 = floatx80_infinity;
     } else {
         int expdif;
 
-        expdif = EXPD(temp) - EXPBIAS;
+        if (EXPD(temp) == 0) {
+            int shift = clz64(temp.l.lower);
+            temp.l.lower <<= shift;
+            expdif = 1 - EXPBIAS - shift;
+            float_raise(float_flag_input_denormal, &env->fp_status);
+        } else {
+            expdif = EXPD(temp) - EXPBIAS;
+        }
         /* DP exponent bias */
         ST0 = int32_to_floatx80(expdif, &env->fp_status);
         fpush(env);
         BIASEXPONENT(temp);
         ST0 = temp.d;
     }
+    merge_exception_flags(env, old_flags);
 }
 
-void helper_fprem1(CPUX86State *env)
+static void helper_fprem_common(CPUX86State *env, bool mod)
 {
-    double st0, st1, dblq, fpsrcop, fptemp;
-    CPU_LDoubleU fpsrcop1, fptemp1;
-    int expdif;
-    signed long long int q;
-
-    st0 = floatx80_to_double(env, ST0);
-    st1 = floatx80_to_double(env, ST1);
-
-    if (isinf(st0) || isnan(st0) || isnan(st1) || (st1 == 0.0)) {
-        ST0 = double_to_floatx80(env, NAN); /* NaN */
-        env->fpus &= ~0x4700; /* (C3,C2,C1,C0) <-- 0000 */
-        return;
-    }
-
-    fpsrcop = st0;
-    fptemp = st1;
-    fpsrcop1.d = ST0;
-    fptemp1.d = ST1;
-    expdif = EXPD(fpsrcop1) - EXPD(fptemp1);
-
-    if (expdif < 0) {
-        /* optimisation? taken from the AMD docs */
-        env->fpus &= ~0x4700; /* (C3,C2,C1,C0) <-- 0000 */
-        /* ST0 is unchanged */
-        return;
-    }
+    uint8_t old_flags = save_exception_flags(env);
+    uint64_t quotient;
+    CPU_LDoubleU temp0, temp1;
+    int exp0, exp1, expdiff;
 
-    if (expdif < 53) {
-        dblq = fpsrcop / fptemp;
-        /* round dblq towards nearest integer */
-        dblq = rint(dblq);
-        st0 = fpsrcop - fptemp * dblq;
+    temp0.d = ST0;
+    temp1.d = ST1;
+    exp0 = EXPD(temp0);
+    exp1 = EXPD(temp1);
 
-        /* convert dblq to q by truncating towards zero */
-        if (dblq < 0.0) {
-            q = (signed long long int)(-dblq);
+    env->fpus &= ~0x4700; /* (C3,C2,C1,C0) <-- 0000 */
+    if (floatx80_is_zero(ST0) || floatx80_is_zero(ST1) ||
+        exp0 == 0x7fff || exp1 == 0x7fff ||
+        floatx80_invalid_encoding(ST0) || floatx80_invalid_encoding(ST1)) {
+        ST0 = floatx80_modrem(ST0, ST1, mod, &quotient, &env->fp_status);
+    } else {
+        if (exp0 == 0) {
+            exp0 = 1 - clz64(temp0.l.lower);
+        }
+        if (exp1 == 0) {
+            exp1 = 1 - clz64(temp1.l.lower);
+        }
+        expdiff = exp0 - exp1;
+        if (expdiff < 64) {
+            ST0 = floatx80_modrem(ST0, ST1, mod, &quotient, &env->fp_status);
+            env->fpus |= (quotient & 0x4) << (8 - 2);  /* (C0) <-- q2 */
+            env->fpus |= (quotient & 0x2) << (14 - 1); /* (C3) <-- q1 */
+            env->fpus |= (quotient & 0x1) << (9 - 0);  /* (C1) <-- q0 */
         } else {
-            q = (signed long long int)dblq;
+            /*
+             * Partial remainder.  This choice of how many bits to
+             * process at once is specified in AMD instruction set
+             * manuals, and empirically is followed by Intel
+             * processors as well; it ensures that the final remainder
+             * operation in a loop does produce the correct low three
+             * bits of the quotient.  AMD manuals specify that the
+             * flags other than C2 are cleared, and empirically Intel
+             * processors clear them as well.
+             */
+            int n = 32 + (expdiff % 32);
+            temp1.d = floatx80_scalbn(temp1.d, expdiff - n, &env->fp_status);
+            ST0 = floatx80_mod(ST0, temp1.d, &env->fp_status);
+            env->fpus |= 0x400;  /* C2 <-- 1 */
         }
-
-        env->fpus &= ~0x4700; /* (C3,C2,C1,C0) <-- 0000 */
-        /* (C0,C3,C1) <-- (q2,q1,q0) */
-        env->fpus |= (q & 0x4) << (8 - 2);  /* (C0) <-- q2 */
-        env->fpus |= (q & 0x2) << (14 - 1); /* (C3) <-- q1 */
-        env->fpus |= (q & 0x1) << (9 - 0);  /* (C1) <-- q0 */
-    } else {
-        env->fpus |= 0x400;  /* C2 <-- 1 */
-        fptemp = pow(2.0, expdif - 50);
-        fpsrcop = (st0 / st1) / fptemp;
-        /* fpsrcop = integer obtained by chopping */
-        fpsrcop = (fpsrcop < 0.0) ?
-                  -(floor(fabs(fpsrcop))) : floor(fpsrcop);
-        st0 -= (st1 * fpsrcop * fptemp);
     }
-    ST0 = double_to_floatx80(env, st0);
+    merge_exception_flags(env, old_flags);
 }
 
-void helper_fprem(CPUX86State *env)
+void helper_fprem1(CPUX86State *env)
 {
-    double st0, st1, dblq, fpsrcop, fptemp;
-    CPU_LDoubleU fpsrcop1, fptemp1;
-    int expdif;
-    signed long long int q;
-
-    st0 = floatx80_to_double(env, ST0);
-    st1 = floatx80_to_double(env, ST1);
+    helper_fprem_common(env, false);
+}
 
-    if (isinf(st0) || isnan(st0) || isnan(st1) || (st1 == 0.0)) {
-        ST0 = double_to_floatx80(env, NAN); /* NaN */
-        env->fpus &= ~0x4700; /* (C3,C2,C1,C0) <-- 0000 */
-        return;
-    }
+void helper_fprem(CPUX86State *env)
+{
+    helper_fprem_common(env, true);
+}
 
-    fpsrcop = st0;
-    fptemp = st1;
-    fpsrcop1.d = ST0;
-    fptemp1.d = ST1;
-    expdif = EXPD(fpsrcop1) - EXPD(fptemp1);
+/* 128-bit significand of log2(e).  */
+#define log2_e_sig_high 0xb8aa3b295c17f0bbULL
+#define log2_e_sig_low 0xbe87fed0691d3e89ULL
 
-    if (expdif < 0) {
-        /* optimisation? taken from the AMD docs */
-        env->fpus &= ~0x4700; /* (C3,C2,C1,C0) <-- 0000 */
-        /* ST0 is unchanged */
-        return;
-    }
-
-    if (expdif < 53) {
-        dblq = fpsrcop / fptemp; /* ST0 / ST1 */
-        /* round dblq towards zero */
-        dblq = (dblq < 0.0) ? ceil(dblq) : floor(dblq);
-        st0 = fpsrcop - fptemp * dblq; /* fpsrcop is ST0 */
+/*
+ * Polynomial coefficients for an approximation to log2((1+x)/(1-x)),
+ * with only odd powers of x used, for x in the interval [2*sqrt(2)-3,
+ * 3-2*sqrt(2)], which corresponds to logarithms of numbers in the
+ * interval [sqrt(2)/2, sqrt(2)].
+ */
+#define fyl2x_coeff_0 make_floatx80(0x4000, 0xb8aa3b295c17f0bcULL)
+#define fyl2x_coeff_0_low make_floatx80(0xbfbf, 0x834972fe2d7bab1bULL)
+#define fyl2x_coeff_1 make_floatx80(0x3ffe, 0xf6384ee1d01febb8ULL)
+#define fyl2x_coeff_2 make_floatx80(0x3ffe, 0x93bb62877cdfa2e3ULL)
+#define fyl2x_coeff_3 make_floatx80(0x3ffd, 0xd30bb153d808f269ULL)
+#define fyl2x_coeff_4 make_floatx80(0x3ffd, 0xa42589eaf451499eULL)
+#define fyl2x_coeff_5 make_floatx80(0x3ffd, 0x864d42c0f8f17517ULL)
+#define fyl2x_coeff_6 make_floatx80(0x3ffc, 0xe3476578adf26272ULL)
+#define fyl2x_coeff_7 make_floatx80(0x3ffc, 0xc506c5f874e6d80fULL)
+#define fyl2x_coeff_8 make_floatx80(0x3ffc, 0xac5cf50cc57d6372ULL)
+#define fyl2x_coeff_9 make_floatx80(0x3ffc, 0xb1ed0066d971a103ULL)
 
-        /* convert dblq to q by truncating towards zero */
-        if (dblq < 0.0) {
-            q = (signed long long int)(-dblq);
-        } else {
-            q = (signed long long int)dblq;
-        }
+/*
+ * Compute an approximation of log2(1+arg), where 1+arg is in the
+ * interval [sqrt(2)/2, sqrt(2)].  It is assumed that when this
+ * function is called, rounding precision is set to 80 and the
+ * round-to-nearest mode is in effect.  arg must not be exactly zero,
+ * and must not be so close to zero that underflow might occur.
+ */
+static void helper_fyl2x_common(CPUX86State *env, floatx80 arg, int32_t *exp,
+                                uint64_t *sig0, uint64_t *sig1)
+{
+    uint64_t arg0_sig = extractFloatx80Frac(arg);
+    int32_t arg0_exp = extractFloatx80Exp(arg);
+    bool arg0_sign = extractFloatx80Sign(arg);
+    bool asign;
+    int32_t dexp, texp, aexp;
+    uint64_t dsig0, dsig1, tsig0, tsig1, rsig0, rsig1, rsig2;
+    uint64_t msig0, msig1, msig2, t2sig0, t2sig1, t2sig2, t2sig3;
+    uint64_t asig0, asig1, asig2, asig3, bsig0, bsig1;
+    floatx80 t2, accum;
 
-        env->fpus &= ~0x4700; /* (C3,C2,C1,C0) <-- 0000 */
-        /* (C0,C3,C1) <-- (q2,q1,q0) */
-        env->fpus |= (q & 0x4) << (8 - 2);  /* (C0) <-- q2 */
-        env->fpus |= (q & 0x2) << (14 - 1); /* (C3) <-- q1 */
-        env->fpus |= (q & 0x1) << (9 - 0);  /* (C1) <-- q0 */
+    /*
+     * Compute an approximation of arg/(2+arg), with extra precision,
+     * as the argument to a polynomial approximation.  The extra
+     * precision is only needed for the first term of the
+     * approximation, with subsequent terms being significantly
+     * smaller; the approximation only uses odd exponents, and the
+     * square of arg/(2+arg) is at most 17-12*sqrt(2) = 0.029....
+     */
+    if (arg0_sign) {
+        dexp = 0x3fff;
+        shift128RightJamming(arg0_sig, 0, dexp - arg0_exp, &dsig0, &dsig1);
+        sub128(0, 0, dsig0, dsig1, &dsig0, &dsig1);
     } else {
-        int N = 32 + (expdif % 32); /* as per AMD docs */
+        dexp = 0x4000;
+        shift128RightJamming(arg0_sig, 0, dexp - arg0_exp, &dsig0, &dsig1);
+        dsig0 |= 0x8000000000000000ULL;
+    }
+    texp = arg0_exp - dexp + 0x3ffe;
+    rsig0 = arg0_sig;
+    rsig1 = 0;
+    rsig2 = 0;
+    if (dsig0 <= rsig0) {
+        shift128Right(rsig0, rsig1, 1, &rsig0, &rsig1);
+        ++texp;
+    }
+    tsig0 = estimateDiv128To64(rsig0, rsig1, dsig0);
+    mul128By64To192(dsig0, dsig1, tsig0, &msig0, &msig1, &msig2);
+    sub192(rsig0, rsig1, rsig2, msig0, msig1, msig2,
+           &rsig0, &rsig1, &rsig2);
+    while ((int64_t) rsig0 < 0) {
+        --tsig0;
+        add192(rsig0, rsig1, rsig2, 0, dsig0, dsig1,
+               &rsig0, &rsig1, &rsig2);
+    }
+    tsig1 = estimateDiv128To64(rsig1, rsig2, dsig0);
+    /*
+     * No need to correct any estimation error in tsig1; even with
+     * such error, it is accurate enough.  Now compute the square of
+     * that approximation.
+     */
+    mul128To256(tsig0, tsig1, tsig0, tsig1,
+                &t2sig0, &t2sig1, &t2sig2, &t2sig3);
+    t2 = normalizeRoundAndPackFloatx80(floatx80_precision_x, false,
+                                       texp + texp - 0x3ffe,
+                                       t2sig0, t2sig1, &env->fp_status);
+
+    /* Compute the lower parts of the polynomial expansion.  */
+    accum = floatx80_mul(fyl2x_coeff_9, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_8, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_7, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_6, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_5, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_4, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_3, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_2, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_1, accum, &env->fp_status);
+    accum = floatx80_mul(accum, t2, &env->fp_status);
+    accum = floatx80_add(fyl2x_coeff_0_low, accum, &env->fp_status);
 
-        env->fpus |= 0x400;  /* C2 <-- 1 */
-        fptemp = pow(2.0, (double)(expdif - N));
-        fpsrcop = (st0 / st1) / fptemp;
-        /* fpsrcop = integer obtained by chopping */
-        fpsrcop = (fpsrcop < 0.0) ?
-                  -(floor(fabs(fpsrcop))) : floor(fpsrcop);
-        st0 -= (st1 * fpsrcop * fptemp);
+    /*
+     * The full polynomial expansion is fyl2x_coeff_0 + accum (where
+     * accum has much lower magnitude, and so, in particular, carry
+     * out of the addition is not possible), multiplied by t.  (This
+     * expansion is only accurate to about 70 bits, not 128 bits.)
+     */
+    aexp = extractFloatx80Exp(fyl2x_coeff_0);
+    asign = extractFloatx80Sign(fyl2x_coeff_0);
+    shift128RightJamming(extractFloatx80Frac(accum), 0,
+                         aexp - extractFloatx80Exp(accum),
+                         &asig0, &asig1);
+    bsig0 = extractFloatx80Frac(fyl2x_coeff_0);
+    bsig1 = 0;
+    if (asign == extractFloatx80Sign(accum)) {
+        add128(bsig0, bsig1, asig0, asig1, &asig0, &asig1);
+    } else {
+        sub128(bsig0, bsig1, asig0, asig1, &asig0, &asig1);
     }
-    ST0 = double_to_floatx80(env, st0);
+    /* Multiply by t to compute the required result.  */
+    mul128To256(asig0, asig1, tsig0, tsig1,
+                &asig0, &asig1, &asig2, &asig3);
+    aexp += texp - 0x3ffe;
+    *exp = aexp;
+    *sig0 = asig0;
+    *sig1 = asig1;
 }
 
 void helper_fyl2xp1(CPUX86State *env)
 {
-    double fptemp = floatx80_to_double(env, ST0);
+    uint8_t old_flags = save_exception_flags(env);
+    uint64_t arg0_sig = extractFloatx80Frac(ST0);
+    int32_t arg0_exp = extractFloatx80Exp(ST0);
+    bool arg0_sign = extractFloatx80Sign(ST0);
+    uint64_t arg1_sig = extractFloatx80Frac(ST1);
+    int32_t arg1_exp = extractFloatx80Exp(ST1);
+    bool arg1_sign = extractFloatx80Sign(ST1);
+
+    if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_silence_nan(ST0, &env->fp_status);
+    } else if (floatx80_is_signaling_nan(ST1, &env->fp_status)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_silence_nan(ST1, &env->fp_status);
+    } else if (floatx80_invalid_encoding(ST0) ||
+               floatx80_invalid_encoding(ST1)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_default_nan(&env->fp_status);
+    } else if (floatx80_is_any_nan(ST0)) {
+        ST1 = ST0;
+    } else if (floatx80_is_any_nan(ST1)) {
+        /* Pass this NaN through.  */
+    } else if (arg0_exp > 0x3ffd ||
+               (arg0_exp == 0x3ffd && arg0_sig > (arg0_sign ?
+                                                  0x95f619980c4336f7ULL :
+                                                  0xd413cccfe7799211ULL))) {
+        /*
+         * Out of range for the instruction (ST0 must have absolute
+         * value less than 1 - sqrt(2)/2 = 0.292..., according to
+         * Intel manuals; AMD manuals allow a range from sqrt(2)/2 - 1
+         * to sqrt(2) - 1, which we allow here), treat as invalid.
+         */
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_default_nan(&env->fp_status);
+    } else if (floatx80_is_zero(ST0) || floatx80_is_zero(ST1) ||
+               arg1_exp == 0x7fff) {
+        /*
+         * One argument is zero, or multiplying by infinity; correct
+         * result is exact and can be obtained by multiplying the
+         * arguments.
+         */
+        ST1 = floatx80_mul(ST0, ST1, &env->fp_status);
+    } else if (arg0_exp < 0x3fb0) {
+        /*
+         * Multiplying both arguments and an extra-precision version
+         * of log2(e) is sufficiently precise.
+         */
+        uint64_t sig0, sig1, sig2;
+        int32_t exp;
+        if (arg0_exp == 0) {
+            normalizeFloatx80Subnormal(arg0_sig, &arg0_exp, &arg0_sig);
+        }
+        if (arg1_exp == 0) {
+            normalizeFloatx80Subnormal(arg1_sig, &arg1_exp, &arg1_sig);
+        }
+        mul128By64To192(log2_e_sig_high, log2_e_sig_low, arg0_sig,
+                        &sig0, &sig1, &sig2);
+        exp = arg0_exp + 1;
+        mul128By64To192(sig0, sig1, arg1_sig, &sig0, &sig1, &sig2);
+        exp += arg1_exp - 0x3ffe;
+        /* This result is inexact.  */
+        sig1 |= 1;
+        ST1 = normalizeRoundAndPackFloatx80(floatx80_precision_x,
+                                            arg0_sign ^ arg1_sign, exp,
+                                            sig0, sig1, &env->fp_status);
+    } else {
+        int32_t aexp;
+        uint64_t asig0, asig1, asig2;
+        FloatRoundMode save_mode = env->fp_status.float_rounding_mode;
+        FloatX80RoundPrec save_prec =
+            env->fp_status.floatx80_rounding_precision;
+        env->fp_status.float_rounding_mode = float_round_nearest_even;
+        env->fp_status.floatx80_rounding_precision = floatx80_precision_x;
+
+        helper_fyl2x_common(env, ST0, &aexp, &asig0, &asig1);
+        /*
+         * Multiply by the second argument to compute the required
+         * result.
+         */
+        if (arg1_exp == 0) {
+            normalizeFloatx80Subnormal(arg1_sig, &arg1_exp, &arg1_sig);
+        }
+        mul128By64To192(asig0, asig1, arg1_sig, &asig0, &asig1, &asig2);
+        aexp += arg1_exp - 0x3ffe;
+        /* This result is inexact.  */
+        asig1 |= 1;
+        env->fp_status.float_rounding_mode = save_mode;
+        ST1 = normalizeRoundAndPackFloatx80(floatx80_precision_x,
+                                            arg0_sign ^ arg1_sign, aexp,
+                                            asig0, asig1, &env->fp_status);
+        env->fp_status.floatx80_rounding_precision = save_prec;
+    }
+    fpop(env);
+    merge_exception_flags(env, old_flags);
+}
 
-    if ((fptemp + 1.0) > 0.0) {
-        fptemp = log(fptemp + 1.0) / log(2.0); /* log2(ST + 1.0) */
-        fptemp *= floatx80_to_double(env, ST1);
-        ST1 = double_to_floatx80(env, fptemp);
-        fpop(env);
+void helper_fyl2x(CPUX86State *env)
+{
+    uint8_t old_flags = save_exception_flags(env);
+    uint64_t arg0_sig = extractFloatx80Frac(ST0);
+    int32_t arg0_exp = extractFloatx80Exp(ST0);
+    bool arg0_sign = extractFloatx80Sign(ST0);
+    uint64_t arg1_sig = extractFloatx80Frac(ST1);
+    int32_t arg1_exp = extractFloatx80Exp(ST1);
+    bool arg1_sign = extractFloatx80Sign(ST1);
+
+    if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_silence_nan(ST0, &env->fp_status);
+    } else if (floatx80_is_signaling_nan(ST1, &env->fp_status)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_silence_nan(ST1, &env->fp_status);
+    } else if (floatx80_invalid_encoding(ST0) ||
+               floatx80_invalid_encoding(ST1)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_default_nan(&env->fp_status);
+    } else if (floatx80_is_any_nan(ST0)) {
+        ST1 = ST0;
+    } else if (floatx80_is_any_nan(ST1)) {
+        /* Pass this NaN through.  */
+    } else if (arg0_sign && !floatx80_is_zero(ST0)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST1 = floatx80_default_nan(&env->fp_status);
+    } else if (floatx80_is_infinity(ST1)) {
+        FloatRelation cmp = floatx80_compare(ST0, floatx80_one,
+                                             &env->fp_status);
+        switch (cmp) {
+        case float_relation_less:
+            ST1 = floatx80_chs(ST1);
+            break;
+        case float_relation_greater:
+            /* Result is infinity of the same sign as ST1.  */
+            break;
+        default:
+            float_raise(float_flag_invalid, &env->fp_status);
+            ST1 = floatx80_default_nan(&env->fp_status);
+            break;
+        }
+    } else if (floatx80_is_infinity(ST0)) {
+        if (floatx80_is_zero(ST1)) {
+            float_raise(float_flag_invalid, &env->fp_status);
+            ST1 = floatx80_default_nan(&env->fp_status);
+        } else if (arg1_sign) {
+            ST1 = floatx80_chs(ST0);
+        } else {
+            ST1 = ST0;
+        }
+    } else if (floatx80_is_zero(ST0)) {
+        if (floatx80_is_zero(ST1)) {
+            float_raise(float_flag_invalid, &env->fp_status);
+            ST1 = floatx80_default_nan(&env->fp_status);
+        } else {
+            /* Result is infinity with opposite sign to ST1.  */
+            float_raise(float_flag_divbyzero, &env->fp_status);
+            ST1 = make_floatx80(arg1_sign ? 0x7fff : 0xffff,
+                                0x8000000000000000ULL);
+        }
+    } else if (floatx80_is_zero(ST1)) {
+        if (floatx80_lt(ST0, floatx80_one, &env->fp_status)) {
+            ST1 = floatx80_chs(ST1);
+        }
+        /* Otherwise, ST1 is already the correct result.  */
+    } else if (floatx80_eq(ST0, floatx80_one, &env->fp_status)) {
+        if (arg1_sign) {
+            ST1 = floatx80_chs(floatx80_zero);
+        } else {
+            ST1 = floatx80_zero;
+        }
     } else {
-        env->fpus &= ~0x4700;
-        env->fpus |= 0x400;
+        int32_t int_exp;
+        floatx80 arg0_m1;
+        FloatRoundMode save_mode = env->fp_status.float_rounding_mode;
+        FloatX80RoundPrec save_prec =
+            env->fp_status.floatx80_rounding_precision;
+        env->fp_status.float_rounding_mode = float_round_nearest_even;
+        env->fp_status.floatx80_rounding_precision = floatx80_precision_x;
+
+        if (arg0_exp == 0) {
+            normalizeFloatx80Subnormal(arg0_sig, &arg0_exp, &arg0_sig);
+        }
+        if (arg1_exp == 0) {
+            normalizeFloatx80Subnormal(arg1_sig, &arg1_exp, &arg1_sig);
+        }
+        int_exp = arg0_exp - 0x3fff;
+        if (arg0_sig > 0xb504f333f9de6484ULL) {
+            ++int_exp;
+        }
+        arg0_m1 = floatx80_sub(floatx80_scalbn(ST0, -int_exp,
+                                               &env->fp_status),
+                               floatx80_one, &env->fp_status);
+        if (floatx80_is_zero(arg0_m1)) {
+            /* Exact power of 2; multiply by ST1.  */
+            env->fp_status.float_rounding_mode = save_mode;
+            ST1 = floatx80_mul(int32_to_floatx80(int_exp, &env->fp_status),
+                               ST1, &env->fp_status);
+        } else {
+            bool asign = extractFloatx80Sign(arg0_m1);
+            int32_t aexp;
+            uint64_t asig0, asig1, asig2;
+            helper_fyl2x_common(env, arg0_m1, &aexp, &asig0, &asig1);
+            if (int_exp != 0) {
+                bool isign = (int_exp < 0);
+                int32_t iexp;
+                uint64_t isig;
+                int shift;
+                int_exp = isign ? -int_exp : int_exp;
+                shift = clz32(int_exp) + 32;
+                isig = int_exp;
+                isig <<= shift;
+                iexp = 0x403e - shift;
+                shift128RightJamming(asig0, asig1, iexp - aexp,
+                                     &asig0, &asig1);
+                if (asign == isign) {
+                    add128(isig, 0, asig0, asig1, &asig0, &asig1);
+                } else {
+                    sub128(isig, 0, asig0, asig1, &asig0, &asig1);
+                }
+                aexp = iexp;
+                asign = isign;
+            }
+            /*
+             * Multiply by the second argument to compute the required
+             * result.
+             */
+            if (arg1_exp == 0) {
+                normalizeFloatx80Subnormal(arg1_sig, &arg1_exp, &arg1_sig);
+            }
+            mul128By64To192(asig0, asig1, arg1_sig, &asig0, &asig1, &asig2);
+            aexp += arg1_exp - 0x3ffe;
+            /* This result is inexact.  */
+            asig1 |= 1;
+            env->fp_status.float_rounding_mode = save_mode;
+            ST1 = normalizeRoundAndPackFloatx80(floatx80_precision_x,
+                                                asign ^ arg1_sign, aexp,
+                                                asig0, asig1, &env->fp_status);
+        }
+
+        env->fp_status.floatx80_rounding_precision = save_prec;
     }
+    fpop(env);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fsqrt(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     if (floatx80_is_neg(ST0)) {
         env->fpus &= ~0x4700;  /* (C3,C2,C1,C0) <-- 0000 */
         env->fpus |= 0x400;
     }
     ST0 = floatx80_sqrt(ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fsincos(CPUX86State *env)
@@ -950,17 +2264,60 @@ void helper_fsincos(CPUX86State *env)
 
 void helper_frndint(CPUX86State *env)
 {
+    uint8_t old_flags = save_exception_flags(env);
     ST0 = floatx80_round_to_int(ST0, &env->fp_status);
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fscale(CPUX86State *env)
 {
-    if (floatx80_is_any_nan(ST1)) {
+    uint8_t old_flags = save_exception_flags(env);
+    if (floatx80_invalid_encoding(ST1) || floatx80_invalid_encoding(ST0)) {
+        float_raise(float_flag_invalid, &env->fp_status);
+        ST0 = floatx80_default_nan(&env->fp_status);
+    } else if (floatx80_is_any_nan(ST1)) {
+        if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
+            float_raise(float_flag_invalid, &env->fp_status);
+        }
         ST0 = ST1;
+        if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
+            float_raise(float_flag_invalid, &env->fp_status);
+            ST0 = floatx80_silence_nan(ST0, &env->fp_status);
+        }
+    } else if (floatx80_is_infinity(ST1) &&
+               !floatx80_invalid_encoding(ST0) &&
+               !floatx80_is_any_nan(ST0)) {
+        if (floatx80_is_neg(ST1)) {
+            if (floatx80_is_infinity(ST0)) {
+                float_raise(float_flag_invalid, &env->fp_status);
+                ST0 = floatx80_default_nan(&env->fp_status);
+            } else {
+                ST0 = (floatx80_is_neg(ST0) ?
+                       floatx80_chs(floatx80_zero) :
+                       floatx80_zero);
+            }
+        } else {
+            if (floatx80_is_zero(ST0)) {
+                float_raise(float_flag_invalid, &env->fp_status);
+                ST0 = floatx80_default_nan(&env->fp_status);
+            } else {
+                ST0 = (floatx80_is_neg(ST0) ?
+                       floatx80_chs(floatx80_infinity) :
+                       floatx80_infinity);
+            }
+        }
     } else {
-        int n = floatx80_to_int32_round_to_zero(ST1, &env->fp_status);
+        int n;
+        FloatX80RoundPrec save = env->fp_status.floatx80_rounding_precision;
+        uint8_t save_flags = get_float_exception_flags(&env->fp_status);
+        set_float_exception_flags(0, &env->fp_status);
+        n = floatx80_to_int32_round_to_zero(ST1, &env->fp_status);
+        set_float_exception_flags(save_flags, &env->fp_status);
+        env->fp_status.floatx80_rounding_precision = floatx80_precision_x;
         ST0 = floatx80_scalbn(ST0, n, &env->fp_status);
+        env->fp_status.floatx80_rounding_precision = save;
     }
+    merge_exception_flags(env, old_flags);
 }
 
 void helper_fsin(CPUX86State *env)
@@ -1010,7 +2367,7 @@ void helper_fxam_ST0(CPUX86State *env)
     if (expdif == MAXEXPD) {
         if (MANTD(temp) == 0x8000000000000000ULL) {
             env->fpus |= 0x500; /* Infinity */
-        } else {
+        } else if (MANTD(temp) & 0x8000000000000000ULL) {
             env->fpus |= 0x100; /* NaN */
         }
     } else if (expdif == 0) {
@@ -1019,7 +2376,7 @@ void helper_fxam_ST0(CPUX86State *env)
         } else {
             env->fpus |= 0x4400; /* Denormal */
         }
-    } else {
+    } else if (MANTD(temp) & 0x8000000000000000ULL) {
         env->fpus |= 0x400;
     }
 }
@@ -1045,13 +2402,12 @@ static void do_fstenv(CPUX86State *env, target_ulong ptr, int data32,
                 /* zero */
                 fptag |= 1;
             } else if (exp == 0 || exp == MAXEXPD
-                       || (mant & (1ULL << 63)) == 0) {
+                       || (mant & (1LL << 63)) == 0) {
                 /* NaNs, infinity, denormal */
                 fptag |= 2;
             }
         }
     }
-
     if (data32) {
         /* 32 bit */
         cpu_stl_data_ra(env, ptr, env->fpuc, retaddr);
@@ -1083,6 +2439,7 @@ static void cpu_set_fpus(CPUX86State *env, uint16_t fpus)
     env->fpstt = (fpus >> 11) & 7;
     env->fpus = fpus & ~0x3800 & ~FPUS_B;
     env->fpus |= env->fpus & FPUS_SE ? FPUS_B : 0;
+#if !defined(CONFIG_USER_ONLY)
     if (!(env->fpus & FPUS_SE)) {
         /*
          * Here the processor deasserts FERR#; in response, the chipset deasserts
@@ -1090,6 +2447,7 @@ static void cpu_set_fpus(CPUX86State *env, uint16_t fpus)
          */
         cpu_clear_ignne(env);
     }
+#endif
 }
 
 static void do_fldenv(CPUX86State *env, target_ulong ptr, int data32,
@@ -1118,38 +2476,50 @@ void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
     do_fldenv(env, ptr, data32, GETPC());
 }
 
-void helper_fsave(CPUX86State *env, target_ulong ptr, int data32)
+static void do_fsave(CPUX86State *env, target_ulong ptr, int data32,
+                     uintptr_t retaddr)
 {
     floatx80 tmp;
     int i;
 
-    do_fstenv(env, ptr, data32, GETPC());
+    do_fstenv(env, ptr, data32, retaddr);
 
-    ptr += (14 << data32);
+    ptr += (target_ulong)14 << data32;
     for (i = 0; i < 8; i++) {
         tmp = ST(i);
-        helper_fstt(env, tmp, ptr, GETPC());
+        do_fstt(env, tmp, ptr, retaddr);
         ptr += 10;
     }
 
     do_fninit(env);
 }
 
-void helper_frstor(CPUX86State *env, target_ulong ptr, int data32)
+void helper_fsave(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fsave(env, ptr, data32, GETPC());
+}
+
+static void do_frstor(CPUX86State *env, target_ulong ptr, int data32,
+                      uintptr_t retaddr)
 {
     floatx80 tmp;
     int i;
 
-    do_fldenv(env, ptr, data32, GETPC());
-    ptr += (14 << data32);
+    do_fldenv(env, ptr, data32, retaddr);
+    ptr += (target_ulong)14 << data32;
 
     for (i = 0; i < 8; i++) {
-        tmp = helper_fldt(env, ptr, GETPC());
+        tmp = do_fldt(env, ptr, retaddr);
         ST(i) = tmp;
         ptr += 10;
     }
 }
 
+void helper_frstor(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_frstor(env, ptr, data32, GETPC());
+}
+
 #define XO(X)  offsetof(X86XSaveArea, X)
 
 static void do_xsave_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra)
@@ -1167,22 +2537,21 @@ static void do_xsave_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra)
     cpu_stw_data_ra(env, ptr + XO(legacy.fsw), fpus, ra);
     cpu_stw_data_ra(env, ptr + XO(legacy.ftw), fptag ^ 0xff, ra);
 
-    /* In 32-bit mode this is eip, sel, dp, sel.
-       In 64-bit mode this is rip, rdp.
-       But in either case we don't write actual data, just zeros.  */
-    cpu_stq_data_ra(env, ptr + XO(legacy.fpip), env->fpip, ra); /* eip+sel; rip */
+    /* In 32-bit mode this is eip, sel; in 64-bit mode this is rip. */
+    cpu_stq_data_ra(env, ptr + XO(legacy.fpip), env->fpip, ra);
     cpu_stq_data_ra(env, ptr + XO(legacy.fpdp), 0, ra); /* edp+sel; rdp */
 
     addr = ptr + XO(legacy.fpregs);
     for (i = 0; i < 8; i++) {
         floatx80 tmp = ST(i);
-        helper_fstt(env, tmp, addr, ra);
+        do_fstt(env, tmp, addr, ra);
         addr += 16;
     }
 }
 
 static void do_xsave_mxcsr(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 {
+    update_mxcsr_from_sse_status(env);
     cpu_stl_data_ra(env, ptr + XO(legacy.mxcsr), env->mxcsr, ra);
     cpu_stl_data_ra(env, ptr + XO(legacy.mxcsr_mask), 0x0000ffff, ra);
 }
@@ -1206,6 +2575,22 @@ static void do_xsave_sse(CPUX86State *env, target_ulong ptr, uintptr_t ra)
     }
 }
 
+static void do_xsave_ymmh(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, nb_xmm_regs;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    for (i = 0; i < nb_xmm_regs; i++, ptr += 16) {
+        cpu_stq_data_ra(env, ptr, env->xmm_regs[i].ZMM_Q(2), ra);
+        cpu_stq_data_ra(env, ptr + 8, env->xmm_regs[i].ZMM_Q(3), ra);
+    }
+}
+
 static void do_xsave_bndregs(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 {
     target_ulong addr = ptr + offsetof(XSaveBNDREG, bnd_regs);
@@ -1230,10 +2615,8 @@ static void do_xsave_pkru(CPUX86State *env, target_ulong ptr, uintptr_t ra)
     cpu_stq_data_ra(env, ptr, env->pkru, ra);
 }
 
-void helper_fxsave(CPUX86State *env, target_ulong ptr)
+static void do_fxsave(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 {
-    uintptr_t ra = GETPC();
-
     /* The operand must be 16 byte aligned */
     if (ptr & 0xf) {
         raise_exception_ra(env, EXCP0D_GPF, ra);
@@ -1252,6 +2635,11 @@ void helper_fxsave(CPUX86State *env, target_ulong ptr)
     }
 }
 
+void helper_fxsave(CPUX86State *env, target_ulong ptr)
+{
+    do_fxsave(env, ptr, GETPC());
+}
+
 static uint64_t get_xinuse(CPUX86State *env)
 {
     uint64_t inuse = -1;
@@ -1295,6 +2683,9 @@ static void do_xsave(CPUX86State *env, target_ulong ptr, uint64_t rfbm,
     if (opt & XSTATE_SSE_MASK) {
         do_xsave_sse(env, ptr, ra);
     }
+    if (opt & XSTATE_YMM_MASK) {
+        do_xsave_ymmh(env, ptr + XO(avx_state), ra);
+    }
     if (opt & XSTATE_BNDREGS_MASK) {
         do_xsave_bndregs(env, ptr + XO(bndreg_state), ra);
     }
@@ -1339,7 +2730,7 @@ static void do_xrstor_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 
     addr = ptr + XO(legacy.fpregs);
     for (i = 0; i < 8; i++) {
-        floatx80 tmp = helper_fldt(env, addr, ra);
+        floatx80 tmp = do_fldt(env, addr, ra);
         ST(i) = tmp;
         addr += 16;
     }
@@ -1369,6 +2760,54 @@ static void do_xrstor_sse(CPUX86State *env, target_ulong ptr, uintptr_t ra)
     }
 }
 
+static void do_clear_sse(CPUX86State *env)
+{
+    int i, nb_xmm_regs;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    for (i = 0; i < nb_xmm_regs; i++) {
+        env->xmm_regs[i].ZMM_Q(0) = 0;
+        env->xmm_regs[i].ZMM_Q(1) = 0;
+    }
+}
+
+static void do_xrstor_ymmh(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, nb_xmm_regs;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    for (i = 0; i < nb_xmm_regs; i++, ptr += 16) {
+        env->xmm_regs[i].ZMM_Q(2) = cpu_ldq_data_ra(env, ptr, ra);
+        env->xmm_regs[i].ZMM_Q(3) = cpu_ldq_data_ra(env, ptr + 8, ra);
+    }
+}
+
+static void do_clear_ymmh(CPUX86State *env)
+{
+    int i, nb_xmm_regs;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    for (i = 0; i < nb_xmm_regs; i++) {
+        env->xmm_regs[i].ZMM_Q(2) = 0;
+        env->xmm_regs[i].ZMM_Q(3) = 0;
+    }
+}
+
 static void do_xrstor_bndregs(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 {
     target_ulong addr = ptr + offsetof(XSaveBNDREG, bnd_regs);
@@ -1394,10 +2833,8 @@ static void do_xrstor_pkru(CPUX86State *env, target_ulong ptr, uintptr_t ra)
     env->pkru = cpu_ldq_data_ra(env, ptr, ra);
 }
 
-void helper_fxrstor(CPUX86State *env, target_ulong ptr)
+static void do_fxrstor(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 {
-    uintptr_t ra = GETPC();
-
     /* The operand must be 16 byte aligned */
     if (ptr & 0xf) {
         raise_exception_ra(env, EXCP0D_GPF, ra);
@@ -1416,9 +2853,13 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr)
     }
 }
 
-void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+void helper_fxrstor(CPUX86State *env, target_ulong ptr)
+{
+    do_fxrstor(env, ptr, GETPC());
+}
+
+static void do_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm, uintptr_t ra)
 {
-    uintptr_t ra = GETPC();
     uint64_t xstate_bv, xcomp_bv, reserve0;
 
     rfbm &= env->xcr0;
@@ -1473,9 +2914,14 @@ void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
         if (xstate_bv & XSTATE_SSE_MASK) {
             do_xrstor_sse(env, ptr, ra);
         } else {
-            /* ??? When AVX is implemented, we may have to be more
-               selective in the clearing.  */
-            memset(env->xmm_regs, 0, sizeof(env->xmm_regs));
+            do_clear_sse(env);
+        }
+    }
+    if (rfbm & XSTATE_YMM_MASK) {
+        if (xstate_bv & XSTATE_YMM_MASK) {
+            do_xrstor_ymmh(env, ptr + XO(avx_state), ra);
+        } else {
+            do_clear_ymmh(env);
         }
     }
     if (rfbm & XSTATE_BNDREGS_MASK) {
@@ -1511,6 +2957,43 @@ void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
 
 #undef XO
 
+void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+{
+    do_xrstor(env, ptr, rfbm, GETPC());
+}
+
+#if defined(CONFIG_USER_ONLY)
+void cpu_x86_fsave(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fsave(env, ptr, data32, 0);
+}
+
+void cpu_x86_frstor(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_frstor(env, ptr, data32, 0);
+}
+
+void cpu_x86_fxsave(CPUX86State *env, target_ulong ptr)
+{
+    do_fxsave(env, ptr, 0);
+}
+
+void cpu_x86_fxrstor(CPUX86State *env, target_ulong ptr)
+{
+    do_fxrstor(env, ptr, 0);
+}
+
+void cpu_x86_xsave(CPUX86State *env, target_ulong ptr)
+{
+    do_xsave(env, ptr, -1, get_xinuse(env), -1, 0);
+}
+
+void cpu_x86_xrstor(CPUX86State *env, target_ulong ptr)
+{
+    do_xrstor(env, ptr, -1, 0);
+}
+#endif
+
 uint64_t helper_xgetbv(CPUX86State *env, uint32_t ecx)
 {
     /* The OS must have enabled XSAVE.  */
@@ -1545,6 +3028,11 @@ void helper_xsetbv(CPUX86State *env, uint32_t ecx, uint64_t mask)
         goto do_gpf;
     }
 
+    /* SSE can be disabled, but only if AVX is disabled too.  */
+    if ((mask & (XSTATE_SSE_MASK | XSTATE_YMM_MASK)) == XSTATE_YMM_MASK) {
+        goto do_gpf;
+    }
+
     /* Disallow enabling unimplemented features.  */
     cpu_x86_cpuid(env, 0x0d, 0, &ena_lo, &dummy, &dummy, &ena_hi);
     ena = ((uint64_t)ena_hi << 32) | ena_lo;
@@ -1560,6 +3048,7 @@ void helper_xsetbv(CPUX86State *env, uint32_t ecx, uint64_t mask)
 
     env->xcr0 = mask;
     cpu_sync_bndcs_hflags(env);
+    cpu_sync_avx_hflag(env);
     return;
 
  do_gpf:
@@ -1570,11 +3059,8 @@ void helper_xsetbv(CPUX86State *env, uint32_t ecx, uint64_t mask)
 /* XXX: optimize by storing fptt and fptags in the static cpu state */
 
 #define SSE_DAZ             0x0040
-#define SSE_RC_MASK         0x6000
-#define SSE_RC_NEAR         0x0000
-#define SSE_RC_DOWN         0x2000
-#define SSE_RC_UP           0x4000
-#define SSE_RC_CHOP         0x6000
+#define SSE_RC_SHIFT        13
+#define SSE_RC_MASK         (3 << SSE_RC_SHIFT)
 #define SSE_FZ              0x8000
 
 void update_mxcsr_status(CPUX86State *env)
@@ -1583,28 +3069,46 @@ void update_mxcsr_status(CPUX86State *env)
     int rnd_type;
 
     /* set rounding mode */
-    switch (mxcsr & SSE_RC_MASK) {
-    default:
-    case SSE_RC_NEAR:
-        rnd_type = float_round_nearest_even;
-        break;
-    case SSE_RC_DOWN:
-        rnd_type = float_round_down;
-        break;
-    case SSE_RC_UP:
-        rnd_type = float_round_up;
-        break;
-    case SSE_RC_CHOP:
-        rnd_type = float_round_to_zero;
-        break;
-    }
-    set_float_rounding_mode(rnd_type, &env->sse_status);
+    rnd_type = (mxcsr & SSE_RC_MASK) >> SSE_RC_SHIFT;
+    set_x86_rounding_mode(rnd_type, &env->sse_status);
+
+    /* Set exception flags.  */
+    set_float_exception_flags((mxcsr & FPUS_IE ? float_flag_invalid : 0) |
+                              (mxcsr & FPUS_ZE ? float_flag_divbyzero : 0) |
+                              (mxcsr & FPUS_OE ? float_flag_overflow : 0) |
+                              (mxcsr & FPUS_UE ? float_flag_underflow : 0) |
+                              (mxcsr & FPUS_PE ? float_flag_inexact : 0),
+                              &env->sse_status);
 
     /* set denormals are zero */
     set_flush_inputs_to_zero((mxcsr & SSE_DAZ) ? 1 : 0, &env->sse_status);
 
     /* set flush to zero */
-    set_flush_to_zero((mxcsr & SSE_FZ) ? 1 : 0, &env->fp_status);
+    set_flush_to_zero((mxcsr & SSE_FZ) ? 1 : 0, &env->sse_status);
+}
+
+void update_mxcsr_from_sse_status(CPUX86State *env)
+{
+    uint8_t flags = get_float_exception_flags(&env->sse_status);
+    /*
+     * The MXCSR denormal flag has opposite semantics to
+     * float_flag_input_denormal (the softfloat code sets that flag
+     * only when flushing input denormals to zero, but SSE sets it
+     * only when not flushing them to zero), so is not converted
+     * here.
+     */
+    env->mxcsr |= ((flags & float_flag_invalid ? FPUS_IE : 0) |
+                   (flags & float_flag_divbyzero ? FPUS_ZE : 0) |
+                   (flags & float_flag_overflow ? FPUS_OE : 0) |
+                   (flags & float_flag_underflow ? FPUS_UE : 0) |
+                   (flags & float_flag_inexact ? FPUS_PE : 0) |
+                   (flags & float_flag_output_denormal ? FPUS_UE | FPUS_PE :
+                    0));
+}
+
+void helper_update_mxcsr(CPUX86State *env)
+{
+    update_mxcsr_from_sse_status(env);
 }
 
 void helper_ldmxcsr(CPUX86State *env, uint32_t val)
@@ -1626,10 +3130,10 @@ void helper_emms(CPUX86State *env)
     *(uint32_t *)(env->fptags + 4) = 0x01010101;
 }
 
-/* XXX: suppress */
 void helper_movq(CPUX86State *env, void *d, void *s)
 {
-    *(uint64_t *)d = *(uint64_t *)s;
+    (void)env;
+    memcpy(d, s, sizeof(MMXReg));
 }
 
 #define SHIFT 0
@@ -1637,3 +3141,6 @@ void helper_movq(CPUX86State *env, void *d, void *s)
 
 #define SHIFT 1
 #include "ops_sse.h"
+
+#define SHIFT 2
+#include "ops_sse.h"
diff --git a/qemu/target/i386/helper-tcg.h b/qemu/target/i386/helper-tcg.h
new file mode 100644
index 0000000000..cd1723389a
--- /dev/null
+++ b/qemu/target/i386/helper-tcg.h
@@ -0,0 +1,116 @@
+/*
+ * TCG specific prototypes for helpers
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef I386_HELPER_TCG_H
+#define I386_HELPER_TCG_H
+
+#include "exec/exec-all.h"
+
+/* Maximum instruction code size */
+#define TARGET_MAX_INSN_SIZE 16
+
+#if defined(TARGET_X86_64)
+# define TCG_PHYS_ADDR_BITS 40
+#else
+# define TCG_PHYS_ADDR_BITS 36
+#endif
+
+QEMU_BUILD_BUG_ON(TCG_PHYS_ADDR_BITS > TARGET_PHYS_ADDR_SPACE_BITS);
+
+/**
+ * x86_cpu_do_interrupt:
+ * @cpu: vCPU the interrupt is to be handled by.
+ */
+void x86_cpu_do_interrupt(CPUState *cpu);
+#ifndef CONFIG_USER_ONLY
+bool x86_cpu_exec_interrupt(CPUState *cpu, int int_req);
+#endif
+
+void breakpoint_handler(CPUState *cs);
+
+/* n must be a constant to be efficient */
+static inline target_long lshift(target_long x, int n)
+{
+    if (n >= 0) {
+        return x << n;
+    } else {
+        return x >> (-n);
+    }
+}
+
+/* translate.c */
+void tcg_x86_init(void);
+
+/* excp_helper.c */
+G_NORETURN void raise_exception(CPUX86State *env, int exception_index);
+G_NORETURN void raise_exception_ra(CPUX86State *env, int exception_index,
+                                   uintptr_t retaddr);
+G_NORETURN void raise_exception_err(CPUX86State *env, int exception_index,
+                                    int error_code);
+G_NORETURN void raise_exception_err_ra(CPUX86State *env, int exception_index,
+                                       int error_code, uintptr_t retaddr);
+G_NORETURN void raise_interrupt(CPUX86State *nenv, int intno, int is_int,
+                                int error_code, int next_eip_addend);
+G_NORETURN void handle_unaligned_access(CPUX86State *env, vaddr vaddr,
+                                        MMUAccessType access_type,
+                                        uintptr_t retaddr);
+#ifdef CONFIG_USER_ONLY
+void x86_cpu_record_sigsegv(CPUState *cs, vaddr addr,
+                            MMUAccessType access_type,
+                            bool maperr, uintptr_t ra);
+void x86_cpu_record_sigbus(CPUState *cs, vaddr addr,
+                           MMUAccessType access_type, uintptr_t ra);
+#else
+bool x86_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
+                      MMUAccessType access_type, int mmu_idx,
+                      bool probe, uintptr_t retaddr);
+G_NORETURN void x86_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr,
+                                            MMUAccessType access_type,
+                                            int mmu_idx, uintptr_t retaddr);
+#endif
+
+/* cc_helper.c */
+extern const uint8_t parity_table[256];
+
+/* misc_helper.c */
+void cpu_load_eflags(CPUX86State *env, int eflags, int update_mask);
+G_NORETURN void do_pause(CPUX86State *env);
+
+/* sysemu/svm_helper.c */
+#ifndef CONFIG_USER_ONLY
+G_NORETURN void cpu_vmexit(CPUX86State *nenv, uint32_t exit_code,
+                           uint64_t exit_info_1, uintptr_t retaddr);
+void do_vmexit(CPUX86State *env);
+#endif
+
+/* seg_helper.c */
+void do_interrupt_x86_hardirq(CPUX86State *env, int intno, int is_hw);
+void do_interrupt_all(X86CPU *cpu, int intno, int is_int,
+                      int error_code, target_ulong next_eip, int is_hw);
+void handle_even_inj(CPUX86State *env, int intno, int is_int,
+                     int error_code, int is_hw, int rm);
+int exception_has_error_code(int intno);
+
+/* smm_helper.c */
+void do_smm_enter(X86CPU *cpu);
+
+/* bpt_helper.c */
+bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update);
+
+#endif /* I386_HELPER_TCG_H */
diff --git a/qemu/target/i386/helper.c b/qemu/target/i386/helper.c
index 8a3540e0e4..b37e4ede28 100644
--- a/qemu/target/i386/helper.c
+++ b/qemu/target/i386/helper.c
@@ -22,6 +22,17 @@
 #include "exec/exec-all.h"
 #include "sysemu/tcg.h"
 
+void cpu_sync_avx_hflag(CPUX86State *env)
+{
+    if ((env->cr[4] & CR4_OSXSAVE_MASK)
+        && (env->xcr0 & (XSTATE_SSE_MASK | XSTATE_YMM_MASK))
+            == (XSTATE_SSE_MASK | XSTATE_YMM_MASK)) {
+        env->hflags |= HF_AVX_EN_MASK;
+    } else {
+        env->hflags &= ~HF_AVX_EN_MASK;
+    }
+}
+
 void cpu_sync_bndcs_hflags(CPUX86State *env)
 {
     uint32_t hflags = env->hflags;
@@ -168,7 +179,7 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
     }
 
     /* Clear bits we're going to recompute.  */
-    hflags = env->hflags & ~(HF_OSFXSR_MASK | HF_SMAP_MASK);
+    hflags = env->hflags & ~(HF_OSFXSR_MASK | HF_SMAP_MASK | HF_UMIP_MASK);
 
     /* SSE handling */
     if (!(env->features[FEAT_1_EDX] & CPUID_SSE)) {
@@ -184,15 +195,25 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
     if (new_cr4 & CR4_SMAP_MASK) {
         hflags |= HF_SMAP_MASK;
     }
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_UMIP)) {
+        new_cr4 &= ~CR4_UMIP_MASK;
+    }
+    if (new_cr4 & CR4_UMIP_MASK) {
+        hflags |= HF_UMIP_MASK;
+    }
 
     if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_PKU)) {
         new_cr4 &= ~CR4_PKE_MASK;
     }
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_PKS)) {
+        new_cr4 &= ~CR4_PKS_MASK;
+    }
 
     env->cr[4] = new_cr4;
     env->hflags = hflags;
 
     cpu_sync_bndcs_hflags(env);
+    cpu_sync_avx_hflag(env);
 }
 
 hwaddr x86_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr,
diff --git a/qemu/target/i386/helper.h b/qemu/target/i386/helper.h
index 399cc0df99..4c69cd943c 100644
--- a/qemu/target/i386/helper.h
+++ b/qemu/target/i386/helper.h
@@ -93,6 +93,7 @@ DEF_HELPER_1(wrmsr, void, env)
 DEF_HELPER_2(check_iob, void, env, i32)
 DEF_HELPER_2(check_iow, void, env, i32)
 DEF_HELPER_2(check_iol, void, env, i32)
+DEF_HELPER_FLAGS_3(check_io, TCG_CALL_NO_WG, void, env, i32, i32)
 DEF_HELPER_3(outb, void, env, i32, i32)
 DEF_HELPER_2(inb, tl, env, i32)
 DEF_HELPER_3(outw, void, env, i32, i32)
@@ -102,6 +103,7 @@ DEF_HELPER_2(inl, tl, env, i32)
 DEF_HELPER_FLAGS_4(bpt_io, TCG_CALL_NO_WG, void, env, i32, i32, tl)
 
 DEF_HELPER_3(svm_check_intercept_param, void, env, i32, i64)
+DEF_HELPER_2(svm_check_intercept, void, env, i32)
 DEF_HELPER_4(svm_check_io, void, env, i32, i32, i32)
 DEF_HELPER_3(vmrun, void, env, int, int)
 DEF_HELPER_1(vmmcall, void, env)
@@ -111,6 +113,7 @@ DEF_HELPER_1(stgi, void, env)
 DEF_HELPER_1(clgi, void, env)
 DEF_HELPER_1(skinit, void, env)
 DEF_HELPER_2(invlpga, void, env, int)
+DEF_HELPER_FLAGS_2(flush_page, TCG_CALL_NO_RWG, void, env, tl)
 
 /* x86 FPU */
 
@@ -210,6 +213,7 @@ DEF_HELPER_FLAGS_2(pext, TCG_CALL_NO_RWG_SE, tl, tl, tl)
 /* MMX/SSE */
 
 DEF_HELPER_2(ldmxcsr, void, env, i32)
+DEF_HELPER_1(update_mxcsr, void, env)
 DEF_HELPER_1(enter_mmx, void, env)
 DEF_HELPER_1(emms, void, env)
 DEF_HELPER_3(movq, void, env, ptr, ptr)
@@ -218,6 +222,8 @@ DEF_HELPER_3(movq, void, env, ptr, ptr)
 #include "ops_sse_header.h"
 #define SHIFT 1
 #include "ops_sse_header.h"
+#define SHIFT 2
+#include "ops_sse_header.h"
 
 DEF_HELPER_3(rclb, tl, env, tl, tl)
 DEF_HELPER_3(rclw, tl, env, tl, tl)
diff --git a/qemu/target/i386/misc_helper.c b/qemu/target/i386/misc_helper.c
index ccc12c8192..613b0c690f 100644
--- a/qemu/target/i386/misc_helper.c
+++ b/qemu/target/i386/misc_helper.c
@@ -179,6 +179,13 @@ void helper_write_crN(CPUX86State *env, int reg, target_ulong t0)
         cpu_x86_update_cr3(env, t0);
         break;
     case 4:
+        if (t0 & cr4_reserved_bits(env)) {
+            raise_exception_ra(env, EXCP0D_GPF, GETPC());
+        }
+        if (((t0 ^ env->cr[4]) & CR4_LA57_MASK) &&
+            (env->hflags & HF_CS64_MASK)) {
+            raise_exception_ra(env, EXCP0D_GPF, GETPC());
+        }
         cpu_x86_update_cr4(env, (uint32_t)t0);
         break;
     case 8:
@@ -211,6 +218,11 @@ void helper_invlpg(CPUX86State *env, target_ulong addr)
     tlb_flush_page(CPU(cpu), addr);
 }
 
+void helper_flush_page(CPUX86State *env, target_ulong addr)
+{
+    tlb_flush_page(env_cpu(env), addr);
+}
+
 void helper_rdtsc(CPUX86State *env)
 {
     uint64_t val;
@@ -315,6 +327,7 @@ void helper_rdpmc(CPUX86State *env)
 
 void helper_wrmsr(CPUX86State *env)
 {
+    CPUState *cs = env_cpu(env);
     uint64_t val;
 
     cpu_svm_check_intercept_param(env, SVM_EXIT_MSR, 1, GETPC());
@@ -448,6 +461,33 @@ void helper_wrmsr(CPUX86State *env)
     case MSR_TSC_AUX:
         env->tsc_aux = val;
         break;
+    case MSR_IA32_XSS: {
+        uint64_t valid;
+
+        valid = ((uint64_t)env->features[FEAT_XSAVE_XSS_HI] << 32) |
+                env->features[FEAT_XSAVE_XSS_LO];
+        env->xss = val & valid;
+        break;
+    }
+    case MSR_IA32_XFD:
+        env->msr_xfd = val;
+        break;
+    case MSR_IA32_XFD_ERR:
+        env->msr_xfd_err = val;
+        break;
+    case MSR_IA32_PKRS:
+        if (val & 0xffffffff00000000ull) {
+            raise_exception_ra(env, EXCP0D_GPF, GETPC());
+        }
+        env->pkrs = val;
+        tlb_flush(cs);
+        break;
+    case MSR_ARCH_LBR_CTL:
+        env->msr_lbr_ctl = val;
+        break;
+    case MSR_ARCH_LBR_DEPTH:
+        env->msr_lbr_depth = val;
+        break;
     case MSR_IA32_MISC_ENABLE:
         env->msr_ia32_misc_enable = val;
         break;
@@ -458,6 +498,27 @@ void helper_wrmsr(CPUX86State *env)
         cpu_sync_bndcs_hflags(env);
         break;
     default:
+        if ((uint32_t)env->regs[R_ECX] >= MSR_ARCH_LBR_FROM_0 &&
+            (uint32_t)env->regs[R_ECX] <
+            MSR_ARCH_LBR_FROM_0 + ARCH_LBR_NR_ENTRIES) {
+            env->lbr_records[(uint32_t)env->regs[R_ECX] -
+                             MSR_ARCH_LBR_FROM_0].from = val;
+            break;
+        }
+        if ((uint32_t)env->regs[R_ECX] >= MSR_ARCH_LBR_TO_0 &&
+            (uint32_t)env->regs[R_ECX] <
+            MSR_ARCH_LBR_TO_0 + ARCH_LBR_NR_ENTRIES) {
+            env->lbr_records[(uint32_t)env->regs[R_ECX] -
+                             MSR_ARCH_LBR_TO_0].to = val;
+            break;
+        }
+        if ((uint32_t)env->regs[R_ECX] >= MSR_ARCH_LBR_INFO_0 &&
+            (uint32_t)env->regs[R_ECX] <
+            MSR_ARCH_LBR_INFO_0 + ARCH_LBR_NR_ENTRIES) {
+            env->lbr_records[(uint32_t)env->regs[R_ECX] -
+                             MSR_ARCH_LBR_INFO_0].info = val;
+            break;
+        }
         if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL
             && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL +
             (4 * env->mcg_cap & 0xff)) {
@@ -609,10 +670,49 @@ void helper_rdmsr(CPUX86State *env)
     case MSR_IA32_BNDCFGS:
         val = env->msr_bndcfgs;
         break;
-     case MSR_IA32_UCODE_REV:
+    case MSR_IA32_XSS:
+        val = env->xss;
+        break;
+    case MSR_IA32_XFD:
+        val = env->msr_xfd;
+        break;
+    case MSR_IA32_XFD_ERR:
+        val = env->msr_xfd_err;
+        break;
+    case MSR_IA32_PKRS:
+        val = env->pkrs;
+        break;
+    case MSR_ARCH_LBR_CTL:
+        val = env->msr_lbr_ctl;
+        break;
+    case MSR_ARCH_LBR_DEPTH:
+        val = env->msr_lbr_depth;
+        break;
+    case MSR_IA32_UCODE_REV:
         val = x86_cpu->ucode_rev;
         break;
     default:
+        if ((uint32_t)env->regs[R_ECX] >= MSR_ARCH_LBR_FROM_0 &&
+            (uint32_t)env->regs[R_ECX] <
+            MSR_ARCH_LBR_FROM_0 + ARCH_LBR_NR_ENTRIES) {
+            val = env->lbr_records[(uint32_t)env->regs[R_ECX] -
+                                   MSR_ARCH_LBR_FROM_0].from;
+            break;
+        }
+        if ((uint32_t)env->regs[R_ECX] >= MSR_ARCH_LBR_TO_0 &&
+            (uint32_t)env->regs[R_ECX] <
+            MSR_ARCH_LBR_TO_0 + ARCH_LBR_NR_ENTRIES) {
+            val = env->lbr_records[(uint32_t)env->regs[R_ECX] -
+                                   MSR_ARCH_LBR_TO_0].to;
+            break;
+        }
+        if ((uint32_t)env->regs[R_ECX] >= MSR_ARCH_LBR_INFO_0 &&
+            (uint32_t)env->regs[R_ECX] <
+            MSR_ARCH_LBR_INFO_0 + ARCH_LBR_NR_ENTRIES) {
+            val = env->lbr_records[(uint32_t)env->regs[R_ECX] -
+                                   MSR_ARCH_LBR_INFO_0].info;
+            break;
+        }
         if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL
             && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL +
             (4 * env->mcg_cap & 0xff)) {
diff --git a/qemu/target/i386/ops_sse.h b/qemu/target/i386/ops_sse.h
index ec1ec745d0..79d167f1db 100644
--- a/qemu/target/i386/ops_sse.h
+++ b/qemu/target/i386/ops_sse.h
@@ -7,7 +7,7 @@
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
+ * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -35,262 +35,206 @@
 #define W(n) ZMM_W(n)
 #define L(n) ZMM_L(n)
 #define Q(n) ZMM_Q(n)
+#if SHIFT == 1
 #define SUFFIX _xmm
+#else
+#define SUFFIX _ymm
+#endif
 #endif
 
-void glue(helper_psrlw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    int shift;
+#define LANE_WIDTH (SHIFT ? 16 : 8)
+#define PACK_WIDTH (LANE_WIDTH / 2)
 
-    if (s->Q(0) > 15) {
-        d->Q(0) = 0;
-#if SHIFT == 1
-        d->Q(1) = 0;
+#if SHIFT == 0
+#define FPSRL(x, c) ((x) >> shift)
+#define FPSRAW(x, c) ((int16_t)(x) >> shift)
+#define FPSRAL(x, c) ((int32_t)(x) >> shift)
+#define FPSLL(x, c) ((x) << shift)
 #endif
+
+void glue(helper_psrlw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
+{
+    int shift;
+    if (c->Q(0) > 15) {
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = 0;
+        }
     } else {
-        shift = s->B(0);
-        d->W(0) >>= shift;
-        d->W(1) >>= shift;
-        d->W(2) >>= shift;
-        d->W(3) >>= shift;
-#if SHIFT == 1
-        d->W(4) >>= shift;
-        d->W(5) >>= shift;
-        d->W(6) >>= shift;
-        d->W(7) >>= shift;
-#endif
+        shift = c->B(0);
+        for (int i = 0; i < 4 << SHIFT; i++) {
+            d->W(i) = FPSRL(s->W(i), shift);
+        }
     }
 }
 
-void glue(helper_psraw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_psllw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
     int shift;
-
-    if (s->Q(0) > 15) {
-        shift = 15;
+    if (c->Q(0) > 15) {
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = 0;
+        }
     } else {
-        shift = s->B(0);
+        shift = c->B(0);
+        for (int i = 0; i < 4 << SHIFT; i++) {
+            d->W(i) = FPSLL(s->W(i), shift);
+        }
     }
-    d->W(0) = (int16_t)d->W(0) >> shift;
-    d->W(1) = (int16_t)d->W(1) >> shift;
-    d->W(2) = (int16_t)d->W(2) >> shift;
-    d->W(3) = (int16_t)d->W(3) >> shift;
-#if SHIFT == 1
-    d->W(4) = (int16_t)d->W(4) >> shift;
-    d->W(5) = (int16_t)d->W(5) >> shift;
-    d->W(6) = (int16_t)d->W(6) >> shift;
-    d->W(7) = (int16_t)d->W(7) >> shift;
-#endif
 }
 
-void glue(helper_psllw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_psraw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
     int shift;
-
-    if (s->Q(0) > 15) {
-        d->Q(0) = 0;
-#if SHIFT == 1
-        d->Q(1) = 0;
-#endif
+    if (c->Q(0) > 15) {
+        shift = 15;
     } else {
-        shift = s->B(0);
-        d->W(0) <<= shift;
-        d->W(1) <<= shift;
-        d->W(2) <<= shift;
-        d->W(3) <<= shift;
-#if SHIFT == 1
-        d->W(4) <<= shift;
-        d->W(5) <<= shift;
-        d->W(6) <<= shift;
-        d->W(7) <<= shift;
-#endif
+        shift = c->B(0);
+    }
+    for (int i = 0; i < 4 << SHIFT; i++) {
+        d->W(i) = FPSRAW(s->W(i), shift);
     }
 }
 
-void glue(helper_psrld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_psrld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
     int shift;
-
-    if (s->Q(0) > 31) {
-        d->Q(0) = 0;
-#if SHIFT == 1
-        d->Q(1) = 0;
-#endif
+    if (c->Q(0) > 31) {
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = 0;
+        }
     } else {
-        shift = s->B(0);
-        d->L(0) >>= shift;
-        d->L(1) >>= shift;
-#if SHIFT == 1
-        d->L(2) >>= shift;
-        d->L(3) >>= shift;
-#endif
+        shift = c->B(0);
+        for (int i = 0; i < 2 << SHIFT; i++) {
+            d->L(i) = FPSRL(s->L(i), shift);
+        }
     }
 }
 
-void glue(helper_psrad, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_pslld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
     int shift;
-
-    if (s->Q(0) > 31) {
-        shift = 31;
+    if (c->Q(0) > 31) {
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = 0;
+        }
     } else {
-        shift = s->B(0);
+        shift = c->B(0);
+        for (int i = 0; i < 2 << SHIFT; i++) {
+            d->L(i) = FPSLL(s->L(i), shift);
+        }
     }
-    d->L(0) = (int32_t)d->L(0) >> shift;
-    d->L(1) = (int32_t)d->L(1) >> shift;
-#if SHIFT == 1
-    d->L(2) = (int32_t)d->L(2) >> shift;
-    d->L(3) = (int32_t)d->L(3) >> shift;
-#endif
 }
 
-void glue(helper_pslld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_psrad, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
     int shift;
-
-    if (s->Q(0) > 31) {
-        d->Q(0) = 0;
-#if SHIFT == 1
-        d->Q(1) = 0;
-#endif
+    if (c->Q(0) > 31) {
+        shift = 31;
     } else {
-        shift = s->B(0);
-        d->L(0) <<= shift;
-        d->L(1) <<= shift;
-#if SHIFT == 1
-        d->L(2) <<= shift;
-        d->L(3) <<= shift;
-#endif
+        shift = c->B(0);
+    }
+    for (int i = 0; i < 2 << SHIFT; i++) {
+        d->L(i) = FPSRAL(s->L(i), shift);
     }
 }
 
-void glue(helper_psrlq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_psrlq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
     int shift;
-
-    if (s->Q(0) > 63) {
-        d->Q(0) = 0;
-#if SHIFT == 1
-        d->Q(1) = 0;
-#endif
+    if (c->Q(0) > 63) {
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = 0;
+        }
     } else {
-        shift = s->B(0);
-        d->Q(0) >>= shift;
-#if SHIFT == 1
-        d->Q(1) >>= shift;
-#endif
+        shift = c->B(0);
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = FPSRL(s->Q(i), shift);
+        }
     }
 }
 
-void glue(helper_psllq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_psllq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
     int shift;
-
-    if (s->Q(0) > 63) {
-        d->Q(0) = 0;
-#if SHIFT == 1
-        d->Q(1) = 0;
-#endif
+    if (c->Q(0) > 63) {
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = 0;
+        }
     } else {
-        shift = s->B(0);
-        d->Q(0) <<= shift;
-#if SHIFT == 1
-        d->Q(1) <<= shift;
-#endif
+        shift = c->B(0);
+        for (int i = 0; i < 1 << SHIFT; i++) {
+            d->Q(i) = FPSLL(s->Q(i), shift);
+        }
     }
 }
 
-#if SHIFT == 1
-void glue(helper_psrldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+#if SHIFT >= 1
+void glue(helper_psrldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
-    int shift, i;
+    int shift, i, j;
 
-    shift = s->L(0);
+    shift = c->L(0);
     if (shift > 16) {
         shift = 16;
     }
-    for (i = 0; i < 16 - shift; i++) {
-        d->B(i) = d->B(i + shift);
-    }
-    for (i = 16 - shift; i < 16; i++) {
-        d->B(i) = 0;
+    for (j = 0; j < 8 << SHIFT; j += LANE_WIDTH) {
+        for (i = 0; i < 16 - shift; i++) {
+            d->B(j + i) = s->B(j + i + shift);
+        }
+        for (i = 16 - shift; i < 16; i++) {
+            d->B(j + i) = 0;
+        }
     }
 }
 
-void glue(helper_pslldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_pslldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c)
 {
-    int shift, i;
+    int shift, i, j;
 
-    shift = s->L(0);
+    shift = c->L(0);
     if (shift > 16) {
         shift = 16;
     }
-    for (i = 15; i >= shift; i--) {
-        d->B(i) = d->B(i - shift);
-    }
-    for (i = 0; i < shift; i++) {
-        d->B(i) = 0;
+    for (j = 0; j < 8 << SHIFT; j += LANE_WIDTH) {
+        for (i = 15; i >= shift; i--) {
+            d->B(j + i) = s->B(j + i - shift);
+        }
+        for (i = 0; i < shift; i++) {
+            d->B(j + i) = 0;
+        }
     }
 }
 #endif
 
-#define SSE_HELPER_B(name, F)                                   \
+#define SSE_HELPER_1(name, elem, num, F)                        \
     void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)   \
     {                                                           \
-        d->B(0) = F(d->B(0), s->B(0));                          \
-        d->B(1) = F(d->B(1), s->B(1));                          \
-        d->B(2) = F(d->B(2), s->B(2));                          \
-        d->B(3) = F(d->B(3), s->B(3));                          \
-        d->B(4) = F(d->B(4), s->B(4));                          \
-        d->B(5) = F(d->B(5), s->B(5));                          \
-        d->B(6) = F(d->B(6), s->B(6));                          \
-        d->B(7) = F(d->B(7), s->B(7));                          \
-        XMM_ONLY(                                               \
-                 d->B(8) = F(d->B(8), s->B(8));                 \
-                 d->B(9) = F(d->B(9), s->B(9));                 \
-                 d->B(10) = F(d->B(10), s->B(10));              \
-                 d->B(11) = F(d->B(11), s->B(11));              \
-                 d->B(12) = F(d->B(12), s->B(12));              \
-                 d->B(13) = F(d->B(13), s->B(13));              \
-                 d->B(14) = F(d->B(14), s->B(14));              \
-                 d->B(15) = F(d->B(15), s->B(15));              \
-                                                        )       \
-            }
+        int n = num;                                            \
+        for (int i = 0; i < n; i++) {                           \
+            d->elem(i) = F(s->elem(i));                         \
+        }                                                       \
+    }
 
-#define SSE_HELPER_W(name, F)                                   \
-    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)   \
+#define SSE_HELPER_2(name, elem, num, F)                        \
+    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)   \
     {                                                           \
-        d->W(0) = F(d->W(0), s->W(0));                          \
-        d->W(1) = F(d->W(1), s->W(1));                          \
-        d->W(2) = F(d->W(2), s->W(2));                          \
-        d->W(3) = F(d->W(3), s->W(3));                          \
-        XMM_ONLY(                                               \
-                 d->W(4) = F(d->W(4), s->W(4));                 \
-                 d->W(5) = F(d->W(5), s->W(5));                 \
-                 d->W(6) = F(d->W(6), s->W(6));                 \
-                 d->W(7) = F(d->W(7), s->W(7));                 \
-                                                        )       \
-            }
+        int n = num;                                            \
+        for (int i = 0; i < n; i++) {                           \
+            d->elem(i) = F(v->elem(i), s->elem(i));             \
+        }                                                       \
+    }
+
+#define SSE_HELPER_B(name, F)                                   \
+    SSE_HELPER_2(name, B, 8 << SHIFT, F)
+
+#define SSE_HELPER_W(name, F)                                   \
+    SSE_HELPER_2(name, W, 4 << SHIFT, F)
 
 #define SSE_HELPER_L(name, F)                                   \
-    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)   \
-    {                                                           \
-        d->L(0) = F(d->L(0), s->L(0));                          \
-        d->L(1) = F(d->L(1), s->L(1));                          \
-        XMM_ONLY(                                               \
-                 d->L(2) = F(d->L(2), s->L(2));                 \
-                 d->L(3) = F(d->L(3), s->L(3));                 \
-                                                        )       \
-            }
+    SSE_HELPER_2(name, L, 2 << SHIFT, F)
 
 #define SSE_HELPER_Q(name, F)                                   \
-    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)   \
-    {                                                           \
-        d->Q(0) = F(d->Q(0), s->Q(0));                          \
-        XMM_ONLY(                                               \
-                 d->Q(1) = F(d->Q(1), s->Q(1));                 \
-                                                        )       \
-            }
+    SSE_HELPER_2(name, Q, 1 << SHIFT, F)
 
 #if SHIFT == 0
 static inline int satub(int x)
@@ -353,17 +297,6 @@ static inline int satsw(int x)
 #define FMAXUB(a, b) ((a) > (b)) ? (a) : (b)
 #define FMAXSW(a, b) ((int16_t)(a) > (int16_t)(b)) ? (a) : (b)
 
-#define FAND(a, b) ((a) & (b))
-#define FANDN(a, b) ((~(a)) & (b))
-#define FOR(a, b) ((a) | (b))
-#define FXOR(a, b) ((a) ^ (b))
-
-#define FCMPGTB(a, b) ((int8_t)(a) > (int8_t)(b) ? -1 : 0)
-#define FCMPGTW(a, b) ((int16_t)(a) > (int16_t)(b) ? -1 : 0)
-#define FCMPGTL(a, b) ((int32_t)(a) > (int32_t)(b) ? -1 : 0)
-#define FCMPEQ(a, b) ((a) == (b) ? -1 : 0)
-
-#define FMULLW(a, b) ((a) * (b))
 #define FMULHRW(a, b) (((int16_t)(a) * (int16_t)(b) + 0x8000) >> 16)
 #define FMULHUW(a, b) ((a) * (b) >> 16)
 #define FMULHW(a, b) ((int16_t)(a) * (int16_t)(b) >> 16)
@@ -371,70 +304,38 @@ static inline int satsw(int x)
 #define FAVG(a, b) (((a) + (b) + 1) >> 1)
 #endif
 
-SSE_HELPER_B(helper_paddb, FADD)
-SSE_HELPER_W(helper_paddw, FADD)
-SSE_HELPER_L(helper_paddl, FADD)
-SSE_HELPER_Q(helper_paddq, FADD)
-
-SSE_HELPER_B(helper_psubb, FSUB)
-SSE_HELPER_W(helper_psubw, FSUB)
-SSE_HELPER_L(helper_psubl, FSUB)
-SSE_HELPER_Q(helper_psubq, FSUB)
-
-SSE_HELPER_B(helper_paddusb, FADDUB)
-SSE_HELPER_B(helper_paddsb, FADDSB)
-SSE_HELPER_B(helper_psubusb, FSUBUB)
-SSE_HELPER_B(helper_psubsb, FSUBSB)
-
-SSE_HELPER_W(helper_paddusw, FADDUW)
-SSE_HELPER_W(helper_paddsw, FADDSW)
-SSE_HELPER_W(helper_psubusw, FSUBUW)
-SSE_HELPER_W(helper_psubsw, FSUBSW)
-
-SSE_HELPER_B(helper_pminub, FMINUB)
-SSE_HELPER_B(helper_pmaxub, FMAXUB)
-
-SSE_HELPER_W(helper_pminsw, FMINSW)
-SSE_HELPER_W(helper_pmaxsw, FMAXSW)
-
-SSE_HELPER_Q(helper_pand, FAND)
-SSE_HELPER_Q(helper_pandn, FANDN)
-SSE_HELPER_Q(helper_por, FOR)
-SSE_HELPER_Q(helper_pxor, FXOR)
-
-SSE_HELPER_B(helper_pcmpgtb, FCMPGTB)
-SSE_HELPER_W(helper_pcmpgtw, FCMPGTW)
-SSE_HELPER_L(helper_pcmpgtl, FCMPGTL)
-
-SSE_HELPER_B(helper_pcmpeqb, FCMPEQ)
-SSE_HELPER_W(helper_pcmpeqw, FCMPEQ)
-SSE_HELPER_L(helper_pcmpeql, FCMPEQ)
+SSE_HELPER_W(helper_pmulhuw, FMULHUW)
+SSE_HELPER_W(helper_pmulhw, FMULHW)
 
-SSE_HELPER_W(helper_pmullw, FMULLW)
 #if SHIFT == 0
-SSE_HELPER_W(helper_pmulhrw, FMULHRW)
+void glue(helper_pmulhrw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+{
+    d->W(0) = FMULHRW(d->W(0), s->W(0));
+    d->W(1) = FMULHRW(d->W(1), s->W(1));
+    d->W(2) = FMULHRW(d->W(2), s->W(2));
+    d->W(3) = FMULHRW(d->W(3), s->W(3));
+}
 #endif
-SSE_HELPER_W(helper_pmulhuw, FMULHUW)
-SSE_HELPER_W(helper_pmulhw, FMULHW)
 
 SSE_HELPER_B(helper_pavgb, FAVG)
 SSE_HELPER_W(helper_pavgw, FAVG)
 
-void glue(helper_pmuludq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_pmuludq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
-    d->Q(0) = (uint64_t)s->L(0) * (uint64_t)d->L(0);
-#if SHIFT == 1
-    d->Q(1) = (uint64_t)s->L(2) * (uint64_t)d->L(2);
-#endif
+    int i;
+
+    for (i = 0; i < (1 << SHIFT); i++) {
+        d->Q(i) = (uint64_t)s->L(i * 2) * (uint64_t)v->L(i * 2);
+    }
 }
 
-void glue(helper_pmaddwd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_pmaddwd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
     int i;
 
     for (i = 0; i < (2 << SHIFT); i++) {
-        d->L(i) = (int16_t)s->W(2 * i) * (int16_t)d->W(2 * i) +
-            (int16_t)s->W(2 * i + 1) * (int16_t)d->W(2 * i + 1);
+        d->L(i) = (int16_t)s->W(2 * i) * (int16_t)v->W(2 * i) +
+            (int16_t)s->W(2 * i + 1) * (int16_t)v->W(2 * i + 1);
     }
 }
 
@@ -448,34 +349,25 @@ static inline int abs1(int a)
     }
 }
 #endif
-void glue(helper_psadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    unsigned int val;
-
-    val = 0;
-    val += abs1(d->B(0) - s->B(0));
-    val += abs1(d->B(1) - s->B(1));
-    val += abs1(d->B(2) - s->B(2));
-    val += abs1(d->B(3) - s->B(3));
-    val += abs1(d->B(4) - s->B(4));
-    val += abs1(d->B(5) - s->B(5));
-    val += abs1(d->B(6) - s->B(6));
-    val += abs1(d->B(7) - s->B(7));
-    d->Q(0) = val;
-#if SHIFT == 1
-    val = 0;
-    val += abs1(d->B(8) - s->B(8));
-    val += abs1(d->B(9) - s->B(9));
-    val += abs1(d->B(10) - s->B(10));
-    val += abs1(d->B(11) - s->B(11));
-    val += abs1(d->B(12) - s->B(12));
-    val += abs1(d->B(13) - s->B(13));
-    val += abs1(d->B(14) - s->B(14));
-    val += abs1(d->B(15) - s->B(15));
-    d->Q(1) = val;
-#endif
+void glue(helper_psadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    int i;
+
+    for (i = 0; i < (1 << SHIFT); i++) {
+        unsigned int val = 0;
+        val += abs1(v->B(8 * i + 0) - s->B(8 * i + 0));
+        val += abs1(v->B(8 * i + 1) - s->B(8 * i + 1));
+        val += abs1(v->B(8 * i + 2) - s->B(8 * i + 2));
+        val += abs1(v->B(8 * i + 3) - s->B(8 * i + 3));
+        val += abs1(v->B(8 * i + 4) - s->B(8 * i + 4));
+        val += abs1(v->B(8 * i + 5) - s->B(8 * i + 5));
+        val += abs1(v->B(8 * i + 6) - s->B(8 * i + 6));
+        val += abs1(v->B(8 * i + 7) - s->B(8 * i + 7));
+        d->Q(i) = val;
+    }
 }
 
+#if SHIFT < 2
 void glue(helper_maskmov, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
                                   target_ulong a0)
 {
@@ -487,128 +379,140 @@ void glue(helper_maskmov, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
         }
     }
 }
-
-void glue(helper_movl_mm_T0, SUFFIX)(Reg *d, uint32_t val)
-{
-    d->L(0) = val;
-    d->L(1) = 0;
-#if SHIFT == 1
-    d->Q(1) = 0;
 #endif
-}
 
-#ifdef TARGET_X86_64
-void glue(helper_movq_mm_T0, SUFFIX)(Reg *d, uint64_t val)
-{
-    d->Q(0) = val;
-#if SHIFT == 1
-    d->Q(1) = 0;
-#endif
-}
-#endif
+#define SHUFFLE4(F, a, b, offset) do {      \
+    r0 = a->F((order & 3) + offset);        \
+    r1 = a->F(((order >> 2) & 3) + offset); \
+    r2 = b->F(((order >> 4) & 3) + offset); \
+    r3 = b->F(((order >> 6) & 3) + offset); \
+    d->F(offset) = r0;                      \
+    d->F(offset + 1) = r1;                  \
+    d->F(offset + 2) = r2;                  \
+    d->F(offset + 3) = r3;                  \
+    } while (0)
 
 #if SHIFT == 0
 void glue(helper_pshufw, SUFFIX)(Reg *d, Reg *s, int order)
 {
-    Reg r;
+    uint16_t r0, r1, r2, r3;
 
-    r.W(0) = s->W(order & 3);
-    r.W(1) = s->W((order >> 2) & 3);
-    r.W(2) = s->W((order >> 4) & 3);
-    r.W(3) = s->W((order >> 6) & 3);
-    *d = r;
+    SHUFFLE4(W, s, s, 0);
 }
 #else
-void helper_shufps(Reg *d, Reg *s, int order)
+void glue(helper_shufps, SUFFIX)(Reg *d, Reg *v, Reg *s, int order)
 {
-    Reg r;
+    uint32_t r0, r1, r2, r3;
+    int i;
 
-    r.L(0) = d->L(order & 3);
-    r.L(1) = d->L((order >> 2) & 3);
-    r.L(2) = s->L((order >> 4) & 3);
-    r.L(3) = s->L((order >> 6) & 3);
-    *d = r;
+    for (i = 0; i < 2 << SHIFT; i += 4) {
+        SHUFFLE4(L, v, s, i);
+    }
 }
 
-void helper_shufpd(Reg *d, Reg *s, int order)
+void glue(helper_shufpd, SUFFIX)(Reg *d, Reg *v, Reg *s, int order)
 {
-    Reg r;
+    uint64_t r0, r1;
+    int i;
 
-    r.Q(0) = d->Q(order & 1);
-    r.Q(1) = s->Q((order >> 1) & 1);
-    *d = r;
+    for (i = 0; i < 1 << SHIFT; i += 2) {
+        r0 = v->Q(((order & 1) & 1) + i);
+        r1 = s->Q(((order >> 1) & 1) + i);
+        d->Q(i) = r0;
+        d->Q(i + 1) = r1;
+        order >>= 2;
+    }
 }
 
 void glue(helper_pshufd, SUFFIX)(Reg *d, Reg *s, int order)
 {
-    Reg r;
+    uint32_t r0, r1, r2, r3;
+    int i;
 
-    r.L(0) = s->L(order & 3);
-    r.L(1) = s->L((order >> 2) & 3);
-    r.L(2) = s->L((order >> 4) & 3);
-    r.L(3) = s->L((order >> 6) & 3);
-    *d = r;
+    for (i = 0; i < 2 << SHIFT; i += 4) {
+        SHUFFLE4(L, s, s, i);
+    }
 }
 
 void glue(helper_pshuflw, SUFFIX)(Reg *d, Reg *s, int order)
 {
-    Reg r;
+    uint16_t r0, r1, r2, r3;
+    int i, j;
 
-    r.W(0) = s->W(order & 3);
-    r.W(1) = s->W((order >> 2) & 3);
-    r.W(2) = s->W((order >> 4) & 3);
-    r.W(3) = s->W((order >> 6) & 3);
-    r.Q(1) = s->Q(1);
-    *d = r;
+    for (i = 0, j = 1; j < 1 << SHIFT; i += 8, j += 2) {
+        SHUFFLE4(W, s, s, i);
+        d->Q(j) = s->Q(j);
+    }
 }
 
 void glue(helper_pshufhw, SUFFIX)(Reg *d, Reg *s, int order)
 {
-    Reg r;
+    uint16_t r0, r1, r2, r3;
+    int i, j;
 
-    r.Q(0) = s->Q(0);
-    r.W(4) = s->W(4 + (order & 3));
-    r.W(5) = s->W(4 + ((order >> 2) & 3));
-    r.W(6) = s->W(4 + ((order >> 4) & 3));
-    r.W(7) = s->W(4 + ((order >> 6) & 3));
-    *d = r;
+    for (i = 4, j = 0; j < 1 << SHIFT; i += 8, j += 2) {
+        d->Q(j) = s->Q(j);
+        SHUFFLE4(W, s, s, i);
+    }
 }
 #endif
 
-#if SHIFT == 1
+#if SHIFT >= 1
 /* FPU ops */
 /* XXX: not accurate */
 
-#define SSE_HELPER_S(name, F)                                           \
-    void helper_ ## name ## ps(CPUX86State *env, Reg *d, Reg *s)        \
+#define SSE_HELPER_P(name, F)                                           \
+    void glue(helper_ ## name ## ps, SUFFIX)(CPUX86State *env,          \
+            Reg *d, Reg *v, Reg *s)                                     \
     {                                                                   \
-        d->ZMM_S(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
-        d->ZMM_S(1) = F(32, d->ZMM_S(1), s->ZMM_S(1));                  \
-        d->ZMM_S(2) = F(32, d->ZMM_S(2), s->ZMM_S(2));                  \
-        d->ZMM_S(3) = F(32, d->ZMM_S(3), s->ZMM_S(3));                  \
+        int i;                                                          \
+        for (i = 0; i < 2 << SHIFT; i++) {                              \
+            d->ZMM_S(i) = F(32, v->ZMM_S(i), s->ZMM_S(i));              \
+        }                                                               \
     }                                                                   \
                                                                         \
-    void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *s)        \
+    void glue(helper_ ## name ## pd, SUFFIX)(CPUX86State *env,          \
+            Reg *d, Reg *v, Reg *s)                                     \
     {                                                                   \
-        d->ZMM_S(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
-    }                                                                   \
+        int i;                                                          \
+        for (i = 0; i < 1 << SHIFT; i++) {                              \
+            d->ZMM_D(i) = F(64, v->ZMM_D(i), s->ZMM_D(i));              \
+        }                                                               \
+    }
+
+#if SHIFT == 1
+
+#define SSE_HELPER_S(name, F)                                           \
+    SSE_HELPER_P(name, F)                                               \
                                                                         \
-    void helper_ ## name ## pd(CPUX86State *env, Reg *d, Reg *s)        \
+    void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *v, Reg *s)\
     {                                                                   \
-        d->ZMM_D(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
-        d->ZMM_D(1) = F(64, d->ZMM_D(1), s->ZMM_D(1));                  \
+        int i;                                                          \
+        d->ZMM_S(0) = F(32, v->ZMM_S(0), s->ZMM_S(0));                  \
+        for (i = 1; i < 2 << SHIFT; i++) {                              \
+            d->ZMM_L(i) = v->ZMM_L(i);                                  \
+        }                                                               \
     }                                                                   \
                                                                         \
-    void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *s)        \
+    void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *v, Reg *s)\
     {                                                                   \
-        d->ZMM_D(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
+        int i;                                                          \
+        d->ZMM_D(0) = F(64, v->ZMM_D(0), s->ZMM_D(0));                  \
+        for (i = 1; i < 1 << SHIFT; i++) {                              \
+            d->ZMM_Q(i) = v->ZMM_Q(i);                                  \
+        }                                                               \
     }
 
+#else
+
+#define SSE_HELPER_S(name, F) SSE_HELPER_P(name, F)
+
+#endif
+
 #define FPU_ADD(size, a, b) float ## size ## _add(a, b, &env->sse_status)
 #define FPU_SUB(size, a, b) float ## size ## _sub(a, b, &env->sse_status)
 #define FPU_MUL(size, a, b) float ## size ## _mul(a, b, &env->sse_status)
 #define FPU_DIV(size, a, b) float ## size ## _div(a, b, &env->sse_status)
-#define FPU_SQRT(size, a, b) float ## size ## _sqrt(b, &env->sse_status)
 
 /* Note that the choice of comparison op here is important to get the
  * special cases right: for min and max Intel specifies that (-0,0),
@@ -625,56 +529,131 @@ SSE_HELPER_S(mul, FPU_MUL)
 SSE_HELPER_S(div, FPU_DIV)
 SSE_HELPER_S(min, FPU_MIN)
 SSE_HELPER_S(max, FPU_MAX)
-SSE_HELPER_S(sqrt, FPU_SQRT)
 
+void glue(helper_sqrtps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+{
+    int i;
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_S(i) = float32_sqrt(s->ZMM_S(i), &env->sse_status);
+    }
+}
+
+void glue(helper_sqrtpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+{
+    int i;
+    for (i = 0; i < 1 << SHIFT; i++) {
+        d->ZMM_D(i) = float64_sqrt(s->ZMM_D(i), &env->sse_status);
+    }
+}
+
+#if SHIFT == 1
+void helper_sqrtss(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    int i;
+    d->ZMM_S(0) = float32_sqrt(s->ZMM_S(0), &env->sse_status);
+    for (i = 1; i < 2 << SHIFT; i++) {
+        d->ZMM_L(i) = v->ZMM_L(i);
+    }
+}
+
+void helper_sqrtsd(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    int i;
+    d->ZMM_D(0) = float64_sqrt(s->ZMM_D(0), &env->sse_status);
+    for (i = 1; i < 1 << SHIFT; i++) {
+        d->ZMM_Q(i) = v->ZMM_Q(i);
+    }
+}
+#endif
 
 /* float to float conversions */
-void helper_cvtps2pd(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_cvtps2pd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
 {
-    float32 s0, s1;
+    int i;
+    for (i = 1 << SHIFT; --i >= 0; ) {
+        d->ZMM_D(i) = float32_to_float64(s->ZMM_S(i), &env->sse_status);
+    }
+}
 
-    s0 = s->ZMM_S(0);
-    s1 = s->ZMM_S(1);
-    d->ZMM_D(0) = float32_to_float64(s0, &env->sse_status);
-    d->ZMM_D(1) = float32_to_float64(s1, &env->sse_status);
+void glue(helper_cvtpd2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+{
+    int i;
+    for (i = 0; i < 1 << SHIFT; i++) {
+         d->ZMM_S(i) = float64_to_float32(s->ZMM_D(i), &env->sse_status);
+    }
+    for (i >>= 1; i < 1 << SHIFT; i++) {
+         d->Q(i) = 0;
+    }
 }
 
-void helper_cvtpd2ps(CPUX86State *env, Reg *d, Reg *s)
+#if SHIFT >= 1
+void glue(helper_cvtph2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status);
-    d->ZMM_S(1) = float64_to_float32(s->ZMM_D(1), &env->sse_status);
-    d->Q(1) = 0;
+    int i;
+
+    for (i = 2 << SHIFT; --i >= 0; ) {
+         d->ZMM_S(i) = float16_to_float32(s->ZMM_H(i), true, &env->sse_status);
+    }
+}
+
+void glue(helper_cvtps2ph, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, int mode)
+{
+    int i;
+    FloatRoundMode prev_rounding_mode = env->sse_status.float_rounding_mode;
+    if (!(mode & (1 << 2))) {
+        set_x86_rounding_mode(mode & 3, &env->sse_status);
+    }
+
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_H(i) = float32_to_float16(s->ZMM_S(i), true, &env->sse_status);
+    }
+    for (i >>= 2; i < 1 << SHIFT; i++) {
+        d->Q(i) = 0;
+    }
+
+    env->sse_status.float_rounding_mode = prev_rounding_mode;
 }
+#endif
 
-void helper_cvtss2sd(CPUX86State *env, Reg *d, Reg *s)
+#if SHIFT == 1
+void helper_cvtss2sd(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
+    int i;
     d->ZMM_D(0) = float32_to_float64(s->ZMM_S(0), &env->sse_status);
+    for (i = 1; i < 1 << SHIFT; i++) {
+        d->ZMM_Q(i) = v->ZMM_Q(i);
+    }
 }
 
-void helper_cvtsd2ss(CPUX86State *env, Reg *d, Reg *s)
+void helper_cvtsd2ss(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
+    int i;
     d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status);
+    for (i = 1; i < 2 << SHIFT; i++) {
+        d->ZMM_L(i) = v->ZMM_L(i);
+    }
 }
+#endif
 
 /* integer to float */
-void helper_cvtdq2ps(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_cvtdq2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->ZMM_S(0) = int32_to_float32(s->ZMM_L(0), &env->sse_status);
-    d->ZMM_S(1) = int32_to_float32(s->ZMM_L(1), &env->sse_status);
-    d->ZMM_S(2) = int32_to_float32(s->ZMM_L(2), &env->sse_status);
-    d->ZMM_S(3) = int32_to_float32(s->ZMM_L(3), &env->sse_status);
+    int i;
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_S(i) = int32_to_float32(s->ZMM_L(i), &env->sse_status);
+    }
 }
 
-void helper_cvtdq2pd(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_cvtdq2pd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
 {
-    int32_t l0, l1;
-
-    l0 = (int32_t)s->ZMM_L(0);
-    l1 = (int32_t)s->ZMM_L(1);
-    d->ZMM_D(0) = int32_to_float64(l0, &env->sse_status);
-    d->ZMM_D(1) = int32_to_float64(l1, &env->sse_status);
+    int i;
+    for (i = 1 << SHIFT; --i >= 0; ) {
+        int32_t l = s->ZMM_L(i);
+        d->ZMM_D(i) = int32_to_float64(l, &env->sse_status);
+    }
 }
 
+#if SHIFT == 1
 void helper_cvtpi2ps(CPUX86State *env, ZMMReg *d, MMXReg *s)
 {
     d->ZMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
@@ -709,8 +688,11 @@ void helper_cvtsq2sd(CPUX86State *env, ZMMReg *d, uint64_t val)
 }
 #endif
 
+#endif
+
 /* float to integer */
 
+#if SHIFT == 1
 /*
  * x86 mandates that we return the indefinite integer value for the result
  * of any float-to-integer conversion that raises the 'invalid' exception.
@@ -741,22 +723,28 @@ WRAP_FLOATCONV(int64_t, float32_to_int64, float32, INT64_MIN)
 WRAP_FLOATCONV(int64_t, float32_to_int64_round_to_zero, float32, INT64_MIN)
 WRAP_FLOATCONV(int64_t, float64_to_int64, float64, INT64_MIN)
 WRAP_FLOATCONV(int64_t, float64_to_int64_round_to_zero, float64, INT64_MIN)
+#endif
 
-void helper_cvtps2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void glue(helper_cvtps2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_L(0) = x86_float32_to_int32(s->ZMM_S(0), &env->sse_status);
-    d->ZMM_L(1) = x86_float32_to_int32(s->ZMM_S(1), &env->sse_status);
-    d->ZMM_L(2) = x86_float32_to_int32(s->ZMM_S(2), &env->sse_status);
-    d->ZMM_L(3) = x86_float32_to_int32(s->ZMM_S(3), &env->sse_status);
+    int i;
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_L(i) = x86_float32_to_int32(s->ZMM_S(i), &env->sse_status);
+    }
 }
 
-void helper_cvtpd2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void glue(helper_cvtpd2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_L(0) = x86_float64_to_int32(s->ZMM_D(0), &env->sse_status);
-    d->ZMM_L(1) = x86_float64_to_int32(s->ZMM_D(1), &env->sse_status);
-    d->ZMM_Q(1) = 0;
+    int i;
+    for (i = 0; i < 1 << SHIFT; i++) {
+        d->ZMM_L(i) = x86_float64_to_int32(s->ZMM_D(i), &env->sse_status);
+    }
+    for (i >>= 1; i < 1 << SHIFT; i++) {
+         d->Q(i) = 0;
+    }
 }
 
+#if SHIFT == 1
 void helper_cvtps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
     d->MMX_L(0) = x86_float32_to_int32(s->ZMM_S(0), &env->sse_status);
@@ -790,23 +778,31 @@ int64_t helper_cvtsd2sq(CPUX86State *env, ZMMReg *s)
     return x86_float64_to_int64(s->ZMM_D(0), &env->sse_status);
 }
 #endif
+#endif
 
 /* float to integer truncated */
-void helper_cvttps2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void glue(helper_cvttps2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_L(0) = x86_float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
-    d->ZMM_L(1) = x86_float32_to_int32_round_to_zero(s->ZMM_S(1), &env->sse_status);
-    d->ZMM_L(2) = x86_float32_to_int32_round_to_zero(s->ZMM_S(2), &env->sse_status);
-    d->ZMM_L(3) = x86_float32_to_int32_round_to_zero(s->ZMM_S(3), &env->sse_status);
+    int i;
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_L(i) = x86_float32_to_int32_round_to_zero(s->ZMM_S(i),
+                                                         &env->sse_status);
+    }
 }
 
-void helper_cvttpd2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void glue(helper_cvttpd2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_L(0) = x86_float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status);
-    d->ZMM_L(1) = x86_float64_to_int32_round_to_zero(s->ZMM_D(1), &env->sse_status);
-    d->ZMM_Q(1) = 0;
+    int i;
+    for (i = 0; i < 1 << SHIFT; i++) {
+        d->ZMM_L(i) = x86_float64_to_int32_round_to_zero(s->ZMM_D(i),
+                                                         &env->sse_status);
+    }
+    for (i >>= 1; i < 1 << SHIFT; i++) {
+         d->Q(i) = 0;
+    }
 }
 
+#if SHIFT == 1
 void helper_cvttps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
     d->MMX_L(0) = x86_float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
@@ -840,43 +836,59 @@ int64_t helper_cvttsd2sq(CPUX86State *env, ZMMReg *s)
     return x86_float64_to_int64_round_to_zero(s->ZMM_D(0), &env->sse_status);
 }
 #endif
+#endif
 
-void helper_rsqrtps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void glue(helper_rsqrtps, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_S(0) = float32_div(float32_one,
-                              float32_sqrt(s->ZMM_S(0), &env->sse_status),
-                              &env->sse_status);
-    d->ZMM_S(1) = float32_div(float32_one,
-                              float32_sqrt(s->ZMM_S(1), &env->sse_status),
-                              &env->sse_status);
-    d->ZMM_S(2) = float32_div(float32_one,
-                              float32_sqrt(s->ZMM_S(2), &env->sse_status),
-                              &env->sse_status);
-    d->ZMM_S(3) = float32_div(float32_one,
-                              float32_sqrt(s->ZMM_S(3), &env->sse_status),
-                              &env->sse_status);
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
+    int i;
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_S(i) = float32_div(float32_one,
+                                  float32_sqrt(s->ZMM_S(i), &env->sse_status),
+                                  &env->sse_status);
+    }
+    set_float_exception_flags(old_flags, &env->sse_status);
 }
 
-void helper_rsqrtss(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+#if SHIFT == 1
+void helper_rsqrtss(CPUX86State *env, ZMMReg *d, ZMMReg *v, ZMMReg *s)
 {
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
+    int i;
     d->ZMM_S(0) = float32_div(float32_one,
                               float32_sqrt(s->ZMM_S(0), &env->sse_status),
                               &env->sse_status);
+    set_float_exception_flags(old_flags, &env->sse_status);
+    for (i = 1; i < 2 << SHIFT; i++) {
+        d->ZMM_L(i) = v->ZMM_L(i);
+    }
 }
+#endif
 
-void helper_rcpps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void glue(helper_rcpps, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status);
-    d->ZMM_S(1) = float32_div(float32_one, s->ZMM_S(1), &env->sse_status);
-    d->ZMM_S(2) = float32_div(float32_one, s->ZMM_S(2), &env->sse_status);
-    d->ZMM_S(3) = float32_div(float32_one, s->ZMM_S(3), &env->sse_status);
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
+    int i;
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_S(i) = float32_div(float32_one, s->ZMM_S(i), &env->sse_status);
+    }
+    set_float_exception_flags(old_flags, &env->sse_status);
 }
 
-void helper_rcpss(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+#if SHIFT == 1
+void helper_rcpss(CPUX86State *env, ZMMReg *d, ZMMReg *v, ZMMReg *s)
 {
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
+    int i;
     d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status);
+    for (i = 1; i < 2 << SHIFT; i++) {
+        d->ZMM_L(i) = v->ZMM_L(i);
+    }
+    set_float_exception_flags(old_flags, &env->sse_status);
 }
+#endif
 
+#if SHIFT == 1
 static inline uint64_t helper_extrq(uint64_t src, int shift, int len)
 {
     uint64_t mask;
@@ -891,7 +903,7 @@ static inline uint64_t helper_extrq(uint64_t src, int shift, int len)
 
 void helper_extrq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), s->ZMM_B(1), s->ZMM_B(0));
+    d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), s->ZMM_B(1) & 63, s->ZMM_B(0) & 63);
 }
 
 void helper_extrq_i(CPUX86State *env, ZMMReg *d, int index, int length)
@@ -899,7 +911,7 @@ void helper_extrq_i(CPUX86State *env, ZMMReg *d, int index, int length)
     d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), index, length);
 }
 
-static inline uint64_t helper_insertq(uint64_t src, int shift, int len)
+static inline uint64_t helper_insertq(uint64_t dest, uint64_t src, int shift, int len)
 {
     uint64_t mask;
 
@@ -908,130 +920,189 @@ static inline uint64_t helper_insertq(uint64_t src, int shift, int len)
     } else {
         mask = (1ULL << len) - 1;
     }
-    return (src & ~(mask << shift)) | ((src & mask) << shift);
+    return (dest & ~(mask << shift)) | ((src & mask) << shift);
 }
 
 void helper_insertq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->ZMM_Q(0) = helper_insertq(s->ZMM_Q(0), s->ZMM_B(9), s->ZMM_B(8));
-}
-
-void helper_insertq_i(CPUX86State *env, ZMMReg *d, int index, int length)
-{
-    d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), index, length);
-}
-
-void helper_haddps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
-{
-    ZMMReg r;
-
-    r.ZMM_S(0) = float32_add(d->ZMM_S(0), d->ZMM_S(1), &env->sse_status);
-    r.ZMM_S(1) = float32_add(d->ZMM_S(2), d->ZMM_S(3), &env->sse_status);
-    r.ZMM_S(2) = float32_add(s->ZMM_S(0), s->ZMM_S(1), &env->sse_status);
-    r.ZMM_S(3) = float32_add(s->ZMM_S(2), s->ZMM_S(3), &env->sse_status);
-    *d = r;
-}
-
-void helper_haddpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
-{
-    ZMMReg r;
-
-    r.ZMM_D(0) = float64_add(d->ZMM_D(0), d->ZMM_D(1), &env->sse_status);
-    r.ZMM_D(1) = float64_add(s->ZMM_D(0), s->ZMM_D(1), &env->sse_status);
-    *d = r;
-}
-
-void helper_hsubps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
-{
-    ZMMReg r;
-
-    r.ZMM_S(0) = float32_sub(d->ZMM_S(0), d->ZMM_S(1), &env->sse_status);
-    r.ZMM_S(1) = float32_sub(d->ZMM_S(2), d->ZMM_S(3), &env->sse_status);
-    r.ZMM_S(2) = float32_sub(s->ZMM_S(0), s->ZMM_S(1), &env->sse_status);
-    r.ZMM_S(3) = float32_sub(s->ZMM_S(2), s->ZMM_S(3), &env->sse_status);
-    *d = r;
+    d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), s->ZMM_Q(0), s->ZMM_B(9) & 63, s->ZMM_B(8) & 63);
 }
 
-void helper_hsubpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void helper_insertq_i(CPUX86State *env, ZMMReg *d, ZMMReg *s, int index, int length)
 {
-    ZMMReg r;
-
-    r.ZMM_D(0) = float64_sub(d->ZMM_D(0), d->ZMM_D(1), &env->sse_status);
-    r.ZMM_D(1) = float64_sub(s->ZMM_D(0), s->ZMM_D(1), &env->sse_status);
-    *d = r;
+    d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), s->ZMM_Q(0), index, length);
 }
+#endif
 
-void helper_addsubps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+#define SSE_HELPER_HPS(name, F)  \
+void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \
+{                                                                 \
+    float32 r[2 << SHIFT];                                        \
+    int i, j, k;                                                  \
+    for (k = 0; k < 2 << SHIFT; k += LANE_WIDTH / 4) {            \
+        for (i = j = 0; j < 4; i++, j += 2) {                     \
+            r[i + k] = F(v->ZMM_S(j + k), v->ZMM_S(j + k + 1), &env->sse_status); \
+        }                                                         \
+        for (j = 0; j < 4; i++, j += 2) {                         \
+            r[i + k] = F(s->ZMM_S(j + k), s->ZMM_S(j + k + 1), &env->sse_status); \
+        }                                                         \
+    }                                                             \
+    for (i = 0; i < 2 << SHIFT; i++) {                            \
+        d->ZMM_S(i) = r[i];                                       \
+    }                                                             \
+}
+
+SSE_HELPER_HPS(haddps, float32_add)
+SSE_HELPER_HPS(hsubps, float32_sub)
+
+#define SSE_HELPER_HPD(name, F)  \
+void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \
+{                                                                 \
+    float64 r[1 << SHIFT];                                        \
+    int i, j, k;                                                  \
+    for (k = 0; k < 1 << SHIFT; k += LANE_WIDTH / 8) {            \
+        for (i = j = 0; j < 2; i++, j += 2) {                     \
+            r[i + k] = F(v->ZMM_D(j + k), v->ZMM_D(j + k + 1), &env->sse_status); \
+        }                                                         \
+        for (j = 0; j < 2; i++, j += 2) {                         \
+            r[i + k] = F(s->ZMM_D(j + k), s->ZMM_D(j + k + 1), &env->sse_status); \
+        }                                                         \
+    }                                                             \
+    for (i = 0; i < 1 << SHIFT; i++) {                            \
+        d->ZMM_D(i) = r[i];                                       \
+    }                                                             \
+}
+
+SSE_HELPER_HPD(haddpd, float64_add)
+SSE_HELPER_HPD(hsubpd, float64_sub)
+
+void glue(helper_addsubps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
-    d->ZMM_S(0) = float32_sub(d->ZMM_S(0), s->ZMM_S(0), &env->sse_status);
-    d->ZMM_S(1) = float32_add(d->ZMM_S(1), s->ZMM_S(1), &env->sse_status);
-    d->ZMM_S(2) = float32_sub(d->ZMM_S(2), s->ZMM_S(2), &env->sse_status);
-    d->ZMM_S(3) = float32_add(d->ZMM_S(3), s->ZMM_S(3), &env->sse_status);
+    int i;
+    for (i = 0; i < 2 << SHIFT; i += 2) {
+        d->ZMM_S(i) = float32_sub(v->ZMM_S(i), s->ZMM_S(i), &env->sse_status);
+        d->ZMM_S(i+1) = float32_add(v->ZMM_S(i+1), s->ZMM_S(i+1), &env->sse_status);
+    }
 }
 
-void helper_addsubpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
+void glue(helper_addsubpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
-    d->ZMM_D(0) = float64_sub(d->ZMM_D(0), s->ZMM_D(0), &env->sse_status);
-    d->ZMM_D(1) = float64_add(d->ZMM_D(1), s->ZMM_D(1), &env->sse_status);
+    int i;
+    for (i = 0; i < 1 << SHIFT; i += 2) {
+        d->ZMM_D(i) = float64_sub(v->ZMM_D(i), s->ZMM_D(i), &env->sse_status);
+        d->ZMM_D(i+1) = float64_add(v->ZMM_D(i+1), s->ZMM_D(i+1), &env->sse_status);
+    }
 }
 
-/* XXX: unordered */
-#define SSE_HELPER_CMP(name, F)                                         \
-    void helper_ ## name ## ps(CPUX86State *env, Reg *d, Reg *s)        \
+#define SSE_HELPER_CMP_P(name, F, C)                                    \
+    void glue(helper_ ## name ## ps, SUFFIX)(CPUX86State *env,          \
+                                             Reg *d, Reg *v, Reg *s)    \
     {                                                                   \
-        d->ZMM_L(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
-        d->ZMM_L(1) = F(32, d->ZMM_S(1), s->ZMM_S(1));                  \
-        d->ZMM_L(2) = F(32, d->ZMM_S(2), s->ZMM_S(2));                  \
-        d->ZMM_L(3) = F(32, d->ZMM_S(3), s->ZMM_S(3));                  \
-    }                                                                   \
-                                                                        \
-    void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *s)        \
-    {                                                                   \
-        d->ZMM_L(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
-    }                                                                   \
-                                                                        \
-    void helper_ ## name ## pd(CPUX86State *env, Reg *d, Reg *s)        \
-    {                                                                   \
-        d->ZMM_Q(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
-        d->ZMM_Q(1) = F(64, d->ZMM_D(1), s->ZMM_D(1));                  \
+        int i;                                                          \
+        for (i = 0; i < 2 << SHIFT; i++) {                              \
+            d->ZMM_L(i) = C(F(32, v->ZMM_S(i), s->ZMM_S(i))) ? -1 : 0;  \
+        }                                                               \
     }                                                                   \
                                                                         \
-    void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *s)        \
+    void glue(helper_ ## name ## pd, SUFFIX)(CPUX86State *env,          \
+                                             Reg *d, Reg *v, Reg *s)    \
     {                                                                   \
-        d->ZMM_Q(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
-    }
-
-#define FPU_CMPEQ(size, a, b)                                           \
-    (float ## size ## _eq_quiet(a, b, &env->sse_status) ? -1 : 0)
-#define FPU_CMPLT(size, a, b)                                           \
-    (float ## size ## _lt(a, b, &env->sse_status) ? -1 : 0)
-#define FPU_CMPLE(size, a, b)                                           \
-    (float ## size ## _le(a, b, &env->sse_status) ? -1 : 0)
-#define FPU_CMPUNORD(size, a, b)                                        \
-    (float ## size ## _unordered_quiet(a, b, &env->sse_status) ? -1 : 0)
-#define FPU_CMPNEQ(size, a, b)                                          \
-    (float ## size ## _eq_quiet(a, b, &env->sse_status) ? 0 : -1)
-#define FPU_CMPNLT(size, a, b)                                          \
-    (float ## size ## _lt(a, b, &env->sse_status) ? 0 : -1)
-#define FPU_CMPNLE(size, a, b)                                          \
-    (float ## size ## _le(a, b, &env->sse_status) ? 0 : -1)
-#define FPU_CMPORD(size, a, b)                                          \
-    (float ## size ## _unordered_quiet(a, b, &env->sse_status) ? 0 : -1)
-
-SSE_HELPER_CMP(cmpeq, FPU_CMPEQ)
-SSE_HELPER_CMP(cmplt, FPU_CMPLT)
-SSE_HELPER_CMP(cmple, FPU_CMPLE)
-SSE_HELPER_CMP(cmpunord, FPU_CMPUNORD)
-SSE_HELPER_CMP(cmpneq, FPU_CMPNEQ)
-SSE_HELPER_CMP(cmpnlt, FPU_CMPNLT)
-SSE_HELPER_CMP(cmpnle, FPU_CMPNLE)
-SSE_HELPER_CMP(cmpord, FPU_CMPORD)
+        int i;                                                          \
+        for (i = 0; i < 1 << SHIFT; i++) {                              \
+            d->ZMM_Q(i) = C(F(64, v->ZMM_D(i), s->ZMM_D(i))) ? -1 : 0;  \
+        }                                                               \
+    }
 
+#if SHIFT == 1
+#define SSE_HELPER_CMP(name, F, C)                                          \
+    SSE_HELPER_CMP_P(name, F, C)                                            \
+    void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *v, Reg *s)    \
+    {                                                                       \
+        int i;                                                              \
+        d->ZMM_L(0) = C(F(32, v->ZMM_S(0), s->ZMM_S(0))) ? -1 : 0;          \
+        for (i = 1; i < 2 << SHIFT; i++) {                                  \
+            d->ZMM_L(i) = v->ZMM_L(i);                                      \
+        }                                                                   \
+    }                                                                       \
+                                                                            \
+    void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *v, Reg *s)    \
+    {                                                                       \
+        int i;                                                              \
+        d->ZMM_Q(0) = C(F(64, v->ZMM_D(0), s->ZMM_D(0))) ? -1 : 0;          \
+        for (i = 1; i < 1 << SHIFT; i++) {                                  \
+            d->ZMM_Q(i) = v->ZMM_Q(i);                                      \
+        }                                                                   \
+    }
+
+static inline bool FPU_EQU(FloatRelation x)
+{
+    return (x == float_relation_equal || x == float_relation_unordered);
+}
+static inline bool FPU_GE(FloatRelation x)
+{
+    return (x == float_relation_equal || x == float_relation_greater);
+}
+#define FPU_EQ(x) (x == float_relation_equal)
+#define FPU_LT(x) (x == float_relation_less)
+#define FPU_LE(x) (x <= float_relation_equal)
+#define FPU_GT(x) (x == float_relation_greater)
+#define FPU_UNORD(x) (x == float_relation_unordered)
+/* We must make sure we evaluate the argument in case it is a signalling NAN */
+#define FPU_FALSE(x) (x == float_relation_equal && 0)
+
+#define FPU_CMPQ(size, a, b) \
+    float ## size ## _compare_quiet(a, b, &env->sse_status)
+#define FPU_CMPS(size, a, b) \
+    float ## size ## _compare(a, b, &env->sse_status)
+
+#else
+#define SSE_HELPER_CMP(name, F, C) SSE_HELPER_CMP_P(name, F, C)
+#endif
+
+SSE_HELPER_CMP(cmpeq, FPU_CMPQ, FPU_EQ)
+SSE_HELPER_CMP(cmplt, FPU_CMPS, FPU_LT)
+SSE_HELPER_CMP(cmple, FPU_CMPS, FPU_LE)
+SSE_HELPER_CMP(cmpunord, FPU_CMPQ,  FPU_UNORD)
+SSE_HELPER_CMP(cmpneq, FPU_CMPQ, !FPU_EQ)
+SSE_HELPER_CMP(cmpnlt, FPU_CMPS, !FPU_LT)
+SSE_HELPER_CMP(cmpnle, FPU_CMPS, !FPU_LE)
+SSE_HELPER_CMP(cmpord, FPU_CMPQ, !FPU_UNORD)
+
+SSE_HELPER_CMP(cmpequ, FPU_CMPQ, FPU_EQU)
+SSE_HELPER_CMP(cmpnge, FPU_CMPS, !FPU_GE)
+SSE_HELPER_CMP(cmpngt, FPU_CMPS, !FPU_GT)
+SSE_HELPER_CMP(cmpfalse, FPU_CMPQ,  FPU_FALSE)
+SSE_HELPER_CMP(cmpnequ, FPU_CMPQ, !FPU_EQU)
+SSE_HELPER_CMP(cmpge, FPU_CMPS, FPU_GE)
+SSE_HELPER_CMP(cmpgt, FPU_CMPS, FPU_GT)
+SSE_HELPER_CMP(cmptrue, FPU_CMPQ,  !FPU_FALSE)
+
+SSE_HELPER_CMP(cmpeqs, FPU_CMPS, FPU_EQ)
+SSE_HELPER_CMP(cmpltq, FPU_CMPQ, FPU_LT)
+SSE_HELPER_CMP(cmpleq, FPU_CMPQ, FPU_LE)
+SSE_HELPER_CMP(cmpunords, FPU_CMPS,  FPU_UNORD)
+SSE_HELPER_CMP(cmpneqq, FPU_CMPS, !FPU_EQ)
+SSE_HELPER_CMP(cmpnltq, FPU_CMPQ, !FPU_LT)
+SSE_HELPER_CMP(cmpnleq, FPU_CMPQ, !FPU_LE)
+SSE_HELPER_CMP(cmpords, FPU_CMPS, !FPU_UNORD)
+
+SSE_HELPER_CMP(cmpequs, FPU_CMPS, FPU_EQU)
+SSE_HELPER_CMP(cmpngeq, FPU_CMPQ, !FPU_GE)
+SSE_HELPER_CMP(cmpngtq, FPU_CMPQ, !FPU_GT)
+SSE_HELPER_CMP(cmpfalses, FPU_CMPS,  FPU_FALSE)
+SSE_HELPER_CMP(cmpnequs, FPU_CMPS, !FPU_EQU)
+SSE_HELPER_CMP(cmpgeq, FPU_CMPQ, FPU_GE)
+SSE_HELPER_CMP(cmpgtq, FPU_CMPQ, FPU_GT)
+SSE_HELPER_CMP(cmptrues, FPU_CMPS,  !FPU_FALSE)
+
+#undef SSE_HELPER_CMP
+
+#if SHIFT == 1
 static const int comis_eflags[4] = {CC_C, CC_Z, 0, CC_Z | CC_P | CC_C};
 
 void helper_ucomiss(CPUX86State *env, Reg *d, Reg *s)
 {
-    int ret;
+    FloatRelation ret;
     float32 s0, s1;
 
     s0 = d->ZMM_S(0);
@@ -1042,7 +1113,7 @@ void helper_ucomiss(CPUX86State *env, Reg *d, Reg *s)
 
 void helper_comiss(CPUX86State *env, Reg *d, Reg *s)
 {
-    int ret;
+    FloatRelation ret;
     float32 s0, s1;
 
     s0 = d->ZMM_S(0);
@@ -1053,7 +1124,7 @@ void helper_comiss(CPUX86State *env, Reg *d, Reg *s)
 
 void helper_ucomisd(CPUX86State *env, Reg *d, Reg *s)
 {
-    int ret;
+    FloatRelation ret;
     float64 d0, d1;
 
     d0 = d->ZMM_D(0);
@@ -1064,7 +1135,7 @@ void helper_ucomisd(CPUX86State *env, Reg *d, Reg *s)
 
 void helper_comisd(CPUX86State *env, Reg *d, Reg *s)
 {
-    int ret;
+    FloatRelation ret;
     float64 d0, d1;
 
     d0 = d->ZMM_D(0);
@@ -1072,205 +1143,154 @@ void helper_comisd(CPUX86State *env, Reg *d, Reg *s)
     ret = float64_compare(d0, d1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
-
-uint32_t helper_movmskps(CPUX86State *env, Reg *s)
-{
-    int b0, b1, b2, b3;
-
-    b0 = s->ZMM_L(0) >> 31;
-    b1 = s->ZMM_L(1) >> 31;
-    b2 = s->ZMM_L(2) >> 31;
-    b3 = s->ZMM_L(3) >> 31;
-    return b0 | (b1 << 1) | (b2 << 2) | (b3 << 3);
-}
-
-uint32_t helper_movmskpd(CPUX86State *env, Reg *s)
-{
-    int b0, b1;
-
-    b0 = s->ZMM_L(1) >> 31;
-    b1 = s->ZMM_L(3) >> 31;
-    return b0 | (b1 << 1);
-}
-
 #endif
 
-uint32_t glue(helper_pmovmskb, SUFFIX)(CPUX86State *env, Reg *s)
+uint32_t glue(helper_movmskps, SUFFIX)(CPUX86State *env, Reg *s)
 {
-    uint32_t val;
+    uint32_t mask;
+    int i;
 
-    val = 0;
-    val |= (s->B(0) >> 7);
-    val |= (s->B(1) >> 6) & 0x02;
-    val |= (s->B(2) >> 5) & 0x04;
-    val |= (s->B(3) >> 4) & 0x08;
-    val |= (s->B(4) >> 3) & 0x10;
-    val |= (s->B(5) >> 2) & 0x20;
-    val |= (s->B(6) >> 1) & 0x40;
-    val |= (s->B(7)) & 0x80;
-#if SHIFT == 1
-    val |= (s->B(8) << 1) & 0x0100;
-    val |= (s->B(9) << 2) & 0x0200;
-    val |= (s->B(10) << 3) & 0x0400;
-    val |= (s->B(11) << 4) & 0x0800;
-    val |= (s->B(12) << 5) & 0x1000;
-    val |= (s->B(13) << 6) & 0x2000;
-    val |= (s->B(14) << 7) & 0x4000;
-    val |= (s->B(15) << 8) & 0x8000;
-#endif
-    return val;
+    mask = 0;
+    for (i = 0; i < 2 << SHIFT; i++) {
+        mask |= (s->ZMM_L(i) >> (31 - i)) & (1 << i);
+    }
+    return mask;
 }
 
-void glue(helper_packsswb, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+uint32_t glue(helper_movmskpd, SUFFIX)(CPUX86State *env, Reg *s)
 {
-    Reg r;
+    uint32_t mask;
+    int i;
 
-    r.B(0) = satsb((int16_t)d->W(0));
-    r.B(1) = satsb((int16_t)d->W(1));
-    r.B(2) = satsb((int16_t)d->W(2));
-    r.B(3) = satsb((int16_t)d->W(3));
-#if SHIFT == 1
-    r.B(4) = satsb((int16_t)d->W(4));
-    r.B(5) = satsb((int16_t)d->W(5));
-    r.B(6) = satsb((int16_t)d->W(6));
-    r.B(7) = satsb((int16_t)d->W(7));
-#endif
-    r.B((4 << SHIFT) + 0) = satsb((int16_t)s->W(0));
-    r.B((4 << SHIFT) + 1) = satsb((int16_t)s->W(1));
-    r.B((4 << SHIFT) + 2) = satsb((int16_t)s->W(2));
-    r.B((4 << SHIFT) + 3) = satsb((int16_t)s->W(3));
-#if SHIFT == 1
-    r.B(12) = satsb((int16_t)s->W(4));
-    r.B(13) = satsb((int16_t)s->W(5));
-    r.B(14) = satsb((int16_t)s->W(6));
-    r.B(15) = satsb((int16_t)s->W(7));
-#endif
-    *d = r;
+    mask = 0;
+    for (i = 0; i < 1 << SHIFT; i++) {
+        mask |= (s->ZMM_Q(i) >> (63 - i)) & (1 << i);
+    }
+    return mask;
 }
 
-void glue(helper_packuswb, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    Reg r;
-
-    r.B(0) = satub((int16_t)d->W(0));
-    r.B(1) = satub((int16_t)d->W(1));
-    r.B(2) = satub((int16_t)d->W(2));
-    r.B(3) = satub((int16_t)d->W(3));
-#if SHIFT == 1
-    r.B(4) = satub((int16_t)d->W(4));
-    r.B(5) = satub((int16_t)d->W(5));
-    r.B(6) = satub((int16_t)d->W(6));
-    r.B(7) = satub((int16_t)d->W(7));
-#endif
-    r.B((4 << SHIFT) + 0) = satub((int16_t)s->W(0));
-    r.B((4 << SHIFT) + 1) = satub((int16_t)s->W(1));
-    r.B((4 << SHIFT) + 2) = satub((int16_t)s->W(2));
-    r.B((4 << SHIFT) + 3) = satub((int16_t)s->W(3));
-#if SHIFT == 1
-    r.B(12) = satub((int16_t)s->W(4));
-    r.B(13) = satub((int16_t)s->W(5));
-    r.B(14) = satub((int16_t)s->W(6));
-    r.B(15) = satub((int16_t)s->W(7));
 #endif
-    *d = r;
-}
-
-void glue(helper_packssdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    Reg r;
 
-    r.W(0) = satsw(d->L(0));
-    r.W(1) = satsw(d->L(1));
-#if SHIFT == 1
-    r.W(2) = satsw(d->L(2));
-    r.W(3) = satsw(d->L(3));
-#endif
-    r.W((2 << SHIFT) + 0) = satsw(s->L(0));
-    r.W((2 << SHIFT) + 1) = satsw(s->L(1));
-#if SHIFT == 1
-    r.W(6) = satsw(s->L(2));
-    r.W(7) = satsw(s->L(3));
-#endif
-    *d = r;
+#define PACK_HELPER_B(name, F) \
+void glue(helper_pack ## name, SUFFIX)(CPUX86State *env,      \
+        Reg *d, Reg *v, Reg *s)                               \
+{                                                             \
+    uint8_t r[PACK_WIDTH * 2];                                \
+    int j, k;                                                 \
+    for (j = 0; j < 4 << SHIFT; j += PACK_WIDTH) {            \
+        for (k = 0; k < PACK_WIDTH; k++) {                    \
+            r[k] = F((int16_t)v->W(j + k));                   \
+        }                                                     \
+        for (k = 0; k < PACK_WIDTH; k++) {                    \
+            r[PACK_WIDTH + k] = F((int16_t)s->W(j + k));      \
+        }                                                     \
+        for (k = 0; k < PACK_WIDTH * 2; k++) {                \
+            d->B(2 * j + k) = r[k];                           \
+        }                                                     \
+    }                                                         \
+}
+
+PACK_HELPER_B(sswb, satsb)
+PACK_HELPER_B(uswb, satub)
+
+void glue(helper_packssdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    uint16_t r[PACK_WIDTH];
+    int j, k;
+
+    for (j = 0; j < 2 << SHIFT; j += PACK_WIDTH / 2) {
+        for (k = 0; k < PACK_WIDTH / 2; k++) {
+            r[k] = satsw(v->L(j + k));
+        }
+        for (k = 0; k < PACK_WIDTH / 2; k++) {
+            r[PACK_WIDTH / 2 + k] = satsw(s->L(j + k));
+        }
+        for (k = 0; k < PACK_WIDTH; k++) {
+            d->W(2 * j + k) = r[k];
+        }
+    }
 }
 
 #define UNPCK_OP(base_name, base)                                       \
                                                                         \
     void glue(helper_punpck ## base_name ## bw, SUFFIX)(CPUX86State *env,\
-                                                        Reg *d, Reg *s) \
+                                                Reg *d, Reg *v, Reg *s) \
     {                                                                   \
-        Reg r;                                                          \
+        uint8_t r[PACK_WIDTH * 2];                                      \
+        int j, i;                                                       \
                                                                         \
-        r.B(0) = d->B((base << (SHIFT + 2)) + 0);                       \
-        r.B(1) = s->B((base << (SHIFT + 2)) + 0);                       \
-        r.B(2) = d->B((base << (SHIFT + 2)) + 1);                       \
-        r.B(3) = s->B((base << (SHIFT + 2)) + 1);                       \
-        r.B(4) = d->B((base << (SHIFT + 2)) + 2);                       \
-        r.B(5) = s->B((base << (SHIFT + 2)) + 2);                       \
-        r.B(6) = d->B((base << (SHIFT + 2)) + 3);                       \
-        r.B(7) = s->B((base << (SHIFT + 2)) + 3);                       \
-        XMM_ONLY(                                                       \
-                 r.B(8) = d->B((base << (SHIFT + 2)) + 4);              \
-                 r.B(9) = s->B((base << (SHIFT + 2)) + 4);              \
-                 r.B(10) = d->B((base << (SHIFT + 2)) + 5);             \
-                 r.B(11) = s->B((base << (SHIFT + 2)) + 5);             \
-                 r.B(12) = d->B((base << (SHIFT + 2)) + 6);             \
-                 r.B(13) = s->B((base << (SHIFT + 2)) + 6);             \
-                 r.B(14) = d->B((base << (SHIFT + 2)) + 7);             \
-                 r.B(15) = s->B((base << (SHIFT + 2)) + 7);             \
-                                                                      ) \
-            *d = r;                                                     \
+        for (j = 0; j < 8 << SHIFT; ) {                                 \
+            int k = j + base * PACK_WIDTH;                              \
+            for (i = 0; i < PACK_WIDTH; i++) {                          \
+                r[2 * i] = v->B(k + i);                                 \
+                r[2 * i + 1] = s->B(k + i);                             \
+            }                                                           \
+            for (i = 0; i < PACK_WIDTH * 2; i++, j++) {                 \
+                d->B(j) = r[i];                                         \
+            }                                                           \
+        }                                                               \
     }                                                                   \
                                                                         \
     void glue(helper_punpck ## base_name ## wd, SUFFIX)(CPUX86State *env,\
-                                                        Reg *d, Reg *s) \
+                                                Reg *d, Reg *v, Reg *s) \
     {                                                                   \
-        Reg r;                                                          \
+        uint16_t r[PACK_WIDTH];                                         \
+        int j, i;                                                       \
                                                                         \
-        r.W(0) = d->W((base << (SHIFT + 1)) + 0);                       \
-        r.W(1) = s->W((base << (SHIFT + 1)) + 0);                       \
-        r.W(2) = d->W((base << (SHIFT + 1)) + 1);                       \
-        r.W(3) = s->W((base << (SHIFT + 1)) + 1);                       \
-        XMM_ONLY(                                                       \
-                 r.W(4) = d->W((base << (SHIFT + 1)) + 2);              \
-                 r.W(5) = s->W((base << (SHIFT + 1)) + 2);              \
-                 r.W(6) = d->W((base << (SHIFT + 1)) + 3);              \
-                 r.W(7) = s->W((base << (SHIFT + 1)) + 3);              \
-                                                                      ) \
-            *d = r;                                                     \
+        for (j = 0; j < 4 << SHIFT; ) {                                 \
+            int k = j + base * PACK_WIDTH / 2;                          \
+            for (i = 0; i < PACK_WIDTH / 2; i++) {                      \
+                r[2 * i] = v->W(k + i);                                 \
+                r[2 * i + 1] = s->W(k + i);                             \
+            }                                                           \
+            for (i = 0; i < PACK_WIDTH; i++, j++) {                     \
+                d->W(j) = r[i];                                         \
+            }                                                           \
+        }                                                               \
     }                                                                   \
                                                                         \
     void glue(helper_punpck ## base_name ## dq, SUFFIX)(CPUX86State *env,\
-                                                        Reg *d, Reg *s) \
+                                                Reg *d, Reg *v, Reg *s) \
     {                                                                   \
-        Reg r;                                                          \
+        uint32_t r[PACK_WIDTH / 2];                                     \
+        int j, i;                                                       \
                                                                         \
-        r.L(0) = d->L((base << SHIFT) + 0);                             \
-        r.L(1) = s->L((base << SHIFT) + 0);                             \
-        XMM_ONLY(                                                       \
-                 r.L(2) = d->L((base << SHIFT) + 1);                    \
-                 r.L(3) = s->L((base << SHIFT) + 1);                    \
-                                                                      ) \
-            *d = r;                                                     \
+        for (j = 0; j < 2 << SHIFT; ) {                                 \
+            int k = j + base * PACK_WIDTH / 4;                          \
+            for (i = 0; i < PACK_WIDTH / 4; i++) {                      \
+                r[2 * i] = v->L(k + i);                                 \
+                r[2 * i + 1] = s->L(k + i);                             \
+            }                                                           \
+            for (i = 0; i < PACK_WIDTH / 2; i++, j++) {                 \
+                d->L(j) = r[i];                                         \
+            }                                                           \
+        }                                                               \
     }                                                                   \
                                                                         \
     XMM_ONLY(                                                           \
-             void glue(helper_punpck ## base_name ## qdq, SUFFIX)(CPUX86State \
-                                                                  *env, \
-                                                                  Reg *d, \
-                                                                  Reg *s) \
+             void glue(helper_punpck ## base_name ## qdq, SUFFIX)(      \
+                        CPUX86State *env, Reg *d, Reg *v, Reg *s)       \
              {                                                          \
-                 Reg r;                                                 \
+                 uint64_t r[2];                                         \
+                 int i;                                                 \
                                                                         \
-                 r.Q(0) = d->Q(base);                                   \
-                 r.Q(1) = s->Q(base);                                   \
-                 *d = r;                                                \
+                 for (i = 0; i < 1 << SHIFT; i += 2) {                  \
+                     r[0] = v->Q(base + i);                             \
+                     r[1] = s->Q(base + i);                             \
+                     d->Q(i) = r[0];                                    \
+                     d->Q(i + 1) = r[1];                                \
+                 }                                                      \
              }                                                          \
                                                                         )
 
 UNPCK_OP(l, 0)
 UNPCK_OP(h, 1)
 
+#undef PACK_WIDTH
+#undef PACK_HELPER_B
+#undef UNPCK_OP
+
+
 /* 3DNow! float ops */
 #if SHIFT == 0
 void helper_pi2fd(CPUX86State *env, MMXReg *d, MMXReg *s)
@@ -1301,11 +1321,11 @@ void helper_pf2iw(CPUX86State *env, MMXReg *d, MMXReg *s)
 
 void helper_pfacc(CPUX86State *env, MMXReg *d, MMXReg *s)
 {
-    MMXReg r;
+    float32 r;
 
-    r.MMX_S(0) = float32_add(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
-    r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
-    *d = r;
+    r = float32_add(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
+    d->MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
+    d->MMX_S(0) = r;
 }
 
 void helper_pfadd(CPUX86State *env, MMXReg *d, MMXReg *s)
@@ -1366,20 +1386,20 @@ void helper_pfmul(CPUX86State *env, MMXReg *d, MMXReg *s)
 
 void helper_pfnacc(CPUX86State *env, MMXReg *d, MMXReg *s)
 {
-    MMXReg r;
+    float32 r;
 
-    r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
-    r.MMX_S(1) = float32_sub(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
-    *d = r;
+    r = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
+    d->MMX_S(1) = float32_sub(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
+    d->MMX_S(0) = r;
 }
 
 void helper_pfpnacc(CPUX86State *env, MMXReg *d, MMXReg *s)
 {
-    MMXReg r;
+    float32 r;
 
-    r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
-    r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
-    *d = r;
+    r = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
+    d->MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
+    d->MMX_S(0) = r;
 }
 
 void helper_pfrcp(CPUX86State *env, MMXReg *d, MMXReg *s)
@@ -1412,120 +1432,95 @@ void helper_pfsubr(CPUX86State *env, MMXReg *d, MMXReg *s)
 
 void helper_pswapd(CPUX86State *env, MMXReg *d, MMXReg *s)
 {
-    MMXReg r;
+    uint32_t r;
 
-    r.MMX_L(0) = s->MMX_L(1);
-    r.MMX_L(1) = s->MMX_L(0);
-    *d = r;
+    r = s->MMX_L(0);
+    d->MMX_L(0) = s->MMX_L(1);
+    d->MMX_L(1) = r;
 }
 #endif
 
 /* SSSE3 op helpers */
-void glue(helper_pshufb, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_pshufb, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
     int i;
-    Reg r;
+#if SHIFT == 0
+    uint8_t r[8];
 
-    for (i = 0; i < (8 << SHIFT); i++) {
-        r.B(i) = (s->B(i) & 0x80) ? 0 : (d->B(s->B(i) & ((8 << SHIFT) - 1)));
+    for (i = 0; i < 8; i++) {
+        r[i] = (s->B(i) & 0x80) ? 0 : (v->B(s->B(i) & 7));
     }
+    for (i = 0; i < 8; i++) {
+        d->B(i) = r[i];
+    }
+#else
+    uint8_t r[8 << SHIFT];
 
-    *d = r;
-}
-
-void glue(helper_phaddw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    d->W(0) = (int16_t)d->W(0) + (int16_t)d->W(1);
-    d->W(1) = (int16_t)d->W(2) + (int16_t)d->W(3);
-    XMM_ONLY(d->W(2) = (int16_t)d->W(4) + (int16_t)d->W(5));
-    XMM_ONLY(d->W(3) = (int16_t)d->W(6) + (int16_t)d->W(7));
-    d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) + (int16_t)s->W(1);
-    d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) + (int16_t)s->W(3);
-    XMM_ONLY(d->W(6) = (int16_t)s->W(4) + (int16_t)s->W(5));
-    XMM_ONLY(d->W(7) = (int16_t)s->W(6) + (int16_t)s->W(7));
-}
-
-void glue(helper_phaddd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    d->L(0) = (int32_t)d->L(0) + (int32_t)d->L(1);
-    XMM_ONLY(d->L(1) = (int32_t)d->L(2) + (int32_t)d->L(3));
-    d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) + (int32_t)s->L(1);
-    XMM_ONLY(d->L(3) = (int32_t)s->L(2) + (int32_t)s->L(3));
-}
-
-void glue(helper_phaddsw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    d->W(0) = satsw((int16_t)d->W(0) + (int16_t)d->W(1));
-    d->W(1) = satsw((int16_t)d->W(2) + (int16_t)d->W(3));
-    XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) + (int16_t)d->W(5)));
-    XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) + (int16_t)d->W(7)));
-    d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) + (int16_t)s->W(1));
-    d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) + (int16_t)s->W(3));
-    XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) + (int16_t)s->W(5)));
-    XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) + (int16_t)s->W(7)));
-}
-
-void glue(helper_pmaddubsw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    d->W(0) = satsw((int8_t)s->B(0) * (uint8_t)d->B(0) +
-                    (int8_t)s->B(1) * (uint8_t)d->B(1));
-    d->W(1) = satsw((int8_t)s->B(2) * (uint8_t)d->B(2) +
-                    (int8_t)s->B(3) * (uint8_t)d->B(3));
-    d->W(2) = satsw((int8_t)s->B(4) * (uint8_t)d->B(4) +
-                    (int8_t)s->B(5) * (uint8_t)d->B(5));
-    d->W(3) = satsw((int8_t)s->B(6) * (uint8_t)d->B(6) +
-                    (int8_t)s->B(7) * (uint8_t)d->B(7));
-#if SHIFT == 1
-    d->W(4) = satsw((int8_t)s->B(8) * (uint8_t)d->B(8) +
-                    (int8_t)s->B(9) * (uint8_t)d->B(9));
-    d->W(5) = satsw((int8_t)s->B(10) * (uint8_t)d->B(10) +
-                    (int8_t)s->B(11) * (uint8_t)d->B(11));
-    d->W(6) = satsw((int8_t)s->B(12) * (uint8_t)d->B(12) +
-                    (int8_t)s->B(13) * (uint8_t)d->B(13));
-    d->W(7) = satsw((int8_t)s->B(14) * (uint8_t)d->B(14) +
-                    (int8_t)s->B(15) * (uint8_t)d->B(15));
+    for (i = 0; i < 8 << SHIFT; i++) {
+        int j = i & ~0xf;
+        r[i] = (s->B(i) & 0x80) ? 0 : v->B(j | (s->B(i) & 0xf));
+    }
+    for (i = 0; i < 8 << SHIFT; i++) {
+        d->B(i) = r[i];
+    }
 #endif
 }
 
-void glue(helper_phsubw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+#define SSE_HELPER_HW(name, F)  \
+void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \
+{                                                          \
+    uint16_t r[4 << SHIFT];                                \
+    int i, j, k;                                           \
+    for (k = 0; k < 4 << SHIFT; k += LANE_WIDTH / 2) {     \
+        for (i = j = 0; j < LANE_WIDTH / 2; i++, j += 2) { \
+            r[i + k] = F(v->W(j + k), v->W(j + k + 1));    \
+        }                                                  \
+        for (j = 0; j < LANE_WIDTH / 2; i++, j += 2) {     \
+            r[i + k] = F(s->W(j + k), s->W(j + k + 1));    \
+        }                                                  \
+    }                                                      \
+    for (i = 0; i < 4 << SHIFT; i++) {                     \
+        d->W(i) = r[i];                                    \
+    }                                                      \
+}
+
+#define SSE_HELPER_HL(name, F)  \
+void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \
+{                                                          \
+    uint32_t r[2 << SHIFT];                                \
+    int i, j, k;                                           \
+    for (k = 0; k < 2 << SHIFT; k += LANE_WIDTH / 4) {     \
+        for (i = j = 0; j < LANE_WIDTH / 4; i++, j += 2) { \
+            r[i + k] = F(v->L(j + k), v->L(j + k + 1));    \
+        }                                                  \
+        for (j = 0; j < LANE_WIDTH / 4; i++, j += 2) {     \
+            r[i + k] = F(s->L(j + k), s->L(j + k + 1));    \
+        }                                                  \
+    }                                                      \
+    for (i = 0; i < 2 << SHIFT; i++) {                     \
+        d->L(i) = r[i];                                    \
+    }                                                      \
+}
+
+SSE_HELPER_HW(phaddw, FADD)
+SSE_HELPER_HW(phsubw, FSUB)
+SSE_HELPER_HW(phaddsw, FADDSW)
+SSE_HELPER_HW(phsubsw, FSUBSW)
+SSE_HELPER_HL(phaddd, FADD)
+SSE_HELPER_HL(phsubd, FSUB)
+
+#undef SSE_HELPER_HW
+#undef SSE_HELPER_HL
+
+void glue(helper_pmaddubsw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
-    d->W(0) = (int16_t)d->W(0) - (int16_t)d->W(1);
-    d->W(1) = (int16_t)d->W(2) - (int16_t)d->W(3);
-    XMM_ONLY(d->W(2) = (int16_t)d->W(4) - (int16_t)d->W(5));
-    XMM_ONLY(d->W(3) = (int16_t)d->W(6) - (int16_t)d->W(7));
-    d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) - (int16_t)s->W(1);
-    d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) - (int16_t)s->W(3);
-    XMM_ONLY(d->W(6) = (int16_t)s->W(4) - (int16_t)s->W(5));
-    XMM_ONLY(d->W(7) = (int16_t)s->W(6) - (int16_t)s->W(7));
-}
-
-void glue(helper_phsubd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    d->L(0) = (int32_t)d->L(0) - (int32_t)d->L(1);
-    XMM_ONLY(d->L(1) = (int32_t)d->L(2) - (int32_t)d->L(3));
-    d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) - (int32_t)s->L(1);
-    XMM_ONLY(d->L(3) = (int32_t)s->L(2) - (int32_t)s->L(3));
-}
-
-void glue(helper_phsubsw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    d->W(0) = satsw((int16_t)d->W(0) - (int16_t)d->W(1));
-    d->W(1) = satsw((int16_t)d->W(2) - (int16_t)d->W(3));
-    XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) - (int16_t)d->W(5)));
-    XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) - (int16_t)d->W(7)));
-    d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) - (int16_t)s->W(1));
-    d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) - (int16_t)s->W(3));
-    XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) - (int16_t)s->W(5)));
-    XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) - (int16_t)s->W(7)));
+    int i;
+    for (i = 0; i < 4 << SHIFT; i++) {
+        d->W(i) = satsw((int8_t)s->B(i * 2) * (uint8_t)v->B(i * 2) +
+                        (int8_t)s->B(i * 2 + 1) * (uint8_t)v->B(i * 2 + 1));
+    }
 }
 
-#define FABSB(_, x) (x > INT8_MAX  ? -(int8_t)x : x)
-#define FABSW(_, x) (x > INT16_MAX ? -(int16_t)x : x)
-#define FABSL(_, x) (x > INT32_MAX ? -(int32_t)x : x)
-SSE_HELPER_B(helper_pabsb, FABSB)
-SSE_HELPER_W(helper_pabsw, FABSW)
-SSE_HELPER_L(helper_pabsd, FABSL)
-
 #define FMULHRSW(d, s) (((int16_t) d * (int16_t)s + 0x4000) >> 15)
 SSE_HELPER_W(helper_pmulhrsw, FMULHRSW)
 
@@ -1536,186 +1531,146 @@ SSE_HELPER_B(helper_psignb, FSIGNB)
 SSE_HELPER_W(helper_psignw, FSIGNW)
 SSE_HELPER_L(helper_psignd, FSIGNL)
 
-void glue(helper_palignr, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
-                                  int32_t shift)
+void glue(helper_palignr, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,
+                                  uint32_t imm)
 {
-    Reg r;
+    int i;
 
     /* XXX could be checked during translation */
-    if (shift >= (16 << SHIFT)) {
-        r.Q(0) = 0;
-        XMM_ONLY(r.Q(1) = 0);
+    if (imm >= (SHIFT ? 32 : 16)) {
+        for (i = 0; i < (1 << SHIFT); i++) {
+            d->Q(i) = 0;
+        }
     } else {
-        shift <<= 3;
+        int shift = imm * 8;
 #define SHR(v, i) (i < 64 && i > -64 ? i > 0 ? v >> (i) : (v << -(i)) : 0)
 #if SHIFT == 0
-        r.Q(0) = SHR(s->Q(0), shift - 0) |
-            SHR(d->Q(0), shift -  64);
+        d->Q(0) = SHR(s->Q(0), shift - 0) |
+            SHR(v->Q(0), shift -  64);
 #else
-        r.Q(0) = SHR(s->Q(0), shift - 0) |
-            SHR(s->Q(1), shift -  64) |
-            SHR(d->Q(0), shift - 128) |
-            SHR(d->Q(1), shift - 192);
-        r.Q(1) = SHR(s->Q(0), shift + 64) |
-            SHR(s->Q(1), shift -   0) |
-            SHR(d->Q(0), shift -  64) |
-            SHR(d->Q(1), shift - 128);
+        for (i = 0; i < (1 << SHIFT); i += 2) {
+            uint64_t r0, r1;
+
+            r0 = SHR(s->Q(i), shift - 0) |
+                 SHR(s->Q(i + 1), shift -  64) |
+                 SHR(v->Q(i), shift - 128) |
+                 SHR(v->Q(i + 1), shift - 192);
+            r1 = SHR(s->Q(i), shift + 64) |
+                 SHR(s->Q(i + 1), shift -   0) |
+                 SHR(v->Q(i), shift -  64) |
+                 SHR(v->Q(i + 1), shift - 128);
+            d->Q(i) = r0;
+            d->Q(i + 1) = r1;
+        }
 #endif
 #undef SHR
     }
-
-    *d = r;
 }
 
-#define XMM0 (env->xmm_regs[0])
+#if SHIFT >= 1
 
-#if SHIFT == 1
 #define SSE_HELPER_V(name, elem, num, F)                                \
-    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)           \
+    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,   \
+                            Reg *m)                                     \
     {                                                                   \
-        d->elem(0) = F(d->elem(0), s->elem(0), XMM0.elem(0));           \
-        d->elem(1) = F(d->elem(1), s->elem(1), XMM0.elem(1));           \
-        if (num > 2) {                                                  \
-            d->elem(2) = F(d->elem(2), s->elem(2), XMM0.elem(2));       \
-            d->elem(3) = F(d->elem(3), s->elem(3), XMM0.elem(3));       \
-            if (num > 4) {                                              \
-                d->elem(4) = F(d->elem(4), s->elem(4), XMM0.elem(4));   \
-                d->elem(5) = F(d->elem(5), s->elem(5), XMM0.elem(5));   \
-                d->elem(6) = F(d->elem(6), s->elem(6), XMM0.elem(6));   \
-                d->elem(7) = F(d->elem(7), s->elem(7), XMM0.elem(7));   \
-                if (num > 8) {                                          \
-                    d->elem(8) = F(d->elem(8), s->elem(8), XMM0.elem(8)); \
-                    d->elem(9) = F(d->elem(9), s->elem(9), XMM0.elem(9)); \
-                    d->elem(10) = F(d->elem(10), s->elem(10), XMM0.elem(10)); \
-                    d->elem(11) = F(d->elem(11), s->elem(11), XMM0.elem(11)); \
-                    d->elem(12) = F(d->elem(12), s->elem(12), XMM0.elem(12)); \
-                    d->elem(13) = F(d->elem(13), s->elem(13), XMM0.elem(13)); \
-                    d->elem(14) = F(d->elem(14), s->elem(14), XMM0.elem(14)); \
-                    d->elem(15) = F(d->elem(15), s->elem(15), XMM0.elem(15)); \
-                }                                                       \
-            }                                                           \
+        int i;                                                          \
+        for (i = 0; i < num; i++) {                                     \
+            d->elem(i) = F(v->elem(i), s->elem(i), m->elem(i));         \
         }                                                               \
     }
 
 #define SSE_HELPER_I(name, elem, num, F)                                \
-    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, uint32_t imm) \
+    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,   \
+                            uint32_t imm)                               \
     {                                                                   \
-        d->elem(0) = F(d->elem(0), s->elem(0), ((imm >> 0) & 1));       \
-        d->elem(1) = F(d->elem(1), s->elem(1), ((imm >> 1) & 1));       \
-        if (num > 2) {                                                  \
-            d->elem(2) = F(d->elem(2), s->elem(2), ((imm >> 2) & 1));   \
-            d->elem(3) = F(d->elem(3), s->elem(3), ((imm >> 3) & 1));   \
-            if (num > 4) {                                              \
-                d->elem(4) = F(d->elem(4), s->elem(4), ((imm >> 4) & 1)); \
-                d->elem(5) = F(d->elem(5), s->elem(5), ((imm >> 5) & 1)); \
-                d->elem(6) = F(d->elem(6), s->elem(6), ((imm >> 6) & 1)); \
-                d->elem(7) = F(d->elem(7), s->elem(7), ((imm >> 7) & 1)); \
-                if (num > 8) {                                          \
-                    d->elem(8) = F(d->elem(8), s->elem(8), ((imm >> 8) & 1)); \
-                    d->elem(9) = F(d->elem(9), s->elem(9), ((imm >> 9) & 1)); \
-                    d->elem(10) = F(d->elem(10), s->elem(10),           \
-                                    ((imm >> 10) & 1));                 \
-                    d->elem(11) = F(d->elem(11), s->elem(11),           \
-                                    ((imm >> 11) & 1));                 \
-                    d->elem(12) = F(d->elem(12), s->elem(12),           \
-                                    ((imm >> 12) & 1));                 \
-                    d->elem(13) = F(d->elem(13), s->elem(13),           \
-                                    ((imm >> 13) & 1));                 \
-                    d->elem(14) = F(d->elem(14), s->elem(14),           \
-                                    ((imm >> 14) & 1));                 \
-                    d->elem(15) = F(d->elem(15), s->elem(15),           \
-                                    ((imm >> 15) & 1));                 \
-                }                                                       \
-            }                                                           \
+        int i;                                                          \
+        for (i = 0; i < num; i++) {                                     \
+            int j = i & 7;                                              \
+            d->elem(i) = F(v->elem(i), s->elem(i), (imm >> j) & 1);     \
         }                                                               \
     }
 
 /* SSE4.1 op helpers */
-#define FBLENDVB(d, s, m) ((m & 0x80) ? s : d)
-#define FBLENDVPS(d, s, m) ((m & 0x80000000) ? s : d)
-#define FBLENDVPD(d, s, m) ((m & 0x8000000000000000LL) ? s : d)
-SSE_HELPER_V(helper_pblendvb, B, 16, FBLENDVB)
-SSE_HELPER_V(helper_blendvps, L, 4, FBLENDVPS)
-SSE_HELPER_V(helper_blendvpd, Q, 2, FBLENDVPD)
+#define FBLENDVB(v, s, m) ((m & 0x80) ? s : v)
+#define FBLENDVPS(v, s, m) ((m & 0x80000000) ? s : v)
+#define FBLENDVPD(v, s, m) ((m & 0x8000000000000000LL) ? s : v)
+SSE_HELPER_V(helper_pblendvb, B, 8 << SHIFT, FBLENDVB)
+SSE_HELPER_V(helper_blendvps, L, 2 << SHIFT, FBLENDVPS)
+SSE_HELPER_V(helper_blendvpd, Q, 1 << SHIFT, FBLENDVPD)
 
 void glue(helper_ptest, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
 {
-    uint64_t zf = (s->Q(0) &  d->Q(0)) | (s->Q(1) &  d->Q(1));
-    uint64_t cf = (s->Q(0) & ~d->Q(0)) | (s->Q(1) & ~d->Q(1));
+    uint64_t zf = 0, cf = 0;
+    int i;
 
+    for (i = 0; i < 1 << SHIFT; i++) {
+        zf |= (s->Q(i) &  d->Q(i));
+        cf |= (s->Q(i) & ~d->Q(i));
+    }
     CC_SRC = (zf ? 0 : CC_Z) | (cf ? 0 : CC_C);
 }
 
-#define SSE_HELPER_F(name, elem, num, F)        \
-    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)     \
-    {                                           \
-        if (num > 2) {                          \
-            if (num > 4) {                      \
-                d->elem(7) = F(7);              \
-                d->elem(6) = F(6);              \
-                d->elem(5) = F(5);              \
-                d->elem(4) = F(4);              \
-            }                                   \
-            d->elem(3) = F(3);                  \
-            d->elem(2) = F(2);                  \
-        }                                       \
-        d->elem(1) = F(1);                      \
-        d->elem(0) = F(0);                      \
-    }
-
-SSE_HELPER_F(helper_pmovsxbw, W, 8, (int8_t) s->B)
-SSE_HELPER_F(helper_pmovsxbd, L, 4, (int8_t) s->B)
-SSE_HELPER_F(helper_pmovsxbq, Q, 2, (int8_t) s->B)
-SSE_HELPER_F(helper_pmovsxwd, L, 4, (int16_t) s->W)
-SSE_HELPER_F(helper_pmovsxwq, Q, 2, (int16_t) s->W)
-SSE_HELPER_F(helper_pmovsxdq, Q, 2, (int32_t) s->L)
-SSE_HELPER_F(helper_pmovzxbw, W, 8, s->B)
-SSE_HELPER_F(helper_pmovzxbd, L, 4, s->B)
-SSE_HELPER_F(helper_pmovzxbq, Q, 2, s->B)
-SSE_HELPER_F(helper_pmovzxwd, L, 4, s->W)
-SSE_HELPER_F(helper_pmovzxwq, Q, 2, s->W)
-SSE_HELPER_F(helper_pmovzxdq, Q, 2, s->L)
-
-void glue(helper_pmuldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    d->Q(0) = (int64_t)(int32_t) d->L(0) * (int32_t) s->L(0);
-    d->Q(1) = (int64_t)(int32_t) d->L(2) * (int32_t) s->L(2);
-}
-
-#define FCMPEQQ(d, s) (d == s ? -1 : 0)
-SSE_HELPER_Q(helper_pcmpeqq, FCMPEQQ)
-
-void glue(helper_packusdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
-{
-    Reg r;
-
-    r.W(0) = satuw((int32_t) d->L(0));
-    r.W(1) = satuw((int32_t) d->L(1));
-    r.W(2) = satuw((int32_t) d->L(2));
-    r.W(3) = satuw((int32_t) d->L(3));
-    r.W(4) = satuw((int32_t) s->L(0));
-    r.W(5) = satuw((int32_t) s->L(1));
-    r.W(6) = satuw((int32_t) s->L(2));
-    r.W(7) = satuw((int32_t) s->L(3));
-    *d = r;
-}
-
-#define FMINSB(d, s) MIN((int8_t)d, (int8_t)s)
-#define FMINSD(d, s) MIN((int32_t)d, (int32_t)s)
-#define FMAXSB(d, s) MAX((int8_t)d, (int8_t)s)
-#define FMAXSD(d, s) MAX((int32_t)d, (int32_t)s)
-SSE_HELPER_B(helper_pminsb, FMINSB)
-SSE_HELPER_L(helper_pminsd, FMINSD)
-SSE_HELPER_W(helper_pminuw, MIN)
-SSE_HELPER_L(helper_pminud, MIN)
-SSE_HELPER_B(helper_pmaxsb, FMAXSB)
-SSE_HELPER_L(helper_pmaxsd, FMAXSD)
-SSE_HELPER_W(helper_pmaxuw, MAX)
-SSE_HELPER_L(helper_pmaxud, MAX)
-
-#define FMULLD(d, s) ((int32_t)d * (int32_t)s)
-SSE_HELPER_L(helper_pmulld, FMULLD)
+#define FMOVSLDUP(i) s->L((i) & ~1)
+#define FMOVSHDUP(i) s->L((i) | 1)
+#define FMOVDLDUP(i) s->Q((i) & ~1)
 
+#define SSE_HELPER_F(name, elem, num, F)                        \
+    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)   \
+    {                                                           \
+        int n = num;                                            \
+        for (int i = n; --i >= 0; ) {                           \
+            d->elem(i) = F(i);                                  \
+        }                                                       \
+    }
+
+#if SHIFT > 0
+SSE_HELPER_F(helper_pmovsxbw, W, 4 << SHIFT, (int8_t) s->B)
+SSE_HELPER_F(helper_pmovsxbd, L, 2 << SHIFT, (int8_t) s->B)
+SSE_HELPER_F(helper_pmovsxbq, Q, 1 << SHIFT, (int8_t) s->B)
+SSE_HELPER_F(helper_pmovsxwd, L, 2 << SHIFT, (int16_t) s->W)
+SSE_HELPER_F(helper_pmovsxwq, Q, 1 << SHIFT, (int16_t) s->W)
+SSE_HELPER_F(helper_pmovsxdq, Q, 1 << SHIFT, (int32_t) s->L)
+SSE_HELPER_F(helper_pmovzxbw, W, 4 << SHIFT, s->B)
+SSE_HELPER_F(helper_pmovzxbd, L, 2 << SHIFT, s->B)
+SSE_HELPER_F(helper_pmovzxbq, Q, 1 << SHIFT, s->B)
+SSE_HELPER_F(helper_pmovzxwd, L, 2 << SHIFT, s->W)
+SSE_HELPER_F(helper_pmovzxwq, Q, 1 << SHIFT, s->W)
+SSE_HELPER_F(helper_pmovzxdq, Q, 1 << SHIFT, s->L)
+SSE_HELPER_F(helper_pmovsldup, L, 2 << SHIFT, FMOVSLDUP)
+SSE_HELPER_F(helper_pmovshdup, L, 2 << SHIFT, FMOVSHDUP)
+SSE_HELPER_F(helper_pmovdldup, Q, 1 << SHIFT, FMOVDLDUP)
+#endif
+
+void glue(helper_pmuldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    int i;
+
+    for (i = 0; i < 1 << SHIFT; i++) {
+        d->Q(i) = (int64_t)(int32_t) v->L(2 * i) * (int32_t) s->L(2 * i);
+    }
+}
+
+void glue(helper_packusdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    uint16_t r[8];
+    int i, j, k;
+
+    for (i = 0, j = 0; i <= 2 << SHIFT; i += 8, j += 4) {
+        r[0] = satuw(v->L(j));
+        r[1] = satuw(v->L(j + 1));
+        r[2] = satuw(v->L(j + 2));
+        r[3] = satuw(v->L(j + 3));
+        r[4] = satuw(s->L(j));
+        r[5] = satuw(s->L(j + 1));
+        r[6] = satuw(s->L(j + 2));
+        r[7] = satuw(s->L(j + 3));
+        for (k = 0; k < 8; k++) {
+            d->W(i + k) = r[k];
+        }
+    }
+}
+
+#if SHIFT == 1
 void glue(helper_phminposuw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
 {
     int idx = 0;
@@ -1747,254 +1702,222 @@ void glue(helper_phminposuw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
     d->L(1) = 0;
     d->Q(1) = 0;
 }
+#endif
 
 void glue(helper_roundps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
                                   uint32_t mode)
 {
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
     signed char prev_rounding_mode;
+    int i;
 
     prev_rounding_mode = env->sse_status.float_rounding_mode;
     if (!(mode & (1 << 2))) {
-        switch (mode & 3) {
-        case 0:
-            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
-            break;
-        case 1:
-            set_float_rounding_mode(float_round_down, &env->sse_status);
-            break;
-        case 2:
-            set_float_rounding_mode(float_round_up, &env->sse_status);
-            break;
-        case 3:
-            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
-            break;
-        }
+        set_x86_rounding_mode(mode & 3, &env->sse_status);
     }
 
-    d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status);
-    d->ZMM_S(1) = float32_round_to_int(s->ZMM_S(1), &env->sse_status);
-    d->ZMM_S(2) = float32_round_to_int(s->ZMM_S(2), &env->sse_status);
-    d->ZMM_S(3) = float32_round_to_int(s->ZMM_S(3), &env->sse_status);
+    for (i = 0; i < 2 << SHIFT; i++) {
+        d->ZMM_S(i) = float32_round_to_int(s->ZMM_S(i), &env->sse_status);
+    }
 
-#if 0 /* TODO */
-    if (mode & (1 << 3)) {
+    if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) {
         set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
                                   ~float_flag_inexact,
                                   &env->sse_status);
     }
-#endif
     env->sse_status.float_rounding_mode = prev_rounding_mode;
 }
 
 void glue(helper_roundpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
                                   uint32_t mode)
 {
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
     signed char prev_rounding_mode;
+    int i;
 
     prev_rounding_mode = env->sse_status.float_rounding_mode;
     if (!(mode & (1 << 2))) {
-        switch (mode & 3) {
-        case 0:
-            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
-            break;
-        case 1:
-            set_float_rounding_mode(float_round_down, &env->sse_status);
-            break;
-        case 2:
-            set_float_rounding_mode(float_round_up, &env->sse_status);
-            break;
-        case 3:
-            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
-            break;
-        }
+        set_x86_rounding_mode(mode & 3, &env->sse_status);
     }
 
-    d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status);
-    d->ZMM_D(1) = float64_round_to_int(s->ZMM_D(1), &env->sse_status);
+    for (i = 0; i < 1 << SHIFT; i++) {
+        d->ZMM_D(i) = float64_round_to_int(s->ZMM_D(i), &env->sse_status);
+    }
 
-#if 0 /* TODO */
-    if (mode & (1 << 3)) {
+    if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) {
         set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
                                   ~float_flag_inexact,
                                   &env->sse_status);
     }
-#endif
     env->sse_status.float_rounding_mode = prev_rounding_mode;
 }
 
-void glue(helper_roundss, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
+#if SHIFT == 1
+void glue(helper_roundss, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,
                                   uint32_t mode)
 {
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
     signed char prev_rounding_mode;
+    int i;
 
     prev_rounding_mode = env->sse_status.float_rounding_mode;
     if (!(mode & (1 << 2))) {
-        switch (mode & 3) {
-        case 0:
-            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
-            break;
-        case 1:
-            set_float_rounding_mode(float_round_down, &env->sse_status);
-            break;
-        case 2:
-            set_float_rounding_mode(float_round_up, &env->sse_status);
-            break;
-        case 3:
-            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
-            break;
-        }
+        set_x86_rounding_mode(mode & 3, &env->sse_status);
     }
 
     d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status);
+    for (i = 1; i < 2 << SHIFT; i++) {
+        d->ZMM_L(i) = v->ZMM_L(i);
+    }
 
-#if 0 /* TODO */
-    if (mode & (1 << 3)) {
+    if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) {
         set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
                                   ~float_flag_inexact,
                                   &env->sse_status);
     }
-#endif
     env->sse_status.float_rounding_mode = prev_rounding_mode;
 }
 
-void glue(helper_roundsd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
+void glue(helper_roundsd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,
                                   uint32_t mode)
 {
+    uint8_t old_flags = get_float_exception_flags(&env->sse_status);
     signed char prev_rounding_mode;
+    int i;
 
     prev_rounding_mode = env->sse_status.float_rounding_mode;
     if (!(mode & (1 << 2))) {
-        switch (mode & 3) {
-        case 0:
-            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
-            break;
-        case 1:
-            set_float_rounding_mode(float_round_down, &env->sse_status);
-            break;
-        case 2:
-            set_float_rounding_mode(float_round_up, &env->sse_status);
-            break;
-        case 3:
-            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
-            break;
-        }
+        set_x86_rounding_mode(mode & 3, &env->sse_status);
     }
 
     d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status);
+    for (i = 1; i < 1 << SHIFT; i++) {
+        d->ZMM_Q(i) = v->ZMM_Q(i);
+    }
 
-#if 0 /* TODO */
-    if (mode & (1 << 3)) {
+    if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) {
         set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
                                   ~float_flag_inexact,
                                   &env->sse_status);
     }
-#endif
     env->sse_status.float_rounding_mode = prev_rounding_mode;
 }
+#endif
 
-#define FBLENDP(d, s, m) (m ? s : d)
-SSE_HELPER_I(helper_blendps, L, 4, FBLENDP)
-SSE_HELPER_I(helper_blendpd, Q, 2, FBLENDP)
-SSE_HELPER_I(helper_pblendw, W, 8, FBLENDP)
+#define FBLENDP(v, s, m) (m ? s : v)
+SSE_HELPER_I(helper_blendps, L, 2 << SHIFT, FBLENDP)
+SSE_HELPER_I(helper_blendpd, Q, 1 << SHIFT, FBLENDP)
+SSE_HELPER_I(helper_pblendw, W, 4 << SHIFT, FBLENDP)
 
-void glue(helper_dpps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, uint32_t mask)
+void glue(helper_dpps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,
+                               uint32_t mask)
 {
-    float32 iresult = float32_zero;
+    float32 prod1, prod2, temp2, temp3, temp4;
+    int i;
 
-    if (mask & (1 << 4)) {
-        iresult = float32_add(iresult,
-                              float32_mul(d->ZMM_S(0), s->ZMM_S(0),
-                                          &env->sse_status),
-                              &env->sse_status);
-    }
-    if (mask & (1 << 5)) {
-        iresult = float32_add(iresult,
-                              float32_mul(d->ZMM_S(1), s->ZMM_S(1),
-                                          &env->sse_status),
-                              &env->sse_status);
-    }
-    if (mask & (1 << 6)) {
-        iresult = float32_add(iresult,
-                              float32_mul(d->ZMM_S(2), s->ZMM_S(2),
-                                          &env->sse_status),
-                              &env->sse_status);
-    }
-    if (mask & (1 << 7)) {
-        iresult = float32_add(iresult,
-                              float32_mul(d->ZMM_S(3), s->ZMM_S(3),
-                                          &env->sse_status),
-                              &env->sse_status);
+    for (i = 0; i < 2 << SHIFT; i += 4) {
+        /*
+         * We must evaluate (A+B)+(C+D), not ((A+B)+C)+D
+         * to correctly round the intermediate results
+         */
+        if (mask & (1 << 4)) {
+            prod1 = float32_mul(v->ZMM_S(i), s->ZMM_S(i), &env->sse_status);
+        } else {
+            prod1 = float32_zero;
+        }
+        if (mask & (1 << 5)) {
+            prod2 = float32_mul(v->ZMM_S(i+1), s->ZMM_S(i+1), &env->sse_status);
+        } else {
+            prod2 = float32_zero;
+        }
+        temp2 = float32_add(prod1, prod2, &env->sse_status);
+        if (mask & (1 << 6)) {
+            prod1 = float32_mul(v->ZMM_S(i+2), s->ZMM_S(i+2), &env->sse_status);
+        } else {
+            prod1 = float32_zero;
+        }
+        if (mask & (1 << 7)) {
+            prod2 = float32_mul(v->ZMM_S(i+3), s->ZMM_S(i+3), &env->sse_status);
+        } else {
+            prod2 = float32_zero;
+        }
+        temp3 = float32_add(prod1, prod2, &env->sse_status);
+        temp4 = float32_add(temp2, temp3, &env->sse_status);
+
+        d->ZMM_S(i) = (mask & (1 << 0)) ? temp4 : float32_zero;
+        d->ZMM_S(i+1) = (mask & (1 << 1)) ? temp4 : float32_zero;
+        d->ZMM_S(i+2) = (mask & (1 << 2)) ? temp4 : float32_zero;
+        d->ZMM_S(i+3) = (mask & (1 << 3)) ? temp4 : float32_zero;
     }
-    d->ZMM_S(0) = (mask & (1 << 0)) ? iresult : float32_zero;
-    d->ZMM_S(1) = (mask & (1 << 1)) ? iresult : float32_zero;
-    d->ZMM_S(2) = (mask & (1 << 2)) ? iresult : float32_zero;
-    d->ZMM_S(3) = (mask & (1 << 3)) ? iresult : float32_zero;
 }
 
-void glue(helper_dppd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, uint32_t mask)
+#if SHIFT == 1
+/* Oddly, there is no ymm version of dppd */
+void glue(helper_dppd, SUFFIX)(CPUX86State *env,
+                               Reg *d, Reg *v, Reg *s, uint32_t mask)
 {
-    float64 iresult = float64_zero;
+    float64 prod1, prod2, temp2;
 
     if (mask & (1 << 4)) {
-        iresult = float64_add(iresult,
-                              float64_mul(d->ZMM_D(0), s->ZMM_D(0),
-                                          &env->sse_status),
-                              &env->sse_status);
+        prod1 = float64_mul(v->ZMM_D(0), s->ZMM_D(0), &env->sse_status);
+    } else {
+        prod1 = float64_zero;
     }
     if (mask & (1 << 5)) {
-        iresult = float64_add(iresult,
-                              float64_mul(d->ZMM_D(1), s->ZMM_D(1),
-                                          &env->sse_status),
-                              &env->sse_status);
+        prod2 = float64_mul(v->ZMM_D(1), s->ZMM_D(1), &env->sse_status);
+    } else {
+        prod2 = float64_zero;
     }
-    d->ZMM_D(0) = (mask & (1 << 0)) ? iresult : float64_zero;
-    d->ZMM_D(1) = (mask & (1 << 1)) ? iresult : float64_zero;
+    temp2 = float64_add(prod1, prod2, &env->sse_status);
+    d->ZMM_D(0) = (mask & (1 << 0)) ? temp2 : float64_zero;
+    d->ZMM_D(1) = (mask & (1 << 1)) ? temp2 : float64_zero;
 }
+#endif
 
-void glue(helper_mpsadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
+void glue(helper_mpsadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,
                                   uint32_t offset)
 {
-    int s0 = (offset & 3) << 2;
-    int d0 = (offset & 4) << 0;
-    int i;
-    Reg r;
-
-    for (i = 0; i < 8; i++, d0++) {
-        r.W(i) = 0;
-        r.W(i) += abs1(d->B(d0 + 0) - s->B(s0 + 0));
-        r.W(i) += abs1(d->B(d0 + 1) - s->B(s0 + 1));
-        r.W(i) += abs1(d->B(d0 + 2) - s->B(s0 + 2));
-        r.W(i) += abs1(d->B(d0 + 3) - s->B(s0 + 3));
+    int i, j;
+    uint16_t r[8];
+
+    for (j = 0; j < 4 << SHIFT; ) {
+        int s0 = (j * 2) + ((offset & 3) << 2);
+        int d0 = (j * 2) + ((offset & 4) << 0);
+        for (i = 0; i < LANE_WIDTH / 2; i++, d0++) {
+            r[i] = 0;
+            r[i] += abs1(v->B(d0 + 0) - s->B(s0 + 0));
+            r[i] += abs1(v->B(d0 + 1) - s->B(s0 + 1));
+            r[i] += abs1(v->B(d0 + 2) - s->B(s0 + 2));
+            r[i] += abs1(v->B(d0 + 3) - s->B(s0 + 3));
+        }
+        for (i = 0; i < LANE_WIDTH / 2; i++, j++) {
+            d->W(j) = r[i];
+        }
+        offset >>= 3;
     }
-
-    *d = r;
 }
 
 /* SSE4.2 op helpers */
-#define FCMPGTQ(d, s) ((int64_t)d > (int64_t)s ? -1 : 0)
-SSE_HELPER_Q(helper_pcmpgtq, FCMPGTQ)
-
+#if SHIFT == 1
 static inline int pcmp_elen(CPUX86State *env, int reg, uint32_t ctrl)
 {
-    int val;
+    target_long val, limit;
 
     /* Presence of REX.W is indicated by a bit higher than 7 set */
     if (ctrl >> 8) {
-        val = abs1((int64_t)env->regs[reg]);
+        val = (target_long)env->regs[reg];
     } else {
-        val = abs1((int32_t)env->regs[reg]);
+        val = (int32_t)env->regs[reg];
     }
-
     if (ctrl & 1) {
-        if (val > 8) {
-            return 8;
-        }
+        limit = 8;
     } else {
-        if (val > 16) {
-            return 16;
-        }
+        limit = 16;
     }
-    return val;
+    if ((val > limit) || (val < -limit)) {
+        return limit;
+    }
+    return abs1(val);
 }
 
 static inline int pcmp_ilen(Reg *r, uint8_t ctrl)
@@ -2030,7 +1953,7 @@ static inline int pcmp_val(Reg *r, uint8_t ctrl, int i)
 }
 
 static inline unsigned pcmpxstrx(CPUX86State *env, Reg *d, Reg *s,
-                                 int8_t ctrl, int valids, int validd)
+                                 uint8_t ctrl, int valids, int validd)
 {
     unsigned int res = 0;
     int v;
@@ -2076,10 +1999,10 @@ static inline unsigned pcmpxstrx(CPUX86State *env, Reg *d, Reg *s,
             res = (2 << upper) - 1;
             break;
         }
-        for (j = valids - validd; j >= 0; j--) {
+        for (j = valids == upper ? valids : valids - validd; j >= 0; j--) {
             res <<= 1;
             v = 1;
-            for (i = validd; i >= 0; i--) {
+            for (i = MIN(valids - j, validd); i >= 0; i--) {
                 v &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i));
             }
             res |= v;
@@ -2196,14 +2119,16 @@ target_ulong helper_crc32(uint32_t crc1, target_ulong msg, uint32_t len)
     return crc;
 }
 
-void glue(helper_pclmulqdq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
-                                    uint32_t ctrl)
+#endif
+
+#if SHIFT == 1
+static void clmulq(uint64_t *dest_l, uint64_t *dest_h,
+                          uint64_t a, uint64_t b)
 {
-    uint64_t ah, al, b, resh, resl;
+    uint64_t al, ah, resh, resl;
 
     ah = 0;
-    al = d->Q((ctrl & 1) != 0);
-    b = s->Q((ctrl & 16) != 0);
+    al = a;
     resh = resl = 0;
 
     while (b) {
@@ -2216,71 +2141,99 @@ void glue(helper_pclmulqdq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
         b >>= 1;
     }
 
-    d->Q(0) = resl;
-    d->Q(1) = resh;
+    *dest_l = resl;
+    *dest_h = resh;
 }
+#endif
 
-void glue(helper_aesdec, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_pclmulqdq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s,
+                                    uint32_t ctrl)
 {
+    uint64_t a, b;
     int i;
-    Reg st = *d;
-    Reg rk = *s;
 
-    for (i = 0 ; i < 4 ; i++) {
-        d->L(i) = rk.L(i) ^ bswap32(AES_Td0[st.B(AES_ishifts[4*i+0])] ^
-                                    AES_Td1[st.B(AES_ishifts[4*i+1])] ^
-                                    AES_Td2[st.B(AES_ishifts[4*i+2])] ^
-                                    AES_Td3[st.B(AES_ishifts[4*i+3])]);
+    for (i = 0; i < 1 << SHIFT; i += 2) {
+        a = v->Q(((ctrl & 1) != 0) + i);
+        b = s->Q(((ctrl & 16) != 0) + i);
+        clmulq(&d->Q(i), &d->Q(i + 1), a, b);
     }
 }
 
-void glue(helper_aesdeclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_aesdec, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
     int i;
-    Reg st = *d;
+    Reg st = *v;
     Reg rk = *s;
 
-    for (i = 0; i < 16; i++) {
-        d->B(i) = rk.B(i) ^ (AES_isbox[st.B(AES_ishifts[i])]);
+    for (i = 0 ; i < 2 << SHIFT ; i++) {
+        int j = i & 3;
+        int k = (i & ~3) << 2;
+        int b0 = AES_ishifts[4 * j + 0] + k;
+        int b1 = AES_ishifts[4 * j + 1] + k;
+        int b2 = AES_ishifts[4 * j + 2] + k;
+        int b3 = AES_ishifts[4 * j + 3] + k;
+
+        d->L(i) = rk.L(i) ^ bswap32(AES_Td0[st.B(b0)] ^
+                                    AES_Td1[st.B(b1)] ^
+                                    AES_Td2[st.B(b2)] ^
+                                    AES_Td3[st.B(b3)]);
     }
 }
 
-void glue(helper_aesenc, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_aesdeclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
     int i;
-    Reg st = *d;
+    Reg st = *v;
     Reg rk = *s;
 
-    for (i = 0 ; i < 4 ; i++) {
-        d->L(i) = rk.L(i) ^ bswap32(AES_Te0[st.B(AES_shifts[4*i+0])] ^
-                                    AES_Te1[st.B(AES_shifts[4*i+1])] ^
-                                    AES_Te2[st.B(AES_shifts[4*i+2])] ^
-                                    AES_Te3[st.B(AES_shifts[4*i+3])]);
+    for (i = 0; i < 8 << SHIFT; i++) {
+        d->B(i) = rk.B(i) ^ (AES_isbox[st.B(AES_ishifts[i & 15] + (i & ~15))]);
     }
 }
 
-void glue(helper_aesenclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+void glue(helper_aesenc, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
 {
     int i;
-    Reg st = *d;
+    Reg st = *v;
     Reg rk = *s;
 
-    for (i = 0; i < 16; i++) {
-        d->B(i) = rk.B(i) ^ (AES_sbox[st.B(AES_shifts[i])]);
+    for (i = 0 ; i < 2 << SHIFT ; i++) {
+        int j = i & 3;
+        int k = (i & ~3) << 2;
+        int b0 = AES_shifts[4 * j + 0] + k;
+        int b1 = AES_shifts[4 * j + 1] + k;
+        int b2 = AES_shifts[4 * j + 2] + k;
+        int b3 = AES_shifts[4 * j + 3] + k;
+
+        d->L(i) = rk.L(i) ^ bswap32(AES_Te0[st.B(b0)] ^
+                                    AES_Te1[st.B(b1)] ^
+                                    AES_Te2[st.B(b2)] ^
+                                    AES_Te3[st.B(b3)]);
     }
+}
 
+void glue(helper_aesenclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    int i;
+    Reg st = *v;
+    Reg rk = *s;
+
+    for (i = 0; i < 8 << SHIFT; i++) {
+        d->B(i) = rk.B(i) ^ (AES_sbox[st.B(AES_shifts[i & 15] + (i & ~15))]);
+    }
 }
 
+#if SHIFT == 1
 void glue(helper_aesimc, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
 {
     int i;
     Reg tmp = *s;
 
     for (i = 0 ; i < 4 ; i++) {
-        d->L(i) = bswap32(AES_imc[tmp.B(4*i+0)][0] ^
-                          AES_imc[tmp.B(4*i+1)][1] ^
-                          AES_imc[tmp.B(4*i+2)][2] ^
-                          AES_imc[tmp.B(4*i+3)][3]);
+        d->L(i) = bswap32(AES_imc[tmp.B(4 * i + 0)][0] ^
+                          AES_imc[tmp.B(4 * i + 1)][1] ^
+                          AES_imc[tmp.B(4 * i + 2)][2] ^
+                          AES_imc[tmp.B(4 * i + 3)][3]);
     }
 }
 
@@ -2298,7 +2251,327 @@ void glue(helper_aeskeygenassist, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
     d->L(3) = (d->L(2) << 24 | d->L(2) >> 8) ^ ctrl;
 }
 #endif
+#endif
+
+#if SHIFT >= 1
+void glue(helper_vpermilpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    uint64_t r0, r1;
+    int i;
+
+    for (i = 0; i < 1 << SHIFT; i += 2) {
+        r0 = v->Q(i + ((s->Q(i) >> 1) & 1));
+        r1 = v->Q(i + ((s->Q(i+1) >> 1) & 1));
+        d->Q(i) = r0;
+        d->Q(i+1) = r1;
+    }
+}
+
+void glue(helper_vpermilps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    uint32_t r0, r1, r2, r3;
+    int i;
+
+    for (i = 0; i < 2 << SHIFT; i += 4) {
+        r0 = v->L(i + (s->L(i) & 3));
+        r1 = v->L(i + (s->L(i+1) & 3));
+        r2 = v->L(i + (s->L(i+2) & 3));
+        r3 = v->L(i + (s->L(i+3) & 3));
+        d->L(i) = r0;
+        d->L(i+1) = r1;
+        d->L(i+2) = r2;
+        d->L(i+3) = r3;
+    }
+}
+
+void glue(helper_vpermilpd_imm, SUFFIX)(Reg *d, Reg *s, uint32_t order)
+{
+    uint64_t r0, r1;
+    int i;
+
+    for (i = 0; i < 1 << SHIFT; i += 2) {
+        r0 = s->Q(i + ((order >> 0) & 1));
+        r1 = s->Q(i + ((order >> 1) & 1));
+        d->Q(i) = r0;
+        d->Q(i+1) = r1;
+
+        order >>= 2;
+    }
+}
+
+void glue(helper_vpermilps_imm, SUFFIX)(Reg *d, Reg *s, uint32_t order)
+{
+    uint32_t r0, r1, r2, r3;
+    int i;
+
+    for (i = 0; i < 2 << SHIFT; i += 4) {
+        r0 = s->L(i + ((order >> 0) & 3));
+        r1 = s->L(i + ((order >> 2) & 3));
+        r2 = s->L(i + ((order >> 4) & 3));
+        r3 = s->L(i + ((order >> 6) & 3));
+        d->L(i) = r0;
+        d->L(i+1) = r1;
+        d->L(i+2) = r2;
+        d->L(i+3) = r3;
+    }
+}
+
+#if SHIFT == 1
+#define FPSRLVD(x, c) (c < 32 ? ((x) >> c) : 0)
+#define FPSRLVQ(x, c) (c < 64 ? ((x) >> c) : 0)
+#define FPSRAVD(x, c) ((int32_t)(x) >> (c < 32 ? c : 31))
+#define FPSRAVQ(x, c) ((int64_t)(x) >> (c < 64 ? c : 63))
+#define FPSLLVD(x, c) (c < 32 ? ((x) << c) : 0)
+#define FPSLLVQ(x, c) (c < 64 ? ((x) << c) : 0)
+#endif
+
+SSE_HELPER_L(helper_vpsrlvd, FPSRLVD)
+SSE_HELPER_L(helper_vpsravd, FPSRAVD)
+SSE_HELPER_L(helper_vpsllvd, FPSLLVD)
+
+SSE_HELPER_Q(helper_vpsrlvq, FPSRLVQ)
+SSE_HELPER_Q(helper_vpsravq, FPSRAVQ)
+SSE_HELPER_Q(helper_vpsllvq, FPSLLVQ)
+
+void glue(helper_vtestps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+{
+    uint32_t zf = 0, cf = 0;
+    int i;
+
+    for (i = 0; i < 2 << SHIFT; i++) {
+        zf |= (s->L(i) &  d->L(i));
+        cf |= (s->L(i) & ~d->L(i));
+    }
+    CC_SRC = ((zf >> 31) ? 0 : CC_Z) | ((cf >> 31) ? 0 : CC_C);
+}
+
+void glue(helper_vtestpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s)
+{
+    uint64_t zf = 0, cf = 0;
+    int i;
+
+    for (i = 0; i < 1 << SHIFT; i++) {
+        zf |= (s->Q(i) &  d->Q(i));
+        cf |= (s->Q(i) & ~d->Q(i));
+    }
+    CC_SRC = ((zf >> 63) ? 0 : CC_Z) | ((cf >> 63) ? 0 : CC_C);
+}
+
+void glue(helper_vpmaskmovd_st, SUFFIX)(CPUX86State *env,
+                                        Reg *v, Reg *s, target_ulong a0)
+{
+    int i;
+
+    for (i = 0; i < (2 << SHIFT); i++) {
+        if (v->L(i) >> 31) {
+            cpu_stl_data_ra(env, a0 + i * 4, s->L(i), GETPC());
+        }
+    }
+}
+
+void glue(helper_vpmaskmovq_st, SUFFIX)(CPUX86State *env,
+                                        Reg *v, Reg *s, target_ulong a0)
+{
+    int i;
+
+    for (i = 0; i < (1 << SHIFT); i++) {
+        if (v->Q(i) >> 63) {
+            cpu_stq_data_ra(env, a0 + i * 8, s->Q(i), GETPC());
+        }
+    }
+}
+
+void glue(helper_vpmaskmovd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    int i;
+
+    for (i = 0; i < (2 << SHIFT); i++) {
+        d->L(i) = (v->L(i) >> 31) ? s->L(i) : 0;
+    }
+}
+
+void glue(helper_vpmaskmovq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s)
+{
+    int i;
+
+    for (i = 0; i < (1 << SHIFT); i++) {
+        d->Q(i) = (v->Q(i) >> 63) ? s->Q(i) : 0;
+    }
+}
+
+void glue(helper_vpgatherdd, SUFFIX)(CPUX86State *env,
+        Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale)
+{
+    int i;
+    for (i = 0; i < (2 << SHIFT); i++) {
+        if (v->L(i) >> 31) {
+            target_ulong addr = a0
+                + ((target_ulong)(int32_t)s->L(i) << scale);
+            d->L(i) = cpu_ldl_data_ra(env, addr, GETPC());
+        }
+        v->L(i) = 0;
+    }
+}
+
+void glue(helper_vpgatherdq, SUFFIX)(CPUX86State *env,
+        Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale)
+{
+    int i;
+    for (i = 0; i < (1 << SHIFT); i++) {
+        if (v->Q(i) >> 63) {
+            target_ulong addr = a0
+                + ((target_ulong)(int32_t)s->L(i) << scale);
+            d->Q(i) = cpu_ldq_data_ra(env, addr, GETPC());
+        }
+        v->Q(i) = 0;
+    }
+}
+
+void glue(helper_vpgatherqd, SUFFIX)(CPUX86State *env,
+        Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale)
+{
+    int i;
+    for (i = 0; i < (1 << SHIFT); i++) {
+        if (v->L(i) >> 31) {
+            target_ulong addr = a0
+                + ((target_ulong)(int64_t)s->Q(i) << scale);
+            d->L(i) = cpu_ldl_data_ra(env, addr, GETPC());
+        }
+        v->L(i) = 0;
+    }
+    for (i /= 2; i < 1 << SHIFT; i++) {
+        d->Q(i) = 0;
+        v->Q(i) = 0;
+    }
+}
+
+void glue(helper_vpgatherqq, SUFFIX)(CPUX86State *env,
+        Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale)
+{
+    int i;
+    for (i = 0; i < (1 << SHIFT); i++) {
+        if (v->Q(i) >> 63) {
+            target_ulong addr = a0
+                + ((target_ulong)(int64_t)s->Q(i) << scale);
+            d->Q(i) = cpu_ldq_data_ra(env, addr, GETPC());
+        }
+        v->Q(i) = 0;
+    }
+}
+#endif
+
+#if SHIFT >= 2
+void helper_vpermdq_ymm(Reg *d, Reg *v, Reg *s, uint32_t order)
+{
+    uint64_t r0, r1, r2, r3;
+
+    switch (order & 3) {
+    case 0:
+        r0 = v->Q(0);
+        r1 = v->Q(1);
+        break;
+    case 1:
+        r0 = v->Q(2);
+        r1 = v->Q(3);
+        break;
+    case 2:
+        r0 = s->Q(0);
+        r1 = s->Q(1);
+        break;
+    case 3:
+        r0 = s->Q(2);
+        r1 = s->Q(3);
+        break;
+    }
+    switch ((order >> 4) & 3) {
+    case 0:
+        r2 = v->Q(0);
+        r3 = v->Q(1);
+        break;
+    case 1:
+        r2 = v->Q(2);
+        r3 = v->Q(3);
+        break;
+    case 2:
+        r2 = s->Q(0);
+        r3 = s->Q(1);
+        break;
+    case 3:
+        r2 = s->Q(2);
+        r3 = s->Q(3);
+        break;
+    }
+    d->Q(0) = r0;
+    d->Q(1) = r1;
+    d->Q(2) = r2;
+    d->Q(3) = r3;
+    if (order & 0x8) {
+        d->Q(0) = 0;
+        d->Q(1) = 0;
+    }
+    if (order & 0x80) {
+        d->Q(2) = 0;
+        d->Q(3) = 0;
+    }
+}
+
+void helper_vpermq_ymm(Reg *d, Reg *s, uint32_t order)
+{
+    uint64_t r0, r1, r2, r3;
+    r0 = s->Q(order & 3);
+    r1 = s->Q((order >> 2) & 3);
+    r2 = s->Q((order >> 4) & 3);
+    r3 = s->Q((order >> 6) & 3);
+    d->Q(0) = r0;
+    d->Q(1) = r1;
+    d->Q(2) = r2;
+    d->Q(3) = r3;
+}
+
+void helper_vpermd_ymm(Reg *d, Reg *v, Reg *s)
+{
+    uint32_t r[8];
+    int i;
+
+    for (i = 0; i < 8; i++) {
+        r[i] = s->L(v->L(i) & 7);
+    }
+    for (i = 0; i < 8; i++) {
+        d->L(i) = r[i];
+    }
+}
+#endif
+
+/* FMA3 op helpers */
+#if SHIFT == 1
+#define SSE_HELPER_FMAS(name, elem, F)                                         \
+    void name(CPUX86State *env, Reg *d, Reg *a, Reg *b, Reg *c, int flags)     \
+    {                                                                          \
+        d->elem(0) = F(a->elem(0), b->elem(0), c->elem(0), flags, &env->sse_status); \
+    }
+#define SSE_HELPER_FMAP(name, elem, num, F)                                    \
+    void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *a, Reg *b, Reg *c,  \
+                            int flags, int flip)                               \
+    {                                                                          \
+        int i;                                                                 \
+        for (i = 0; i < num; i++) {                                            \
+            d->elem(i) = F(a->elem(i), b->elem(i), c->elem(i), flags, &env->sse_status); \
+            flags ^= flip;                                                     \
+        }                                                                      \
+    }
+
+SSE_HELPER_FMAS(helper_fma4ss,  ZMM_S, float32_muladd)
+SSE_HELPER_FMAS(helper_fma4sd,  ZMM_D, float64_muladd)
+#endif
+
+#if SHIFT >= 1
+SSE_HELPER_FMAP(helper_fma4ps,  ZMM_S, 2 << SHIFT, float32_muladd)
+SSE_HELPER_FMAP(helper_fma4pd,  ZMM_D, 1 << SHIFT, float64_muladd)
+#endif
+
+#undef SSE_HELPER_S
 
+#undef LANE_WIDTH
 #undef SHIFT
 #undef XMM_ONLY
 #undef Reg
diff --git a/qemu/target/i386/ops_sse_header.h b/qemu/target/i386/ops_sse_header.h
index 094aafc573..4bd51d8a92 100644
--- a/qemu/target/i386/ops_sse_header.h
+++ b/qemu/target/i386/ops_sse_header.h
@@ -6,7 +6,7 @@
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
+ * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -21,7 +21,11 @@
 #define SUFFIX _mmx
 #else
 #define Reg ZMMReg
+#if SHIFT == 1
 #define SUFFIX _xmm
+#else
+#define SUFFIX _ymm
+#endif
 #endif
 
 #define dh_alias_Reg ptr
@@ -30,78 +34,41 @@
 #define dh_ctype_Reg Reg *
 #define dh_ctype_ZMMReg ZMMReg *
 #define dh_ctype_MMXReg MMXReg *
+#define dh_typecode_Reg dh_typecode_ptr
+#define dh_typecode_ZMMReg dh_typecode_ptr
+#define dh_typecode_MMXReg dh_typecode_ptr
 #define dh_is_signed_Reg dh_is_signed_ptr
 #define dh_is_signed_ZMMReg dh_is_signed_ptr
 #define dh_is_signed_MMXReg dh_is_signed_ptr
 
-DEF_HELPER_3(glue(psrlw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psraw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psllw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psrld, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psrad, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pslld, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psrlq, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psllq, SUFFIX), void, env, Reg, Reg)
-
-#if SHIFT == 1
-DEF_HELPER_3(glue(psrldq, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pslldq, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_4(glue(psrlw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psraw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psllw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psrld, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psrad, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(pslld, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psrlq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psllq, SUFFIX), void, env, Reg, Reg, Reg)
+
+#if SHIFT >= 1
+DEF_HELPER_4(glue(psrldq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(pslldq, SUFFIX), void, env, Reg, Reg, Reg)
 #endif
 
 #define SSE_HELPER_B(name, F)\
-    DEF_HELPER_3(glue(name, SUFFIX), void, env, Reg, Reg)
+    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
 
 #define SSE_HELPER_W(name, F)\
-    DEF_HELPER_3(glue(name, SUFFIX), void, env, Reg, Reg)
+    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
 
 #define SSE_HELPER_L(name, F)\
-    DEF_HELPER_3(glue(name, SUFFIX), void, env, Reg, Reg)
+    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
 
 #define SSE_HELPER_Q(name, F)\
-    DEF_HELPER_3(glue(name, SUFFIX), void, env, Reg, Reg)
-
-SSE_HELPER_B(paddb, FADD)
-SSE_HELPER_W(paddw, FADD)
-SSE_HELPER_L(paddl, FADD)
-SSE_HELPER_Q(paddq, FADD)
-
-SSE_HELPER_B(psubb, FSUB)
-SSE_HELPER_W(psubw, FSUB)
-SSE_HELPER_L(psubl, FSUB)
-SSE_HELPER_Q(psubq, FSUB)
-
-SSE_HELPER_B(paddusb, FADDUB)
-SSE_HELPER_B(paddsb, FADDSB)
-SSE_HELPER_B(psubusb, FSUBUB)
-SSE_HELPER_B(psubsb, FSUBSB)
-
-SSE_HELPER_W(paddusw, FADDUW)
-SSE_HELPER_W(paddsw, FADDSW)
-SSE_HELPER_W(psubusw, FSUBUW)
-SSE_HELPER_W(psubsw, FSUBSW)
-
-SSE_HELPER_B(pminub, FMINUB)
-SSE_HELPER_B(pmaxub, FMAXUB)
-
-SSE_HELPER_W(pminsw, FMINSW)
-SSE_HELPER_W(pmaxsw, FMAXSW)
-
-SSE_HELPER_Q(pand, FAND)
-SSE_HELPER_Q(pandn, FANDN)
-SSE_HELPER_Q(por, FOR)
-SSE_HELPER_Q(pxor, FXOR)
-
-SSE_HELPER_B(pcmpgtb, FCMPGTB)
-SSE_HELPER_W(pcmpgtw, FCMPGTW)
-SSE_HELPER_L(pcmpgtl, FCMPGTL)
-
-SSE_HELPER_B(pcmpeqb, FCMPEQ)
-SSE_HELPER_W(pcmpeqw, FCMPEQ)
-SSE_HELPER_L(pcmpeql, FCMPEQ)
+    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
 
-SSE_HELPER_W(pmullw, FMULLW)
 #if SHIFT == 0
-SSE_HELPER_W(pmulhrw, FMULHRW)
+DEF_HELPER_3(glue(pmulhrw, SUFFIX), void, env, Reg, Reg)
 #endif
 SSE_HELPER_W(pmulhuw, FMULHUW)
 SSE_HELPER_W(pmulhw, FMULHW)
@@ -109,51 +76,74 @@ SSE_HELPER_W(pmulhw, FMULHW)
 SSE_HELPER_B(pavgb, FAVG)
 SSE_HELPER_W(pavgw, FAVG)
 
-DEF_HELPER_3(glue(pmuludq, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmaddwd, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_4(glue(pmuludq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(pmaddwd, SUFFIX), void, env, Reg, Reg, Reg)
 
-DEF_HELPER_3(glue(psadbw, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_4(glue(psadbw, SUFFIX), void, env, Reg, Reg, Reg)
+#if SHIFT < 2
 DEF_HELPER_4(glue(maskmov, SUFFIX), void, env, Reg, Reg, tl)
-DEF_HELPER_2(glue(movl_mm_T0, SUFFIX), void, Reg, i32)
-#ifdef TARGET_X86_64
-DEF_HELPER_2(glue(movq_mm_T0, SUFFIX), void, Reg, i64)
 #endif
 
 #if SHIFT == 0
 DEF_HELPER_3(glue(pshufw, SUFFIX), void, Reg, Reg, int)
 #else
-DEF_HELPER_3(shufps, void, Reg, Reg, int)
-DEF_HELPER_3(shufpd, void, Reg, Reg, int)
 DEF_HELPER_3(glue(pshufd, SUFFIX), void, Reg, Reg, int)
 DEF_HELPER_3(glue(pshuflw, SUFFIX), void, Reg, Reg, int)
 DEF_HELPER_3(glue(pshufhw, SUFFIX), void, Reg, Reg, int)
 #endif
 
-#if SHIFT == 1
+#if SHIFT >= 1
 /* FPU ops */
 /* XXX: not accurate */
 
-#define SSE_HELPER_S(name, F)                            \
-    DEF_HELPER_3(name ## ps, void, env, Reg, Reg)        \
-    DEF_HELPER_3(name ## ss, void, env, Reg, Reg)        \
-    DEF_HELPER_3(name ## pd, void, env, Reg, Reg)        \
-    DEF_HELPER_3(name ## sd, void, env, Reg, Reg)
-
-SSE_HELPER_S(add, FPU_ADD)
-SSE_HELPER_S(sub, FPU_SUB)
-SSE_HELPER_S(mul, FPU_MUL)
-SSE_HELPER_S(div, FPU_DIV)
-SSE_HELPER_S(min, FPU_MIN)
-SSE_HELPER_S(max, FPU_MAX)
-SSE_HELPER_S(sqrt, FPU_SQRT)
-
-
-DEF_HELPER_3(cvtps2pd, void, env, Reg, Reg)
-DEF_HELPER_3(cvtpd2ps, void, env, Reg, Reg)
-DEF_HELPER_3(cvtss2sd, void, env, Reg, Reg)
-DEF_HELPER_3(cvtsd2ss, void, env, Reg, Reg)
-DEF_HELPER_3(cvtdq2ps, void, env, Reg, Reg)
-DEF_HELPER_3(cvtdq2pd, void, env, Reg, Reg)
+#define SSE_HELPER_P4(name)                                             \
+    DEF_HELPER_4(glue(name ## ps, SUFFIX), void, env, Reg, Reg, Reg)    \
+    DEF_HELPER_4(glue(name ## pd, SUFFIX), void, env, Reg, Reg, Reg)
+
+#define SSE_HELPER_P3(name, ...)                                        \
+    DEF_HELPER_3(glue(name ## ps, SUFFIX), void, env, Reg, Reg)         \
+    DEF_HELPER_3(glue(name ## pd, SUFFIX), void, env, Reg, Reg)
+
+#if SHIFT == 1
+#define SSE_HELPER_S4(name)                                             \
+    SSE_HELPER_P4(name)                                                 \
+    DEF_HELPER_4(name ## ss, void, env, Reg, Reg, Reg)                  \
+    DEF_HELPER_4(name ## sd, void, env, Reg, Reg, Reg)
+#define SSE_HELPER_S3(name)                                             \
+    SSE_HELPER_P3(name)                                                 \
+    DEF_HELPER_4(name ## ss, void, env, Reg, Reg, Reg)                  \
+    DEF_HELPER_4(name ## sd, void, env, Reg, Reg, Reg)
+#else
+#define SSE_HELPER_S4(name, ...) SSE_HELPER_P4(name)
+#define SSE_HELPER_S3(name, ...) SSE_HELPER_P3(name)
+#endif
+
+DEF_HELPER_4(glue(shufps, SUFFIX), void, Reg, Reg, Reg, int)
+DEF_HELPER_4(glue(shufpd, SUFFIX), void, Reg, Reg, Reg, int)
+
+SSE_HELPER_S4(add)
+SSE_HELPER_S4(sub)
+SSE_HELPER_S4(mul)
+SSE_HELPER_S4(div)
+SSE_HELPER_S4(min)
+SSE_HELPER_S4(max)
+
+SSE_HELPER_S3(sqrt)
+
+DEF_HELPER_3(glue(cvtps2pd, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_3(glue(cvtpd2ps, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_3(glue(cvtdq2ps, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_3(glue(cvtdq2pd, SUFFIX), void, env, Reg, Reg)
+
+DEF_HELPER_3(glue(cvtps2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(glue(cvtpd2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
+
+DEF_HELPER_3(glue(cvttps2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(glue(cvttpd2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
+
+#if SHIFT == 1
+DEF_HELPER_4(cvtss2sd, void, env, Reg, Reg, Reg)
+DEF_HELPER_4(cvtsd2ss, void, env, Reg, Reg, Reg)
 DEF_HELPER_3(cvtpi2ps, void, env, ZMMReg, MMXReg)
 DEF_HELPER_3(cvtpi2pd, void, env, ZMMReg, MMXReg)
 DEF_HELPER_3(cvtsi2ss, void, env, ZMMReg, i32)
@@ -164,8 +154,6 @@ DEF_HELPER_3(cvtsq2ss, void, env, ZMMReg, i64)
 DEF_HELPER_3(cvtsq2sd, void, env, ZMMReg, i64)
 #endif
 
-DEF_HELPER_3(cvtps2dq, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(cvtpd2dq, void, env, ZMMReg, ZMMReg)
 DEF_HELPER_3(cvtps2pi, void, env, MMXReg, ZMMReg)
 DEF_HELPER_3(cvtpd2pi, void, env, MMXReg, ZMMReg)
 DEF_HELPER_2(cvtss2si, s32, env, ZMMReg)
@@ -175,8 +163,6 @@ DEF_HELPER_2(cvtss2sq, s64, env, ZMMReg)
 DEF_HELPER_2(cvtsd2sq, s64, env, ZMMReg)
 #endif
 
-DEF_HELPER_3(cvttps2dq, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(cvttpd2dq, void, env, ZMMReg, ZMMReg)
 DEF_HELPER_3(cvttps2pi, void, env, MMXReg, ZMMReg)
 DEF_HELPER_3(cvttpd2pi, void, env, MMXReg, ZMMReg)
 DEF_HELPER_2(cvttss2si, s32, env, ZMMReg)
@@ -185,60 +171,87 @@ DEF_HELPER_2(cvttsd2si, s32, env, ZMMReg)
 DEF_HELPER_2(cvttss2sq, s64, env, ZMMReg)
 DEF_HELPER_2(cvttsd2sq, s64, env, ZMMReg)
 #endif
+#endif
 
-DEF_HELPER_3(rsqrtps, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(rsqrtss, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(rcpps, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(rcpss, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(glue(rsqrtps, SUFFIX), void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(glue(rcpps, SUFFIX), void, env, ZMMReg, ZMMReg)
+
+#if SHIFT == 1
+DEF_HELPER_4(rsqrtss, void, env, ZMMReg, ZMMReg, ZMMReg)
+DEF_HELPER_4(rcpss, void, env, ZMMReg, ZMMReg, ZMMReg)
 DEF_HELPER_3(extrq_r, void, env, ZMMReg, ZMMReg)
 DEF_HELPER_4(extrq_i, void, env, ZMMReg, int, int)
 DEF_HELPER_3(insertq_r, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_4(insertq_i, void, env, ZMMReg, int, int)
-DEF_HELPER_3(haddps, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(haddpd, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(hsubps, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(hsubpd, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(addsubps, void, env, ZMMReg, ZMMReg)
-DEF_HELPER_3(addsubpd, void, env, ZMMReg, ZMMReg)
-
-#define SSE_HELPER_CMP(name, F)                           \
-    DEF_HELPER_3(name ## ps, void, env, Reg, Reg)         \
-    DEF_HELPER_3(name ## ss, void, env, Reg, Reg)         \
-    DEF_HELPER_3(name ## pd, void, env, Reg, Reg)         \
-    DEF_HELPER_3(name ## sd, void, env, Reg, Reg)
-
-SSE_HELPER_CMP(cmpeq, FPU_CMPEQ)
-SSE_HELPER_CMP(cmplt, FPU_CMPLT)
-SSE_HELPER_CMP(cmple, FPU_CMPLE)
-SSE_HELPER_CMP(cmpunord, FPU_CMPUNORD)
-SSE_HELPER_CMP(cmpneq, FPU_CMPNEQ)
-SSE_HELPER_CMP(cmpnlt, FPU_CMPNLT)
-SSE_HELPER_CMP(cmpnle, FPU_CMPNLE)
-SSE_HELPER_CMP(cmpord, FPU_CMPORD)
+DEF_HELPER_5(insertq_i, void, env, ZMMReg, ZMMReg, int, int)
+#endif
+
+SSE_HELPER_P4(hadd)
+SSE_HELPER_P4(hsub)
+SSE_HELPER_P4(addsub)
+
+#define SSE_HELPER_CMP(name, F, C) SSE_HELPER_S4(name)
+
+SSE_HELPER_CMP(cmpeq, FPU_CMPQ, FPU_EQ)
+SSE_HELPER_CMP(cmplt, FPU_CMPS, FPU_LT)
+SSE_HELPER_CMP(cmple, FPU_CMPS, FPU_LE)
+SSE_HELPER_CMP(cmpunord, FPU_CMPQ,  FPU_UNORD)
+SSE_HELPER_CMP(cmpneq, FPU_CMPQ, !FPU_EQ)
+SSE_HELPER_CMP(cmpnlt, FPU_CMPS, !FPU_LT)
+SSE_HELPER_CMP(cmpnle, FPU_CMPS, !FPU_LE)
+SSE_HELPER_CMP(cmpord, FPU_CMPQ, !FPU_UNORD)
+
+SSE_HELPER_CMP(cmpequ, FPU_CMPQ, FPU_EQU)
+SSE_HELPER_CMP(cmpnge, FPU_CMPS, !FPU_GE)
+SSE_HELPER_CMP(cmpngt, FPU_CMPS, !FPU_GT)
+SSE_HELPER_CMP(cmpfalse, FPU_CMPQ,  FPU_FALSE)
+SSE_HELPER_CMP(cmpnequ, FPU_CMPQ, !FPU_EQU)
+SSE_HELPER_CMP(cmpge, FPU_CMPS, FPU_GE)
+SSE_HELPER_CMP(cmpgt, FPU_CMPS, FPU_GT)
+SSE_HELPER_CMP(cmptrue, FPU_CMPQ,  !FPU_FALSE)
+
+SSE_HELPER_CMP(cmpeqs, FPU_CMPS, FPU_EQ)
+SSE_HELPER_CMP(cmpltq, FPU_CMPQ, FPU_LT)
+SSE_HELPER_CMP(cmpleq, FPU_CMPQ, FPU_LE)
+SSE_HELPER_CMP(cmpunords, FPU_CMPS,  FPU_UNORD)
+SSE_HELPER_CMP(cmpneqq, FPU_CMPS, !FPU_EQ)
+SSE_HELPER_CMP(cmpnltq, FPU_CMPQ, !FPU_LT)
+SSE_HELPER_CMP(cmpnleq, FPU_CMPQ, !FPU_LE)
+SSE_HELPER_CMP(cmpords, FPU_CMPS, !FPU_UNORD)
+
+SSE_HELPER_CMP(cmpequs, FPU_CMPS, FPU_EQU)
+SSE_HELPER_CMP(cmpngeq, FPU_CMPQ, !FPU_GE)
+SSE_HELPER_CMP(cmpngtq, FPU_CMPQ, !FPU_GT)
+SSE_HELPER_CMP(cmpfalses, FPU_CMPS,  FPU_FALSE)
+SSE_HELPER_CMP(cmpnequs, FPU_CMPS, !FPU_EQU)
+SSE_HELPER_CMP(cmpgeq, FPU_CMPQ, FPU_GE)
+SSE_HELPER_CMP(cmpgtq, FPU_CMPQ, FPU_GT)
+SSE_HELPER_CMP(cmptrues, FPU_CMPS,  !FPU_FALSE)
 
+#if SHIFT == 1
 DEF_HELPER_3(ucomiss, void, env, Reg, Reg)
 DEF_HELPER_3(comiss, void, env, Reg, Reg)
 DEF_HELPER_3(ucomisd, void, env, Reg, Reg)
 DEF_HELPER_3(comisd, void, env, Reg, Reg)
-DEF_HELPER_2(movmskps, i32, env, Reg)
-DEF_HELPER_2(movmskpd, i32, env, Reg)
 #endif
 
-DEF_HELPER_2(glue(pmovmskb, SUFFIX), i32, env, Reg)
-DEF_HELPER_3(glue(packsswb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(packuswb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(packssdw, SUFFIX), void, env, Reg, Reg)
-#define UNPCK_OP(base_name, base)                                       \
-    DEF_HELPER_3(glue(punpck ## base_name ## bw, SUFFIX), void, env, Reg, Reg) \
-    DEF_HELPER_3(glue(punpck ## base_name ## wd, SUFFIX), void, env, Reg, Reg) \
-    DEF_HELPER_3(glue(punpck ## base_name ## dq, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_2(glue(movmskps, SUFFIX), i32, env, Reg)
+DEF_HELPER_2(glue(movmskpd, SUFFIX), i32, env, Reg)
+#endif
+
+DEF_HELPER_4(glue(packsswb, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(packuswb, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(packssdw, SUFFIX), void, env, Reg, Reg, Reg)
+#define UNPCK_OP(name, base)                                       \
+    DEF_HELPER_4(glue(punpck ## name ## bw, SUFFIX), void, env, Reg, Reg, Reg) \
+    DEF_HELPER_4(glue(punpck ## name ## wd, SUFFIX), void, env, Reg, Reg, Reg) \
+    DEF_HELPER_4(glue(punpck ## name ## dq, SUFFIX), void, env, Reg, Reg, Reg)
 
 UNPCK_OP(l, 0)
 UNPCK_OP(h, 1)
 
-#if SHIFT == 1
-DEF_HELPER_3(glue(punpcklqdq, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(punpckhqdq, SUFFIX), void, env, Reg, Reg)
+#if SHIFT >= 1
+DEF_HELPER_4(glue(punpcklqdq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(punpckhqdq, SUFFIX), void, env, Reg, Reg, Reg)
 #endif
 
 /* 3DNow! float ops */
@@ -265,28 +278,25 @@ DEF_HELPER_3(pswapd, void, env, MMXReg, MMXReg)
 #endif
 
 /* SSSE3 op helpers */
-DEF_HELPER_3(glue(phaddw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(phaddd, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(phaddsw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(phsubw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(phsubd, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(phsubsw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pabsb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pabsw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pabsd, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmaddubsw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmulhrsw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pshufb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psignb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psignw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(psignd, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_4(glue(palignr, SUFFIX), void, env, Reg, Reg, s32)
+DEF_HELPER_4(glue(phaddw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(phaddd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(phaddsw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(phsubw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(phsubd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(phsubsw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(pmaddubsw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(pmulhrsw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(pshufb, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psignb, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psignw, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(psignd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_5(glue(palignr, SUFFIX), void, env, Reg, Reg, Reg, i32)
 
 /* SSE4.1 op helpers */
-#if SHIFT == 1
-DEF_HELPER_3(glue(pblendvb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(blendvps, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(blendvpd, SUFFIX), void, env, Reg, Reg)
+#if SHIFT >= 1
+DEF_HELPER_5(glue(pblendvb, SUFFIX), void, env, Reg, Reg, Reg, Reg)
+DEF_HELPER_5(glue(blendvps, SUFFIX), void, env, Reg, Reg, Reg, Reg)
+DEF_HELPER_5(glue(blendvpd, SUFFIX), void, env, Reg, Reg, Reg, Reg)
 DEF_HELPER_3(glue(ptest, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_3(glue(pmovsxbw, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_3(glue(pmovsxbd, SUFFIX), void, env, Reg, Reg)
@@ -300,34 +310,32 @@ DEF_HELPER_3(glue(pmovzxbq, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_3(glue(pmovzxwd, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_3(glue(pmovzxwq, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_3(glue(pmovzxdq, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmuldq, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pcmpeqq, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(packusdw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pminsb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pminsd, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pminuw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pminud, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmaxsb, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmaxsd, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmaxuw, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmaxud, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(pmulld, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_3(glue(pmovsldup, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_3(glue(pmovshdup, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_3(glue(pmovdldup, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_4(glue(pmuldq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(packusdw, SUFFIX), void, env, Reg, Reg, Reg)
+#if SHIFT == 1
 DEF_HELPER_3(glue(phminposuw, SUFFIX), void, env, Reg, Reg)
+#endif
 DEF_HELPER_4(glue(roundps, SUFFIX), void, env, Reg, Reg, i32)
 DEF_HELPER_4(glue(roundpd, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(roundss, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(roundsd, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(blendps, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(blendpd, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(pblendw, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(dpps, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(dppd, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(mpsadbw, SUFFIX), void, env, Reg, Reg, i32)
+#if SHIFT == 1
+DEF_HELPER_5(roundss_xmm, void, env, Reg, Reg, Reg, i32)
+DEF_HELPER_5(roundsd_xmm, void, env, Reg, Reg, Reg, i32)
+#endif
+DEF_HELPER_5(glue(blendps, SUFFIX), void, env, Reg, Reg, Reg, i32)
+DEF_HELPER_5(glue(blendpd, SUFFIX), void, env, Reg, Reg, Reg, i32)
+DEF_HELPER_5(glue(pblendw, SUFFIX), void, env, Reg, Reg, Reg, i32)
+DEF_HELPER_5(glue(dpps, SUFFIX), void, env, Reg, Reg, Reg, i32)
+#if SHIFT == 1
+DEF_HELPER_5(glue(dppd, SUFFIX), void, env, Reg, Reg, Reg, i32)
+#endif
+DEF_HELPER_5(glue(mpsadbw, SUFFIX), void, env, Reg, Reg, Reg, i32)
 #endif
 
 /* SSE4.2 op helpers */
 #if SHIFT == 1
-DEF_HELPER_3(glue(pcmpgtq, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_4(glue(pcmpestri, SUFFIX), void, env, Reg, Reg, i32)
 DEF_HELPER_4(glue(pcmpestrm, SUFFIX), void, env, Reg, Reg, i32)
 DEF_HELPER_4(glue(pcmpistri, SUFFIX), void, env, Reg, Reg, i32)
@@ -336,14 +344,62 @@ DEF_HELPER_3(crc32, tl, i32, tl, i32)
 #endif
 
 /* AES-NI op helpers */
+#if SHIFT >= 1
+DEF_HELPER_4(glue(aesdec, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(aesdeclast, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(aesenc, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(aesenclast, SUFFIX), void, env, Reg, Reg, Reg)
 #if SHIFT == 1
-DEF_HELPER_3(glue(aesdec, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(aesdeclast, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(aesenc, SUFFIX), void, env, Reg, Reg)
-DEF_HELPER_3(glue(aesenclast, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_3(glue(aesimc, SUFFIX), void, env, Reg, Reg)
 DEF_HELPER_4(glue(aeskeygenassist, SUFFIX), void, env, Reg, Reg, i32)
-DEF_HELPER_4(glue(pclmulqdq, SUFFIX), void, env, Reg, Reg, i32)
+#endif
+DEF_HELPER_5(glue(pclmulqdq, SUFFIX), void, env, Reg, Reg, Reg, i32)
+#endif
+
+/* F16C helpers */
+#if SHIFT >= 1
+DEF_HELPER_3(glue(cvtph2ps, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_4(glue(cvtps2ph, SUFFIX), void, env, Reg, Reg, int)
+#endif
+
+/* FMA3 helpers */
+#if SHIFT == 1
+DEF_HELPER_6(fma4ss, void, env, Reg, Reg, Reg, Reg, int)
+DEF_HELPER_6(fma4sd, void, env, Reg, Reg, Reg, Reg, int)
+#endif
+
+#if SHIFT >= 1
+DEF_HELPER_7(glue(fma4ps, SUFFIX), void, env, Reg, Reg, Reg, Reg, int, int)
+DEF_HELPER_7(glue(fma4pd, SUFFIX), void, env, Reg, Reg, Reg, Reg, int, int)
+#endif
+
+/* AVX helpers */
+#if SHIFT >= 1
+DEF_HELPER_4(glue(vpermilpd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(vpermilps, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_3(glue(vpermilpd_imm, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(vpermilps_imm, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_4(glue(vpsrlvd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(vpsravd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(vpsllvd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(vpsrlvq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(vpsravq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(vpsllvq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_3(glue(vtestps, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_3(glue(vtestpd, SUFFIX), void, env, Reg, Reg)
+DEF_HELPER_4(glue(vpmaskmovd_st, SUFFIX), void, env, Reg, Reg, tl)
+DEF_HELPER_4(glue(vpmaskmovq_st, SUFFIX), void, env, Reg, Reg, tl)
+DEF_HELPER_4(glue(vpmaskmovd, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_4(glue(vpmaskmovq, SUFFIX), void, env, Reg, Reg, Reg)
+DEF_HELPER_6(glue(vpgatherdd, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
+DEF_HELPER_6(glue(vpgatherdq, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
+DEF_HELPER_6(glue(vpgatherqd, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
+DEF_HELPER_6(glue(vpgatherqq, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
+#if SHIFT == 2
+DEF_HELPER_3(vpermd_ymm, void, Reg, Reg, Reg)
+DEF_HELPER_4(vpermdq_ymm, void, Reg, Reg, Reg, i32)
+DEF_HELPER_3(vpermq_ymm, void, Reg, Reg, i32)
+#endif
 #endif
 
 #undef SHIFT
@@ -354,6 +410,9 @@ DEF_HELPER_4(glue(pclmulqdq, SUFFIX), void, env, Reg, Reg, i32)
 #undef SSE_HELPER_W
 #undef SSE_HELPER_L
 #undef SSE_HELPER_Q
-#undef SSE_HELPER_S
+#undef SSE_HELPER_S3
+#undef SSE_HELPER_S4
+#undef SSE_HELPER_P3
+#undef SSE_HELPER_P4
 #undef SSE_HELPER_CMP
 #undef UNPCK_OP
diff --git a/qemu/target/i386/seg_helper.c b/qemu/target/i386/seg_helper.c
index 97447fee38..ffcb551326 100644
--- a/qemu/target/i386/seg_helper.c
+++ b/qemu/target/i386/seg_helper.c
@@ -2647,3 +2647,8 @@ void helper_check_iol(CPUX86State *env, uint32_t t0)
 {
     check_io(env, t0, 4, GETPC());
 }
+
+void helper_check_io(CPUX86State *env, uint32_t addr, uint32_t size)
+{
+    check_io(env, addr, size, GETPC());
+}
diff --git a/qemu/target/i386/seg_helper.h b/qemu/target/i386/seg_helper.h
new file mode 100644
index 0000000000..ebf1035277
--- /dev/null
+++ b/qemu/target/i386/seg_helper.h
@@ -0,0 +1,66 @@
+/*
+ *  x86 segmentation related helpers macros
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef SEG_HELPER_H
+#define SEG_HELPER_H
+
+//#define DEBUG_PCALL
+
+#ifdef DEBUG_PCALL
+# define LOG_PCALL(...) qemu_log_mask(CPU_LOG_PCALL, ## __VA_ARGS__)
+# define LOG_PCALL_STATE(cpu)                                  \
+    log_cpu_state_mask(CPU_LOG_PCALL, (cpu), CPU_DUMP_CCOP)
+#else
+# define LOG_PCALL(...) do { } while (0)
+# define LOG_PCALL_STATE(cpu) do { } while (0)
+#endif
+
+/*
+ * TODO: Convert callers to compute cpu_mmu_index_kernel once
+ * and use *_mmuidx_ra directly.
+ */
+#define cpu_ldub_kernel_ra(e, p, r) \
+    cpu_ldub_mmuidx_ra(e, p, cpu_mmu_index_kernel(e), r)
+#define cpu_lduw_kernel_ra(e, p, r) \
+    cpu_lduw_mmuidx_ra(e, p, cpu_mmu_index_kernel(e), r)
+#define cpu_ldl_kernel_ra(e, p, r) \
+    cpu_ldl_mmuidx_ra(e, p, cpu_mmu_index_kernel(e), r)
+#define cpu_ldq_kernel_ra(e, p, r) \
+    cpu_ldq_mmuidx_ra(e, p, cpu_mmu_index_kernel(e), r)
+
+#define cpu_stb_kernel_ra(e, p, v, r) \
+    cpu_stb_mmuidx_ra(e, p, v, cpu_mmu_index_kernel(e), r)
+#define cpu_stw_kernel_ra(e, p, v, r) \
+    cpu_stw_mmuidx_ra(e, p, v, cpu_mmu_index_kernel(e), r)
+#define cpu_stl_kernel_ra(e, p, v, r) \
+    cpu_stl_mmuidx_ra(e, p, v, cpu_mmu_index_kernel(e), r)
+#define cpu_stq_kernel_ra(e, p, v, r) \
+    cpu_stq_mmuidx_ra(e, p, v, cpu_mmu_index_kernel(e), r)
+
+#define cpu_ldub_kernel(e, p)    cpu_ldub_kernel_ra(e, p, 0)
+#define cpu_lduw_kernel(e, p)    cpu_lduw_kernel_ra(e, p, 0)
+#define cpu_ldl_kernel(e, p)     cpu_ldl_kernel_ra(e, p, 0)
+#define cpu_ldq_kernel(e, p)     cpu_ldq_kernel_ra(e, p, 0)
+
+#define cpu_stb_kernel(e, p, v)  cpu_stb_kernel_ra(e, p, v, 0)
+#define cpu_stw_kernel(e, p, v)  cpu_stw_kernel_ra(e, p, v, 0)
+#define cpu_stl_kernel(e, p, v)  cpu_stl_kernel_ra(e, p, v, 0)
+#define cpu_stq_kernel(e, p, v)  cpu_stq_kernel_ra(e, p, v, 0)
+
+#endif /* SEG_HELPER_H */
diff --git a/qemu/target/i386/svm_helper.c b/qemu/target/i386/svm_helper.c
index ade26593a3..a65d58ed39 100644
--- a/qemu/target/i386/svm_helper.c
+++ b/qemu/target/i386/svm_helper.c
@@ -513,6 +513,11 @@ void helper_svm_check_intercept_param(CPUX86State *env, uint32_t type,
     cpu_svm_check_intercept_param(env, type, param, GETPC());
 }
 
+void helper_svm_check_intercept(CPUX86State *env, uint32_t type)
+{
+    cpu_svm_check_intercept_param(env, type, 0, GETPC());
+}
+
 void helper_svm_check_io(CPUX86State *env, uint32_t port, uint32_t param,
                          uint32_t next_eip_addend)
 {
diff --git a/qemu/target/i386/translate.c b/qemu/target/i386/translate.c
index f410803e12..75a6c68507 100644
--- a/qemu/target/i386/translate.c
+++ b/qemu/target/i386/translate.c
@@ -6,7 +6,7 @@
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
+ * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -22,8 +22,10 @@
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "tcg/tcg-op.h"
+#include "tcg/tcg-op-gvec.h"
 #include "exec/cpu_ldst.h"
 #include "exec/translator.h"
+#include "fpu/softfloat.h"
 
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
@@ -38,16 +40,7 @@
 #define PREFIX_DATA   0x08
 #define PREFIX_ADR    0x10
 #define PREFIX_VEX    0x20
-
-#ifdef TARGET_X86_64
-#define CODE64(s) ((s)->code64)
-#define REX_X(s) ((s)->rex_x)
-#define REX_B(s) ((s)->rex_b)
-#else
-#define CODE64(s) 0
-#define REX_X(s) 0
-#define REX_B(s) 0
-#endif
+#define PREFIX_REX    0x40
 
 #ifdef TARGET_X86_64
 # define ctztl  ctz64
@@ -57,6 +50,10 @@
 # define clztl  clz32
 #endif
 
+#ifndef TARGET_TB_PCREL
+# define TARGET_TB_PCREL 0
+#endif
+
 /* For a switch indexed by MODRM, match all memory operands for a given OP.  */
 #define CASE_MODRM_MEM_OP(OP) \
     case (0 << 6) | (OP << 3) | 0: \
@@ -118,54 +115,69 @@
     case (3 << 6) | (OP << 3) | 6: \
     case (3 << 6) | (OP << 3) | 7
 
+#if 0
+#define MACRO_TEST   1
+#endif
+
 #include "exec/gen-icount.h"
 
+#define cpu_cc_dst     (tcg_ctx->cpu_cc_dst)
+#define cpu_cc_src     (tcg_ctx->cpu_cc_src)
+#define cpu_cc_src2    (tcg_ctx->cpu_cc_src2)
+#define cpu_eip        (tcg_ctx->cpu_eip)
+#define cpu_cc_op      (tcg_ctx->cpu_cc_op)
+#define cpu_regs       (tcg_ctx->cpu_regs)
+#define cpu_seg_base   (tcg_ctx->cpu_seg_base)
+#define cpu_bndl       (tcg_ctx->cpu_bndl)
+#define cpu_bndu       (tcg_ctx->cpu_bndu)
+#define cpu_env        (tcg_ctx->cpu_env)
+
 typedef struct DisasContext {
     DisasContextBase base;
 
-    /* current insn context */
-    int override; /* -1 if no override */
-    int prefix;
+    target_ulong pc;       /* pc = eip + cs_base */
+    target_ulong cs_base;  /* base of CS segment */
+    target_ulong pc_save;
+
     MemOp aflag;
     MemOp dflag;
-    target_ulong pc_start;
-    target_ulong pc; /* pc = eip + cs_base */
-    /* current block context */
-    target_ulong cs_base; /* base of CS segment */
-    int pe;     /* protected mode */
-    int code32; /* 32 bit code segment */
-#ifdef TARGET_X86_64
-    int lma;    /* long mode active */
-    int code64; /* 64 bit code segment */
-    int rex_x, rex_b;
+
+    int8_t override; /* -1 if no override, else R_CS, R_DS, etc */
+    uint8_t prefix;
+
+    bool has_modrm;
+    uint8_t modrm;
+
+#ifndef CONFIG_USER_ONLY
+    uint8_t cpl;   /* code priv level */
+    uint8_t iopl;  /* i/o priv level */
 #endif
-    int vex_l;  /* vex vector length */
-    int vex_v;  /* vex vvvv register, without 1's complement.  */
-    int ss32;   /* 32 bit stack segment */
-    CCOp cc_op;  /* current CC operation */
-    CCOp last_cc_op;  /* Unicorn: last CC operation. Save this to see if cc_op has changed */
-    bool cc_op_dirty;
+    uint8_t vex_l;  /* vex vector length */
+    uint8_t vex_v;  /* vex vvvv register, without 1's complement.  */
+    uint8_t popl_esp_hack; /* for correct popl with esp base handling */
+    uint8_t rip_offset; /* only used in x86_64, but left for simplicity */
+
 #ifdef TARGET_X86_64
-    bool x86_64_hregs;
+    uint8_t rex_r;
+    uint8_t rex_x;
+    uint8_t rex_b;
 #endif
-    int addseg; /* non zero if either DS/ES/SS have a non zero base */
-    int f_st;   /* currently unused */
-    int vm86;   /* vm86 mode */
-    int cpl;
-    int iopl;
-    int tf;     /* TF cpu flag */
-    int jmp_opt; /* use direct block chaining for direct jumps */
-    int repz_opt; /* optimize jumps within repz instructions */
+    bool vex_w; /* used by AVX even on 32-bit processors */
+    bool jmp_opt; /* use direct block chaining for direct jumps */
+    bool repz_opt; /* optimize jumps within repz instructions */
+    bool cc_op_dirty;
+
+    CCOp cc_op;  /* current CC operation */
     int mem_index; /* select memory access functions */
-    uint64_t flags; /* all execution flags */
-    int popl_esp_hack; /* for correct popl with esp base handling */
-    int rip_offset; /* only used in x86_64, but left for simplicity */
+    uint32_t flags; /* all execution flags */
     int cpuid_features;
     int cpuid_ext_features;
     int cpuid_ext2_features;
     int cpuid_ext3_features;
     int cpuid_7_0_ebx_features;
+    int cpuid_7_0_ecx_features;
     int cpuid_xsave_features;
+    struct uc_struct *uc;
 
     /* TCG local temps */
     TCGv cc_srcT;
@@ -176,24 +188,124 @@ typedef struct DisasContext {
     /* TCG local register indexes (only used inside old micro ops) */
     TCGv tmp0;
     TCGv tmp4;
-    TCGv_ptr ptr0;
-    TCGv_ptr ptr1;
     TCGv_i32 tmp2_i32;
     TCGv_i32 tmp3_i32;
     TCGv_i64 tmp1_i64;
 
     sigjmp_buf jmpbuf;
-
-    // Unicorn
-    struct uc_struct *uc;
-    target_ulong prev_pc; /* save address of the previous instruction */
+    TCGOp *prev_insn_end;
+    target_ulong prev_pc;
 } DisasContext;
 
+#define DISAS_EOB_ONLY         DISAS_TARGET_0
+#define DISAS_EOB_NEXT         DISAS_TARGET_1
+#define DISAS_EOB_INHIBIT_IRQ  DISAS_TARGET_2
+#define DISAS_JUMP             DISAS_TARGET_3
+
+/* The environment in which user-only runs is constrained. */
+#ifdef CONFIG_USER_ONLY
+#define PE(S)     true
+#define CPL(S)    3
+#define IOPL(S)   0
+#define SVME(S)   false
+#define GUEST(S)  false
+#else
+#define PE(S)     (((S)->flags & HF_PE_MASK) != 0)
+#define CPL(S)    ((S)->cpl)
+#define IOPL(S)   ((S)->iopl)
+#define SVME(S)   (((S)->flags & HF_SVME_MASK) != 0)
+#define GUEST(S)  (((S)->flags & HF_GUEST_MASK) != 0)
+#endif
+#if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
+#define VM86(S)   false
+#define CODE32(S) true
+#define SS32(S)   true
+#define ADDSEG(S) false
+#else
+#define VM86(S)   (((S)->flags & HF_VM_MASK) != 0)
+#define CODE32(S) (((S)->flags & HF_CS32_MASK) != 0)
+#define SS32(S)   (((S)->flags & HF_SS32_MASK) != 0)
+#define ADDSEG(S) (((S)->flags & HF_ADDSEG_MASK) != 0)
+#endif
+#if !defined(TARGET_X86_64)
+#define CODE64(S) false
+#define LMA(S)    false
+#elif defined(CONFIG_USER_ONLY)
+#define CODE64(S) true
+#define LMA(S)    true
+#else
+#define CODE64(S) (((S)->flags & HF_CS64_MASK) != 0)
+#define LMA(S)    (((S)->flags & HF_LMA_MASK) != 0)
+#endif
+
+static void clear_rex_prefix(DisasContext *s, int *prefixes)
+{
+#ifdef TARGET_X86_64
+    if (CODE64(s)) {
+        *prefixes &= ~PREFIX_REX;
+        s->vex_w = 0;
+        s->rex_r = 0;
+        s->rex_x = 0;
+        s->rex_b = 0;
+    }
+#endif
+}
+
+#ifdef TARGET_X86_64
+#define REX_PREFIX(S)  (((S)->prefix & PREFIX_REX) != 0)
+#define REX_W(S)       ((S)->vex_w)
+#define REX_R(S)       ((S)->rex_r + 0)
+#define REX_X(S)       ((S)->rex_x + 0)
+#define REX_B(S)       ((S)->rex_b + 0)
+#else
+#define REX_PREFIX(S)  false
+#define REX_W(S)       false
+#define REX_R(S)       0
+#define REX_X(S)       0
+#define REX_B(S)       0
+#endif
+
+/*
+ * Many sysemu-only helpers are not reachable for user-only.
+ * Define stub generators here, so that we need not either sprinkle
+ * ifdefs through the translator, nor provide the helper function.
+ */
+#define STUB_HELPER(NAME, ...) \
+    static inline void gen_helper_##NAME(__VA_ARGS__) \
+    { qemu_build_not_reached(); }
+
+#ifdef CONFIG_USER_ONLY
+STUB_HELPER(clgi, TCGv_env env)
+STUB_HELPER(flush_page, TCGv_env env, TCGv addr)
+STUB_HELPER(hlt, TCGv_env env, TCGv_i32 pc_ofs)
+STUB_HELPER(inb, TCGv ret, TCGv_env env, TCGv_i32 port)
+STUB_HELPER(inw, TCGv ret, TCGv_env env, TCGv_i32 port)
+STUB_HELPER(inl, TCGv ret, TCGv_env env, TCGv_i32 port)
+STUB_HELPER(monitor, TCGv_env env, TCGv addr)
+STUB_HELPER(mwait, TCGv_env env, TCGv_i32 pc_ofs)
+STUB_HELPER(outb, TCGv_env env, TCGv_i32 port, TCGv_i32 val)
+STUB_HELPER(outw, TCGv_env env, TCGv_i32 port, TCGv_i32 val)
+STUB_HELPER(outl, TCGv_env env, TCGv_i32 port, TCGv_i32 val)
+STUB_HELPER(rdmsr, TCGv_env env)
+STUB_HELPER(read_crN, TCGv ret, TCGv_env env, TCGv_i32 reg)
+STUB_HELPER(get_dr, TCGv ret, TCGv_env env, TCGv_i32 reg)
+STUB_HELPER(set_dr, TCGv_env env, TCGv_i32 reg, TCGv val)
+STUB_HELPER(stgi, TCGv_env env)
+STUB_HELPER(svm_check_intercept, TCGv_env env, TCGv_i32 type)
+STUB_HELPER(vmload, TCGv_env env, TCGv_i32 aflag)
+STUB_HELPER(vmmcall, TCGv_env env)
+STUB_HELPER(vmrun, TCGv_env env, TCGv_i32 aflag, TCGv_i32 pc_ofs)
+STUB_HELPER(vmsave, TCGv_env env, TCGv_i32 aflag)
+STUB_HELPER(write_crN, TCGv_env env, TCGv_i32 reg, TCGv val)
+STUB_HELPER(wrmsr, TCGv_env env)
+#endif
+
 static void gen_eob(DisasContext *s);
-static void gen_jr(DisasContext *s, TCGv dest);
-static void gen_jmp(DisasContext *s, target_ulong eip);
-static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num);
-static void gen_op(DisasContext *s, int op, MemOp ot, int d);
+static void gen_jr(DisasContext *s);
+static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num);
+static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num);
+static void gen_op(DisasContext *s1, int op, MemOp ot, int d);
+static void gen_exception_gpf(DisasContext *s);
 
 /* i386 arith/logic operations */
 enum {
@@ -253,71 +365,28 @@ enum {
     USES_CC_SRCT = 8,
 };
 
+#define CC_OP_LIVE_BWLQ(OP, LIVE) \
+    [CC_OP_##OP##B] = LIVE,       \
+    [CC_OP_##OP##W] = LIVE,       \
+    [CC_OP_##OP##L] = LIVE,       \
+    [CC_OP_##OP##Q] = LIVE
+
 /* Bit set if the global variable is live after setting CC_OP to X.  */
 static const uint8_t cc_op_live[CC_OP_NB] = {
     [CC_OP_DYNAMIC] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
     [CC_OP_EFLAGS] = USES_CC_SRC,
-
-    [CC_OP_MULB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_MULW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_MULL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_MULQ] = USES_CC_DST | USES_CC_SRC,
-
-    [CC_OP_ADDB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_ADDW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_ADDL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_ADDQ] = USES_CC_DST | USES_CC_SRC,
-
-    [CC_OP_ADCB] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-    [CC_OP_ADCW] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-    [CC_OP_ADCL] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-    [CC_OP_ADCQ] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-
-    [CC_OP_SUBB] = USES_CC_DST | USES_CC_SRC | USES_CC_SRCT,
-    [CC_OP_SUBW] = USES_CC_DST | USES_CC_SRC | USES_CC_SRCT,
-    [CC_OP_SUBL] = USES_CC_DST | USES_CC_SRC | USES_CC_SRCT,
-    [CC_OP_SUBQ] = USES_CC_DST | USES_CC_SRC | USES_CC_SRCT,
-
-    [CC_OP_SBBB] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-    [CC_OP_SBBW] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-    [CC_OP_SBBL] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-    [CC_OP_SBBQ] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
-
-    [CC_OP_LOGICB] = USES_CC_DST,
-    [CC_OP_LOGICW] = USES_CC_DST,
-    [CC_OP_LOGICL] = USES_CC_DST,
-    [CC_OP_LOGICQ] = USES_CC_DST,
-
-    [CC_OP_INCB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_INCW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_INCL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_INCQ] = USES_CC_DST | USES_CC_SRC,
-
-    [CC_OP_DECB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_DECW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_DECL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_DECQ] = USES_CC_DST | USES_CC_SRC,
-
-    [CC_OP_SHLB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_SHLW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_SHLL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_SHLQ] = USES_CC_DST | USES_CC_SRC,
-
-    [CC_OP_SARB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_SARW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_SARL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_SARQ] = USES_CC_DST | USES_CC_SRC,
-
-    [CC_OP_BMILGB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_BMILGW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_BMILGL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_BMILGQ] = USES_CC_DST | USES_CC_SRC,
-
-    [CC_OP_BLSIB] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_BLSIW] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_BLSIL] = USES_CC_DST | USES_CC_SRC,
-    [CC_OP_BLSIQ] = USES_CC_DST | USES_CC_SRC,
-
+    CC_OP_LIVE_BWLQ(MUL, USES_CC_DST | USES_CC_SRC),
+    CC_OP_LIVE_BWLQ(ADD, USES_CC_DST | USES_CC_SRC),
+    CC_OP_LIVE_BWLQ(ADC, USES_CC_DST | USES_CC_SRC | USES_CC_SRC2),
+    CC_OP_LIVE_BWLQ(SUB, USES_CC_DST | USES_CC_SRC | USES_CC_SRCT),
+    CC_OP_LIVE_BWLQ(SBB, USES_CC_DST | USES_CC_SRC | USES_CC_SRC2),
+    CC_OP_LIVE_BWLQ(LOGIC, USES_CC_DST),
+    CC_OP_LIVE_BWLQ(INC, USES_CC_DST | USES_CC_SRC),
+    CC_OP_LIVE_BWLQ(DEC, USES_CC_DST | USES_CC_SRC),
+    CC_OP_LIVE_BWLQ(SHL, USES_CC_DST | USES_CC_SRC),
+    CC_OP_LIVE_BWLQ(SAR, USES_CC_DST | USES_CC_SRC),
+    CC_OP_LIVE_BWLQ(BMILG, USES_CC_DST | USES_CC_SRC),
+    CC_OP_LIVE_BWLQ(BLSI, USES_CC_DST | USES_CC_SRC),
     [CC_OP_ADCX] = USES_CC_DST | USES_CC_SRC,
     [CC_OP_ADOX] = USES_CC_SRC | USES_CC_SRC2,
     [CC_OP_ADCOX] = USES_CC_DST | USES_CC_SRC | USES_CC_SRC2,
@@ -325,16 +394,12 @@ static const uint8_t cc_op_live[CC_OP_NB] = {
     [CC_OP_POPCNT] = USES_CC_SRC,
 };
 
-static inline void gen_jmp_im(DisasContext *s, target_ulong pc);
+#undef CC_OP_LIVE_BWLQ
 
 static void set_cc_op(DisasContext *s, CCOp op)
 {
-    int dead;
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv_i32 cpu_cc_op = tcg_ctx->cpu_cc_op;
-    TCGv cpu_cc_dst = tcg_ctx->cpu_cc_dst;
-    TCGv cpu_cc_src = tcg_ctx->cpu_cc_src;
-    TCGv cpu_cc_src2 = tcg_ctx->cpu_cc_src2;
+    int dead;
 
     if (s->cc_op == op) {
         return;
@@ -371,10 +436,8 @@ static void set_cc_op(DisasContext *s, CCOp op)
 
 static void gen_update_cc_op(DisasContext *s)
 {
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
     if (s->cc_op_dirty) {
-        TCGContext *tcg_ctx = s->uc->tcg_ctx;
-        TCGv_i32 cpu_cc_op = tcg_ctx->cpu_cc_op;
-
         tcg_gen_movi_i32(tcg_ctx, cpu_cc_op, s->cc_op);
         s->cc_op_dirty = false;
     }
@@ -390,7 +453,7 @@ static void gen_update_cc_op(DisasContext *s)
 
 #endif /* !TARGET_X86_64 */
 
-#if defined(HOST_WORDS_BIGENDIAN)
+#if HOST_BIG_ENDIAN
 #define REG_B_OFFSET (sizeof(target_ulong) - 1)
 #define REG_H_OFFSET (sizeof(target_ulong) - 2)
 #define REG_W_OFFSET (sizeof(target_ulong) - 2)
@@ -412,14 +475,10 @@ static void gen_update_cc_op(DisasContext *s)
  */
 static inline bool byte_reg_is_xH(DisasContext *s, int reg)
 {
-    if (reg < 4) {
-        return false;
-    }
-#ifdef TARGET_X86_64
-    if (reg >= 8 || s->x86_64_hregs) {
+    /* Any time the REX prefix is present, byte registers are uniform */
+    if (reg < 4 || REX_PREFIX(s)) {
         return false;
     }
-#endif
     return true;
 }
 
@@ -436,7 +495,7 @@ static inline MemOp mo_pushpop(DisasContext *s, MemOp ot)
 /* Select the size of the stack pointer.  */
 static inline MemOp mo_stacksize(DisasContext *s)
 {
-    return CODE64(s) ? MO_64 : s->ss32 ? MO_32 : MO_16;
+    return CODE64(s) ? MO_64 : SS32(s) ? MO_32 : MO_16;
 }
 
 /* Select only size 64 else 32.  Used for SSE operand sizes.  */
@@ -463,132 +522,105 @@ static inline MemOp mo_b_d32(int b, MemOp ot)
     return b & 1 ? (ot == MO_16 ? MO_16 : MO_32) : MO_8;
 }
 
-static void gen_op_mov_reg_v(DisasContext *s, MemOp ot, int reg, TCGv t0)
+/* Compute the result of writing t0 to the OT-sized register REG.
+ *
+ * If DEST is NULL, store the result into the register and return the
+ * register's TCGv.
+ *
+ * If DEST is not NULL, store the result into DEST and return the
+ * register's TCGv.
+ */
+static TCGv gen_op_deposit_reg_v(DisasContext *s, MemOp ot, int reg, TCGv dest, TCGv t0)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     switch(ot) {
     case MO_8:
-        if (!byte_reg_is_xH(s, reg)) {
-            tcg_gen_deposit_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], tcg_ctx->cpu_regs[reg], t0, 0, 8);
-        } else {
-            tcg_gen_deposit_tl(tcg_ctx, tcg_ctx->cpu_regs[reg - 4], tcg_ctx->cpu_regs[reg - 4], t0, 8, 8);
+        if (byte_reg_is_xH(s, reg)) {
+            dest = dest ? dest : cpu_regs[reg - 4];
+            tcg_gen_deposit_tl(tcg_ctx, dest, cpu_regs[reg - 4], t0, 8, 8);
+            return cpu_regs[reg - 4];
         }
+        dest = dest ? dest : cpu_regs[reg];
+        tcg_gen_deposit_tl(tcg_ctx, dest, cpu_regs[reg], t0, 0, 8);
         break;
     case MO_16:
-        tcg_gen_deposit_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], tcg_ctx->cpu_regs[reg], t0, 0, 16);
+        dest = dest ? dest : cpu_regs[reg];
+        tcg_gen_deposit_tl(tcg_ctx, dest, cpu_regs[reg], t0, 0, 16);
         break;
     case MO_32:
         /* For x86_64, this sets the higher half of register to zero.
            For i386, this is equivalent to a mov. */
-        tcg_gen_ext32u_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], t0);
+        dest = dest ? dest : cpu_regs[reg];
+        tcg_gen_ext32u_tl(tcg_ctx, dest, t0);
         break;
 #ifdef TARGET_X86_64
     case MO_64:
-        tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], t0);
+        dest = dest ? dest : cpu_regs[reg];
+        tcg_gen_mov_tl(tcg_ctx, dest, t0);
         break;
 #endif
     default:
         tcg_abort();
     }
+    return cpu_regs[reg];
 }
 
-static void gen_op_update_cmpxchg_acc(DisasContext *s, MemOp ot, TCGv oldv,
-                                      TCGv cmpv)
+static void gen_op_mov_reg_v(DisasContext *s, MemOp ot, int reg, TCGv t0)
 {
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv new_acc = tcg_temp_new(tcg_ctx);
-
-    switch(ot) {
-    case MO_8:
-        tcg_gen_deposit_tl(tcg_ctx, new_acc, tcg_ctx->cpu_regs[R_EAX],
-                           oldv, 0, 8);
-        break;
-    case MO_16:
-        tcg_gen_deposit_tl(tcg_ctx, new_acc, tcg_ctx->cpu_regs[R_EAX],
-                           oldv, 0, 16);
-        break;
-    case MO_32:
-        tcg_gen_ext32u_tl(tcg_ctx, new_acc, oldv);
-        break;
-#ifdef TARGET_X86_64
-    case MO_64:
-        tcg_gen_mov_tl(tcg_ctx, new_acc, oldv);
-        break;
-#endif
-    default:
-        tcg_abort();
-    }
-
-    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, tcg_ctx->cpu_regs[R_EAX],
-                       oldv, cmpv, tcg_ctx->cpu_regs[R_EAX], new_acc);
-    tcg_temp_free(tcg_ctx, new_acc);
+    gen_op_deposit_reg_v(s, ot, reg, NULL, t0);
 }
 
 static inline
 void gen_op_mov_v_reg(DisasContext *s, MemOp ot, TCGv t0, int reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     if (ot == MO_8 && byte_reg_is_xH(s, reg)) {
-        tcg_gen_extract_tl(tcg_ctx, t0, tcg_ctx->cpu_regs[reg - 4], 8, 8);
+        tcg_gen_extract_tl(tcg_ctx, t0, cpu_regs[reg - 4], 8, 8);
     } else {
-        tcg_gen_mov_tl(tcg_ctx, t0, tcg_ctx->cpu_regs[reg]);
+        tcg_gen_mov_tl(tcg_ctx, t0, cpu_regs[reg]);
     }
 }
 
 static void gen_add_A0_im(DisasContext *s, int val)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     tcg_gen_addi_tl(tcg_ctx, s->A0, s->A0, val);
     if (!CODE64(s)) {
         tcg_gen_ext32u_tl(tcg_ctx, s->A0, s->A0);
     }
 }
 
-static inline void gen_op_jmp_v(TCGContext *tcg_ctx, TCGv dest)
+static inline void gen_op_jmp_v(DisasContext *s, TCGv dest)
 {
-    tcg_gen_st_tl(tcg_ctx, dest, tcg_ctx->cpu_env, offsetof(CPUX86State, eip));
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_mov_tl(tcg_ctx, cpu_eip, dest);
+    s->pc_save = -1;
 }
 
 static inline
 void gen_op_add_reg_im(DisasContext *s, MemOp size, int reg, int32_t val)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_addi_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_regs[reg], val);
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, cpu_regs[reg], val);
     gen_op_mov_reg_v(s, size, reg, s->tmp0);
 }
 
 static inline void gen_op_add_reg_T0(DisasContext *s, MemOp size, int reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_add_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_regs[reg], s->T0);
+    tcg_gen_add_tl(tcg_ctx, s->tmp0, cpu_regs[reg], s->T0);
     gen_op_mov_reg_v(s, size, reg, s->tmp0);
 }
 
-static inline void gen_sync_pc(TCGContext *ctx, uint64_t pc) {
-    TCGv v = tcg_temp_new(ctx);
-
-    tcg_gen_movi_tl(ctx, v, pc);
-    gen_op_jmp_v(ctx, v);
-
-    tcg_temp_free(ctx, v);
-}
-
 static inline void gen_op_ld_v(DisasContext *s, int idx, TCGv t0, TCGv a0)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     tcg_gen_qemu_ld_tl(tcg_ctx, t0, a0, s->mem_index, idx | MO_LE);
 }
 
 static inline void gen_op_st_v(DisasContext *s, int idx, TCGv t0, TCGv a0)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     tcg_gen_qemu_st_tl(tcg_ctx, t0, a0, s->mem_index, idx | MO_LE);
 }
 
@@ -601,12 +633,98 @@ static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d)
     }
 }
 
-static inline void gen_jmp_im(DisasContext *s, target_ulong pc)
+static void gen_update_eip_cur(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(s->pc_save != -1);
+    if (TARGET_TB_PCREL) {
+        tcg_gen_addi_tl(tcg_ctx, cpu_eip, cpu_eip, s->base.pc_next - s->pc_save);
+    } else if (CODE64(s)) {
+        tcg_gen_movi_tl(tcg_ctx, cpu_eip, s->base.pc_next);
+    } else {
+        tcg_gen_movi_tl(tcg_ctx, cpu_eip, (uint32_t)(s->base.pc_next - s->cs_base));
+    }
+    s->pc_save = s->base.pc_next;
+}
+
+static void gen_update_eip_next(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(s->pc_save != -1);
+    if (TARGET_TB_PCREL) {
+        tcg_gen_addi_tl(tcg_ctx, cpu_eip, cpu_eip, s->pc - s->pc_save);
+    } else if (CODE64(s)) {
+        tcg_gen_movi_tl(tcg_ctx, cpu_eip, s->pc);
+    } else {
+        tcg_gen_movi_tl(tcg_ctx, cpu_eip, (uint32_t)(s->pc - s->cs_base));
+    }
+    s->pc_save = s->pc;
+}
+
+static int cur_insn_len(DisasContext *s)
+{
+    return s->pc - s->base.pc_next;
+}
+
+static TCGv_i32 cur_insn_len_i32(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    return tcg_constant_i32(tcg_ctx, cur_insn_len(s));
+}
+
+static TCGv_i32 eip_next_i32(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(s->pc_save != -1);
+    /*
+     * This function has two users: lcall_real (always 16-bit mode), and
+     * iret_protected (16, 32, or 64-bit mode).  IRET only uses the value
+     * when EFLAGS.NT is set, which is illegal in 64-bit mode, which is
+     * why passing a 32-bit value isn't broken.  To avoid using this where
+     * we shouldn't, return -1 in 64-bit mode so that execution goes into
+     * the weeds quickly.
+     */
+    if (CODE64(s)) {
+        return tcg_constant_i32(tcg_ctx, -1);
+    }
+    if (TARGET_TB_PCREL) {
+        TCGv_i32 ret = tcg_temp_new_i32(tcg_ctx);
+        tcg_gen_trunc_tl_i32(tcg_ctx, ret, cpu_eip);
+        tcg_gen_addi_i32(tcg_ctx, ret, ret, s->pc - s->pc_save);
+        return ret;
+    } else {
+        return tcg_constant_i32(tcg_ctx, s->pc - s->cs_base);
+    }
+}
+
+static TCGv eip_next_tl(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(s->pc_save != -1);
+    if (TARGET_TB_PCREL) {
+        TCGv ret = tcg_temp_new(tcg_ctx);
+        tcg_gen_addi_tl(tcg_ctx, ret, cpu_eip, s->pc - s->pc_save);
+        return ret;
+    } else if (CODE64(s)) {
+        return tcg_constant_tl(tcg_ctx, s->pc);
+    } else {
+        return tcg_constant_tl(tcg_ctx, (uint32_t)(s->pc - s->cs_base));
+    }
+}
 
-    tcg_gen_movi_tl(tcg_ctx, s->tmp0, pc);
-    gen_op_jmp_v(tcg_ctx, s->tmp0);
+static TCGv eip_cur_tl(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    assert(s->pc_save != -1);
+    if (TARGET_TB_PCREL) {
+        TCGv ret = tcg_temp_new(tcg_ctx);
+        tcg_gen_addi_tl(tcg_ctx, ret, cpu_eip, s->base.pc_next - s->pc_save);
+        return ret;
+    } else if (CODE64(s)) {
+        return tcg_constant_tl(tcg_ctx, s->base.pc_next);
+    } else {
+        return tcg_constant_tl(tcg_ctx, (uint32_t)(s->base.pc_next - s->cs_base));
+    }
 }
 
 /* Compute SEG:REG into A0.  SEG is selected from the override segment
@@ -616,7 +734,6 @@ static void gen_lea_v_seg(DisasContext *s, MemOp aflag, TCGv a0,
                           int def_seg, int ovr_seg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     switch (aflag) {
 #ifdef TARGET_X86_64
     case MO_64:
@@ -628,7 +745,7 @@ static void gen_lea_v_seg(DisasContext *s, MemOp aflag, TCGv a0,
 #endif
     case MO_32:
         /* 32 bit address */
-        if (ovr_seg < 0 && s->addseg) {
+        if (ovr_seg < 0 && ADDSEG(s)) {
             ovr_seg = def_seg;
         }
         if (ovr_seg < 0) {
@@ -641,7 +758,7 @@ static void gen_lea_v_seg(DisasContext *s, MemOp aflag, TCGv a0,
         tcg_gen_ext16u_tl(tcg_ctx, s->A0, a0);
         a0 = s->A0;
         if (ovr_seg < 0) {
-            if (s->addseg) {
+            if (ADDSEG(s)) {
                 ovr_seg = def_seg;
             } else {
                 return;
@@ -653,7 +770,7 @@ static void gen_lea_v_seg(DisasContext *s, MemOp aflag, TCGv a0,
     }
 
     if (ovr_seg >= 0) {
-        TCGv seg = tcg_ctx->cpu_seg_base[ovr_seg];
+        TCGv seg = cpu_seg_base[ovr_seg];
 
         if (aflag == MO_64) {
             tcg_gen_add_tl(tcg_ctx, s->A0, a0, seg);
@@ -670,23 +787,26 @@ static void gen_lea_v_seg(DisasContext *s, MemOp aflag, TCGv a0,
 static inline void gen_string_movl_A0_ESI(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    gen_lea_v_seg(s, s->aflag, tcg_ctx->cpu_regs[R_ESI], R_DS, s->override);
+
+    gen_lea_v_seg(s, s->aflag, cpu_regs[R_ESI], R_DS, s->override);
 }
 
 static inline void gen_string_movl_A0_EDI(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    gen_lea_v_seg(s, s->aflag, tcg_ctx->cpu_regs[R_EDI], R_ES, -1);
+
+    gen_lea_v_seg(s, s->aflag, cpu_regs[R_EDI], R_ES, -1);
 }
 
 static inline void gen_op_movl_T0_Dshift(DisasContext *s, MemOp ot)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    tcg_gen_ld32s_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, df));
+    tcg_gen_ld32s_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, df));
     tcg_gen_shli_tl(tcg_ctx, s->T0, s->T0, ot);
 };
 
-static TCGv gen_ext_tl(TCGContext *tcg_ctx, TCGv dst, TCGv src, MemOp size, bool sign)
+static TCGv gen_ext_tl(TCGContext *tcg_ctx, TCGv dst, TCGv src,
+                       MemOp size, bool sign)
 {
     switch (size) {
     case MO_8:
@@ -727,96 +847,95 @@ static void gen_exts(TCGContext *tcg_ctx, MemOp ot, TCGv reg)
     gen_ext_tl(tcg_ctx, reg, reg, ot, true);
 }
 
-static inline
-void gen_op_jnz_ecx(DisasContext *s, MemOp size, TCGLabel *label1)
+static void gen_op_j_ecx(DisasContext *s, TCGCond cond, TCGLabel *label1)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    tcg_gen_mov_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_regs[R_ECX]);
-    gen_extu(tcg_ctx, size, s->tmp0);
-    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_NE, s->tmp0, 0, label1);
+    tcg_gen_mov_tl(tcg_ctx, s->tmp0, cpu_regs[R_ECX]);
+    gen_extu(tcg_ctx, s->aflag, s->tmp0);
+    tcg_gen_brcondi_tl(tcg_ctx, cond, s->tmp0, 0, label1);
 }
 
-static inline
-void gen_op_jz_ecx(DisasContext *s, MemOp size, TCGLabel *label1)
+static inline void gen_op_jz_ecx(DisasContext *s, TCGLabel *label1)
 {
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    tcg_gen_mov_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_regs[R_ECX]);
-    gen_extu(tcg_ctx, size, s->tmp0);
-    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, s->tmp0, 0, label1);
+    gen_op_j_ecx(s, TCG_COND_EQ, label1);
+}
+
+static inline void gen_op_jnz_ecx(DisasContext *s, TCGLabel *label1)
+{
+    gen_op_j_ecx(s, TCG_COND_NE, label1);
 }
 
-static void gen_helper_in_func(TCGContext *tcg_ctx, MemOp ot, TCGv v, TCGv_i32 n)
+static void gen_helper_in_func(TCGContext *tcg_ctx, MemOp ot, TCGv v,
+                               TCGv_i32 n)
 {
     switch (ot) {
     case MO_8:
-        gen_helper_inb(tcg_ctx, v, tcg_ctx->cpu_env, n);
+        gen_helper_inb(tcg_ctx, v, cpu_env, n);
         break;
     case MO_16:
-        gen_helper_inw(tcg_ctx, v, tcg_ctx->cpu_env, n);
+        gen_helper_inw(tcg_ctx, v, cpu_env, n);
         break;
     case MO_32:
-        gen_helper_inl(tcg_ctx, v, tcg_ctx->cpu_env, n);
+        gen_helper_inl(tcg_ctx, v, cpu_env, n);
         break;
     default:
         tcg_abort();
     }
 }
 
-static void gen_helper_out_func(TCGContext *tcg_ctx, MemOp ot, TCGv_i32 v, TCGv_i32 n)
+static void gen_helper_out_func(TCGContext *tcg_ctx, MemOp ot, TCGv_i32 v,
+                                TCGv_i32 n)
 {
     switch (ot) {
     case MO_8:
-        gen_helper_outb(tcg_ctx, tcg_ctx->cpu_env, v, n);
+        gen_helper_outb(tcg_ctx, cpu_env, v, n);
         break;
     case MO_16:
-        gen_helper_outw(tcg_ctx, tcg_ctx->cpu_env, v, n);
+        gen_helper_outw(tcg_ctx, cpu_env, v, n);
         break;
     case MO_32:
-        gen_helper_outl(tcg_ctx, tcg_ctx->cpu_env, v, n);
+        gen_helper_outl(tcg_ctx, cpu_env, v, n);
         break;
     default:
         tcg_abort();
     }
 }
 
-static void gen_check_io(DisasContext *s, MemOp ot, target_ulong cur_eip,
+/*
+ * Validate that access to [port, port + 1<<ot) is allowed.
+ * Raise #GP, or VMM exit if not.
+ */
+static bool gen_check_io(DisasContext *s, MemOp ot, TCGv_i32 port,
                          uint32_t svm_flags)
 {
-    // Unicorn: allow all I/O instructions
-    return;
-
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    target_ulong next_eip;
-
-    if (s->pe && (s->cpl > s->iopl || s->vm86)) {
-        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-        switch (ot) {
-        case MO_8:
-            gen_helper_check_iob(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
-            break;
-        case MO_16:
-            gen_helper_check_iow(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
-            break;
-        case MO_32:
-            gen_helper_check_iol(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
-            break;
-        default:
-            tcg_abort();
-        }
+#ifdef CONFIG_USER_ONLY
+    /*
+     * We do not implement the ioperm(2) syscall, so the TSS check
+     * will always fail.
+     */
+    gen_exception_gpf(s);
+    return false;
+#else
+    if (PE(s) && (CPL(s) > IOPL(s) || VM86(s))) {
+        gen_helper_check_io(tcg_ctx, cpu_env, port, tcg_constant_i32(tcg_ctx, 1 << ot));
     }
-    if(s->flags & HF_GUEST_MASK) {
+    if (GUEST(s)) {
         gen_update_cc_op(s);
-        gen_jmp_im(s, cur_eip);
-        svm_flags |= (1 << (4 + ot));
-        next_eip = s->pc - s->cs_base;
-        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-        gen_helper_svm_check_io(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32,
-                                tcg_const_i32(tcg_ctx, svm_flags),
-                                tcg_const_i32(tcg_ctx, next_eip - cur_eip));
+        gen_update_eip_cur(s);
+        if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) {
+            svm_flags |= SVM_IOIO_REP_MASK;
+        }
+        svm_flags |= 1 << (SVM_IOIO_SIZE_SHIFT + ot);
+        gen_helper_svm_check_io(tcg_ctx, cpu_env, port,
+                                tcg_constant_i32(tcg_ctx, svm_flags),
+                                cur_insn_len_i32(s));
     }
+    return true;
+#endif
 }
 
-static inline void gen_movs(DisasContext *s, MemOp ot)
+static void gen_movs(DisasContext *s, MemOp ot)
 {
     gen_string_movl_A0_ESI(s);
     gen_op_ld_v(s, ot, s->T0, s->A0);
@@ -830,36 +949,35 @@ static inline void gen_movs(DisasContext *s, MemOp ot)
 static void gen_op_update1_cc(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
 }
 
 static void gen_op_update2_cc(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T1);
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T1);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
 }
 
 static void gen_op_update3_cc(DisasContext *s, TCGv reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, reg);
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T1);
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_src2, reg);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T1);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
 }
 
 static inline void gen_op_testl_T0_T1_cc(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    tcg_gen_and_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0, s->T1);
+    tcg_gen_and_tl(tcg_ctx, cpu_cc_dst, s->T0, s->T1);
 }
 
 static void gen_op_update_neg_cc(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
-    tcg_gen_neg_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
+    tcg_gen_neg_tl(tcg_ctx, cpu_cc_src, s->T0);
     tcg_gen_movi_tl(tcg_ctx, s->cc_srcT, 0);
 }
 
@@ -874,15 +992,15 @@ static void gen_compute_eflags(DisasContext *s)
         return;
     }
     if (s->cc_op == CC_OP_CLR) {
-        tcg_gen_movi_tl(tcg_ctx, tcg_ctx->cpu_cc_src, CC_Z | CC_P);
+        tcg_gen_movi_tl(tcg_ctx, cpu_cc_src, CC_Z | CC_P);
         set_cc_op(s, CC_OP_EFLAGS);
         return;
     }
 
     zero = NULL;
-    dst = tcg_ctx->cpu_cc_dst;
-    src1 = tcg_ctx->cpu_cc_src;
-    src2 = tcg_ctx->cpu_cc_src2;
+    dst = cpu_cc_dst;
+    src1 = cpu_cc_src;
+    src2 = cpu_cc_src2;
 
     /* Take care to not read values that are not live.  */
     live = cc_op_live[s->cc_op] & ~USES_CC_SRCT;
@@ -901,7 +1019,7 @@ static void gen_compute_eflags(DisasContext *s)
     }
 
     gen_update_cc_op(s);
-    gen_helper_cc_compute_all(tcg_ctx, tcg_ctx->cpu_cc_src, dst, src1, src2, tcg_ctx->cpu_cc_op);
+    gen_helper_cc_compute_all(tcg_ctx, cpu_cc_src, dst, src1, src2, cpu_cc_op);
     set_cc_op(s, CC_OP_EFLAGS);
 
     if (dead) {
@@ -909,6 +1027,16 @@ static void gen_compute_eflags(DisasContext *s)
     }
 }
 
+static void sync_eflags(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    gen_update_cc_op(s);
+    gen_helper_read_eflags(tcg_ctx, s->T0, cpu_env);
+    tcg_gen_st_tl(tcg_ctx, s->T0, cpu_env,
+                  offsetof(CPUX86State, eflags));
+}
+
 typedef struct CCPrepare {
     TCGCond cond;
     TCGv reg;
@@ -927,104 +1055,68 @@ static CCPrepare gen_prepare_eflags_c(DisasContext *s, TCGv reg)
     int size, shift;
 
     switch (s->cc_op) {
-    case CC_OP_SUBB:
-    case CC_OP_SUBW:
-    case CC_OP_SUBL:
-    case CC_OP_SUBQ:
+    case CC_OP_SUBB: case CC_OP_SUBW: case CC_OP_SUBL: case CC_OP_SUBQ:
         /* (DATA_TYPE)CC_SRCT < (DATA_TYPE)CC_SRC */
         size = s->cc_op - CC_OP_SUBB;
-        t1 = gen_ext_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_cc_src, size, false);
+        t1 = gen_ext_tl(tcg_ctx, s->tmp0, cpu_cc_src, size, false);
         /* If no temporary was used, be careful not to alias t1 and t0.  */
-        t0 = t1 == tcg_ctx->cpu_cc_src ? s->tmp0 : reg;
+        t0 = t1 == cpu_cc_src ? s->tmp0 : reg;
         tcg_gen_mov_tl(tcg_ctx, t0, s->cc_srcT);
         gen_extu(tcg_ctx, size, t0);
         goto add_sub;
 
-    case CC_OP_ADDB:
-    case CC_OP_ADDW:
-    case CC_OP_ADDL:
-    case CC_OP_ADDQ:
+    case CC_OP_ADDB: case CC_OP_ADDW: case CC_OP_ADDL: case CC_OP_ADDQ:
         /* (DATA_TYPE)CC_DST < (DATA_TYPE)CC_SRC */
         size = s->cc_op - CC_OP_ADDB;
-        t1 = gen_ext_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_cc_src, size, false);
-        t0 = gen_ext_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_dst, size, false);
+        t1 = gen_ext_tl(tcg_ctx, s->tmp0, cpu_cc_src, size, false);
+        t0 = gen_ext_tl(tcg_ctx, reg, cpu_cc_dst, size, false);
     add_sub:
         return (CCPrepare) { .cond = TCG_COND_LTU, .reg = t0,
                              .reg2 = t1, .mask = -1, .use_reg2 = true };
 
-    case CC_OP_LOGICB:
-    case CC_OP_LOGICW:
-    case CC_OP_LOGICL:
-    case CC_OP_LOGICQ:
+    case CC_OP_LOGICB: case CC_OP_LOGICW: case CC_OP_LOGICL: case CC_OP_LOGICQ:
     case CC_OP_CLR:
     case CC_OP_POPCNT:
         return (CCPrepare) { .cond = TCG_COND_NEVER, .mask = -1 };
 
-    case CC_OP_INCB:
-    case CC_OP_INCW:
-    case CC_OP_INCL:
-    case CC_OP_INCQ:
-
-    case CC_OP_DECB:
-    case CC_OP_DECW:
-    case CC_OP_DECL:
-    case CC_OP_DECQ:
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src,
+    case CC_OP_INCB: case CC_OP_INCW: case CC_OP_INCL: case CC_OP_INCQ:
+    case CC_OP_DECB: case CC_OP_DECW: case CC_OP_DECL: case CC_OP_DECQ:
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
                              .mask = -1, .no_setcond = true };
 
-    case CC_OP_SHLB:
-    case CC_OP_SHLW:
-    case CC_OP_SHLL:
-    case CC_OP_SHLQ:
+    case CC_OP_SHLB: case CC_OP_SHLW: case CC_OP_SHLL: case CC_OP_SHLQ:
         /* (CC_SRC >> (DATA_BITS - 1)) & 1 */
         size = s->cc_op - CC_OP_SHLB;
         shift = (8 << size) - 1;
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src,
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
                              .mask = (target_ulong)1 << shift };
 
-    case CC_OP_MULB:
-    case CC_OP_MULW:
-    case CC_OP_MULL:
-    case CC_OP_MULQ:
+    case CC_OP_MULB: case CC_OP_MULW: case CC_OP_MULL: case CC_OP_MULQ:
         return (CCPrepare) { .cond = TCG_COND_NE,
-                             .reg = tcg_ctx->cpu_cc_src, .mask = -1 };
+                             .reg = cpu_cc_src, .mask = -1 };
 
-    case CC_OP_BMILGB:
-    case CC_OP_BMILGW:
-    case CC_OP_BMILGL:
-    case CC_OP_BMILGQ:
+    case CC_OP_BMILGB: case CC_OP_BMILGW: case CC_OP_BMILGL: case CC_OP_BMILGQ:
         size = s->cc_op - CC_OP_BMILGB;
-        t0 = gen_ext_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_src, size, false);
+        t0 = gen_ext_tl(tcg_ctx, reg, cpu_cc_src, size, false);
         return (CCPrepare) { .cond = TCG_COND_EQ, .reg = t0, .mask = -1 };
 
-    case CC_OP_BLSIB:
-    case CC_OP_BLSIW:
-    case CC_OP_BLSIL:
-    case CC_OP_BLSIQ:
-        size = s->cc_op - CC_OP_BLSIB;
-        t0 = gen_ext_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_src, size, false);
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = t0, .mask = -1 };
-
     case CC_OP_ADCX:
     case CC_OP_ADCOX:
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_dst,
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_dst,
                              .mask = -1, .no_setcond = true };
 
     case CC_OP_EFLAGS:
-    case CC_OP_SARB:
-    case CC_OP_SARW:
-    case CC_OP_SARL:
-    case CC_OP_SARQ:
+    case CC_OP_SARB: case CC_OP_SARW: case CC_OP_SARL: case CC_OP_SARQ:
         /* CC_SRC & 1 */
         return (CCPrepare) { .cond = TCG_COND_NE,
-                             .reg = tcg_ctx->cpu_cc_src, .mask = CC_C };
+                             .reg = cpu_cc_src, .mask = CC_C };
 
     default:
        /* The need to compute only C from CC_OP_DYNAMIC is important
           in efficiently implementing e.g. INC at the start of a TB.  */
        gen_update_cc_op(s);
-       gen_helper_cc_compute_c(tcg_ctx, reg, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_cc_src,
-                               tcg_ctx->cpu_cc_src2, tcg_ctx->cpu_cc_op);
+       gen_helper_cc_compute_c(tcg_ctx, reg, cpu_cc_dst, cpu_cc_src,
+                               cpu_cc_src2, cpu_cc_op);
        return (CCPrepare) { .cond = TCG_COND_NE, .reg = reg,
                             .mask = -1, .no_setcond = true };
     }
@@ -1034,9 +1126,8 @@ static CCPrepare gen_prepare_eflags_c(DisasContext *s, TCGv reg)
 static CCPrepare gen_prepare_eflags_p(DisasContext *s, TCGv reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     gen_compute_eflags(s);
-    return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src,
+    return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
                          .mask = CC_P };
 }
 
@@ -1044,7 +1135,6 @@ static CCPrepare gen_prepare_eflags_p(DisasContext *s, TCGv reg)
 static CCPrepare gen_prepare_eflags_s(DisasContext *s, TCGv reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     switch (s->cc_op) {
     case CC_OP_DYNAMIC:
         gen_compute_eflags(s);
@@ -1053,7 +1143,7 @@ static CCPrepare gen_prepare_eflags_s(DisasContext *s, TCGv reg)
     case CC_OP_ADCX:
     case CC_OP_ADOX:
     case CC_OP_ADCOX:
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src,
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
                              .mask = CC_S };
     case CC_OP_CLR:
     case CC_OP_POPCNT:
@@ -1061,7 +1151,7 @@ static CCPrepare gen_prepare_eflags_s(DisasContext *s, TCGv reg)
     default:
         {
             MemOp size = (s->cc_op - CC_OP_ADDB) & 3;
-            TCGv t0 = gen_ext_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_dst, size, true);
+            TCGv t0 = gen_ext_tl(tcg_ctx, reg, cpu_cc_dst, size, true);
             return (CCPrepare) { .cond = TCG_COND_LT, .reg = t0, .mask = -1 };
         }
     }
@@ -1071,18 +1161,17 @@ static CCPrepare gen_prepare_eflags_s(DisasContext *s, TCGv reg)
 static CCPrepare gen_prepare_eflags_o(DisasContext *s, TCGv reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     switch (s->cc_op) {
     case CC_OP_ADOX:
     case CC_OP_ADCOX:
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src2,
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src2,
                              .mask = -1, .no_setcond = true };
     case CC_OP_CLR:
     case CC_OP_POPCNT:
         return (CCPrepare) { .cond = TCG_COND_NEVER, .mask = -1 };
     default:
         gen_compute_eflags(s);
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src,
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
                              .mask = CC_O };
     }
 }
@@ -1091,7 +1180,6 @@ static CCPrepare gen_prepare_eflags_o(DisasContext *s, TCGv reg)
 static CCPrepare gen_prepare_eflags_z(DisasContext *s, TCGv reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     switch (s->cc_op) {
     case CC_OP_DYNAMIC:
         gen_compute_eflags(s);
@@ -1100,17 +1188,17 @@ static CCPrepare gen_prepare_eflags_z(DisasContext *s, TCGv reg)
     case CC_OP_ADCX:
     case CC_OP_ADOX:
     case CC_OP_ADCOX:
-        return (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src,
+        return (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
                              .mask = CC_Z };
     case CC_OP_CLR:
         return (CCPrepare) { .cond = TCG_COND_ALWAYS, .mask = -1 };
     case CC_OP_POPCNT:
-        return (CCPrepare) { .cond = TCG_COND_EQ, .reg = tcg_ctx->cpu_cc_src,
+        return (CCPrepare) { .cond = TCG_COND_EQ, .reg = cpu_cc_src,
                              .mask = -1 };
     default:
         {
             MemOp size = (s->cc_op - CC_OP_ADDB) & 3;
-            TCGv t0 = gen_ext_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_dst, size, false);
+            TCGv t0 = gen_ext_tl(tcg_ctx, reg, cpu_cc_dst, size, false);
             return (CCPrepare) { .cond = TCG_COND_EQ, .reg = t0, .mask = -1 };
         }
     }
@@ -1130,17 +1218,14 @@ static CCPrepare gen_prepare_cc(DisasContext *s, int b, TCGv reg)
     jcc_op = (b >> 1) & 7;
 
     switch (s->cc_op) {
-    case CC_OP_SUBB:
-    case CC_OP_SUBW:
-    case CC_OP_SUBL:
-    case CC_OP_SUBQ:
+    case CC_OP_SUBB: case CC_OP_SUBW: case CC_OP_SUBL: case CC_OP_SUBQ:
         /* We optimize relational operators for the cmp/jcc case.  */
         size = s->cc_op - CC_OP_SUBB;
         switch (jcc_op) {
         case JCC_BE:
             tcg_gen_mov_tl(tcg_ctx, s->tmp4, s->cc_srcT);
             gen_extu(tcg_ctx, size, s->tmp4);
-            t0 = gen_ext_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_cc_src, size, false);
+            t0 = gen_ext_tl(tcg_ctx, s->tmp0, cpu_cc_src, size, false);
             cc = (CCPrepare) { .cond = TCG_COND_LEU, .reg = s->tmp4,
                                .reg2 = t0, .mask = -1, .use_reg2 = true };
             break;
@@ -1153,7 +1238,7 @@ static CCPrepare gen_prepare_cc(DisasContext *s, int b, TCGv reg)
         fast_jcc_l:
             tcg_gen_mov_tl(tcg_ctx, s->tmp4, s->cc_srcT);
             gen_exts(tcg_ctx, size, s->tmp4);
-            t0 = gen_ext_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_cc_src, size, true);
+            t0 = gen_ext_tl(tcg_ctx, s->tmp0, cpu_cc_src, size, true);
             cc = (CCPrepare) { .cond = cond, .reg = s->tmp4,
                                .reg2 = t0, .mask = -1, .use_reg2 = true };
             break;
@@ -1178,7 +1263,7 @@ static CCPrepare gen_prepare_cc(DisasContext *s, int b, TCGv reg)
             break;
         case JCC_BE:
             gen_compute_eflags(s);
-            cc = (CCPrepare) { .cond = TCG_COND_NE, .reg = tcg_ctx->cpu_cc_src,
+            cc = (CCPrepare) { .cond = TCG_COND_NE, .reg = cpu_cc_src,
                                .mask = CC_Z | CC_C };
             break;
         case JCC_S:
@@ -1189,22 +1274,22 @@ static CCPrepare gen_prepare_cc(DisasContext *s, int b, TCGv reg)
             break;
         case JCC_L:
             gen_compute_eflags(s);
-            if (reg == tcg_ctx->cpu_cc_src) {
+            if (reg == cpu_cc_src) {
                 reg = s->tmp0;
             }
-            tcg_gen_shri_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_src, 4); /* CC_O -> CC_S */
-            tcg_gen_xor_tl(tcg_ctx, reg, reg, tcg_ctx->cpu_cc_src);
+            tcg_gen_shri_tl(tcg_ctx, reg, cpu_cc_src, 4); /* CC_O -> CC_S */
+            tcg_gen_xor_tl(tcg_ctx, reg, reg, cpu_cc_src);
             cc = (CCPrepare) { .cond = TCG_COND_NE, .reg = reg,
                                .mask = CC_S };
             break;
         default:
         case JCC_LE:
             gen_compute_eflags(s);
-            if (reg == tcg_ctx->cpu_cc_src) {
+            if (reg == cpu_cc_src) {
                 reg = s->tmp0;
             }
-            tcg_gen_shri_tl(tcg_ctx, reg, tcg_ctx->cpu_cc_src, 4); /* CC_O -> CC_S */
-            tcg_gen_xor_tl(tcg_ctx, reg, reg, tcg_ctx->cpu_cc_src);
+            tcg_gen_shri_tl(tcg_ctx, reg, cpu_cc_src, 4); /* CC_O -> CC_S */
+            tcg_gen_xor_tl(tcg_ctx, reg, reg, cpu_cc_src);
             cc = (CCPrepare) { .cond = TCG_COND_NE, .reg = reg,
                                .mask = CC_S | CC_Z };
             break;
@@ -1295,19 +1380,19 @@ static inline void gen_jcc1(DisasContext *s, int b, TCGLabel *l1)
 
 /* XXX: does not work with gdbstub "ice" single step - not a
    serious problem */
-static TCGLabel *gen_jz_ecx_string(DisasContext *s, target_ulong next_eip)
+static TCGLabel *gen_jz_ecx_string(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     TCGLabel *l1 = gen_new_label(tcg_ctx);
     TCGLabel *l2 = gen_new_label(tcg_ctx);
-    gen_op_jnz_ecx(s, s->aflag, l1);
+    gen_op_jnz_ecx(s, l1);
     gen_set_label(tcg_ctx, l2);
-    gen_jmp_tb(s, next_eip, 1);
+    gen_jmp_rel_csize(s, 0, 1);
     gen_set_label(tcg_ctx, l1);
     return l2;
 }
 
-static inline void gen_stos(DisasContext *s, MemOp ot)
+static void gen_stos(DisasContext *s, MemOp ot)
 {
     gen_op_mov_v_reg(s, MO_32, s->T0, R_EAX);
     gen_string_movl_A0_EDI(s);
@@ -1316,7 +1401,7 @@ static inline void gen_stos(DisasContext *s, MemOp ot)
     gen_op_add_reg_T0(s, s->aflag, R_EDI);
 }
 
-static inline void gen_lods(DisasContext *s, MemOp ot)
+static void gen_lods(DisasContext *s, MemOp ot)
 {
     gen_string_movl_A0_ESI(s);
     gen_op_ld_v(s, ot, s->T0, s->A0);
@@ -1325,7 +1410,7 @@ static inline void gen_lods(DisasContext *s, MemOp ot)
     gen_op_add_reg_T0(s, s->aflag, R_ESI);
 }
 
-static inline void gen_scas(DisasContext *s, MemOp ot)
+static void gen_scas(DisasContext *s, MemOp ot)
 {
     gen_string_movl_A0_EDI(s);
     gen_op_ld_v(s, ot, s->T1, s->A0);
@@ -1334,7 +1419,7 @@ static inline void gen_scas(DisasContext *s, MemOp ot)
     gen_op_add_reg_T0(s, s->aflag, R_EDI);
 }
 
-static inline void gen_cmps(DisasContext *s, MemOp ot)
+static void gen_cmps(DisasContext *s, MemOp ot)
 {
     gen_string_movl_A0_EDI(s);
     gen_op_ld_v(s, ot, s->T1, s->A0);
@@ -1348,100 +1433,93 @@ static inline void gen_cmps(DisasContext *s, MemOp ot)
 static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     if (s->flags & HF_IOBPT_MASK) {
-        TCGv_i32 t_size = tcg_const_i32(tcg_ctx, 1 << ot);
-        TCGv t_next = tcg_const_tl(tcg_ctx, s->pc - s->cs_base);
-
-        gen_helper_bpt_io(tcg_ctx, tcg_ctx->cpu_env, t_port, t_size, t_next);
-        tcg_temp_free_i32(tcg_ctx, t_size);
-        tcg_temp_free(tcg_ctx, t_next);
+#ifdef CONFIG_USER_ONLY
+        /* user-mode cpu should not be in IOBPT mode */
+        g_assert_not_reached();
+#else
+        TCGv_i32 t_size = tcg_constant_i32(tcg_ctx, 1 << ot);
+        TCGv t_next = eip_next_tl(s);
+        gen_helper_bpt_io(tcg_ctx, cpu_env, t_port, t_size, t_next);
+#endif /* CONFIG_USER_ONLY */
     }
 }
 
-
-static inline void gen_ins(DisasContext *s, MemOp ot)
+static void gen_ins(DisasContext *s, MemOp ot)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-        gen_io_start(tcg_ctx);
-    }
     gen_string_movl_A0_EDI(s);
     /* Note: we must do this dummy write first to be restartable in
        case of page fault. */
     tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
     gen_op_st_v(s, ot, s->T0, s->A0);
-    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[R_EDX]);
+    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_EDX]);
     tcg_gen_andi_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, 0xffff);
     gen_helper_in_func(tcg_ctx, ot, s->T0, s->tmp2_i32);
     gen_op_st_v(s, ot, s->T0, s->A0);
     gen_op_movl_T0_Dshift(s, ot);
     gen_op_add_reg_T0(s, s->aflag, R_EDI);
     gen_bpt_io(s, s->tmp2_i32, ot);
-    if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-        gen_io_end(tcg_ctx);
-    }
 }
 
-static inline void gen_outs(DisasContext *s, MemOp ot)
+static void gen_outs(DisasContext *s, MemOp ot)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-        gen_io_start(tcg_ctx);
-    }
     gen_string_movl_A0_ESI(s);
     gen_op_ld_v(s, ot, s->T0, s->A0);
 
-    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[R_EDX]);
+    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_EDX]);
     tcg_gen_andi_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, 0xffff);
     tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, s->T0);
     gen_helper_out_func(tcg_ctx, ot, s->tmp2_i32, s->tmp3_i32);
     gen_op_movl_T0_Dshift(s, ot);
     gen_op_add_reg_T0(s, s->aflag, R_ESI);
     gen_bpt_io(s, s->tmp2_i32, ot);
-    if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-        gen_io_end(tcg_ctx);
+}
+
+/* Generate jumps to current or next instruction */
+static void gen_repz(DisasContext *s, MemOp ot,
+                     void (*fn)(DisasContext *s, MemOp ot))
+{
+    TCGLabel *l2;
+    gen_update_cc_op(s);
+    l2 = gen_jz_ecx_string(s);
+    fn(s, ot);
+    gen_op_add_reg_im(s, s->aflag, R_ECX, -1);
+    /*
+     * A loop would cause two single step exceptions if ECX = 1
+     * before rep string_insn
+     */
+    if (s->repz_opt) {
+        gen_op_jz_ecx(s, l2);
     }
+    gen_jmp_rel_csize(s, -cur_insn_len(s), 0);
 }
 
-/* same method as Valgrind : we generate jumps to current or next
-   instruction */
-#define GEN_REPZ(op)                                                          \
-static inline void gen_repz_ ## op(DisasContext *s, MemOp ot,              \
-                                 target_ulong cur_eip, target_ulong next_eip) \
-{                                                                             \
-    TCGLabel *l2;                                                             \
-    gen_update_cc_op(s);                                                      \
-    l2 = gen_jz_ecx_string(s, next_eip);                                      \
-    gen_ ## op(s, ot);                                                        \
-    gen_op_add_reg_im(s, s->aflag, R_ECX, -1);                                \
-    /* a loop would cause two single step exceptions if ECX = 1               \
-       before rep string_insn */                                              \
-    if (s->repz_opt)                                                          \
-        gen_op_jz_ecx(s, s->aflag, l2);                                       \
-    gen_jmp(s, cur_eip);                                                      \
-}
-
-#define GEN_REPZ2(op)                                                         \
-static inline void gen_repz_ ## op(DisasContext *s, MemOp ot,              \
-                                   target_ulong cur_eip,                      \
-                                   target_ulong next_eip,                     \
-                                   int nz)                                    \
-{                                                                             \
-    TCGLabel *l2;                                                             \
-    gen_update_cc_op(s);                                                      \
-    l2 = gen_jz_ecx_string(s, next_eip);                                      \
-    gen_ ## op(s, ot);                                                        \
-    gen_op_add_reg_im(s, s->aflag, R_ECX, -1);                                \
-    gen_update_cc_op(s);                                                      \
-    gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2);                                 \
-    if (s->repz_opt)                                                          \
-        gen_op_jz_ecx(s, s->aflag, l2);                                       \
-    gen_jmp(s, cur_eip);                                                      \
+#define GEN_REPZ(op) \
+    static inline void gen_repz_ ## op(DisasContext *s, MemOp ot) \
+    { gen_repz(s, ot, gen_##op); }
+
+static void gen_repz2(DisasContext *s, MemOp ot, int nz,
+                      void (*fn)(DisasContext *s, MemOp ot))
+{
+    TCGLabel *l2;
+    gen_update_cc_op(s);
+    l2 = gen_jz_ecx_string(s);
+    fn(s, ot);
+    gen_op_add_reg_im(s, s->aflag, R_ECX, -1);
+    gen_update_cc_op(s);
+    gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2);
+    if (s->repz_opt) {
+        gen_op_jz_ecx(s, l2);
+    }
+    gen_jmp_rel_csize(s, -cur_insn_len(s), 0);
 }
 
+#define GEN_REPZ2(op) \
+    static inline void gen_repz_ ## op(DisasContext *s, MemOp ot, int nz) \
+    { gen_repz2(s, ot, nz, gen_##op); }
+
 GEN_REPZ(movs)
 GEN_REPZ(stos)
 GEN_REPZ(lods)
@@ -1454,28 +1532,28 @@ static void gen_helper_fp_arith_ST0_FT0(TCGContext *tcg_ctx, int op)
 {
     switch (op) {
     case 0:
-        gen_helper_fadd_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fadd_ST0_FT0(tcg_ctx, cpu_env);
         break;
     case 1:
-        gen_helper_fmul_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fmul_ST0_FT0(tcg_ctx, cpu_env);
         break;
     case 2:
-        gen_helper_fcom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fcom_ST0_FT0(tcg_ctx, cpu_env);
         break;
     case 3:
-        gen_helper_fcom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fcom_ST0_FT0(tcg_ctx, cpu_env);
         break;
     case 4:
-        gen_helper_fsub_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fsub_ST0_FT0(tcg_ctx, cpu_env);
         break;
     case 5:
-        gen_helper_fsubr_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fsubr_ST0_FT0(tcg_ctx, cpu_env);
         break;
     case 6:
-        gen_helper_fdiv_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fdiv_ST0_FT0(tcg_ctx, cpu_env);
         break;
     case 7:
-        gen_helper_fdivr_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_fdivr_ST0_FT0(tcg_ctx, cpu_env);
         break;
     }
 }
@@ -1486,33 +1564,32 @@ static void gen_helper_fp_arith_STN_ST0(TCGContext *tcg_ctx, int op, int opreg)
     TCGv_i32 tmp = tcg_const_i32(tcg_ctx, opreg);
     switch (op) {
     case 0:
-        gen_helper_fadd_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tmp);
+        gen_helper_fadd_STN_ST0(tcg_ctx, cpu_env, tmp);
         break;
     case 1:
-        gen_helper_fmul_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tmp);
+        gen_helper_fmul_STN_ST0(tcg_ctx, cpu_env, tmp);
         break;
     case 4:
-        gen_helper_fsubr_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tmp);
+        gen_helper_fsubr_STN_ST0(tcg_ctx, cpu_env, tmp);
         break;
     case 5:
-        gen_helper_fsub_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tmp);
+        gen_helper_fsub_STN_ST0(tcg_ctx, cpu_env, tmp);
         break;
     case 6:
-        gen_helper_fdivr_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tmp);
+        gen_helper_fdivr_STN_ST0(tcg_ctx, cpu_env, tmp);
         break;
     case 7:
-        gen_helper_fdiv_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tmp);
+        gen_helper_fdiv_STN_ST0(tcg_ctx, cpu_env, tmp);
         break;
     }
 }
 
-static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip)
+static void gen_exception(DisasContext *s, int trapno)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     gen_update_cc_op(s);
-    gen_jmp_im(s, cur_eip);
-    gen_helper_raise_exception(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, trapno));
+    gen_update_eip_cur(s);
+    gen_helper_raise_exception(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, trapno));
     s->base.is_jmp = DISAS_NORETURN;
 }
 
@@ -1520,17 +1597,51 @@ static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip)
    the instruction is known, but it isn't allowed in the current cpu mode.  */
 static void gen_illegal_opcode(DisasContext *s)
 {
-    gen_exception(s, EXCP06_ILLOP, s->pc_start - s->cs_base);
+    gen_exception(s, EXCP06_ILLOP);
+}
+
+/* Generate #GP for the current instruction. */
+static void gen_exception_gpf(DisasContext *s)
+{
+    gen_exception(s, EXCP0D_GPF);
+}
+
+/* Check for cpl == 0; if not, raise #GP and return false. */
+static bool check_cpl0(DisasContext *s)
+{
+    if (CPL(s) == 0) {
+        return true;
+    }
+    gen_exception_gpf(s);
+    return false;
+}
+
+/* If vm86, check for iopl == 3; if not, raise #GP and return false. */
+static bool check_vm86_iopl(DisasContext *s)
+{
+    if (!VM86(s) || IOPL(s) == 3) {
+        return true;
+    }
+    gen_exception_gpf(s);
+    return false;
+}
+
+/* Check for iopl allowing access; if not, raise #GP and return false. */
+static bool check_iopl(DisasContext *s)
+{
+    if (VM86(s) ? IOPL(s) == 3 : CPL(s) <= IOPL(s)) {
+        return true;
+    }
+    gen_exception_gpf(s);
+    return false;
 }
 
 /* if d == OR_TMP0, it means memory operand (address in A0) */
 static void gen_op(DisasContext *s1, int op, MemOp ot, int d)
 {
     TCGContext *tcg_ctx = s1->uc->tcg_ctx;
-    uc_engine *uc = s1->uc;
-
     /* Invalid lock prefix when destination is not memory or OP_CMPL. */
-    if ((d != OR_TMP0 || op == OP_CMPL) && s1->prefix & PREFIX_LOCK){
+    if ((d != OR_TMP0 || op == OP_CMPL) && s1->prefix & PREFIX_LOCK) {
         gen_illegal_opcode(s1);
         return;
     }
@@ -1592,20 +1703,24 @@ static void gen_op(DisasContext *s1, int op, MemOp ot, int d)
             tcg_gen_sub_tl(tcg_ctx, s1->T0, s1->T0, s1->T1);
             gen_op_st_rm_T0_A0(s1, ot, d);
         }
-        
-        if (HOOK_EXISTS_BOUNDED(uc, UC_HOOK_TCG_OPCODE, s1->pc_start)) {
+        if (HOOK_EXISTS_BOUNDED(s1->uc, UC_HOOK_TCG_OPCODE,
+                                s1->base.pc_next)) {
             struct hook *hook;
             HOOK_FOREACH_VAR_DECLARE;
-            HOOK_FOREACH(uc, hook, UC_HOOK_TCG_OPCODE) {
-                if (hook->to_delete)
+
+            HOOK_FOREACH(s1->uc, hook, UC_HOOK_TCG_OPCODE) {
+                if (hook->to_delete) {
                     continue;
-                if (hook->op == UC_TCG_OP_SUB && (hook->op_flags & UC_TCG_OP_FLAG_DIRECT) ) {
-                    // TCGv is just an offset to tcg_ctx so it's safe to do so.
-                    gen_uc_traceopcode(tcg_ctx, hook, (TCGv_i64)s1->T0, (TCGv_i64)s1->T1, 1 << ((ot & MO_SIZE) + 3), uc, s1->pc_start);
+                }
+                if (hook->op == UC_TCG_OP_SUB &&
+                    (hook->op_flags & UC_TCG_OP_FLAG_DIRECT)) {
+                    gen_uc_traceopcode(tcg_ctx, hook, (TCGv_i64)s1->T0,
+                                       (TCGv_i64)s1->T1,
+                                       1 << ((ot & MO_SIZE) + 3), s1->uc,
+                                       s1->base.pc_next);
                 }
             }
         }
-
         gen_op_update2_cc(s1);
         set_cc_op(s1, CC_OP_SUBB + ot);
         break;
@@ -1644,23 +1759,27 @@ static void gen_op(DisasContext *s1, int op, MemOp ot, int d)
         set_cc_op(s1, CC_OP_LOGICB + ot);
         break;
     case OP_CMPL:
-        tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s1->T1);
+        tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s1->T1);
         tcg_gen_mov_tl(tcg_ctx, s1->cc_srcT, s1->T0);
-        tcg_gen_sub_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s1->T0, s1->T1);
-
-        if (HOOK_EXISTS_BOUNDED(uc, UC_HOOK_TCG_OPCODE, s1->pc_start)) {
+        tcg_gen_sub_tl(tcg_ctx, cpu_cc_dst, s1->T0, s1->T1);
+        if (HOOK_EXISTS_BOUNDED(s1->uc, UC_HOOK_TCG_OPCODE,
+                                s1->base.pc_next)) {
             struct hook *hook;
             HOOK_FOREACH_VAR_DECLARE;
-            HOOK_FOREACH(uc, hook, UC_HOOK_TCG_OPCODE) {
-                if (hook->to_delete)
+
+            HOOK_FOREACH(s1->uc, hook, UC_HOOK_TCG_OPCODE) {
+                if (hook->to_delete) {
                     continue;
-                if (hook->op == UC_TCG_OP_SUB && (hook->op_flags & UC_TCG_OP_FLAG_CMP) ) {
-                    // TCGv is just an offset to tcg_ctx so it's safe to do so.
-                    gen_uc_traceopcode(tcg_ctx, hook, (TCGv_i64)s1->T0, (TCGv_i64)s1->T1, 1 << ((ot & MO_SIZE) + 3), uc, s1->pc_start);
+                }
+                if (hook->op == UC_TCG_OP_SUB &&
+                    (hook->op_flags & UC_TCG_OP_FLAG_CMP)) {
+                    gen_uc_traceopcode(tcg_ctx, hook, (TCGv_i64)s1->T0,
+                                       (TCGv_i64)s1->T1,
+                                       1 << ((ot & MO_SIZE) + 3), s1->uc,
+                                       s1->base.pc_next);
                 }
             }
         }
-
         set_cc_op(s1, CC_OP_SUBB + ot);
         break;
     }
@@ -1670,7 +1789,6 @@ static void gen_op(DisasContext *s1, int op, MemOp ot, int d)
 static void gen_inc(DisasContext *s1, MemOp ot, int d, int c)
 {
     TCGContext *tcg_ctx = s1->uc->tcg_ctx;
-
     if (s1->prefix & PREFIX_LOCK) {
         if (d != OR_TMP0) {
             /* Lock prefix when destination is not memory */
@@ -1690,8 +1808,8 @@ static void gen_inc(DisasContext *s1, MemOp ot, int d, int c)
         gen_op_st_rm_T0_A0(s1, ot, d);
     }
 
-    gen_compute_eflags_c(s1, tcg_ctx->cpu_cc_src);
-    tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s1->T0);
+    gen_compute_eflags_c(s1, cpu_cc_src);
+    tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s1->T0);
     set_cc_op(s1, (c > 0 ? CC_OP_INCB : CC_OP_DECB) + ot);
 }
 
@@ -1707,23 +1825,23 @@ static void gen_shift_flags(DisasContext *s, MemOp ot, TCGv result,
        need to not disrupt the current contents.  */
     z_tl = tcg_const_tl(tcg_ctx, 0);
     if (cc_op_live[s->cc_op] & USES_CC_DST) {
-        tcg_gen_movcond_tl(tcg_ctx, TCG_COND_NE, tcg_ctx->cpu_cc_dst, count, z_tl,
-                           result, tcg_ctx->cpu_cc_dst);
+        tcg_gen_movcond_tl(tcg_ctx, TCG_COND_NE, cpu_cc_dst, count, z_tl,
+                           result, cpu_cc_dst);
     } else {
-        tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, result);
+        tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, result);
     }
     if (cc_op_live[s->cc_op] & USES_CC_SRC) {
-        tcg_gen_movcond_tl(tcg_ctx, TCG_COND_NE, tcg_ctx->cpu_cc_src, count, z_tl,
-                           shm1, tcg_ctx->cpu_cc_src);
+        tcg_gen_movcond_tl(tcg_ctx, TCG_COND_NE, cpu_cc_src, count, z_tl,
+                           shm1, cpu_cc_src);
     } else {
-        tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, shm1);
+        tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, shm1);
     }
     tcg_temp_free(tcg_ctx, z_tl);
 
     /* Get the two potential CC_OP values into temporaries.  */
     tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, (is_right ? CC_OP_SARB : CC_OP_SHLB) + ot);
     if (s->cc_op == CC_OP_DYNAMIC) {
-        oldop = tcg_ctx->cpu_cc_op;
+        oldop = cpu_cc_op;
     } else {
         tcg_gen_movi_i32(tcg_ctx, s->tmp3_i32, s->cc_op);
         oldop = s->tmp3_i32;
@@ -1733,7 +1851,7 @@ static void gen_shift_flags(DisasContext *s, MemOp ot, TCGv result,
     z32 = tcg_const_i32(tcg_ctx, 0);
     s32 = tcg_temp_new_i32(tcg_ctx);
     tcg_gen_trunc_tl_i32(tcg_ctx, s32, count);
-    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_NE, tcg_ctx->cpu_cc_op, s32, z32, s->tmp2_i32, oldop);
+    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_NE, cpu_cc_op, s32, z32, s->tmp2_i32, oldop);
     tcg_temp_free_i32(tcg_ctx, z32);
     tcg_temp_free_i32(tcg_ctx, s32);
 
@@ -1813,8 +1931,8 @@ static void gen_shift_rm_im(DisasContext *s, MemOp ot, int op1, int op2,
 
     /* update eflags if non zero shift */
     if (op2 != 0) {
-        tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->tmp4);
-        tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+        tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->tmp4);
+        tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
         set_cc_op(s, (is_right ? CC_OP_SARB : CC_OP_SHLB) + ot);
     }
 }
@@ -1877,15 +1995,15 @@ static void gen_rot_rm_T1(DisasContext *s, MemOp ot, int op1, int is_right)
        since we've computed the flags into CC_SRC, these variables are
        currently dead.  */
     if (is_right) {
-        tcg_gen_shri_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, s->T0, mask - 1);
-        tcg_gen_shri_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0, mask);
-        tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_cc_dst, 1);
+        tcg_gen_shri_tl(tcg_ctx, cpu_cc_src2, s->T0, mask - 1);
+        tcg_gen_shri_tl(tcg_ctx, cpu_cc_dst, s->T0, mask);
+        tcg_gen_andi_tl(tcg_ctx, cpu_cc_dst, cpu_cc_dst, 1);
     } else {
-        tcg_gen_shri_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, s->T0, mask);
-        tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0, 1);
+        tcg_gen_shri_tl(tcg_ctx, cpu_cc_src2, s->T0, mask);
+        tcg_gen_andi_tl(tcg_ctx, cpu_cc_dst, s->T0, 1);
     }
-    tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, tcg_ctx->cpu_cc_src2, 1);
-    tcg_gen_xor_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, tcg_ctx->cpu_cc_src2, tcg_ctx->cpu_cc_dst);
+    tcg_gen_andi_tl(tcg_ctx, cpu_cc_src2, cpu_cc_src2, 1);
+    tcg_gen_xor_tl(tcg_ctx, cpu_cc_src2, cpu_cc_src2, cpu_cc_dst);
 
     /* Now conditionally store the new CC_OP value.  If the shift count
        is 0 we keep the CC_OP_EFLAGS setting so that only CC_SRC is live.
@@ -1896,7 +2014,7 @@ static void gen_rot_rm_T1(DisasContext *s, MemOp ot, int op1, int is_right)
     tcg_gen_trunc_tl_i32(tcg_ctx, t1, s->T1);
     tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, CC_OP_ADCOX);
     tcg_gen_movi_i32(tcg_ctx, s->tmp3_i32, CC_OP_EFLAGS);
-    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_NE, tcg_ctx->cpu_cc_op, t1, t0,
+    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_NE, cpu_cc_op, t1, t0,
                         s->tmp2_i32, s->tmp3_i32);
     tcg_temp_free_i32(tcg_ctx, t0);
     tcg_temp_free_i32(tcg_ctx, t1);
@@ -1970,15 +2088,15 @@ static void gen_rot_rm_im(DisasContext *s, MemOp ot, int op1, int op2,
            since we've computed the flags into CC_SRC, these variables are
            currently dead.  */
         if (is_right) {
-            tcg_gen_shri_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, s->T0, mask - 1);
-            tcg_gen_shri_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0, mask);
-            tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_cc_dst, 1);
+            tcg_gen_shri_tl(tcg_ctx, cpu_cc_src2, s->T0, mask - 1);
+            tcg_gen_shri_tl(tcg_ctx, cpu_cc_dst, s->T0, mask);
+            tcg_gen_andi_tl(tcg_ctx, cpu_cc_dst, cpu_cc_dst, 1);
         } else {
-            tcg_gen_shri_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, s->T0, mask);
-            tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0, 1);
+            tcg_gen_shri_tl(tcg_ctx, cpu_cc_src2, s->T0, mask);
+            tcg_gen_andi_tl(tcg_ctx, cpu_cc_dst, s->T0, 1);
         }
-        tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, tcg_ctx->cpu_cc_src2, 1);
-        tcg_gen_xor_tl(tcg_ctx, tcg_ctx->cpu_cc_src2, tcg_ctx->cpu_cc_src2, tcg_ctx->cpu_cc_dst);
+        tcg_gen_andi_tl(tcg_ctx, cpu_cc_src2, cpu_cc_src2, 1);
+        tcg_gen_xor_tl(tcg_ctx, cpu_cc_src2, cpu_cc_src2, cpu_cc_dst);
         set_cc_op(s, CC_OP_ADCOX);
     }
 }
@@ -1988,9 +2106,8 @@ static void gen_rotc_rm_T1(DisasContext *s, MemOp ot, int op1,
                            int is_right)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     gen_compute_eflags(s);
-    // assert(s->cc_op == CC_OP_EFLAGS);
+    assert(s->cc_op == CC_OP_EFLAGS);
 
     /* load */
     if (op1 == OR_TMP0)
@@ -2001,17 +2118,17 @@ static void gen_rotc_rm_T1(DisasContext *s, MemOp ot, int op1,
     if (is_right) {
         switch (ot) {
         case MO_8:
-            gen_helper_rcrb(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rcrb(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
         case MO_16:
-            gen_helper_rcrw(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rcrw(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
         case MO_32:
-            gen_helper_rcrl(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rcrl(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_helper_rcrq(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rcrq(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
 #endif
         default:
@@ -2020,17 +2137,17 @@ static void gen_rotc_rm_T1(DisasContext *s, MemOp ot, int op1,
     } else {
         switch (ot) {
         case MO_8:
-            gen_helper_rclb(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rclb(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
         case MO_16:
-            gen_helper_rclw(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rclw(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
         case MO_32:
-            gen_helper_rcll(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rcll(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_helper_rclq(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->T0, s->T1);
+            gen_helper_rclq(tcg_ctx, s->T0, cpu_env, s->T0, s->T1);
             break;
 #endif
         default:
@@ -2071,9 +2188,12 @@ static void gen_shiftd_rm_T1(DisasContext *s, MemOp ot, int op1,
         } else {
             tcg_gen_deposit_tl(tcg_ctx, s->T1, s->T0, s->T1, 16, 16);
         }
-        /* FALLTHRU */
-#ifdef TARGET_X86_64
+        /*
+         * If TARGET_X86_64 defined then fall through into MO_32 case,
+         * otherwise fall through default case.
+         */
     case MO_32:
+#ifdef TARGET_X86_64
         /* Concatenate the two 32-bit values and use a 64-bit shift.  */
         tcg_gen_subi_tl(tcg_ctx, s->tmp0, count, 1);
         if (is_right) {
@@ -2157,7 +2277,6 @@ static void gen_shift(DisasContext *s1, int op, MemOp ot, int d, int s)
 static void gen_shifti(DisasContext *s1, int op, MemOp ot, int d, int c)
 {
     TCGContext *tcg_ctx = s1->uc->tcg_ctx;
-
     switch(op) {
     case OP_ROL:
         gen_rot_rm_im(s1, ot, d, c, 0);
@@ -2189,8 +2308,14 @@ static uint64_t advance_pc(CPUX86State *env, DisasContext *s, int num_bytes)
 {
     uint64_t pc = s->pc;
 
+    /* This is a subsequent insn that crosses a page boundary.  */
+    if (s->base.num_insns > 1 &&
+        !is_same_page(&s->base, s->pc + num_bytes - 1)) {
+        siglongjmp(s->jmpbuf, 2);
+    }
+
     s->pc += num_bytes;
-    if (unlikely(s->pc - s->pc_start > X86_MAX_INSN_LENGTH)) {
+    if (unlikely(cur_insn_len(s) > X86_MAX_INSN_LENGTH)) {
         /* If the instruction's 16th byte is on a different page than the 1st, a
          * page fault on the second page wins over the general protection fault
          * caused by the instruction being too long.
@@ -2209,28 +2334,38 @@ static uint64_t advance_pc(CPUX86State *env, DisasContext *s, int num_bytes)
 
 static inline uint8_t x86_ldub_code(CPUX86State *env, DisasContext *s)
 {
-    return translator_ldub(env->uc->tcg_ctx, env, advance_pc(env, s, 1));
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    return translator_ldub(tcg_ctx, env, advance_pc(env, s, 1));
 }
 
 static inline int16_t x86_ldsw_code(CPUX86State *env, DisasContext *s)
 {
-    return translator_ldsw(env->uc->tcg_ctx, env, advance_pc(env, s, 2));
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    return translator_ldsw(tcg_ctx, env, advance_pc(env, s, 2));
 }
 
 static inline uint16_t x86_lduw_code(CPUX86State *env, DisasContext *s)
 {
-    return translator_lduw(env->uc->tcg_ctx, env, advance_pc(env, s, 2));
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    return translator_lduw(tcg_ctx, env, advance_pc(env, s, 2));
 }
 
 static inline uint32_t x86_ldl_code(CPUX86State *env, DisasContext *s)
 {
-    return translator_ldl(env->uc->tcg_ctx, env, advance_pc(env, s, 4));
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    return translator_ldl(tcg_ctx, env, advance_pc(env, s, 4));
 }
 
 #ifdef TARGET_X86_64
 static inline uint64_t x86_ldq_code(CPUX86State *env, DisasContext *s)
 {
-    return translator_ldq(env->uc->tcg_ctx, env, advance_pc(env, s, 8));
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    return translator_ldq(tcg_ctx, env, advance_pc(env, s, 8));
 }
 #endif
 
@@ -2368,27 +2503,32 @@ static AddressParts gen_lea_modrm_0(CPUX86State *env, DisasContext *s,
 }
 
 /* Compute the address, with a minimum number of TCG ops.  */
-static TCGv gen_lea_modrm_1(DisasContext *s, AddressParts a)
+static TCGv gen_lea_modrm_1(DisasContext *s, AddressParts a, bool is_vsib)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     TCGv ea = NULL;
 
-    if (a.index >= 0) {
+    if (a.index >= 0 && !is_vsib) {
         if (a.scale == 0) {
-            ea = tcg_ctx->cpu_regs[a.index];
+            ea = cpu_regs[a.index];
         } else {
-            tcg_gen_shli_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[a.index], a.scale);
+            tcg_gen_shli_tl(tcg_ctx, s->A0, cpu_regs[a.index], a.scale);
             ea = s->A0;
         }
         if (a.base >= 0) {
-            tcg_gen_add_tl(tcg_ctx, s->A0, ea, tcg_ctx->cpu_regs[a.base]);
+            tcg_gen_add_tl(tcg_ctx, s->A0, ea, cpu_regs[a.base]);
             ea = s->A0;
         }
     } else if (a.base >= 0) {
-        ea = tcg_ctx->cpu_regs[a.base];
+        ea = cpu_regs[a.base];
     }
     if (!ea) {
-        tcg_gen_movi_tl(tcg_ctx, s->A0, a.disp);
+        if (TARGET_TB_PCREL && a.base == -2) {
+            /* With cpu_eip ~= pc_save, the expression is pc-relative. */
+            tcg_gen_addi_tl(tcg_ctx, s->A0, cpu_eip, a.disp - s->pc_save);
+        } else {
+            tcg_gen_movi_tl(tcg_ctx, s->A0, a.disp);
+        }
         ea = s->A0;
     } else if (a.disp != 0) {
         tcg_gen_addi_tl(tcg_ctx, s->A0, ea, a.disp);
@@ -2401,7 +2541,7 @@ static TCGv gen_lea_modrm_1(DisasContext *s, AddressParts a)
 static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm)
 {
     AddressParts a = gen_lea_modrm_0(env, s, modrm);
-    TCGv ea = gen_lea_modrm_1(s, a);
+    TCGv ea = gen_lea_modrm_1(s, a, false);
     gen_lea_v_seg(s, s->aflag, ea, a.def_seg, s->override);
 }
 
@@ -2415,7 +2555,8 @@ static void gen_bndck(CPUX86State *env, DisasContext *s, int modrm,
                       TCGCond cond, TCGv_i64 bndv)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv ea = gen_lea_modrm_1(s, gen_lea_modrm_0(env, s, modrm));
+    AddressParts a = gen_lea_modrm_0(env, s, modrm);
+    TCGv ea = gen_lea_modrm_1(s, a, false);
 
     tcg_gen_extu_tl_i64(tcg_ctx, s->tmp1_i64, ea);
     if (!CODE64(s)) {
@@ -2423,7 +2564,7 @@ static void gen_bndck(CPUX86State *env, DisasContext *s, int modrm,
     }
     tcg_gen_setcond_i64(tcg_ctx, cond, s->tmp1_i64, s->tmp1_i64, bndv);
     tcg_gen_extrl_i64_i32(tcg_ctx, s->tmp2_i32, s->tmp1_i64);
-    gen_helper_bndck(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+    gen_helper_bndck(tcg_ctx, cpu_env, s->tmp2_i32);
 }
 
 /* used for LEA and MOV AX, mem */
@@ -2465,9 +2606,9 @@ static void gen_ldst_modrm(CPUX86State *env, DisasContext *s, int modrm,
     }
 }
 
-static inline uint32_t insn_get(CPUX86State *env, DisasContext *s, MemOp ot)
+static target_ulong insn_get_addr(CPUX86State *env, DisasContext *s, MemOp ot)
 {
-    uint32_t ret;
+    target_ulong ret;
 
     switch (ot) {
     case MO_8:
@@ -2477,77 +2618,85 @@ static inline uint32_t insn_get(CPUX86State *env, DisasContext *s, MemOp ot)
         ret = x86_lduw_code(env, s);
         break;
     case MO_32:
+        ret = x86_ldl_code(env, s);
+        break;
 #ifdef TARGET_X86_64
     case MO_64:
-#endif
-        ret = x86_ldl_code(env, s);
+        ret = x86_ldq_code(env, s);
         break;
+#endif
     default:
-        tcg_abort();
+        g_assert_not_reached();
     }
     return ret;
 }
 
-static inline int insn_const_size(MemOp ot)
+static inline uint32_t insn_get(CPUX86State *env, DisasContext *s, MemOp ot)
 {
-    if (ot <= MO_32) {
-        return 1 << ot;
-    } else {
-        return 4;
-    }
-}
+    uint32_t ret;
 
-static inline bool use_goto_tb(DisasContext *s, target_ulong pc)
-{
-    return (pc & TARGET_PAGE_MASK) == (s->base.tb->pc & TARGET_PAGE_MASK) ||
-           (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK);
+    switch (ot) {
+    case MO_8:
+        ret = x86_ldub_code(env, s);
+        break;
+    case MO_16:
+        ret = x86_lduw_code(env, s);
+        break;
+    case MO_32:
+#ifdef TARGET_X86_64
+    case MO_64:
+#endif
+        ret = x86_ldl_code(env, s);
+        break;
+    default:
+        tcg_abort();
+    }
+    return ret;
 }
 
-static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip)
+static target_long insn_get_signed(CPUX86State *env, DisasContext *s, MemOp ot)
 {
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    target_ulong pc = s->cs_base + eip;
+    target_long ret;
 
-    if (use_goto_tb(s, pc))  {
-        /* jump to same page: we can use a direct jump */
-        tcg_gen_goto_tb(tcg_ctx, tb_num);
-        gen_jmp_im(s, eip);
-        tcg_gen_exit_tb(tcg_ctx, s->base.tb, tb_num);
-        s->base.is_jmp = DISAS_NORETURN;
-    } else {
-        /* jump to another page */
-        gen_jmp_im(s, eip);
-        gen_jr(s, s->tmp0);
+    switch (ot) {
+    case MO_8:
+        ret = (int8_t) x86_ldub_code(env, s);
+        break;
+    case MO_16:
+        ret = (int16_t) x86_lduw_code(env, s);
+        break;
+    case MO_32:
+        ret = (int32_t) x86_ldl_code(env, s);
+        break;
+#ifdef TARGET_X86_64
+    case MO_64:
+        ret = x86_ldq_code(env, s);
+        break;
+#endif
+    default:
+        g_assert_not_reached();
     }
+    return ret;
 }
 
-static inline void gen_jcc(DisasContext *s, int b,
-                           target_ulong val, target_ulong next_eip)
+static inline int insn_const_size(MemOp ot)
 {
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGLabel *l1, *l2;
-
-    if (s->jmp_opt) {
-        l1 = gen_new_label(tcg_ctx);
-        gen_jcc1(s, b, l1);
-
-        gen_goto_tb(s, 0, next_eip);
-
-        gen_set_label(tcg_ctx, l1);
-        gen_goto_tb(s, 1, val);
+    if (ot <= MO_32) {
+        return 1 << ot;
     } else {
-        l1 = gen_new_label(tcg_ctx);
-        l2 = gen_new_label(tcg_ctx);
-        gen_jcc1(s, b, l1);
+        return 4;
+    }
+}
 
-        gen_jmp_im(s, next_eip);
-        tcg_gen_br(tcg_ctx, l2);
+static void gen_jcc(DisasContext *s, int b, int diff)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGLabel *l1 = gen_new_label(tcg_ctx);
 
-        gen_set_label(tcg_ctx, l1);
-        gen_jmp_im(s, val);
-        gen_set_label(tcg_ctx, l2);
-        gen_eob(s);
-    }
+    gen_jcc1(s, b, l1);
+    gen_jmp_rel_csize(s, 0, 1);
+    gen_set_label(tcg_ctx, l1);
+    gen_jmp_rel(s, s->dflag, diff, 0);
 }
 
 static void gen_cmovcc1(CPUX86State *env, DisasContext *s, MemOp ot, int b,
@@ -2569,7 +2718,7 @@ static void gen_cmovcc1(CPUX86State *env, DisasContext *s, MemOp ot, int b,
     }
 
     tcg_gen_movcond_tl(tcg_ctx, cc.cond, s->T0, cc.reg, cc.reg2,
-                       s->T0, tcg_ctx->cpu_regs[reg]);
+                       s->T0, cpu_regs[reg]);
     gen_op_mov_reg_v(s, ot, reg, s->T0);
 
     if (cc.mask != -1) {
@@ -2580,72 +2729,55 @@ static void gen_cmovcc1(CPUX86State *env, DisasContext *s, MemOp ot, int b,
     }
 }
 
-static inline void gen_op_movl_T0_seg(DisasContext *s, int seg_reg)
+static inline void gen_op_movl_T0_seg(DisasContext *s, X86Seg seg_reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
+    tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env,
                      offsetof(CPUX86State,segs[seg_reg].selector));
 }
 
-static inline void gen_op_movl_seg_T0_vm(DisasContext *s, int seg_reg)
+static inline void gen_op_movl_seg_T0_vm(DisasContext *s, X86Seg seg_reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     tcg_gen_ext16u_tl(tcg_ctx, s->T0, s->T0);
-    tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
+    tcg_gen_st32_tl(tcg_ctx, s->T0, cpu_env,
                     offsetof(CPUX86State,segs[seg_reg].selector));
-    tcg_gen_shli_tl(tcg_ctx, tcg_ctx->cpu_seg_base[seg_reg], s->T0, 4);
+    tcg_gen_shli_tl(tcg_ctx, cpu_seg_base[seg_reg], s->T0, 4);
 }
 
 /* move T0 to seg_reg and compute if the CPU state may change. Never
    call this function with seg_reg == R_CS */
-static void gen_movl_seg_T0(DisasContext *s, int seg_reg)
+static void gen_movl_seg_T0(DisasContext *s, X86Seg seg_reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    if (s->pe && !s->vm86) {
+    if (PE(s) && !VM86(s)) {
         tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-        gen_helper_load_seg(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, seg_reg), s->tmp2_i32);
+        gen_helper_load_seg(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, seg_reg), s->tmp2_i32);
         /* abort translation because the addseg value may change or
            because ss32 may change. For R_SS, translation must always
            stop as a special handling must be done to disable hardware
            interrupts for the next instruction */
-        if (seg_reg == R_SS || (s->code32 && seg_reg < R_FS)) {
-            s->base.is_jmp = DISAS_TOO_MANY;
+        if (seg_reg == R_SS) {
+            s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ;
+        } else if (CODE32(s) && seg_reg < R_FS) {
+            s->base.is_jmp = DISAS_EOB_NEXT;
         }
     } else {
         gen_op_movl_seg_T0_vm(s, seg_reg);
         if (seg_reg == R_SS) {
-            s->base.is_jmp = DISAS_TOO_MANY;
+            s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ;
         }
     }
 }
 
-static inline int svm_is_rep(int prefixes)
-{
-    return ((prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) ? 8 : 0);
-}
-
-static inline void
-gen_svm_check_intercept_param(DisasContext *s, target_ulong pc_start,
-                              uint32_t type, uint64_t param)
+static void gen_svm_check_intercept(DisasContext *s, uint32_t type)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     /* no SVM activated; fast case */
-    if (likely(!(s->flags & HF_GUEST_MASK)))
+    if (likely(!GUEST(s))) {
         return;
-    gen_update_cc_op(s);
-    gen_jmp_im(s, pc_start - s->cs_base);
-    gen_helper_svm_check_intercept_param(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, type),
-                                         tcg_const_i64(tcg_ctx, param));
-}
-
-static inline void
-gen_svm_check_intercept(DisasContext *s, target_ulong pc_start, uint64_t type)
-{
-    gen_svm_check_intercept_param(s, pc_start, type, 0);
+    }
+    gen_helper_svm_check_intercept(tcg_ctx, cpu_env, tcg_constant_i32(tcg_ctx, type));
 }
 
 static inline void gen_stack_update(DisasContext *s, int addend)
@@ -2662,10 +2794,10 @@ static void gen_push_v(DisasContext *s, TCGv val)
     int size = 1 << d_ot;
     TCGv new_esp = s->A0;
 
-    tcg_gen_subi_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[R_ESP], size);
+    tcg_gen_subi_tl(tcg_ctx, s->A0, cpu_regs[R_ESP], size);
 
     if (!CODE64(s)) {
-        if (s->addseg) {
+        if (ADDSEG(s)) {
             new_esp = s->tmp4;
             tcg_gen_mov_tl(tcg_ctx, new_esp, s->A0);
         }
@@ -2680,10 +2812,9 @@ static void gen_push_v(DisasContext *s, TCGv val)
 static MemOp gen_pop_T0(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
     MemOp d_ot = mo_pushpop(s, s->dflag);
 
-    gen_lea_v_seg(s, mo_stacksize(s), tcg_ctx->cpu_regs[R_ESP], R_SS, -1);
+    gen_lea_v_seg(s, mo_stacksize(s), cpu_regs[R_ESP], R_SS, -1);
     gen_op_ld_v(s, d_ot, s->T0, s->A0);
 
     return d_ot;
@@ -2697,22 +2828,21 @@ static inline void gen_pop_update(DisasContext *s, MemOp ot)
 static inline void gen_stack_A0(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    gen_lea_v_seg(s, s->ss32 ? MO_32 : MO_16, tcg_ctx->cpu_regs[R_ESP], R_SS, -1);
+    gen_lea_v_seg(s, SS32(s) ? MO_32 : MO_16, cpu_regs[R_ESP], R_SS, -1);
 }
 
 static void gen_pusha(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    MemOp s_ot = s->ss32 ? MO_32 : MO_16;
+    MemOp s_ot = SS32(s) ? MO_32 : MO_16;
     MemOp d_ot = s->dflag;
     int size = 1 << d_ot;
     int i;
 
     for (i = 0; i < 8; i++) {
-        tcg_gen_addi_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[R_ESP], (i - 8) * size);
+        tcg_gen_addi_tl(tcg_ctx, s->A0, cpu_regs[R_ESP], (i - 8) * size);
         gen_lea_v_seg(s, s_ot, s->A0, R_SS, -1);
-        gen_op_st_v(s, d_ot, tcg_ctx->cpu_regs[7 - i], s->A0);
+        gen_op_st_v(s, d_ot, cpu_regs[7 - i], s->A0);
     }
 
     gen_stack_update(s, -8 * size);
@@ -2721,7 +2851,7 @@ static void gen_pusha(DisasContext *s)
 static void gen_popa(DisasContext *s)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    MemOp s_ot = s->ss32 ? MO_32 : MO_16;
+    MemOp s_ot = SS32(s) ? MO_32 : MO_16;
     MemOp d_ot = s->dflag;
     int size = 1 << d_ot;
     int i;
@@ -2731,7 +2861,7 @@ static void gen_popa(DisasContext *s)
         if (7 - i == R_ESP) {
             continue;
         }
-        tcg_gen_addi_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[R_ESP], i * size);
+        tcg_gen_addi_tl(tcg_ctx, s->A0, cpu_regs[R_ESP], i * size);
         gen_lea_v_seg(s, s_ot, s->A0, R_SS, -1);
         gen_op_ld_v(s, d_ot, s->T0, s->A0);
         gen_op_mov_reg_v(s, d_ot, 7 - i, s->T0);
@@ -2744,13 +2874,13 @@ static void gen_enter(DisasContext *s, int esp_addend, int level)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     MemOp d_ot = mo_pushpop(s, s->dflag);
-    MemOp a_ot = CODE64(s) ? MO_64 : s->ss32 ? MO_32 : MO_16;
+    MemOp a_ot = CODE64(s) ? MO_64 : SS32(s) ? MO_32 : MO_16;
     int size = 1 << d_ot;
 
     /* Push BP; compute FrameTemp into T1.  */
-    tcg_gen_subi_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[R_ESP], size);
+    tcg_gen_subi_tl(tcg_ctx, s->T1, cpu_regs[R_ESP], size);
     gen_lea_v_seg(s, a_ot, s->T1, R_SS, -1);
-    gen_op_st_v(s, d_ot, tcg_ctx->cpu_regs[R_EBP], s->A0);
+    gen_op_st_v(s, d_ot, cpu_regs[R_EBP], s->A0);
 
     level &= 31;
     if (level != 0) {
@@ -2758,7 +2888,7 @@ static void gen_enter(DisasContext *s, int esp_addend, int level)
 
         /* Copy level-1 pointers from the previous frame.  */
         for (i = 1; i < level; ++i) {
-            tcg_gen_subi_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[R_EBP], size * i);
+            tcg_gen_subi_tl(tcg_ctx, s->A0, cpu_regs[R_EBP], size * i);
             gen_lea_v_seg(s, a_ot, s->A0, R_SS, -1);
             gen_op_ld_v(s, d_ot, s->tmp0, s->A0);
 
@@ -2774,7 +2904,7 @@ static void gen_enter(DisasContext *s, int esp_addend, int level)
     }
 
     /* Copy the FrameTemp value to EBP.  */
-    gen_op_mov_reg_v(s, a_ot, R_EBP, s->T1);
+    gen_op_mov_reg_v(s, d_ot, R_EBP, s->T1);
 
     /* Compute the final value of ESP.  */
     tcg_gen_subi_tl(tcg_ctx, s->T1, s->T1, esp_addend + size * level);
@@ -2787,10 +2917,10 @@ static void gen_leave(DisasContext *s)
     MemOp d_ot = mo_pushpop(s, s->dflag);
     MemOp a_ot = mo_stacksize(s);
 
-    gen_lea_v_seg(s, a_ot, tcg_ctx->cpu_regs[R_EBP], R_SS, -1);
+    gen_lea_v_seg(s, a_ot, cpu_regs[R_EBP], R_SS, -1);
     gen_op_ld_v(s, d_ot, s->T0, s->A0);
 
-    tcg_gen_addi_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[R_EBP], 1ULL << d_ot);
+    tcg_gen_addi_tl(tcg_ctx, s->T1, cpu_regs[R_EBP], 1 << d_ot);
 
     gen_op_mov_reg_v(s, d_ot, R_EBP, s->T0);
     gen_op_mov_reg_v(s, a_ot, R_ESP, s->T1);
@@ -2802,18 +2932,31 @@ static void gen_leave(DisasContext *s)
 static void gen_unknown_opcode(CPUX86State *env, DisasContext *s)
 {
     gen_illegal_opcode(s);
+
+    if (qemu_loglevel_mask(LOG_UNIMP)) {
+        FILE *logfile = qemu_log_trylock();
+        if (logfile) {
+            target_ulong pc = s->base.pc_next, end = s->pc;
+
+            fprintf(logfile, "ILLOPC: " TARGET_FMT_lx ":", pc);
+            for (; pc < end; ++pc) {
+                fprintf(logfile, " %02x", cpu_ldub_code(env, pc));
+            }
+            fprintf(logfile, "\n");
+            qemu_log_unlock(logfile);
+        }
+    }
 }
 
 /* an interrupt is different from an exception because of the
    privilege checks */
-static void gen_interrupt(DisasContext *s, int intno,
-                          target_ulong cur_eip, target_ulong next_eip)
+static void gen_interrupt(DisasContext *s, int intno)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     gen_update_cc_op(s);
-    gen_jmp_im(s, cur_eip);
-    gen_helper_raise_interrupt(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, intno),
-                               tcg_const_i32(tcg_ctx, next_eip - cur_eip));
+    gen_update_eip_cur(s);
+    gen_helper_raise_interrupt(tcg_ctx, cpu_env, tcg_constant_i32(tcg_ctx, intno),
+                               cur_insn_len_i32(s));
     s->base.is_jmp = DISAS_NORETURN;
 }
 
@@ -2821,21 +2964,21 @@ static void gen_debug(DisasContext *s, target_ulong cur_eip)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
 
+    (void)cur_eip;
     gen_update_cc_op(s);
-    gen_jmp_im(s, cur_eip);
-    gen_helper_debug(tcg_ctx, tcg_ctx->cpu_env);
+    gen_update_eip_cur(s);
+    gen_helper_debug(tcg_ctx, cpu_env);
     s->base.is_jmp = DISAS_NORETURN;
 }
 
 static void gen_set_hflag(DisasContext *s, uint32_t mask)
 {
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
     if ((s->flags & mask) == 0) {
-        TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
         TCGv_i32 t = tcg_temp_new_i32(tcg_ctx);
-        tcg_gen_ld_i32(tcg_ctx, t, tcg_ctx->cpu_env, offsetof(CPUX86State, hflags));
+        tcg_gen_ld_i32(tcg_ctx, t, cpu_env, offsetof(CPUX86State, hflags));
         tcg_gen_ori_i32(tcg_ctx, t, t, mask);
-        tcg_gen_st_i32(tcg_ctx, t, tcg_ctx->cpu_env, offsetof(CPUX86State, hflags));
+        tcg_gen_st_i32(tcg_ctx, t, cpu_env, offsetof(CPUX86State, hflags));
         tcg_temp_free_i32(tcg_ctx, t);
         s->flags |= mask;
     }
@@ -2843,2238 +2986,433 @@ static void gen_set_hflag(DisasContext *s, uint32_t mask)
 
 static void gen_reset_hflag(DisasContext *s, uint32_t mask)
 {
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
     if (s->flags & mask) {
-        TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
         TCGv_i32 t = tcg_temp_new_i32(tcg_ctx);
-        tcg_gen_ld_i32(tcg_ctx, t, tcg_ctx->cpu_env, offsetof(CPUX86State, hflags));
+        tcg_gen_ld_i32(tcg_ctx, t, cpu_env, offsetof(CPUX86State, hflags));
         tcg_gen_andi_i32(tcg_ctx, t, t, ~mask);
-        tcg_gen_st_i32(tcg_ctx, t, tcg_ctx->cpu_env, offsetof(CPUX86State, hflags));
-        tcg_temp_free_i32(tcg_ctx, t);
-        s->flags &= ~mask;
-    }
-}
-
-/* Clear BND registers during legacy branches.  */
-static void gen_bnd_jmp(DisasContext *s)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    /* Clear the registers only if BND prefix is missing, MPX is enabled,
-       and if the BNDREGs are known to be in use (non-zero) already.
-       The helper itself will check BNDPRESERVE at runtime.  */
-    if ((s->prefix & PREFIX_REPNZ) == 0
-        && (s->flags & HF_MPX_EN_MASK) != 0
-        && (s->flags & HF_MPX_IU_MASK) != 0) {
-        gen_helper_bnd_jmp(tcg_ctx, tcg_ctx->cpu_env);
-    }
-}
-
-/* Generate an end of block. Trace exception is also generated if needed.
-   If INHIBIT, set HF_INHIBIT_IRQ_MASK if it isn't already set.
-   If RECHECK_TF, emit a rechecking helper for #DB, ignoring the state of
-   S->TF.  This is used by the syscall/sysret insns.  */
-static void
-do_gen_eob_worker(DisasContext *s, bool inhibit, bool recheck_tf, bool jr)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    gen_update_cc_op(s);
-
-    /* If several instructions disable interrupts, only the first does it.  */
-    if (inhibit && !(s->flags & HF_INHIBIT_IRQ_MASK)) {
-        gen_set_hflag(s, HF_INHIBIT_IRQ_MASK);
-    } else {
-        gen_reset_hflag(s, HF_INHIBIT_IRQ_MASK);
-    }
-
-    if (s->base.tb->flags & HF_RF_MASK) {
-        gen_helper_reset_rf(tcg_ctx, tcg_ctx->cpu_env);
-    }
-    if (s->base.singlestep_enabled) {
-        gen_helper_debug(tcg_ctx, tcg_ctx->cpu_env);
-    } else if (recheck_tf) {
-        gen_helper_rechecking_single_step(tcg_ctx, tcg_ctx->cpu_env);
-        tcg_gen_exit_tb(tcg_ctx, NULL, 0);
-    } else if (s->tf) {
-        gen_helper_single_step(tcg_ctx, tcg_ctx->cpu_env);
-    } else if (jr) {
-        tcg_gen_lookup_and_goto_ptr(tcg_ctx);
-    } else {
-        tcg_gen_exit_tb(tcg_ctx, NULL, 0);
-    }
-    s->base.is_jmp = DISAS_NORETURN;
-}
-
-static inline void
-gen_eob_worker(DisasContext *s, bool inhibit, bool recheck_tf)
-{
-    do_gen_eob_worker(s, inhibit, recheck_tf, false);
-}
-
-/* End of block.
-   If INHIBIT, set HF_INHIBIT_IRQ_MASK if it isn't already set.  */
-static void gen_eob_inhibit_irq(DisasContext *s, bool inhibit)
-{
-    gen_eob_worker(s, inhibit, false);
-}
-
-/* End of block, resetting the inhibit irq flag.  */
-static void gen_eob(DisasContext *s)
-{
-    gen_eob_worker(s, false, false);
-}
-
-/* Jump to register */
-static void gen_jr(DisasContext *s, TCGv dest)
-{
-    do_gen_eob_worker(s, false, false, true);
-}
-
-/* generate a jump to eip. No segment change must happen before as a
-   direct call to the next block may occur */
-static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num)
-{
-    gen_update_cc_op(s);
-    set_cc_op(s, CC_OP_DYNAMIC);
-    if (s->jmp_opt) {
-        gen_goto_tb(s, tb_num, eip);
-    } else {
-        gen_jmp_im(s, eip);
-        gen_eob(s);
-    }
-}
-
-static void gen_jmp(DisasContext *s, target_ulong eip)
-{
-    gen_jmp_tb(s, eip, 0);
-}
-
-static inline void gen_ldq_env_A0(DisasContext *s, int offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEQ);
-    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, offset);
-}
-
-static inline void gen_stq_env_A0(DisasContext *s, int offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, offset);
-    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEQ);
-}
-
-static inline void gen_ldo_env_A0(DisasContext *s, int offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    int mem_index = s->mem_index;
-
-    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, mem_index, MO_LEQ);
-    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, offset + offsetof(ZMMReg, ZMM_Q(0)));
-    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 8);
-    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEQ);
-    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, offset + offsetof(ZMMReg, ZMM_Q(1)));
-}
-
-static inline void gen_sto_env_A0(DisasContext *s, int offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    int mem_index = s->mem_index;
-
-    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, offset + offsetof(ZMMReg, ZMM_Q(0)));
-    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, mem_index, MO_LEQ);
-    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 8);
-    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, offset + offsetof(ZMMReg, ZMM_Q(1)));
-    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEQ);
-}
-
-static inline void gen_op_movo(DisasContext *s, int d_offset, int s_offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, s_offset + offsetof(ZMMReg, ZMM_Q(0)));
-    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, d_offset + offsetof(ZMMReg, ZMM_Q(0)));
-    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, s_offset + offsetof(ZMMReg, ZMM_Q(1)));
-    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, d_offset + offsetof(ZMMReg, ZMM_Q(1)));
-}
-
-static inline void gen_op_movq(DisasContext *s, int d_offset, int s_offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, s_offset);
-    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, d_offset);
-}
-
-static inline void gen_op_movl(DisasContext *s, int d_offset, int s_offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, s_offset);
-    tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, d_offset);
-}
-
-static inline void gen_op_movq_env_0(DisasContext *s, int d_offset)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    tcg_gen_movi_i64(tcg_ctx, s->tmp1_i64, 0);
-    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, d_offset);
-}
-
-typedef void (*SSEFunc_i_ep)(TCGContext *s, TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg);
-typedef void (*SSEFunc_l_ep)(TCGContext *s, TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg);
-typedef void (*SSEFunc_0_epi)(TCGContext *s, TCGv_ptr env, TCGv_ptr reg, TCGv_i32 val);
-typedef void (*SSEFunc_0_epl)(TCGContext *s, TCGv_ptr env, TCGv_ptr reg, TCGv_i64 val);
-typedef void (*SSEFunc_0_epp)(TCGContext *s, TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b);
-typedef void (*SSEFunc_0_eppi)(TCGContext *s, TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
-                               TCGv_i32 val);
-typedef void (*SSEFunc_0_ppi)(TCGContext *s, TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val);
-typedef void (*SSEFunc_0_eppt)(TCGContext *s, TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b,
-                               TCGv val);
-
-#define SSE_SPECIAL ((void *)1)
-#define SSE_DUMMY ((void *)2)
-
-#define MMX_OP2(x) { gen_helper_ ## x ## _mmx, gen_helper_ ## x ## _xmm }
-#define SSE_FOP(x) { gen_helper_ ## x ## ps, gen_helper_ ## x ## pd, \
-                     gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, }
-
-static const SSEFunc_0_epp sse_op_table1[256][4] = {
-    /* 3DNow! extensions */
-    [0x0e] = { SSE_DUMMY }, /* femms */
-    [0x0f] = { SSE_DUMMY }, /* pf... */
-    /* pure SSE operations */
-    [0x10] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movups, movupd, movss, movsd */
-    [0x11] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movups, movupd, movss, movsd */
-    [0x12] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movlps, movlpd, movsldup, movddup */
-    [0x13] = { SSE_SPECIAL, SSE_SPECIAL },  /* movlps, movlpd */
-    [0x14] = { gen_helper_punpckldq_xmm, gen_helper_punpcklqdq_xmm },
-    [0x15] = { gen_helper_punpckhdq_xmm, gen_helper_punpckhqdq_xmm },
-    [0x16] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL },  /* movhps, movhpd, movshdup */
-    [0x17] = { SSE_SPECIAL, SSE_SPECIAL },  /* movhps, movhpd */
-
-    [0x28] = { SSE_SPECIAL, SSE_SPECIAL },  /* movaps, movapd */
-    [0x29] = { SSE_SPECIAL, SSE_SPECIAL },  /* movaps, movapd */
-    [0x2a] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvtpi2ps, cvtpi2pd, cvtsi2ss, cvtsi2sd */
-    [0x2b] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movntps, movntpd, movntss, movntsd */
-    [0x2c] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvttps2pi, cvttpd2pi, cvttsd2si, cvttss2si */
-    [0x2d] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvtps2pi, cvtpd2pi, cvtsd2si, cvtss2si */
-    [0x2e] = { gen_helper_ucomiss, gen_helper_ucomisd },
-    [0x2f] = { gen_helper_comiss, gen_helper_comisd },
-    [0x50] = { SSE_SPECIAL, SSE_SPECIAL }, /* movmskps, movmskpd */
-    [0x51] = SSE_FOP(sqrt),
-    [0x52] = { gen_helper_rsqrtps, NULL, gen_helper_rsqrtss, NULL },
-    [0x53] = { gen_helper_rcpps, NULL, gen_helper_rcpss, NULL },
-    [0x54] = { gen_helper_pand_xmm, gen_helper_pand_xmm }, /* andps, andpd */
-    [0x55] = { gen_helper_pandn_xmm, gen_helper_pandn_xmm }, /* andnps, andnpd */
-    [0x56] = { gen_helper_por_xmm, gen_helper_por_xmm }, /* orps, orpd */
-    [0x57] = { gen_helper_pxor_xmm, gen_helper_pxor_xmm }, /* xorps, xorpd */
-    [0x58] = SSE_FOP(add),
-    [0x59] = SSE_FOP(mul),
-    [0x5a] = { gen_helper_cvtps2pd, gen_helper_cvtpd2ps,
-               gen_helper_cvtss2sd, gen_helper_cvtsd2ss },
-    [0x5b] = { gen_helper_cvtdq2ps, gen_helper_cvtps2dq, gen_helper_cvttps2dq },
-    [0x5c] = SSE_FOP(sub),
-    [0x5d] = SSE_FOP(min),
-    [0x5e] = SSE_FOP(div),
-    [0x5f] = SSE_FOP(max),
-
-    [0xc2] = SSE_FOP(cmpeq),
-    [0xc6] = { (SSEFunc_0_epp)gen_helper_shufps,
-               (SSEFunc_0_epp)gen_helper_shufpd }, /* XXX: casts */
-
-    /* SSSE3, SSE4, MOVBE, CRC32, BMI1, BMI2, ADX.  */
-    [0x38] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL },
-    [0x3a] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL },
-
-    /* MMX ops and their SSE extensions */
-    [0x60] = MMX_OP2(punpcklbw),
-    [0x61] = MMX_OP2(punpcklwd),
-    [0x62] = MMX_OP2(punpckldq),
-    [0x63] = MMX_OP2(packsswb),
-    [0x64] = MMX_OP2(pcmpgtb),
-    [0x65] = MMX_OP2(pcmpgtw),
-    [0x66] = MMX_OP2(pcmpgtl),
-    [0x67] = MMX_OP2(packuswb),
-    [0x68] = MMX_OP2(punpckhbw),
-    [0x69] = MMX_OP2(punpckhwd),
-    [0x6a] = MMX_OP2(punpckhdq),
-    [0x6b] = MMX_OP2(packssdw),
-    [0x6c] = { NULL, gen_helper_punpcklqdq_xmm },
-    [0x6d] = { NULL, gen_helper_punpckhqdq_xmm },
-    [0x6e] = { SSE_SPECIAL, SSE_SPECIAL }, /* movd mm, ea */
-    [0x6f] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movq, movdqa, , movqdu */
-    [0x70] = { (SSEFunc_0_epp)gen_helper_pshufw_mmx,
-               (SSEFunc_0_epp)gen_helper_pshufd_xmm,
-               (SSEFunc_0_epp)gen_helper_pshufhw_xmm,
-               (SSEFunc_0_epp)gen_helper_pshuflw_xmm }, /* XXX: casts */
-    [0x71] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftw */
-    [0x72] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftd */
-    [0x73] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftq */
-    [0x74] = MMX_OP2(pcmpeqb),
-    [0x75] = MMX_OP2(pcmpeqw),
-    [0x76] = MMX_OP2(pcmpeql),
-    [0x77] = { SSE_DUMMY }, /* emms */
-    [0x78] = { NULL, SSE_SPECIAL, NULL, SSE_SPECIAL }, /* extrq_i, insertq_i */
-    [0x79] = { NULL, gen_helper_extrq_r, NULL, gen_helper_insertq_r },
-    [0x7c] = { NULL, gen_helper_haddpd, NULL, gen_helper_haddps },
-    [0x7d] = { NULL, gen_helper_hsubpd, NULL, gen_helper_hsubps },
-    [0x7e] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movd, movd, , movq */
-    [0x7f] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movq, movdqa, movdqu */
-    [0xc4] = { SSE_SPECIAL, SSE_SPECIAL }, /* pinsrw */
-    [0xc5] = { SSE_SPECIAL, SSE_SPECIAL }, /* pextrw */
-    [0xd0] = { NULL, gen_helper_addsubpd, NULL, gen_helper_addsubps },
-    [0xd1] = MMX_OP2(psrlw),
-    [0xd2] = MMX_OP2(psrld),
-    [0xd3] = MMX_OP2(psrlq),
-    [0xd4] = MMX_OP2(paddq),
-    [0xd5] = MMX_OP2(pmullw),
-    [0xd6] = { NULL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL },
-    [0xd7] = { SSE_SPECIAL, SSE_SPECIAL }, /* pmovmskb */
-    [0xd8] = MMX_OP2(psubusb),
-    [0xd9] = MMX_OP2(psubusw),
-    [0xda] = MMX_OP2(pminub),
-    [0xdb] = MMX_OP2(pand),
-    [0xdc] = MMX_OP2(paddusb),
-    [0xdd] = MMX_OP2(paddusw),
-    [0xde] = MMX_OP2(pmaxub),
-    [0xdf] = MMX_OP2(pandn),
-    [0xe0] = MMX_OP2(pavgb),
-    [0xe1] = MMX_OP2(psraw),
-    [0xe2] = MMX_OP2(psrad),
-    [0xe3] = MMX_OP2(pavgw),
-    [0xe4] = MMX_OP2(pmulhuw),
-    [0xe5] = MMX_OP2(pmulhw),
-    [0xe6] = { NULL, gen_helper_cvttpd2dq, gen_helper_cvtdq2pd, gen_helper_cvtpd2dq },
-    [0xe7] = { SSE_SPECIAL , SSE_SPECIAL },  /* movntq, movntq */
-    [0xe8] = MMX_OP2(psubsb),
-    [0xe9] = MMX_OP2(psubsw),
-    [0xea] = MMX_OP2(pminsw),
-    [0xeb] = MMX_OP2(por),
-    [0xec] = MMX_OP2(paddsb),
-    [0xed] = MMX_OP2(paddsw),
-    [0xee] = MMX_OP2(pmaxsw),
-    [0xef] = MMX_OP2(pxor),
-    [0xf0] = { NULL, NULL, NULL, SSE_SPECIAL }, /* lddqu */
-    [0xf1] = MMX_OP2(psllw),
-    [0xf2] = MMX_OP2(pslld),
-    [0xf3] = MMX_OP2(psllq),
-    [0xf4] = MMX_OP2(pmuludq),
-    [0xf5] = MMX_OP2(pmaddwd),
-    [0xf6] = MMX_OP2(psadbw),
-    [0xf7] = { (SSEFunc_0_epp)gen_helper_maskmov_mmx,
-               (SSEFunc_0_epp)gen_helper_maskmov_xmm }, /* XXX: casts */
-    [0xf8] = MMX_OP2(psubb),
-    [0xf9] = MMX_OP2(psubw),
-    [0xfa] = MMX_OP2(psubl),
-    [0xfb] = MMX_OP2(psubq),
-    [0xfc] = MMX_OP2(paddb),
-    [0xfd] = MMX_OP2(paddw),
-    [0xfe] = MMX_OP2(paddl),
-};
-
-static const SSEFunc_0_epp sse_op_table2[3 * 8][2] = {
-    [0 + 2] = MMX_OP2(psrlw),
-    [0 + 4] = MMX_OP2(psraw),
-    [0 + 6] = MMX_OP2(psllw),
-    [8 + 2] = MMX_OP2(psrld),
-    [8 + 4] = MMX_OP2(psrad),
-    [8 + 6] = MMX_OP2(pslld),
-    [16 + 2] = MMX_OP2(psrlq),
-    [16 + 3] = { NULL, gen_helper_psrldq_xmm },
-    [16 + 6] = MMX_OP2(psllq),
-    [16 + 7] = { NULL, gen_helper_pslldq_xmm },
-};
-
-static const SSEFunc_0_epi sse_op_table3ai[] = {
-    gen_helper_cvtsi2ss,
-    gen_helper_cvtsi2sd
-};
-
-#ifdef TARGET_X86_64
-static const SSEFunc_0_epl sse_op_table3aq[] = {
-    gen_helper_cvtsq2ss,
-    gen_helper_cvtsq2sd
-};
-#endif
-
-static const SSEFunc_i_ep sse_op_table3bi[] = {
-    gen_helper_cvttss2si,
-    gen_helper_cvtss2si,
-    gen_helper_cvttsd2si,
-    gen_helper_cvtsd2si
-};
-
-#ifdef TARGET_X86_64
-static const SSEFunc_l_ep sse_op_table3bq[] = {
-    gen_helper_cvttss2sq,
-    gen_helper_cvtss2sq,
-    gen_helper_cvttsd2sq,
-    gen_helper_cvtsd2sq
-};
-#endif
-
-static const SSEFunc_0_epp sse_op_table4[8][4] = {
-    SSE_FOP(cmpeq),
-    SSE_FOP(cmplt),
-    SSE_FOP(cmple),
-    SSE_FOP(cmpunord),
-    SSE_FOP(cmpneq),
-    SSE_FOP(cmpnlt),
-    SSE_FOP(cmpnle),
-    SSE_FOP(cmpord),
-};
-
-static const SSEFunc_0_epp sse_op_table5[256] = {
-    [0x0c] = gen_helper_pi2fw,
-    [0x0d] = gen_helper_pi2fd,
-    [0x1c] = gen_helper_pf2iw,
-    [0x1d] = gen_helper_pf2id,
-    [0x8a] = gen_helper_pfnacc,
-    [0x8e] = gen_helper_pfpnacc,
-    [0x90] = gen_helper_pfcmpge,
-    [0x94] = gen_helper_pfmin,
-    [0x96] = gen_helper_pfrcp,
-    [0x97] = gen_helper_pfrsqrt,
-    [0x9a] = gen_helper_pfsub,
-    [0x9e] = gen_helper_pfadd,
-    [0xa0] = gen_helper_pfcmpgt,
-    [0xa4] = gen_helper_pfmax,
-    [0xa6] = gen_helper_movq, /* pfrcpit1; no need to actually increase precision */
-    [0xa7] = gen_helper_movq, /* pfrsqit1 */
-    [0xaa] = gen_helper_pfsubr,
-    [0xae] = gen_helper_pfacc,
-    [0xb0] = gen_helper_pfcmpeq,
-    [0xb4] = gen_helper_pfmul,
-    [0xb6] = gen_helper_movq, /* pfrcpit2 */
-    [0xb7] = gen_helper_pmulhrw_mmx,
-    [0xbb] = gen_helper_pswapd,
-    [0xbf] = gen_helper_pavgb_mmx /* pavgusb */
-};
-
-struct SSEOpHelper_epp {
-    SSEFunc_0_epp op[2];
-    uint32_t ext_mask;
-};
-
-struct SSEOpHelper_eppi {
-    SSEFunc_0_eppi op[2];
-    uint32_t ext_mask;
-};
-
-#define SSSE3_OP(x) { MMX_OP2(x), CPUID_EXT_SSSE3 }
-#define SSE41_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, CPUID_EXT_SSE41 }
-#define SSE42_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, CPUID_EXT_SSE42 }
-#define SSE41_SPECIAL { { NULL, SSE_SPECIAL }, CPUID_EXT_SSE41 }
-#define PCLMULQDQ_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, \
-        CPUID_EXT_PCLMULQDQ }
-#define AESNI_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, CPUID_EXT_AES }
-
-static const struct SSEOpHelper_epp sse_op_table6[256] = {
-    [0x00] = SSSE3_OP(pshufb),
-    [0x01] = SSSE3_OP(phaddw),
-    [0x02] = SSSE3_OP(phaddd),
-    [0x03] = SSSE3_OP(phaddsw),
-    [0x04] = SSSE3_OP(pmaddubsw),
-    [0x05] = SSSE3_OP(phsubw),
-    [0x06] = SSSE3_OP(phsubd),
-    [0x07] = SSSE3_OP(phsubsw),
-    [0x08] = SSSE3_OP(psignb),
-    [0x09] = SSSE3_OP(psignw),
-    [0x0a] = SSSE3_OP(psignd),
-    [0x0b] = SSSE3_OP(pmulhrsw),
-    [0x10] = SSE41_OP(pblendvb),
-    [0x14] = SSE41_OP(blendvps),
-    [0x15] = SSE41_OP(blendvpd),
-    [0x17] = SSE41_OP(ptest),
-    [0x1c] = SSSE3_OP(pabsb),
-    [0x1d] = SSSE3_OP(pabsw),
-    [0x1e] = SSSE3_OP(pabsd),
-    [0x20] = SSE41_OP(pmovsxbw),
-    [0x21] = SSE41_OP(pmovsxbd),
-    [0x22] = SSE41_OP(pmovsxbq),
-    [0x23] = SSE41_OP(pmovsxwd),
-    [0x24] = SSE41_OP(pmovsxwq),
-    [0x25] = SSE41_OP(pmovsxdq),
-    [0x28] = SSE41_OP(pmuldq),
-    [0x29] = SSE41_OP(pcmpeqq),
-    [0x2a] = SSE41_SPECIAL, /* movntqda */
-    [0x2b] = SSE41_OP(packusdw),
-    [0x30] = SSE41_OP(pmovzxbw),
-    [0x31] = SSE41_OP(pmovzxbd),
-    [0x32] = SSE41_OP(pmovzxbq),
-    [0x33] = SSE41_OP(pmovzxwd),
-    [0x34] = SSE41_OP(pmovzxwq),
-    [0x35] = SSE41_OP(pmovzxdq),
-    [0x37] = SSE42_OP(pcmpgtq),
-    [0x38] = SSE41_OP(pminsb),
-    [0x39] = SSE41_OP(pminsd),
-    [0x3a] = SSE41_OP(pminuw),
-    [0x3b] = SSE41_OP(pminud),
-    [0x3c] = SSE41_OP(pmaxsb),
-    [0x3d] = SSE41_OP(pmaxsd),
-    [0x3e] = SSE41_OP(pmaxuw),
-    [0x3f] = SSE41_OP(pmaxud),
-    [0x40] = SSE41_OP(pmulld),
-    [0x41] = SSE41_OP(phminposuw),
-    [0xdb] = AESNI_OP(aesimc),
-    [0xdc] = AESNI_OP(aesenc),
-    [0xdd] = AESNI_OP(aesenclast),
-    [0xde] = AESNI_OP(aesdec),
-    [0xdf] = AESNI_OP(aesdeclast),
-};
-
-static const struct SSEOpHelper_eppi sse_op_table7[256] = {
-    [0x08] = SSE41_OP(roundps),
-    [0x09] = SSE41_OP(roundpd),
-    [0x0a] = SSE41_OP(roundss),
-    [0x0b] = SSE41_OP(roundsd),
-    [0x0c] = SSE41_OP(blendps),
-    [0x0d] = SSE41_OP(blendpd),
-    [0x0e] = SSE41_OP(pblendw),
-    [0x0f] = SSSE3_OP(palignr),
-    [0x14] = SSE41_SPECIAL, /* pextrb */
-    [0x15] = SSE41_SPECIAL, /* pextrw */
-    [0x16] = SSE41_SPECIAL, /* pextrd/pextrq */
-    [0x17] = SSE41_SPECIAL, /* extractps */
-    [0x20] = SSE41_SPECIAL, /* pinsrb */
-    [0x21] = SSE41_SPECIAL, /* insertps */
-    [0x22] = SSE41_SPECIAL, /* pinsrd/pinsrq */
-    [0x40] = SSE41_OP(dpps),
-    [0x41] = SSE41_OP(dppd),
-    [0x42] = SSE41_OP(mpsadbw),
-    [0x44] = PCLMULQDQ_OP(pclmulqdq),
-    [0x60] = SSE42_OP(pcmpestrm),
-    [0x61] = SSE42_OP(pcmpestri),
-    [0x62] = SSE42_OP(pcmpistrm),
-    [0x63] = SSE42_OP(pcmpistri),
-    [0xdf] = AESNI_OP(aeskeygenassist),
-};
-
-static void gen_sse(CPUX86State *env, DisasContext *s, int b,
-                    target_ulong pc_start, int rex_r)
-{
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    int b1, op1_offset, op2_offset, is_xmm, val;
-    int modrm, mod, rm, reg;
-    SSEFunc_0_epp sse_fn_epp;
-    SSEFunc_0_eppi sse_fn_eppi;
-    SSEFunc_0_ppi sse_fn_ppi;
-    SSEFunc_0_eppt sse_fn_eppt;
-    MemOp ot;
-
-    b &= 0xff;
-    if (s->prefix & PREFIX_DATA)
-        b1 = 1;
-    else if (s->prefix & PREFIX_REPZ)
-        b1 = 2;
-    else if (s->prefix & PREFIX_REPNZ)
-        b1 = 3;
-    else
-        b1 = 0;
-    sse_fn_epp = sse_op_table1[b][b1];
-    if (!sse_fn_epp) {
-        goto unknown_op;
-    }
-    if ((b <= 0x5f && b >= 0x10) || b == 0xc6 || b == 0xc2) {
-        is_xmm = 1;
-    } else {
-        if (b1 == 0) {
-            /* MMX case */
-            is_xmm = 0;
-        } else {
-            is_xmm = 1;
-        }
-    }
-    /* simple MMX/SSE operation */
-    if (s->flags & HF_TS_MASK) {
-        gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
-        return;
-    }
-    if (s->flags & HF_EM_MASK) {
-    illegal_op:
-        gen_illegal_opcode(s);
-        return;
-    }
-    if (is_xmm
-        && !(s->flags & HF_OSFXSR_MASK)
-        && ((b != 0x38 && b != 0x3a) || (s->prefix & PREFIX_DATA))) {
-        goto unknown_op;
-    }
-    if (b == 0x0e) {
-        if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW)) {
-            /* If we were fully decoding this we might use illegal_op.  */
-            goto unknown_op;
-        }
-        /* femms */
-        gen_helper_emms(tcg_ctx, tcg_ctx->cpu_env);
-        return;
-    }
-    if (b == 0x77) {
-        /* emms */
-        gen_helper_emms(tcg_ctx, tcg_ctx->cpu_env);
-        return;
-    }
-    /* prepare MMX state (XXX: optimize by storing fptt and fptags in
-       the static cpu state) */
-    if (!is_xmm) {
-        gen_helper_enter_mmx(tcg_ctx, tcg_ctx->cpu_env);
-    }
-
-    modrm = x86_ldub_code(env, s);
-    reg = ((modrm >> 3) & 7);
-    if (is_xmm)
-        reg |= rex_r;
-    mod = (modrm >> 6) & 3;
-    /* VEX.L (256 bit) encodings are not supported */
-    if (s->vex_l != 0) {
-        goto illegal_op; // perhaps it should be unknown_op?
-    }
-    if (sse_fn_epp == SSE_SPECIAL) {
-        b |= (b1 << 8);
-        switch(b) {
-        case 0x0e7: /* movntq */
-            if (mod == 3) {
-                goto illegal_op;
-            }
-            gen_lea_modrm(env, s, modrm);
-            gen_stq_env_A0(s, offsetof(CPUX86State, fpregs[reg].mmx));
-            break;
-        case 0x1e7: /* movntdq */
-        case 0x02b: /* movntps */
-        case 0x12b: /* movntps */
-            if (mod == 3)
-                goto illegal_op;
-            gen_lea_modrm(env, s, modrm);
-            gen_sto_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
-            break;
-        case 0x3f0: /* lddqu */
-            if (mod == 3)
-                goto illegal_op;
-            gen_lea_modrm(env, s, modrm);
-            gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
-            break;
-        case 0x22b: /* movntss */
-        case 0x32b: /* movntsd */
-            if (mod == 3)
-                goto illegal_op;
-            gen_lea_modrm(env, s, modrm);
-            if (b1 & 1) {
-                gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-            } else {
-                tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State,
-                    xmm_regs[reg].ZMM_L(0)));
-                gen_op_st_v(s, MO_32, s->T0, s->A0);
-            }
-            break;
-        case 0x6e: /* movd mm, ea */
-#ifdef TARGET_X86_64
-            if (s->dflag == MO_64) {
-                gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0);
-                tcg_gen_st_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                              offsetof(CPUX86State, fpregs[reg].mmx));
-            } else
-#endif
-            {
-                gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0);
-                tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State,fpregs[reg].mmx));
-                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                gen_helper_movl_mm_T0_mmx(tcg_ctx, s->ptr0, s->tmp2_i32);
-            }
-            break;
-        case 0x16e: /* movd xmm, ea */
-#ifdef TARGET_X86_64
-            if (s->dflag == MO_64) {
-                gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0);
-                tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State,xmm_regs[reg]));
-                gen_helper_movq_mm_T0_xmm(tcg_ctx, s->ptr0, s->T0);
-            } else
-#endif
-            {
-                gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0);
-                tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State,xmm_regs[reg]));
-                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                gen_helper_movl_mm_T0_xmm(tcg_ctx, s->ptr0, s->tmp2_i32);
-            }
-            break;
-        case 0x6f: /* movq mm, ea */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldq_env_A0(s, offsetof(CPUX86State, fpregs[reg].mmx));
-            } else {
-                rm = (modrm & 7);
-                tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env,
-                               offsetof(CPUX86State,fpregs[rm].mmx));
-                tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env,
-                               offsetof(CPUX86State,fpregs[reg].mmx));
-            }
-            break;
-        case 0x010: /* movups */
-        case 0x110: /* movupd */
-        case 0x028: /* movaps */
-        case 0x128: /* movapd */
-        case 0x16f: /* movdqa xmm, ea */
-        case 0x26f: /* movdqu xmm, ea */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movo(s, offsetof(CPUX86State, xmm_regs[reg]),
-                            offsetof(CPUX86State,xmm_regs[rm]));
-            }
-            break;
-        case 0x210: /* movss xmm, ea */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, MO_32, s->T0, s->A0);
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_regs[reg].ZMM_L(0)));
-                tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_regs[reg].ZMM_L(1)));
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_regs[reg].ZMM_L(2)));
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_regs[reg].ZMM_L(3)));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)));
-            }
-            break;
-        case 0x310: /* movsd xmm, ea */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-                tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_regs[reg].ZMM_L(2)));
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_regs[reg].ZMM_L(3)));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
-            }
-            break;
-        case 0x012: /* movlps */
-        case 0x112: /* movlpd */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-            } else {
-                /* movhlps */
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(1)));
-            }
-            break;
-        case 0x212: /* movsldup */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)));
-                gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(2)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(2)));
-            }
-            gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(1)),
-                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
-            gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(3)),
-                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)));
-            break;
-        case 0x312: /* movddup */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
-            }
-            gen_op_movq(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(1)),
-                        offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
-            break;
-        case 0x016: /* movhps */
-        case 0x116: /* movhpd */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(1)));
-            } else {
-                /* movlhps */
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(1)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
-            }
-            break;
-        case 0x216: /* movshdup */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(1)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(1)));
-                gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(3)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(3)));
-            }
-            gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(0)),
-                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)));
-            gen_op_movl(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_L(2)),
-                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)));
-            break;
-        case 0x178:
-        case 0x378:
-            {
-                int bit_index, field_length;
-
-                if (b1 == 1 && reg != 0)
-                    goto illegal_op;
-                field_length = x86_ldub_code(env, s) & 0x3F;
-                bit_index = x86_ldub_code(env, s) & 0x3F;
-                tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                    offsetof(CPUX86State,xmm_regs[reg]));
-                if (b1 == 1)
-                    gen_helper_extrq_i(tcg_ctx, tcg_ctx->cpu_env, s->ptr0,
-                                       tcg_const_i32(tcg_ctx, bit_index),
-                                       tcg_const_i32(tcg_ctx, field_length));
-                else
-                    gen_helper_insertq_i(tcg_ctx, tcg_ctx->cpu_env, s->ptr0,
-                                         tcg_const_i32(tcg_ctx, bit_index),
-                                         tcg_const_i32(tcg_ctx, field_length));
-            }
-            break;
-        case 0x7e: /* movd ea, mm */
-#ifdef TARGET_X86_64
-            if (s->dflag == MO_64) {
-                tcg_gen_ld_i64(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                               offsetof(CPUX86State,fpregs[reg].mmx));
-                gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1);
-            } else
-#endif
-            {
-                tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0)));
-                gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1);
-            }
-            break;
-        case 0x17e: /* movd ea, xmm */
-#ifdef TARGET_X86_64
-            if (s->dflag == MO_64) {
-                tcg_gen_ld_i64(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                               offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
-                gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1);
-            } else
-#endif
-            {
-                tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
-                gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1);
-            }
-            break;
-        case 0x27e: /* movq xmm, ea */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
-            }
-            gen_op_movq_env_0(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(1)));
-            break;
-        case 0x7f: /* movq ea, mm */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_stq_env_A0(s, offsetof(CPUX86State, fpregs[reg].mmx));
-            } else {
-                rm = (modrm & 7);
-                gen_op_movq(s, offsetof(CPUX86State, fpregs[rm].mmx),
-                            offsetof(CPUX86State,fpregs[reg].mmx));
-            }
-            break;
-        case 0x011: /* movups */
-        case 0x111: /* movupd */
-        case 0x029: /* movaps */
-        case 0x129: /* movapd */
-        case 0x17f: /* movdqa ea, xmm */
-        case 0x27f: /* movdqu ea, xmm */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_sto_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movo(s, offsetof(CPUX86State, xmm_regs[rm]),
-                            offsetof(CPUX86State,xmm_regs[reg]));
-            }
-            break;
-        case 0x211: /* movss ea, xmm */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State, xmm_regs[reg].ZMM_L(0)));
-                gen_op_st_v(s, MO_32, s->T0, s->A0);
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(s, offsetof(CPUX86State, xmm_regs[rm].ZMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
-            }
-            break;
-        case 0x311: /* movsd ea, xmm */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(s, offsetof(CPUX86State, xmm_regs[rm].ZMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
-            }
-            break;
-        case 0x013: /* movlps */
-        case 0x113: /* movlpd */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-            } else {
-                goto illegal_op;
-            }
-            break;
-        case 0x017: /* movhps */
-        case 0x117: /* movhpd */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(1)));
-            } else {
-                goto illegal_op;
-            }
-            break;
-        case 0x71: /* shift mm, im */
-        case 0x72:
-        case 0x73:
-        case 0x171: /* shift xmm, im */
-        case 0x172:
-        case 0x173:
-            if (b1 >= 2) {
-                goto unknown_op;
-            }
-            val = x86_ldub_code(env, s);
-            if (is_xmm) {
-                tcg_gen_movi_tl(tcg_ctx, s->T0, val);
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_t0.ZMM_L(0)));
-                tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, xmm_t0.ZMM_L(1)));
-                op1_offset = offsetof(CPUX86State,xmm_t0);
-            } else {
-                tcg_gen_movi_tl(tcg_ctx, s->T0, val);
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, mmx_t0.MMX_L(0)));
-                tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
-                tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, mmx_t0.MMX_L(1)));
-                op1_offset = offsetof(CPUX86State,mmx_t0);
-            }
-            sse_fn_epp = sse_op_table2[((b - 1) & 3) * 8 +
-                                       (((modrm >> 3)) & 7)][b1];
-            if (!sse_fn_epp) {
-                goto unknown_op;
-            }
-            if (is_xmm) {
-                rm = (modrm & 7) | REX_B(s);
-                op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
-            } else {
-                rm = (modrm & 7);
-                op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
-            }
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op2_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op1_offset);
-            sse_fn_epp(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-            break;
-        case 0x050: /* movmskps */
-            rm = (modrm & 7) | REX_B(s);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                             offsetof(CPUX86State,xmm_regs[rm]));
-            gen_helper_movmskps(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, s->ptr0);
-            tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], s->tmp2_i32);
-            break;
-        case 0x150: /* movmskpd */
-            rm = (modrm & 7) | REX_B(s);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                             offsetof(CPUX86State,xmm_regs[rm]));
-            gen_helper_movmskpd(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, s->ptr0);
-            tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], s->tmp2_i32);
-            break;
-        case 0x02a: /* cvtpi2ps */
-        case 0x12a: /* cvtpi2pd */
-            gen_helper_enter_mmx(tcg_ctx, tcg_ctx->cpu_env);
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                op2_offset = offsetof(CPUX86State,mmx_t0);
-                gen_ldq_env_A0(s, op2_offset);
-            } else {
-                rm = (modrm & 7);
-                op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
-            }
-            op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            switch(b >> 8) {
-            case 0x0:
-                gen_helper_cvtpi2ps(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-                break;
-            default:
-            case 0x1:
-                gen_helper_cvtpi2pd(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-                break;
-            }
-            break;
-        case 0x22a: /* cvtsi2ss */
-        case 0x32a: /* cvtsi2sd */
-            ot = mo_64_32(s->dflag);
-            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-            op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            if (ot == MO_32) {
-                SSEFunc_0_epi sse_fn_epi = sse_op_table3ai[(b >> 8) & 1];
-                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                sse_fn_epi(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->tmp2_i32);
-            } else {
-#ifdef TARGET_X86_64
-                SSEFunc_0_epl sse_fn_epl = sse_op_table3aq[(b >> 8) & 1];
-                sse_fn_epl(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->T0);
-#else
-                goto illegal_op;
-#endif
-            }
-            break;
-        case 0x02c: /* cvttps2pi */
-        case 0x12c: /* cvttpd2pi */
-        case 0x02d: /* cvtps2pi */
-        case 0x12d: /* cvtpd2pi */
-            gen_helper_enter_mmx(tcg_ctx, tcg_ctx->cpu_env);
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                op2_offset = offsetof(CPUX86State,xmm_t0);
-                gen_ldo_env_A0(s, op2_offset);
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
-            }
-            op1_offset = offsetof(CPUX86State,fpregs[reg & 7].mmx);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            switch(b) {
-            case 0x02c:
-                gen_helper_cvttps2pi(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-                break;
-            case 0x12c:
-                gen_helper_cvttpd2pi(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-                break;
-            case 0x02d:
-                gen_helper_cvtps2pi(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-                break;
-            case 0x12d:
-                gen_helper_cvtpd2pi(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-                break;
-            }
-            break;
-        case 0x22c: /* cvttss2si */
-        case 0x32c: /* cvttsd2si */
-        case 0x22d: /* cvtss2si */
-        case 0x32d: /* cvtsd2si */
-            ot = mo_64_32(s->dflag);
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                if ((b >> 8) & 1) {
-                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.ZMM_Q(0)));
-                } else {
-                    gen_op_ld_v(s, MO_32, s->T0, s->A0);
-                    tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                    offsetof(CPUX86State, xmm_t0.ZMM_L(0)));
-                }
-                op2_offset = offsetof(CPUX86State,xmm_t0);
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
-            }
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op2_offset);
-            if (ot == MO_32) {
-                SSEFunc_i_ep sse_fn_i_ep =
-                    sse_op_table3bi[((b >> 7) & 2) | (b & 1)];
-                sse_fn_i_ep(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, s->ptr0);
-                tcg_gen_extu_i32_tl(tcg_ctx, s->T0, s->tmp2_i32);
-            } else {
-#ifdef TARGET_X86_64
-                SSEFunc_l_ep sse_fn_l_ep =
-                    sse_op_table3bq[((b >> 7) & 2) | (b & 1)];
-                sse_fn_l_ep(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->ptr0);
-#else
-                goto illegal_op;
-#endif
-            }
-            gen_op_mov_reg_v(s, ot, reg, s->T0);
-            break;
-        case 0xc4: /* pinsrw */
-        case 0x1c4:
-            s->rip_offset = 1;
-            gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-            val = x86_ldub_code(env, s);
-            if (b1) {
-                val &= 7;
-                tcg_gen_st16_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State,xmm_regs[reg].ZMM_W(val)));
-            } else {
-                val &= 3;
-                tcg_gen_st16_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State,fpregs[reg].mmx.MMX_W(val)));
-            }
-            break;
-        case 0xc5: /* pextrw */
-        case 0x1c5:
-            if (mod != 3)
-                goto illegal_op;
-            ot = mo_64_32(s->dflag);
-            val = x86_ldub_code(env, s);
-            if (b1) {
-                val &= 7;
-                rm = (modrm & 7) | REX_B(s);
-                tcg_gen_ld16u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State,xmm_regs[rm].ZMM_W(val)));
-            } else {
-                val &= 3;
-                rm = (modrm & 7);
-                tcg_gen_ld16u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State,fpregs[rm].mmx.MMX_W(val)));
-            }
-            reg = ((modrm >> 3) & 7) | rex_r;
-            gen_op_mov_reg_v(s, ot, reg, s->T0);
-            break;
-        case 0x1d6: /* movq ea, xmm */
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].ZMM_Q(0)));
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(s, offsetof(CPUX86State, xmm_regs[rm].ZMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
-                gen_op_movq_env_0(s,
-                                  offsetof(CPUX86State, xmm_regs[rm].ZMM_Q(1)));
-            }
-            break;
-        case 0x2d6: /* movq2dq */
-            gen_helper_enter_mmx(tcg_ctx, tcg_ctx->cpu_env);
-            rm = (modrm & 7);
-            gen_op_movq(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(0)),
-                        offsetof(CPUX86State,fpregs[rm].mmx));
-            gen_op_movq_env_0(s, offsetof(CPUX86State, xmm_regs[reg].ZMM_Q(1)));
-            break;
-        case 0x3d6: /* movdq2q */
-            gen_helper_enter_mmx(tcg_ctx, tcg_ctx->cpu_env);
-            rm = (modrm & 7) | REX_B(s);
-            gen_op_movq(s, offsetof(CPUX86State, fpregs[reg & 7].mmx),
-                        offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
-            break;
-        case 0xd7: /* pmovmskb */
-        case 0x1d7:
-            if (mod != 3)
-                goto illegal_op;
-            if (b1) {
-                rm = (modrm & 7) | REX_B(s);
-                tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State, xmm_regs[rm]));
-                gen_helper_pmovmskb_xmm(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, s->ptr0);
-            } else {
-                rm = (modrm & 7);
-                tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env,
-                                 offsetof(CPUX86State, fpregs[rm].mmx));
-                gen_helper_pmovmskb_mmx(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, s->ptr0);
-            }
-            reg = ((modrm >> 3) & 7) | rex_r;
-            tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], s->tmp2_i32);
-            break;
-
-        case 0x138:
-        case 0x038:
-            b = modrm;
-            if ((b & 0xf0) == 0xf0) {
-                goto do_0f_38_fx;
-            }
-            modrm = x86_ldub_code(env, s);
-            rm = modrm & 7;
-            reg = ((modrm >> 3) & 7) | rex_r;
-            mod = (modrm >> 6) & 3;
-            if (b1 >= 2) {
-                goto unknown_op;
-            }
-
-            sse_fn_epp = sse_op_table6[b].op[b1];
-            if (!sse_fn_epp) {
-                goto unknown_op;
-            }
-            if (!(s->cpuid_ext_features & sse_op_table6[b].ext_mask))
-                goto illegal_op;
-
-            if (b1) {
-                op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
-                if (mod == 3) {
-                    op2_offset = offsetof(CPUX86State,xmm_regs[rm | REX_B(s)]);
-                } else {
-                    op2_offset = offsetof(CPUX86State,xmm_t0);
-                    gen_lea_modrm(env, s, modrm);
-                    switch (b) {
-                    case 0x20: case 0x30: /* pmovsxbw, pmovzxbw */
-                    case 0x23: case 0x33: /* pmovsxwd, pmovzxwd */
-                    case 0x25: case 0x35: /* pmovsxdq, pmovzxdq */
-                        gen_ldq_env_A0(s, op2_offset +
-                                        offsetof(ZMMReg, ZMM_Q(0)));
-                        break;
-                    case 0x21: case 0x31: /* pmovsxbd, pmovzxbd */
-                    case 0x24: case 0x34: /* pmovsxwq, pmovzxwq */
-                        tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
-                                            s->mem_index, MO_LEUL);
-                        tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, op2_offset +
-                                        offsetof(ZMMReg, ZMM_L(0)));
-                        break;
-                    case 0x22: case 0x32: /* pmovsxbq, pmovzxbq */
-                        tcg_gen_qemu_ld_tl(tcg_ctx, s->tmp0, s->A0,
-                                           s->mem_index, MO_LEUW);
-                        tcg_gen_st16_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_env, op2_offset +
-                                        offsetof(ZMMReg, ZMM_W(0)));
-                        break;
-                    case 0x2a:            /* movntqda */
-                        gen_ldo_env_A0(s, op1_offset);
-                        return;
-                    default:
-                        gen_ldo_env_A0(s, op2_offset);
-                    }
-                }
-            } else {
-                op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
-                if (mod == 3) {
-                    op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
-                } else {
-                    op2_offset = offsetof(CPUX86State,mmx_t0);
-                    gen_lea_modrm(env, s, modrm);
-                    gen_ldq_env_A0(s, op2_offset);
-                }
-            }
-            if (sse_fn_epp == SSE_SPECIAL) {
-                goto unknown_op;
-            }
-
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            sse_fn_epp(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-
-            if (b == 0x17) {
-                set_cc_op(s, CC_OP_EFLAGS);
-            }
-            break;
-
-        case 0x238:
-        case 0x338:
-        do_0f_38_fx:
-            /* Various integer extensions at 0f 38 f[0-f].  */
-            b = modrm | (b1 << 8);
-            modrm = x86_ldub_code(env, s);
-            reg = ((modrm >> 3) & 7) | rex_r;
-
-            switch (b) {
-            case 0x3f0: /* crc32 Gd,Eb */
-            case 0x3f1: /* crc32 Gd,Ey */
-            do_crc32:
-                if (!(s->cpuid_ext_features & CPUID_EXT_SSE42)) {
-                    goto illegal_op;
-                }
-                if ((b & 0xff) == 0xf0) {
-                    ot = MO_8;
-                } else if (s->dflag != MO_64) {
-                    ot = (s->prefix & PREFIX_DATA ? MO_16 : MO_32);
-                } else {
-                    ot = MO_64;
-                }
-
-                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[reg]);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                gen_helper_crc32(tcg_ctx, s->T0, s->tmp2_i32,
-                                 s->T0, tcg_const_i32(tcg_ctx, 8 << ot));
-
-                ot = mo_64_32(s->dflag);
-                gen_op_mov_reg_v(s, ot, reg, s->T0);
-                break;
-
-            case 0x1f0: /* crc32 or movbe */
-            case 0x1f1:
-                /* For these insns, the f3 prefix is supposed to have priority
-                   over the 66 prefix, but that's not what we implement above
-                   setting b1.  */
-                if (s->prefix & PREFIX_REPNZ) {
-                    goto do_crc32;
-                }
-                /* FALLTHRU */
-            case 0x0f0: /* movbe Gy,My */
-            case 0x0f1: /* movbe My,Gy */
-                if (!(s->cpuid_ext_features & CPUID_EXT_MOVBE)) {
-                    goto illegal_op;
-                }
-                if (s->dflag != MO_64) {
-                    ot = (s->prefix & PREFIX_DATA ? MO_16 : MO_32);
-                } else {
-                    ot = MO_64;
-                }
-
-                gen_lea_modrm(env, s, modrm);
-                if ((b & 1) == 0) {
-                    tcg_gen_qemu_ld_tl(tcg_ctx, s->T0, s->A0,
-                                       s->mem_index, ot | MO_BE);
-                    gen_op_mov_reg_v(s, ot, reg, s->T0);
-                } else {
-                    tcg_gen_qemu_st_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], s->A0,
-                                       s->mem_index, ot | MO_BE);
-                }
-                break;
-
-            case 0x0f2: /* andn Gy, By, Ey */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI1)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                tcg_gen_andc_tl(tcg_ctx, s->T0, s->T0, tcg_ctx->cpu_regs[s->vex_v]);
-                gen_op_mov_reg_v(s, ot, reg, s->T0);
-                gen_op_update1_cc(s);
-                set_cc_op(s, CC_OP_LOGICB + ot);
-                break;
-
-            case 0x0f7: /* bextr Gy, Ey, By */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI1)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                {
-                    TCGv bound, zero;
-
-                    gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                    /* Extract START, and shift the operand.
-                       Shifts larger than operand size get zeros.  */
-                    tcg_gen_ext8u_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[s->vex_v]);
-                    tcg_gen_shr_tl(tcg_ctx, s->T0, s->T0, s->A0);
-
-                    bound = tcg_const_tl(tcg_ctx, ot == MO_64 ? 63 : 31);
-                    zero = tcg_const_tl(tcg_ctx, 0);
-                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_LEU, s->T0, s->A0, bound,
-                                       s->T0, zero);
-                    tcg_temp_free(tcg_ctx, zero);
-
-                    /* Extract the LEN into a mask.  Lengths larger than
-                       operand size get all ones.  */
-                    tcg_gen_extract_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[s->vex_v], 8, 8);
-                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_LEU, s->A0, s->A0, bound,
-                                       s->A0, bound);
-                    tcg_temp_free(tcg_ctx, bound);
-                    tcg_gen_movi_tl(tcg_ctx, s->T1, 1);
-                    tcg_gen_shl_tl(tcg_ctx, s->T1, s->T1, s->A0);
-                    tcg_gen_subi_tl(tcg_ctx, s->T1, s->T1, 1);
-                    tcg_gen_and_tl(tcg_ctx, s->T0, s->T0, s->T1);
-
-                    gen_op_mov_reg_v(s, ot, reg, s->T0);
-                    gen_op_update1_cc(s);
-                    set_cc_op(s, CC_OP_LOGICB + ot);
-                }
-                break;
-
-            case 0x0f5: /* bzhi Gy, Ey, By */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI2)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                tcg_gen_ext8u_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
-                {
-                    TCGv bound = tcg_const_tl(tcg_ctx, ot == MO_64 ? 63 : 31);
-                    TCGv zero = tcg_const_tl(tcg_ctx, 0);
-                    /* Note that since we're using BMILG (in order to get O
-                       cleared) we need to store the inverse into C.  */
-                    tcg_gen_setcond_tl(tcg_ctx, TCG_COND_LEU, tcg_ctx->cpu_cc_src,
-                                       s->T1, bound);
-                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_GTU, s->T1, s->T1,
-                                       bound, bound, s->T1);
-                    tcg_temp_free(tcg_ctx, bound);
-                    tcg_gen_movi_tl(tcg_ctx, s->A0, -1);
-                    tcg_gen_shl_tl(tcg_ctx, s->A0, s->A0, s->T1);
-                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, s->A0,
-                                       tcg_ctx->cpu_cc_src, zero,
-                                       zero, s->A0);
-                    tcg_temp_free(tcg_ctx, zero);
-                }
-                tcg_gen_andc_tl(tcg_ctx, s->T0, s->T0, s->A0);
-                gen_op_mov_reg_v(s, ot, reg, s->T0);
-                gen_op_update1_cc(s);
-                set_cc_op(s, CC_OP_BMILGB + ot);
-                break;
-
-            case 0x3f6: /* mulx By, Gy, rdx, Ey */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI2)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                switch (ot) {
-                default:
-                    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, tcg_ctx->cpu_regs[R_EDX]);
-                    tcg_gen_mulu2_i32(tcg_ctx, s->tmp2_i32, s->tmp3_i32,
-                                      s->tmp2_i32, s->tmp3_i32);
-                    tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[s->vex_v], s->tmp2_i32);
-                    tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], s->tmp3_i32);
-                    break;
-#ifdef TARGET_X86_64
-                case MO_64:
-                    tcg_gen_mulu2_i64(tcg_ctx, s->T0, s->T1,
-                                      s->T0, tcg_ctx->cpu_regs[R_EDX]);
-                    tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_regs[s->vex_v], s->T0);
-                    tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_regs[reg], s->T1);
-                    break;
-#endif
-                }
-                break;
-
-            case 0x3f5: /* pdep Gy, By, Ey */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI2)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                if (ot == MO_64) {
-                    tcg_gen_mov_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
-                } else {
-                    /* Keep the helper within the 32-bit operand size. */
-                    tcg_gen_ext32u_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
-                    tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
-                }
-                gen_helper_pdep(tcg_ctx, tcg_ctx->cpu_regs[reg], s->T1, s->T0);
-                break;
-
-            case 0x2f5: /* pext Gy, By, Ey */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI2)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                /* Note that by zero-extending the source operand, we
-                   automatically handle zero-extending the result.  */
-                if (ot == MO_64) {
-                    tcg_gen_mov_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
-                } else {
-                    tcg_gen_ext32u_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v]);
-                }
-                gen_helper_pext(tcg_ctx, tcg_ctx->cpu_regs[reg], s->T1, s->T0);
-                break;
-
-            case 0x1f6: /* adcx Gy, Ey */
-            case 0x2f6: /* adox Gy, Ey */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_ADX)) {
-                    goto illegal_op;
-                } else {
-                    TCGv carry_in, carry_out, zero;
-                    int end_op;
-
-                    ot = mo_64_32(s->dflag);
-                    gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-
-                    /* Re-use the carry-out from a previous round.  */
-                    carry_in = NULL;
-                    carry_out = (b == 0x1f6 ? tcg_ctx->cpu_cc_dst : tcg_ctx->cpu_cc_src2);
-                    switch (s->cc_op) {
-                    case CC_OP_ADCX:
-                        if (b == 0x1f6) {
-                            carry_in = tcg_ctx->cpu_cc_dst;
-                            end_op = CC_OP_ADCX;
-                        } else {
-                            end_op = CC_OP_ADCOX;
-                        }
-                        break;
-                    case CC_OP_ADOX:
-                        if (b == 0x1f6) {
-                            end_op = CC_OP_ADCOX;
-                        } else {
-                            carry_in = tcg_ctx->cpu_cc_src2;
-                            end_op = CC_OP_ADOX;
-                        }
-                        break;
-                    case CC_OP_ADCOX:
-                        end_op = CC_OP_ADCOX;
-                        carry_in = carry_out;
-                        break;
-                    default:
-                        end_op = (b == 0x1f6 ? CC_OP_ADCX : CC_OP_ADOX);
-                        break;
-                    }
-                    /* If we can't reuse carry-out, get it out of EFLAGS.  */
-                    if (!carry_in) {
-                        if (s->cc_op != CC_OP_ADCX && s->cc_op != CC_OP_ADOX) {
-                            gen_compute_eflags(s);
-                        }
-                        carry_in = s->tmp0;
-                        tcg_gen_extract_tl(tcg_ctx, carry_in, tcg_ctx->cpu_cc_src,
-                                           ctz32(b == 0x1f6 ? CC_C : CC_O), 1);
-                    }
-
-                    switch (ot) {
-#ifdef TARGET_X86_64
-                    case MO_32:
-                        /* If we know TL is 64-bit, and we want a 32-bit
-                           result, just do everything in 64-bit arithmetic.  */
-                        tcg_gen_ext32u_i64(tcg_ctx, tcg_ctx->cpu_regs[reg], tcg_ctx->cpu_regs[reg]);
-                        tcg_gen_ext32u_i64(tcg_ctx, s->T0, s->T0);
-                        tcg_gen_add_i64(tcg_ctx, s->T0, s->T0, tcg_ctx->cpu_regs[reg]);
-                        tcg_gen_add_i64(tcg_ctx, s->T0, s->T0, carry_in);
-                        tcg_gen_ext32u_i64(tcg_ctx, tcg_ctx->cpu_regs[reg], s->T0);
-                        tcg_gen_shri_i64(tcg_ctx, carry_out, s->T0, 32);
-                        break;
-#endif
-                    default:
-                        /* Otherwise compute the carry-out in two steps.  */
-                        zero = tcg_const_tl(tcg_ctx, 0);
-                        tcg_gen_add2_tl(tcg_ctx, s->T0, carry_out,
-                                        s->T0, zero,
-                                        carry_in, zero);
-                        tcg_gen_add2_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], carry_out,
-                                        tcg_ctx->cpu_regs[reg], carry_out,
-                                        s->T0, zero);
-                        tcg_temp_free(tcg_ctx, zero);
-                        break;
-                    }
-                    set_cc_op(s, end_op);
-                }
-                break;
-
-            case 0x1f7: /* shlx Gy, Ey, By */
-            case 0x2f7: /* sarx Gy, Ey, By */
-            case 0x3f7: /* shrx Gy, Ey, By */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI2)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                if (ot == MO_64) {
-                    tcg_gen_andi_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v], 63);
-                } else {
-                    tcg_gen_andi_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[s->vex_v], 31);
-                }
-                if (b == 0x1f7) {
-                    tcg_gen_shl_tl(tcg_ctx, s->T0, s->T0, s->T1);
-                } else if (b == 0x2f7) {
-                    if (ot != MO_64) {
-                        tcg_gen_ext32s_tl(tcg_ctx, s->T0, s->T0);
-                    }
-                    tcg_gen_sar_tl(tcg_ctx, s->T0, s->T0, s->T1);
-                } else {
-                    if (ot != MO_64) {
-                        tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
-                    }
-                    tcg_gen_shr_tl(tcg_ctx, s->T0, s->T0, s->T1);
-                }
-                gen_op_mov_reg_v(s, ot, reg, s->T0);
-                break;
-
-            case 0x0f3:
-            case 0x1f3:
-            case 0x2f3:
-            case 0x3f3: /* Group 17 */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI1)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0);
-                switch (reg & 7) {
-                case 1: /* blsr By,Ey */
-                    tcg_gen_subi_tl(tcg_ctx, s->T1, s->T0, 1);
-                    tcg_gen_and_tl(tcg_ctx, s->T0, s->T0, s->T1);
-                    break;
-                case 2: /* blsmsk By,Ey */
-                    tcg_gen_subi_tl(tcg_ctx, s->T1, s->T0, 1);
-                    tcg_gen_xor_tl(tcg_ctx, s->T0, s->T0, s->T1);
-                    break;
-                case 3: /* blsi By, Ey */
-                    tcg_gen_neg_tl(tcg_ctx, s->T1, s->T0);
-                    tcg_gen_and_tl(tcg_ctx, s->T0, s->T0, s->T1);
-                    break;
-                default:
-                    goto unknown_op;
-                }
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
-                gen_op_mov_reg_v(s, ot, s->vex_v, s->T0);
-                if ((reg & 7) == 3) {
-                    set_cc_op(s, CC_OP_BLSIB + ot);
-                } else {
-                    set_cc_op(s, CC_OP_BMILGB + ot);
-                }
-                break;
-
-            default:
-                goto unknown_op;
-            }
-            break;
-
-        case 0x03a:
-        case 0x13a:
-            b = modrm;
-            modrm = x86_ldub_code(env, s);
-            rm = modrm & 7;
-            reg = ((modrm >> 3) & 7) | rex_r;
-            mod = (modrm >> 6) & 3;
-            if (b1 >= 2) {
-                goto unknown_op;
-            }
-
-            sse_fn_eppi = sse_op_table7[b].op[b1];
-            if (!sse_fn_eppi) {
-                goto unknown_op;
-            }
-            if (!(s->cpuid_ext_features & sse_op_table7[b].ext_mask))
-                goto illegal_op;
-
-            s->rip_offset = 1;
-
-            if (sse_fn_eppi == SSE_SPECIAL) {
-                ot = mo_64_32(s->dflag);
-                rm = (modrm & 7) | REX_B(s);
-                if (mod != 3)
-                    gen_lea_modrm(env, s, modrm);
-                reg = ((modrm >> 3) & 7) | rex_r;
-                val = x86_ldub_code(env, s);
-                switch (b) {
-                case 0x14: /* pextrb */
-                    tcg_gen_ld8u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].ZMM_B(val & 15)));
-                    if (mod == 3) {
-                        gen_op_mov_reg_v(s, ot, rm, s->T0);
-                    } else {
-                        tcg_gen_qemu_st_tl(tcg_ctx, s->T0, s->A0,
-                                           s->mem_index, MO_UB);
-                    }
-                    break;
-                case 0x15: /* pextrw */
-                    tcg_gen_ld16u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].ZMM_W(val & 7)));
-                    if (mod == 3) {
-                        gen_op_mov_reg_v(s, ot, rm, s->T0);
-                    } else {
-                        tcg_gen_qemu_st_tl(tcg_ctx, s->T0, s->A0,
-                                           s->mem_index, MO_LEUW);
-                    }
-                    break;
-                case 0x16:
-                    if (ot == MO_32) { /* pextrd */
-                        tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                        offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_L(val & 3)));
-                        if (mod == 3) {
-                            tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[rm], s->tmp2_i32);
-                        } else {
-                            tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
-                                                s->mem_index, MO_LEUL);
-                        }
-                    } else { /* pextrq */
-#ifdef TARGET_X86_64
-                        tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env,
-                                        offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_Q(val & 1)));
-                        if (mod == 3) {
-                            tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_regs[rm], s->tmp1_i64);
-                        } else {
-                            tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0,
-                                                s->mem_index, MO_LEQ);
-                        }
-#else
-                        goto illegal_op;
-#endif
-                    }
-                    break;
-                case 0x17: /* extractps */
-                    tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].ZMM_L(val & 3)));
-                    if (mod == 3) {
-                        gen_op_mov_reg_v(s, ot, rm, s->T0);
-                    } else {
-                        tcg_gen_qemu_st_tl(tcg_ctx, s->T0, s->A0,
-                                           s->mem_index, MO_LEUL);
-                    }
-                    break;
-                case 0x20: /* pinsrb */
-                    if (mod == 3) {
-                        gen_op_mov_v_reg(s, MO_32, s->T0, rm);
-                    } else {
-                        tcg_gen_qemu_ld_tl(tcg_ctx, s->T0, s->A0,
-                                           s->mem_index, MO_UB);
-                    }
-                    tcg_gen_st8_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].ZMM_B(val & 15)));
-                    break;
-                case 0x21: /* insertps */
-                    if (mod == 3) {
-                        tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                        offsetof(CPUX86State,xmm_regs[rm]
-                                                .ZMM_L((val >> 6) & 3)));
-                    } else {
-                        tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
-                                            s->mem_index, MO_LEUL);
-                    }
-                    tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                    offsetof(CPUX86State,xmm_regs[reg]
-                                            .ZMM_L((val >> 4) & 3)));
-                    if ((val >> 0) & 1)
-                        tcg_gen_st_i32(tcg_ctx, tcg_const_i32(tcg_ctx, 0 /*float32_zero*/),
-                                        tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_L(0)));
-                    if ((val >> 1) & 1)
-                        tcg_gen_st_i32(tcg_ctx, tcg_const_i32(tcg_ctx, 0 /*float32_zero*/),
-                                        tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_L(1)));
-                    if ((val >> 2) & 1)
-                        tcg_gen_st_i32(tcg_ctx, tcg_const_i32(tcg_ctx, 0 /*float32_zero*/),
-                                        tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_L(2)));
-                    if ((val >> 3) & 1)
-                        tcg_gen_st_i32(tcg_ctx, tcg_const_i32(tcg_ctx, 0 /*float32_zero*/),
-                                        tcg_ctx->cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_L(3)));
-                    break;
-                case 0x22:
-                    if (ot == MO_32) { /* pinsrd */
-                        if (mod == 3) {
-                            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[rm]);
-                        } else {
-                            tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
-                                                s->mem_index, MO_LEUL);
-                        }
-                        tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                        offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_L(val & 3)));
-                    } else { /* pinsrq */
-#ifdef TARGET_X86_64
-                        if (mod == 3) {
-                            gen_op_mov_v_reg(s, ot, s->tmp1_i64, rm);
-                        } else {
-                            tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0,
-                                                s->mem_index, MO_LEQ);
-                        }
-                        tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env,
-                                        offsetof(CPUX86State,
-                                                xmm_regs[reg].ZMM_Q(val & 1)));
-#else
-                        goto illegal_op;
-#endif
-                    }
-                    break;
-                }
-                return;
-            }
+        tcg_gen_st_i32(tcg_ctx, t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_temp_free_i32(tcg_ctx, t);
+        s->flags &= ~mask;
+    }
+}
 
-            if (b1) {
-                op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
-                if (mod == 3) {
-                    op2_offset = offsetof(CPUX86State,xmm_regs[rm | REX_B(s)]);
-                } else {
-                    op2_offset = offsetof(CPUX86State,xmm_t0);
-                    gen_lea_modrm(env, s, modrm);
-                    gen_ldo_env_A0(s, op2_offset);
-                }
-            } else {
-                op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
-                if (mod == 3) {
-                    op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
-                } else {
-                    op2_offset = offsetof(CPUX86State,mmx_t0);
-                    gen_lea_modrm(env, s, modrm);
-                    gen_ldq_env_A0(s, op2_offset);
-                }
-            }
-            val = x86_ldub_code(env, s);
+static void gen_set_eflags(DisasContext *s, target_ulong mask)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv t = tcg_temp_new(tcg_ctx);
 
-            if ((b & 0xfc) == 0x60) { /* pcmpXstrX */
-                set_cc_op(s, CC_OP_EFLAGS);
+    tcg_gen_ld_tl(tcg_ctx, t, cpu_env, offsetof(CPUX86State, eflags));
+    tcg_gen_ori_tl(tcg_ctx, t, t, mask);
+    tcg_gen_st_tl(tcg_ctx, t, cpu_env, offsetof(CPUX86State, eflags));
+    tcg_temp_free(tcg_ctx, t);
+}
 
-                if (s->dflag == MO_64) {
-                    /* The helper must use entire 64-bit gp registers */
-                    val |= 1 << 8;
-                }
-            }
+static void gen_reset_eflags(DisasContext *s, target_ulong mask)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv t = tcg_temp_new(tcg_ctx);
 
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            sse_fn_eppi(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1, tcg_const_i32(tcg_ctx, val));
-            break;
+    tcg_gen_ld_tl(tcg_ctx, t, cpu_env, offsetof(CPUX86State, eflags));
+    tcg_gen_andi_tl(tcg_ctx, t, t, ~mask);
+    tcg_gen_st_tl(tcg_ctx, t, cpu_env, offsetof(CPUX86State, eflags));
+    tcg_temp_free(tcg_ctx, t);
+}
 
-        case 0x33a:
-            /* Various integer extensions at 0f 3a f[0-f].  */
-            b = modrm | (b1 << 8);
-            modrm = x86_ldub_code(env, s);
-            reg = ((modrm >> 3) & 7) | rex_r;
+/* Clear BND registers during legacy branches.  */
+static void gen_bnd_jmp(DisasContext *s)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    /* Clear the registers only if BND prefix is missing, MPX is enabled,
+       and if the BNDREGs are known to be in use (non-zero) already.
+       The helper itself will check BNDPRESERVE at runtime.  */
+    if ((s->prefix & PREFIX_REPNZ) == 0
+        && (s->flags & HF_MPX_EN_MASK) != 0
+        && (s->flags & HF_MPX_IU_MASK) != 0) {
+        gen_helper_bnd_jmp(tcg_ctx, cpu_env);
+    }
+}
 
-            switch (b) {
-            case 0x3f0: /* rorx Gy,Ey, Ib */
-                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI2)
-                    || !(s->prefix & PREFIX_VEX)
-                    || s->vex_l != 0) {
-                    goto illegal_op;
-                }
-                ot = mo_64_32(s->dflag);
-                s->rip_offset = 1;
-                gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                b = x86_ldub_code(env, s);
-                if (ot == MO_64) {
-                    tcg_gen_rotri_tl(tcg_ctx, s->T0, s->T0, b & 63);
-                } else {
-                    tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                    tcg_gen_rotri_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, b & 31);
-                    tcg_gen_extu_i32_tl(tcg_ctx, s->T0, s->tmp2_i32);
-                }
-                gen_op_mov_reg_v(s, ot, reg, s->T0);
-                break;
+/* Generate an end of block. Trace exception is also generated if needed.
+   If INHIBIT, set HF_INHIBIT_IRQ_MASK if it isn't already set.
+   If RECHECK_TF, emit a rechecking helper for #DB, ignoring the state of
+   S->TF.  This is used by the syscall/sysret insns.  */
+static void
+do_gen_eob_worker(DisasContext *s, bool inhibit, bool recheck_tf, bool jr)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    bool inhibit_reset;
 
-            default:
-                goto unknown_op;
-            }
-            break;
+    gen_update_cc_op(s);
 
-        default:
-        unknown_op:
-            gen_unknown_opcode(env, s);
-            return;
-        }
+    /* If several instructions disable interrupts, only the first does it.  */
+    inhibit_reset = false;
+    if (s->flags & HF_INHIBIT_IRQ_MASK) {
+        gen_reset_hflag(s, HF_INHIBIT_IRQ_MASK);
+        inhibit_reset = true;
+    } else if (inhibit) {
+        gen_set_hflag(s, HF_INHIBIT_IRQ_MASK);
+    }
+
+    if (s->base.tb->flags & HF_RF_MASK) {
+        gen_reset_eflags(s, RF_MASK);
+    }
+    if (recheck_tf) {
+        gen_helper_rechecking_single_step(tcg_ctx, cpu_env);
+        tcg_gen_exit_tb(tcg_ctx, NULL, 0);
+    } else if ((s->flags & HF_TF_MASK) && !inhibit) {
+        gen_helper_single_step(tcg_ctx, cpu_env);
+    } else if (jr &&
+               /* give irqs a chance to happen */
+               !inhibit_reset) {
+        tcg_gen_lookup_and_goto_ptr(tcg_ctx);
     } else {
-        /* generic MMX or SSE operation */
-        switch(b) {
-        case 0x70: /* pshufx insn */
-        case 0xc6: /* pshufx insn */
-        case 0xc2: /* compare insns */
-            s->rip_offset = 1;
-            break;
-        default:
-            break;
-        }
-        if (is_xmm) {
-            op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
-            if (mod != 3) {
-                int sz = 4;
+        tcg_gen_exit_tb(tcg_ctx, NULL, 0);
+    }
+    s->base.is_jmp = DISAS_NORETURN;
+}
 
-                gen_lea_modrm(env, s, modrm);
-                op2_offset = offsetof(CPUX86State,xmm_t0);
-
-                switch (b) {
-                case 0x50:
-                case 0x51:
-                case 0x52:
-                case 0x53:
-                case 0x54:
-                case 0x55:
-                case 0x56:
-                case 0x57:
-                case 0x58:
-                case 0x59:
-                case 0x5a:
-
-                case 0x5c:
-                case 0x5d:
-                case 0x5e:
-                case 0x5f:
-
-                case 0xc2:
-                    /* Most sse scalar operations.  */
-                    if (b1 == 2) {
-                        sz = 2;
-                    } else if (b1 == 3) {
-                        sz = 3;
-                    }
-                    break;
+static inline void
+gen_eob_worker(DisasContext *s, bool inhibit, bool recheck_tf)
+{
+    do_gen_eob_worker(s, inhibit, recheck_tf, false);
+}
 
-                case 0x2e:  /* ucomis[sd] */
-                case 0x2f:  /* comis[sd] */
-                    if (b1 == 0) {
-                        sz = 2;
-                    } else {
-                        sz = 3;
-                    }
-                    break;
-                }
+/* End of block.
+   If INHIBIT, set HF_INHIBIT_IRQ_MASK if it isn't already set.  */
+static void gen_eob_inhibit_irq(DisasContext *s, bool inhibit)
+{
+    gen_eob_worker(s, inhibit, false);
+}
 
-                switch (sz) {
-                case 2:
-                    /* 32 bit access */
-                    gen_op_ld_v(s, MO_32, s->T0, s->A0);
-                    tcg_gen_st32_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
-                                    offsetof(CPUX86State,xmm_t0.ZMM_L(0)));
-                    break;
-                case 3:
-                    /* 64 bit access */
-                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.ZMM_D(0)));
-                    break;
-                default:
-                    /* 128 bit access */
-                    gen_ldo_env_A0(s, op2_offset);
-                    break;
-                }
-            } else {
-                rm = (modrm & 7) | REX_B(s);
-                op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
+/* End of block, resetting the inhibit irq flag.  */
+static void gen_eob(DisasContext *s)
+{
+    gen_eob_worker(s, false, false);
+}
+
+/* Jump to register */
+static void gen_jr(DisasContext *s)
+{
+    do_gen_eob_worker(s, false, false, true);
+}
+
+/* Jump to eip+diff, truncating the result to OT. */
+static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    bool use_goto_tb = s->jmp_opt;
+    target_ulong mask = -1;
+    target_ulong new_pc = s->pc + diff;
+    target_ulong new_eip = new_pc - s->cs_base;
+
+    /* In 64-bit mode, operand size is fixed at 64 bits. */
+    if (!CODE64(s)) {
+        if (ot == MO_16) {
+            mask = 0xffff;
+            if (TARGET_TB_PCREL && CODE32(s)) {
+                use_goto_tb = false;
             }
         } else {
-            op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
-            if (mod != 3) {
-                gen_lea_modrm(env, s, modrm);
-                op2_offset = offsetof(CPUX86State,mmx_t0);
-                gen_ldq_env_A0(s, op2_offset);
-            } else {
-                rm = (modrm & 7);
-                op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
-            }
+            mask = 0xffffffff;
         }
-        switch(b) {
-        case 0x0f: /* 3DNow! data insns */
-            val = x86_ldub_code(env, s);
-            sse_fn_epp = sse_op_table5[val];
-            if (!sse_fn_epp) {
-                goto unknown_op;
-            }
-            if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW)) {
-                goto illegal_op;
-            }
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            sse_fn_epp(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-            break;
-        case 0x70: /* pshufx insn */
-        case 0xc6: /* pshufx insn */
-            val = x86_ldub_code(env, s);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            /* XXX: introduce a new table? */
-            sse_fn_ppi = (SSEFunc_0_ppi)sse_fn_epp;
-            sse_fn_ppi(tcg_ctx, s->ptr0, s->ptr1, tcg_const_i32(tcg_ctx, val));
-            break;
-        case 0xc2:
-            /* compare insns */
-            val = x86_ldub_code(env, s);
-            if (val >= 8)
-                goto unknown_op;
-            sse_fn_epp = sse_op_table4[val][b1];
-
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            sse_fn_epp(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-            break;
-        case 0xf7:
-            /* maskmov : we must prepare A0 */
-            if (mod != 3)
-                goto illegal_op;
-            tcg_gen_mov_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[R_EDI]);
-            gen_extu(tcg_ctx, s->aflag, s->A0);
-            gen_add_A0_ds_seg(s);
+    }
+    new_eip &= mask;
 
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            /* XXX: introduce a new table? */
-            sse_fn_eppt = (SSEFunc_0_eppt)sse_fn_epp;
-            sse_fn_eppt(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1, s->A0);
-            break;
-        default:
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr0, tcg_ctx->cpu_env, op1_offset);
-            tcg_gen_addi_ptr(tcg_ctx, s->ptr1, tcg_ctx->cpu_env, op2_offset);
-            sse_fn_epp(tcg_ctx, tcg_ctx->cpu_env, s->ptr0, s->ptr1);
-            break;
+    gen_update_cc_op(s);
+    set_cc_op(s, CC_OP_DYNAMIC);
+
+    if (TARGET_TB_PCREL) {
+        tcg_gen_addi_tl(tcg_ctx, cpu_eip, cpu_eip, new_pc - s->pc_save);
+        /*
+         * If we can prove the branch does not leave the page and we have
+         * no extra masking to apply (data16 branch in code32, see above),
+         * then we have also proven that the addition does not wrap.
+         */
+        if (!use_goto_tb || !is_same_page(&s->base, new_pc)) {
+            tcg_gen_andi_tl(tcg_ctx, cpu_eip, cpu_eip, mask);
+            use_goto_tb = false;
         }
-        if (b == 0x2e || b == 0x2f) {
-            set_cc_op(s, CC_OP_EFLAGS);
+    } else if (!CODE64(s)) {
+        new_pc = (uint32_t)(new_eip + s->cs_base);
+    }
+
+    if (use_goto_tb && translator_use_goto_tb(&s->base, new_pc)) {
+        /* jump to same page: we can use a direct jump */
+        tcg_gen_goto_tb(tcg_ctx, tb_num);
+        if (!TARGET_TB_PCREL) {
+            tcg_gen_movi_tl(tcg_ctx, cpu_eip, new_eip);
+        }
+        tcg_gen_exit_tb(tcg_ctx, s->base.tb, tb_num);
+        s->base.is_jmp = DISAS_NORETURN;
+    } else {
+        if (!TARGET_TB_PCREL) {
+            tcg_gen_movi_tl(tcg_ctx, cpu_eip, new_eip);
+        }
+        if (s->jmp_opt) {
+            gen_jr(s);   /* jump to another page */
+        } else {
+            gen_eob(s);  /* exit to main loop */
         }
     }
 }
 
-// Unicorn: sync EFLAGS on demand
-static void sync_eflags(DisasContext *s, TCGContext *tcg_ctx)
+/* Jump to eip+diff, truncating to the current code size. */
+static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num)
 {
-    gen_update_cc_op(s);
-    gen_helper_read_eflags(tcg_ctx, s->T0, tcg_ctx->cpu_env);
-    tcg_gen_st_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, eflags));
+    /* CODE64 ignores the OT argument, so we need not consider it. */
+    gen_jmp_rel(s, CODE32(s) ? MO_32 : MO_16, diff, tb_num);
+}
+
+static inline void gen_ldq_env_A0(DisasContext *s, int offset)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset);
+}
+
+static inline void gen_stq_env_A0(DisasContext *s, int offset)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset);
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ);
+}
+
+static inline void gen_ldo_env_A0(DisasContext *s, int offset, bool align)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int mem_index = s->mem_index;
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, mem_index,
+                        MO_LEUQ | (align ? MO_ALIGN_16 : 0));
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 8);
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
+}
+
+static inline void gen_sto_env_A0(DisasContext *s, int offset, bool align)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int mem_index = s->mem_index;
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, mem_index,
+                        MO_LEUQ | (align ? MO_ALIGN_16 : 0));
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 8);
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
+}
+
+static void gen_ldy_env_A0(DisasContext *s, int offset, bool align)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int mem_index = s->mem_index;
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, mem_index,
+                        MO_LEUQ | (align ? MO_ALIGN_32 : 0));
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(0)));
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 8);
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(1)));
+
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 16);
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(2)));
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 24);
+    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
+    tcg_gen_st_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(3)));
+}
+
+static void gen_sty_env_A0(DisasContext *s, int offset, bool align)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int mem_index = s->mem_index;
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(0)));
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, mem_index,
+                        MO_LEUQ | (align ? MO_ALIGN_32 : 0));
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 8);
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(1)));
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 16);
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(2)));
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
+    tcg_gen_addi_tl(tcg_ctx, s->tmp0, s->A0, 24);
+    tcg_gen_ld_i64(tcg_ctx, s->tmp1_i64, cpu_env, offset + offsetof(YMMReg, YMM_Q(3)));
+    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->tmp0, mem_index, MO_LEUQ);
 }
 
+#include "decode-new.h"
+#include "emit.c.inc"
+#include "decode-new.c.inc"
+
 /* convert one instruction. s->base.is_jmp is set if the translation must
    be stopped. Return the next pc value */
-static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
+static bool disas_insn(DisasContext *s, CPUState *cpu)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     CPUX86State *env = cpu->env_ptr;
-    int b, prefixes, prefix_count;
+    int b, prefixes;
     int shift;
     MemOp ot, aflag, dflag;
     int modrm, reg, rm, mod, op, opreg, val;
-    target_ulong next_eip, tval;
-    int rex_w, rex_r, rex_byte, rex_index;
+    bool orig_cc_op_dirty = s->cc_op_dirty;
+    CCOp orig_cc_op = s->cc_op;
+    target_ulong orig_pc_save = s->pc_save;
     target_ulong pc_start = s->base.pc_next;
     TCGOp *tcg_op, *prev_op = NULL;
     bool insn_hook = false;
 
-    s->pc_start = tcg_ctx->pc_start = s->pc = pc_start;
-    s->prefix = 0;
-
-    s->uc = env->uc;
-
-    // Unicorn: end address tells us to stop emulation
-    if (uc_addr_is_exit(env->uc, s->pc)) {
-        // imitate the HLT instruction
-        gen_update_cc_op(s);
-        gen_sync_pc(tcg_ctx, pc_start - s->cs_base);
-        gen_helper_hlt(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->pc - pc_start));
-        s->base.is_jmp = DISAS_NORETURN;
-        return s->pc;
-    }
-
-    // Unicorn: callback might need to access to EFLAGS,
-    // or want to stop emulation immediately
-    if (HOOK_EXISTS_BOUNDED(env->uc, UC_HOOK_CODE, pc_start)) {
-        if (s->last_cc_op != s->cc_op) {
-            sync_eflags(s, tcg_ctx);
-            s->last_cc_op = s->cc_op;
-        }
-
-        // Sync PC in advance
-        gen_sync_pc(tcg_ctx, pc_start - s->cs_base);
-
-        // save the last operand
+    s->pc = s->base.pc_next;
+    if (HOOK_EXISTS_BOUNDED(s->uc, UC_HOOK_CODE, pc_start)) {
+        sync_eflags(s);
+        gen_update_eip_cur(s);
         prev_op = tcg_last_op(tcg_ctx);
         insn_hook = true;
-        gen_uc_tracecode(tcg_ctx, 0xf1f1f1f1, UC_HOOK_CODE_IDX, env->uc, pc_start);
-
+        gen_uc_tracecode(tcg_ctx, 0xf1f1f1f1, UC_HOOK_CODE_IDX, s->uc,
+                         pc_start);
         check_exit_request(tcg_ctx);
-
-        // Unicorn: Previous hook might change eflags to any state, let's sync it
         gen_compute_eflags(s);
     }
 
     s->override = -1;
-
 #ifdef TARGET_X86_64
+    s->rex_r = 0;
     s->rex_x = 0;
     s->rex_b = 0;
-    s->x86_64_hregs = false;
 #endif
     s->rip_offset = 0; /* for relative ip address */
     s->vex_l = 0;
     s->vex_v = 0;
-    if (sigsetjmp(s->jmpbuf, 0) != 0) {
-        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        return s->pc;
+    s->vex_w = false;
+    switch (sigsetjmp(s->jmpbuf, 0)) {
+    case 0:
+        break;
+    case 1:
+        gen_exception_gpf(s);
+        return true;
+    case 2:
+        /* Restore state that may affect the next instruction. */
+        s->pc = s->base.pc_next;
+        /*
+         * TODO: These save/restore can be removed after the table-based
+         * decoder is complete; we will be decoding the insn completely
+         * before any code generation that might affect these variables.
+         */
+        s->cc_op_dirty = orig_cc_op_dirty;
+        s->cc_op = orig_cc_op;
+        s->pc_save = orig_pc_save;
+        /* END TODO */
+        s->base.num_insns--;
+        tcg_remove_ops_after(tcg_ctx, s->prev_insn_end);
+        s->base.is_jmp = DISAS_TOO_MANY;
+        return false;
+    default:
+        g_assert_not_reached();
     }
 
     prefixes = 0;
-    rex_w = -1;
-    rex_r = 0;
-    rex_byte = 0;
-    rex_index = -1;
-    prefix_count = 0;
 
  next_byte:
+    s->prefix = prefixes;
     b = x86_ldub_code(env, s);
     /* Collect prefixes.  */
     switch (b) {
+    default:
+        break;
+    case 0x0f:
+        b = x86_ldub_code(env, s) + 0x100;
+        break;
     case 0xf3:
+        clear_rex_prefix(s, &prefixes);
         prefixes |= PREFIX_REPZ;
-        prefix_count++;
+        prefixes &= ~PREFIX_REPNZ;
         goto next_byte;
     case 0xf2:
+        clear_rex_prefix(s, &prefixes);
         prefixes |= PREFIX_REPNZ;
-        prefix_count++;
+        prefixes &= ~PREFIX_REPZ;
         goto next_byte;
     case 0xf0:
+        clear_rex_prefix(s, &prefixes);
         prefixes |= PREFIX_LOCK;
-        prefix_count++;
         goto next_byte;
     case 0x2e:
+        clear_rex_prefix(s, &prefixes);
         s->override = R_CS;
-        prefix_count++;
         goto next_byte;
     case 0x36:
+        clear_rex_prefix(s, &prefixes);
         s->override = R_SS;
-        prefix_count++;
         goto next_byte;
     case 0x3e:
+        clear_rex_prefix(s, &prefixes);
         s->override = R_DS;
-        prefix_count++;
         goto next_byte;
     case 0x26:
+        clear_rex_prefix(s, &prefixes);
         s->override = R_ES;
-        prefix_count++;
         goto next_byte;
     case 0x64:
+        clear_rex_prefix(s, &prefixes);
         s->override = R_FS;
-        prefix_count++;
         goto next_byte;
     case 0x65:
+        clear_rex_prefix(s, &prefixes);
         s->override = R_GS;
-        prefix_count++;
         goto next_byte;
     case 0x66:
+        clear_rex_prefix(s, &prefixes);
         prefixes |= PREFIX_DATA;
-        prefix_count++;
         goto next_byte;
     case 0x67:
+        clear_rex_prefix(s, &prefixes);
         prefixes |= PREFIX_ADR;
-        prefix_count++;
         goto next_byte;
 #ifdef TARGET_X86_64
-    case 0x40:
-    case 0x41:
-    case 0x42:
-    case 0x43:
-    case 0x44:
-    case 0x45:
-    case 0x46:
-    case 0x47:
-    case 0x48:
-    case 0x49:
-    case 0x4a:
-    case 0x4b:
-    case 0x4c:
-    case 0x4d:
-    case 0x4e:
-    case 0x4f:
+    case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47: case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
         if (CODE64(s)) {
-            rex_byte = b;
-            rex_index = prefix_count;
-            prefix_count++;
+            /* REX prefix */
+            prefixes |= PREFIX_REX;
+            s->vex_w = (b >> 3) & 1;
+            s->rex_r = (b & 0x4) << 1;
+            s->rex_x = (b & 0x2) << 2;
+            s->rex_b = (b & 0x1) << 3;
             goto next_byte;
         }
         break;
 #endif
     case 0xc5: /* 2-byte VEX */
     case 0xc4: /* 3-byte VEX */
-        /* VEX prefixes cannot be used except in 32-bit mode.
-           Otherwise the instruction is LES or LDS.  */
-        if (s->code32 && !s->vm86) {
-            static const int pp_prefix[4] = {
-                0, PREFIX_DATA, PREFIX_REPZ, PREFIX_REPNZ
-            };
-            int vex3, vex2 = x86_ldub_code(env, s);
+        if (CODE32(s) && !VM86(s)) {
+            int vex2 = x86_ldub_code(env, s);
+            s->pc--; /* rewind the advance_pc() x86_ldub_code() did */
 
             if (!CODE64(s) && (vex2 & 0xc0) != 0xc0) {
                 /* 4.1.4.6: In 32-bit mode, bits [7:6] must be 11b,
                    otherwise the instruction is LES or LDS.  */
-                s->pc--; /* rewind the advance_pc() x86_ldub_code() did */
                 break;
             }
-
-            /* 4.1.1-4.1.3: No preceding lock, 66, f2, f3, or rex prefixes. */
-            if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ
-                            | PREFIX_LOCK | PREFIX_DATA)) {
-                goto illegal_op;
-            }
-#ifdef TARGET_X86_64
-            if (rex_byte != 0) {
-                goto illegal_op;
-            }
-#endif
-            rex_r = (~vex2 >> 4) & 8;
-            if (b == 0xc5) {
-                /* 2-byte VEX prefix: RVVVVlpp, implied 0f leading opcode byte */
-                vex3 = vex2;
-                b = x86_ldub_code(env, s) | 0x100;
-            } else {
-                /* 3-byte VEX prefix: RXBmmmmm wVVVVlpp */
-#ifdef TARGET_X86_64
-                s->rex_x = (~vex2 >> 3) & 8;
-                s->rex_b = (~vex2 >> 2) & 8;
-#endif
-                vex3 = x86_ldub_code(env, s);
-                rex_w = (vex3 >> 7) & 1;
-                switch (vex2 & 0x1f) {
-                case 0x01: /* Implied 0f leading opcode bytes.  */
-                    b = x86_ldub_code(env, s) | 0x100;
-                    break;
-                case 0x02: /* Implied 0f 38 leading opcode bytes.  */
-                    b = 0x138;
-                    break;
-                case 0x03: /* Implied 0f 3a leading opcode bytes.  */
-                    b = 0x13a;
-                    break;
-                default:   /* Reserved for future use.  */
-                    goto unknown_op;
-                }
-            }
-            s->vex_v = (~vex3 >> 3) & 0xf;
-            s->vex_l = (vex3 >> 2) & 1;
-            prefixes |= pp_prefix[vex3 & 3] | PREFIX_VEX;
+            disas_insn_new(s, cpu, b);
+            return s->pc;
         }
-        prefix_count++;
         break;
     }
 
     /* Post-process prefixes.  */
     if (CODE64(s)) {
-        /* 2.2.1: A REX prefix is ignored when it does not immediately precede the opcode byte */
-        if (rex_byte != 0 && rex_index + 1 == prefix_count) {
-            /* REX prefix */
-            rex_w = (rex_byte >> 3) & 1;
-            rex_r = (rex_byte & 0x4) << 1;
-            s->rex_x = (rex_byte & 0x2) << 2;
-            REX_B(s) = (rex_byte & 0x1) << 3;
-            /* select uniform byte register addressing */
-            s->x86_64_hregs = true;
-        }
-
         /* In 64-bit mode, the default data size is 32-bit.  Select 64-bit
            data with rex_w, and 16-bit data with 0x66; rex_w takes precedence
            over 0x66 if both are present.  */
-        dflag = (rex_w > 0 ? MO_64 : prefixes & PREFIX_DATA ? MO_16 : MO_32);
+        dflag = (REX_W(s) ? MO_64 : prefixes & PREFIX_DATA ? MO_16 : MO_32);
         /* In 64-bit mode, 0x67 selects 32-bit addressing.  */
         aflag = (prefixes & PREFIX_ADR ? MO_32 : MO_64);
     } else {
         /* In 16/32-bit mode, 0x66 selects the opposite data size.  */
-        if (s->code32 ^ ((prefixes & PREFIX_DATA) != 0)) {
+        if (CODE32(s) ^ ((prefixes & PREFIX_DATA) != 0)) {
             dflag = MO_32;
         } else {
             dflag = MO_16;
         }
         /* In 16/32-bit mode, 0x67 selects the opposite addressing.  */
-        if (s->code32 ^ ((prefixes & PREFIX_ADR) != 0)) {
+        if (CODE32(s) ^ ((prefixes & PREFIX_ADR) != 0)) {
             aflag = MO_32;
         }  else {
             aflag = MO_16;
@@ -5086,71 +3424,17 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     s->dflag = dflag;
 
     /* now check op code */
- reswitch:
-    switch(b) {
-    case 0x0f:
-        /**************************/
-        /* extended op code */
-        b = x86_ldub_code(env, s) | 0x100;
-        goto reswitch;
-
+    switch (b) {
         /**************************/
         /* arith & logic */
-    case 0x00:
-    case 0x01:
-    case 0x02:
-    case 0x03:
-    case 0x04:
-    case 0x05:
-
-    case 0x08:
-    case 0x09:
-    case 0x0a:
-    case 0x0b:
-    case 0x0c:
-    case 0x0d:
-
-    case 0x10:
-    case 0x11:
-    case 0x12:
-    case 0x13:
-    case 0x14:
-    case 0x15:
-
-    case 0x18:
-    case 0x19:
-    case 0x1a:
-    case 0x1b:
-    case 0x1c:
-    case 0x1d:
-
-    case 0x20:
-    case 0x21:
-    case 0x22:
-    case 0x23:
-    case 0x24:
-    case 0x25:
-
-    case 0x28:
-    case 0x29:
-    case 0x2a:
-    case 0x2b:
-    case 0x2c:
-    case 0x2d:
-
-    case 0x30:
-    case 0x31:
-    case 0x32:
-    case 0x33:
-    case 0x34:
-    case 0x35:
-
-    case 0x38:
-    case 0x39:
-    case 0x3a:
-    case 0x3b:
-    case 0x3c:
-    case 0x3d:
+    case 0x0: case 0x1: case 0x2: case 0x3: case 0x4: case 0x5:
+    case 0x8: case 0x9: case 0xa: case 0xb: case 0xc: case 0xd:
+    case 0x10: case 0x11: case 0x12: case 0x13: case 0x14: case 0x15:
+    case 0x18: case 0x19: case 0x1a: case 0x1b: case 0x1c: case 0x1d:
+    case 0x20: case 0x21: case 0x22: case 0x23: case 0x24: case 0x25:
+    case 0x28: case 0x29: case 0x2a: case 0x2b: case 0x2c: case 0x2d:
+    case 0x30: case 0x31: case 0x32: case 0x33: case 0x34: case 0x35:
+    case 0x38: case 0x39: case 0x3a: case 0x3b: case 0x3c: case 0x3d:
         {
             int op, f, val;
             op = (b >> 3) & 7;
@@ -5161,7 +3445,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             switch(f) {
             case 0: /* OP Ev, Gv */
                 modrm = x86_ldub_code(env, s);
-                reg = ((modrm >> 3) & 7) | rex_r;
+                reg = ((modrm >> 3) & 7) | REX_R(s);
                 mod = (modrm >> 6) & 3;
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
@@ -5183,7 +3467,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             case 1: /* OP Gv, Ev */
                 modrm = x86_ldub_code(env, s);
                 mod = (modrm >> 6) & 3;
-                reg = ((modrm >> 3) & 7) | rex_r;
+                reg = ((modrm >> 3) & 7) | REX_R(s);
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
                     gen_lea_modrm(env, s, modrm);
@@ -5250,25 +3534,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
 
         /**************************/
         /* inc, dec, and other misc arith */
-    case 0x40: /* inc Gv */
-    case 0x41: /* inc Gv */
-    case 0x42: /* inc Gv */
-    case 0x43: /* inc Gv */
-    case 0x44: /* inc Gv */
-    case 0x45: /* inc Gv */
-    case 0x46: /* inc Gv */
-    case 0x47: /* inc Gv */
+    case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47: /* inc Gv */
         ot = dflag;
         gen_inc(s, ot, OR_EAX + (b & 7), 1);
         break;
-    case 0x48: /* dec Gv */
-    case 0x49: /* dec Gv */
-    case 0x4a: /* dec Gv */
-    case 0x4b: /* dec Gv */
-    case 0x4c: /* dec Gv */
-    case 0x4d: /* dec Gv */
-    case 0x4e: /* dec Gv */
-    case 0x4f: /* dec Gv */
+    case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f: /* dec Gv */
         ot = dflag;
         gen_inc(s, ot, OR_EAX + (b & 7), -1);
         break;
@@ -5345,7 +3615,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
 
                 tcg_temp_free(tcg_ctx, t2);
                 tcg_temp_free(tcg_ctx, a0);
-                tcg_gen_mov_tl(tcg_ctx, s->T0, t0);
+                tcg_gen_neg_tl(tcg_ctx, s->T0, t0);
                 tcg_temp_free(tcg_ctx, t0);
             } else {
                 tcg_gen_neg_tl(tcg_ctx, s->T0, s->T0);
@@ -5367,8 +3637,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 /* XXX: use 32 bit mul which could be faster */
                 tcg_gen_mul_tl(tcg_ctx, s->T0, s->T0, s->T1);
                 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
-                tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0, 0xff00);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
+                tcg_gen_andi_tl(tcg_ctx, cpu_cc_src, s->T0, 0xff00);
                 set_cc_op(s, CC_OP_MULB);
                 break;
             case MO_16:
@@ -5378,30 +3648,30 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 /* XXX: use 32 bit mul which could be faster */
                 tcg_gen_mul_tl(tcg_ctx, s->T0, s->T0, s->T1);
                 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
                 tcg_gen_shri_tl(tcg_ctx, s->T0, s->T0, 16);
                 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T0);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T0);
                 set_cc_op(s, CC_OP_MULW);
                 break;
             default:
             case MO_32:
                 tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, tcg_ctx->cpu_regs[R_EAX]);
+                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, cpu_regs[R_EAX]);
                 tcg_gen_mulu2_i32(tcg_ctx, s->tmp2_i32, s->tmp3_i32,
                                   s->tmp2_i32, s->tmp3_i32);
-                tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[R_EAX], s->tmp2_i32);
-                tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[R_EDX], s->tmp3_i32);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_regs[R_EAX]);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_regs[R_EDX]);
+                tcg_gen_extu_i32_tl(tcg_ctx, cpu_regs[R_EAX], s->tmp2_i32);
+                tcg_gen_extu_i32_tl(tcg_ctx, cpu_regs[R_EDX], s->tmp3_i32);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, cpu_regs[R_EAX]);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, cpu_regs[R_EDX]);
                 set_cc_op(s, CC_OP_MULL);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                tcg_gen_mulu2_i64(tcg_ctx, tcg_ctx->cpu_regs[R_EAX], tcg_ctx->cpu_regs[R_EDX],
-                                  s->T0, tcg_ctx->cpu_regs[R_EAX]);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_regs[R_EAX]);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_regs[R_EDX]);
+                tcg_gen_mulu2_i64(tcg_ctx, cpu_regs[R_EAX], cpu_regs[R_EDX],
+                                  s->T0, cpu_regs[R_EAX]);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, cpu_regs[R_EAX]);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, cpu_regs[R_EDX]);
                 set_cc_op(s, CC_OP_MULQ);
                 break;
 #endif
@@ -5416,9 +3686,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 /* XXX: use 32 bit mul which could be faster */
                 tcg_gen_mul_tl(tcg_ctx, s->T0, s->T0, s->T1);
                 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
                 tcg_gen_ext8s_tl(tcg_ctx, s->tmp0, s->T0);
-                tcg_gen_sub_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0, s->tmp0);
+                tcg_gen_sub_tl(tcg_ctx, cpu_cc_src, s->T0, s->tmp0);
                 set_cc_op(s, CC_OP_MULB);
                 break;
             case MO_16:
@@ -5428,9 +3698,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 /* XXX: use 32 bit mul which could be faster */
                 tcg_gen_mul_tl(tcg_ctx, s->T0, s->T0, s->T1);
                 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
                 tcg_gen_ext16s_tl(tcg_ctx, s->tmp0, s->T0);
-                tcg_gen_sub_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0, s->tmp0);
+                tcg_gen_sub_tl(tcg_ctx, cpu_cc_src, s->T0, s->tmp0);
                 tcg_gen_shri_tl(tcg_ctx, s->T0, s->T0, 16);
                 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T0);
                 set_cc_op(s, CC_OP_MULW);
@@ -5438,24 +3708,24 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             default:
             case MO_32:
                 tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, tcg_ctx->cpu_regs[R_EAX]);
+                tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, cpu_regs[R_EAX]);
                 tcg_gen_muls2_i32(tcg_ctx, s->tmp2_i32, s->tmp3_i32,
                                   s->tmp2_i32, s->tmp3_i32);
-                tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[R_EAX], s->tmp2_i32);
-                tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[R_EDX], s->tmp3_i32);
+                tcg_gen_extu_i32_tl(tcg_ctx, cpu_regs[R_EAX], s->tmp2_i32);
+                tcg_gen_extu_i32_tl(tcg_ctx, cpu_regs[R_EDX], s->tmp3_i32);
                 tcg_gen_sari_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, 31);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_regs[R_EAX]);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, cpu_regs[R_EAX]);
                 tcg_gen_sub_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, s->tmp3_i32);
-                tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->tmp2_i32);
+                tcg_gen_extu_i32_tl(tcg_ctx, cpu_cc_src, s->tmp2_i32);
                 set_cc_op(s, CC_OP_MULL);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                tcg_gen_muls2_i64(tcg_ctx, tcg_ctx->cpu_regs[R_EAX], tcg_ctx->cpu_regs[R_EDX],
-                                  s->T0, tcg_ctx->cpu_regs[R_EAX]);
-                tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_regs[R_EAX]);
-                tcg_gen_sari_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_regs[R_EAX], 63);
-                tcg_gen_sub_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_regs[R_EDX]);
+                tcg_gen_muls2_i64(tcg_ctx, cpu_regs[R_EAX], cpu_regs[R_EDX],
+                                  s->T0, cpu_regs[R_EAX]);
+                tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, cpu_regs[R_EAX]);
+                tcg_gen_sari_tl(tcg_ctx, cpu_cc_src, cpu_regs[R_EAX], 63);
+                tcg_gen_sub_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, cpu_regs[R_EDX]);
                 set_cc_op(s, CC_OP_MULQ);
                 break;
 #endif
@@ -5464,18 +3734,18 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         case 6: /* div */
             switch(ot) {
             case MO_8:
-                gen_helper_divb_AL(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_divb_AL(tcg_ctx, cpu_env, s->T0);
                 break;
             case MO_16:
-                gen_helper_divw_AX(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_divw_AX(tcg_ctx, cpu_env, s->T0);
                 break;
             default:
             case MO_32:
-                gen_helper_divl_EAX(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_divl_EAX(tcg_ctx, cpu_env, s->T0);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_helper_divq_EAX(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_divq_EAX(tcg_ctx, cpu_env, s->T0);
                 break;
 #endif
             }
@@ -5483,18 +3753,18 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         case 7: /* idiv */
             switch(ot) {
             case MO_8:
-                gen_helper_idivb_AL(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_idivb_AL(tcg_ctx, cpu_env, s->T0);
                 break;
             case MO_16:
-                gen_helper_idivw_AX(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_idivw_AX(tcg_ctx, cpu_env, s->T0);
                 break;
             default:
             case MO_32:
-                gen_helper_idivl_EAX(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_idivl_EAX(tcg_ctx, cpu_env, s->T0);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_helper_idivq_EAX(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_idivq_EAX(tcg_ctx, cpu_env, s->T0);
                 break;
 #endif
             }
@@ -5520,7 +3790,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 /* operand size for jumps is 64 bit */
                 ot = MO_64;
             } else if (op == 3 || op == 5) {
-                ot = dflag != MO_16 ? MO_32 + (rex_w == 1) : MO_16;
+                ot = dflag != MO_16 ? MO_32 + REX_W(s) : MO_16;
             } else if (op == 6) {
                 /* default push size is 64 bit */
                 ot = mo_pushpop(s, dflag);
@@ -5554,12 +3824,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             if (dflag == MO_16) {
                 tcg_gen_ext16u_tl(tcg_ctx, s->T0, s->T0);
             }
-            next_eip = s->pc - s->cs_base;
-            tcg_gen_movi_tl(tcg_ctx, s->T1, next_eip);
-            gen_push_v(s, s->T1);
-            gen_op_jmp_v(tcg_ctx, s->T0);
+            gen_push_v(s, eip_next_tl(s));
+            gen_op_jmp_v(s, s->T0);
             gen_bnd_jmp(s);
-            gen_jr(s, s->T0);
+            s->base.is_jmp = DISAS_JUMP;
             break;
         case 3: /* lcall Ev */
             if (mod == 3) {
@@ -5569,27 +3837,26 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_add_A0_im(s, 1 << ot);
             gen_op_ld_v(s, MO_16, s->T0, s->A0);
         do_lcall:
-            if (s->pe && !s->vm86) {
+            if (PE(s) && !VM86(s)) {
                 tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                gen_helper_lcall_protected(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32, s->T1,
-                                           tcg_const_i32(tcg_ctx, dflag - 1),
-                                           tcg_const_tl(tcg_ctx, s->pc - s->cs_base));
+                gen_helper_lcall_protected(tcg_ctx, cpu_env, s->tmp2_i32, s->T1,
+                                           tcg_constant_i32(tcg_ctx, dflag - 1),
+                                           eip_next_tl(s));
             } else {
                 tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                gen_helper_lcall_real(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32, s->T1,
-                                      tcg_const_i32(tcg_ctx, dflag - 1),
-                                      tcg_const_i32(tcg_ctx, s->pc - s->cs_base));
+                gen_helper_lcall_real(tcg_ctx, cpu_env, s->tmp2_i32, s->T1,
+                                      tcg_constant_i32(tcg_ctx, dflag - 1),
+                                      eip_next_i32(s));
             }
-            tcg_gen_ld_tl(tcg_ctx, s->tmp4, tcg_ctx->cpu_env, offsetof(CPUX86State, eip));
-            gen_jr(s, s->tmp4);
+            s->base.is_jmp = DISAS_JUMP;
             break;
         case 4: /* jmp Ev */
             if (dflag == MO_16) {
                 tcg_gen_ext16u_tl(tcg_ctx, s->T0, s->T0);
             }
-            gen_op_jmp_v(tcg_ctx, s->T0);
+            gen_op_jmp_v(s, s->T0);
             gen_bnd_jmp(s);
-            gen_jr(s, s->T0);
+            s->base.is_jmp = DISAS_JUMP;
             break;
         case 5: /* ljmp Ev */
             if (mod == 3) {
@@ -5599,16 +3866,15 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_add_A0_im(s, 1 << ot);
             gen_op_ld_v(s, MO_16, s->T0, s->A0);
         do_ljmp:
-            if (s->pe && !s->vm86) {
+            if (PE(s) && !VM86(s)) {
                 tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                gen_helper_ljmp_protected(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32, s->T1,
-                                          tcg_const_tl(tcg_ctx, s->pc - s->cs_base));
+                gen_helper_ljmp_protected(tcg_ctx, cpu_env, s->tmp2_i32, s->T1,
+                                          eip_next_tl(s));
             } else {
                 gen_op_movl_seg_T0_vm(s, R_CS);
-                gen_op_jmp_v(tcg_ctx, s->T1);
+                gen_op_jmp_v(s, s->T1);
             }
-            tcg_gen_ld_tl(tcg_ctx, s->tmp4, tcg_ctx->cpu_env, offsetof(CPUX86State, eip));
-            gen_jr(s, s->tmp4);
+            s->base.is_jmp = DISAS_JUMP;
             break;
         case 6: /* push Ev */
             gen_push_v(s, s->T0);
@@ -5623,7 +3889,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         ot = mo_b_d(b, dflag);
 
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_v_reg(s, ot, s->T1, reg);
@@ -5695,7 +3961,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x6b:
         ot = dflag;
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         if (b == 0x69)
             s->rip_offset = insn_const_size(ot);
         else if (b == 0x6b)
@@ -5713,10 +3979,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         switch (ot) {
 #ifdef TARGET_X86_64
         case MO_64:
-            tcg_gen_muls2_i64(tcg_ctx, tcg_ctx->cpu_regs[reg], s->T1, s->T0, s->T1);
-            tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_regs[reg]);
-            tcg_gen_sari_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_dst, 63);
-            tcg_gen_sub_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, s->T1);
+            tcg_gen_muls2_i64(tcg_ctx, cpu_regs[reg], s->T1, s->T0, s->T1);
+            tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, cpu_regs[reg]);
+            tcg_gen_sari_tl(tcg_ctx, cpu_cc_src, cpu_cc_dst, 63);
+            tcg_gen_sub_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, s->T1);
             break;
 #endif
         case MO_32:
@@ -5724,20 +3990,20 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, s->T1);
             tcg_gen_muls2_i32(tcg_ctx, s->tmp2_i32, s->tmp3_i32,
                               s->tmp2_i32, s->tmp3_i32);
-            tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_regs[reg], s->tmp2_i32);
+            tcg_gen_extu_i32_tl(tcg_ctx, cpu_regs[reg], s->tmp2_i32);
             tcg_gen_sari_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, 31);
-            tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, tcg_ctx->cpu_regs[reg]);
+            tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, cpu_regs[reg]);
             tcg_gen_sub_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32, s->tmp3_i32);
-            tcg_gen_extu_i32_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->tmp2_i32);
+            tcg_gen_extu_i32_tl(tcg_ctx, cpu_cc_src, s->tmp2_i32);
             break;
         default:
             tcg_gen_ext16s_tl(tcg_ctx, s->T0, s->T0);
             tcg_gen_ext16s_tl(tcg_ctx, s->T1, s->T1);
             /* XXX: use 32 bit mul which could be faster */
             tcg_gen_mul_tl(tcg_ctx, s->T0, s->T0, s->T1);
-            tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+            tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
             tcg_gen_ext16s_tl(tcg_ctx, s->tmp0, s->T0);
-            tcg_gen_sub_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0, s->tmp0);
+            tcg_gen_sub_tl(tcg_ctx, cpu_cc_src, s->T0, s->tmp0);
             gen_op_mov_reg_v(s, ot, reg, s->T0);
             break;
         }
@@ -5747,7 +4013,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x1c1: /* xadd Ev, Gv */
         ot = mo_b_d(b, dflag);
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         mod = (modrm >> 6) & 3;
         gen_op_mov_v_reg(s, ot, s->T0, reg);
         if (mod == 3) {
@@ -5775,18 +4041,18 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x1b0:
     case 0x1b1: /* cmpxchg Ev, Gv */
         {
-            TCGv oldv, newv, cmpv;
+            TCGv oldv, newv, cmpv, dest;
 
             ot = mo_b_d(b, dflag);
             modrm = x86_ldub_code(env, s);
-            reg = ((modrm >> 3) & 7) | rex_r;
+            reg = ((modrm >> 3) & 7) | REX_R(s);
             mod = (modrm >> 6) & 3;
             oldv = tcg_temp_new(tcg_ctx);
             newv = tcg_temp_new(tcg_ctx);
             cmpv = tcg_temp_new(tcg_ctx);
             gen_op_mov_v_reg(s, ot, newv, reg);
-            tcg_gen_mov_tl(tcg_ctx, cmpv, tcg_ctx->cpu_regs[R_EAX]);
-
+            tcg_gen_mov_tl(tcg_ctx, cmpv, cpu_regs[R_EAX]);
+            gen_extu(tcg_ctx, ot, cmpv);
             if (s->prefix & PREFIX_LOCK) {
                 if (mod == 3) {
                     goto illegal_op;
@@ -5794,37 +4060,46 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 gen_lea_modrm(env, s, modrm);
                 tcg_gen_atomic_cmpxchg_tl(tcg_ctx, oldv, s->A0, cmpv, newv,
                                           s->mem_index, ot | MO_LE);
-                gen_extu(tcg_ctx, ot, oldv);
-                gen_extu(tcg_ctx, ot, cmpv);
-                gen_op_update_cmpxchg_acc(s, ot, oldv, cmpv);
             } else {
                 if (mod == 3) {
                     rm = (modrm & 7) | REX_B(s);
                     gen_op_mov_v_reg(s, ot, oldv, rm);
+                    gen_extu(tcg_ctx, ot, oldv);
+
+                    /*
+                     * Unlike the memory case, where "the destination operand receives
+                     * a write cycle without regard to the result of the comparison",
+                     * rm must not be touched altogether if the write fails, including
+                     * not zero-extending it on 64-bit processors.  So, precompute
+                     * the result of a successful writeback and perform the movcond
+                     * directly on cpu_regs.  Also need to write accumulator first, in
+                     * case rm is part of RAX too.
+                     */
+                    dest = gen_op_deposit_reg_v(s, ot, rm, newv, newv);
+                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, dest, oldv, cmpv, newv, dest);
                 } else {
                     gen_lea_modrm(env, s, modrm);
                     gen_op_ld_v(s, ot, oldv, s->A0);
-                    rm = 0; /* avoid warning */
-                }
-                gen_extu(tcg_ctx, ot, oldv);
-                gen_extu(tcg_ctx, ot, cmpv);
-                /* store value = (old == cmp ? new : old);  */
-                tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, newv, oldv, cmpv, newv, oldv);
-                if (mod == 3) {
-                    gen_op_update_cmpxchg_acc(s, ot, oldv, cmpv);
-                    gen_op_mov_reg_v(s, ot, rm, newv);
-                } else {
-                    /* Perform an unconditional store cycle like physical cpu;
-                       must be before changing accumulator to ensure
-                       idempotency if the store faults and the instruction
-                       is restarted */
+
+                    /*
+                     * Perform an unconditional store cycle like physical cpu;
+                     * must be before changing accumulator to ensure
+                     * idempotency if the store faults and the instruction
+                     * is restarted
+                     */
+                    tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, newv, oldv, cmpv, newv, oldv);
                     gen_op_st_v(s, ot, newv, s->A0);
-                    gen_op_update_cmpxchg_acc(s, ot, oldv, cmpv);
                 }
             }
-            tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, oldv);
+	    /*
+	     * Write EAX only if the cmpxchg fails; reuse newv as the destination,
+	     * since it's dead here.
+	     */
+            dest = gen_op_deposit_reg_v(s, ot, R_EAX, newv, oldv);
+            tcg_gen_movcond_tl(tcg_ctx, TCG_COND_EQ, dest, oldv, cmpv, dest, newv);
+            tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, oldv);
             tcg_gen_mov_tl(tcg_ctx, s->cc_srcT, cmpv);
-            tcg_gen_sub_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, cmpv, oldv);
+            tcg_gen_sub_tl(tcg_ctx, cpu_cc_dst, cmpv, oldv);
             set_cc_op(s, CC_OP_SUBB + ot);
             tcg_temp_free(tcg_ctx, oldv);
             tcg_temp_free(tcg_ctx, newv);
@@ -5847,9 +4122,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 gen_lea_modrm(env, s, modrm);
                 if ((s->prefix & PREFIX_LOCK) &&
                     (tb_cflags(s->base.tb) & CF_PARALLEL)) {
-                    gen_helper_cmpxchg16b(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+                    gen_helper_cmpxchg16b(tcg_ctx, cpu_env, s->A0);
                 } else {
-                    gen_helper_cmpxchg16b_unlocked(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+                    gen_helper_cmpxchg16b_unlocked(tcg_ctx, cpu_env, s->A0);
                 }
                 set_cc_op(s, CC_OP_EFLAGS);
                 break;
@@ -5861,9 +4136,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_lea_modrm(env, s, modrm);
             if ((s->prefix & PREFIX_LOCK) &&
                 (tb_cflags(s->base.tb) & CF_PARALLEL)) {
-                gen_helper_cmpxchg8b(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+                gen_helper_cmpxchg8b(tcg_ctx, cpu_env, s->A0);
             } else {
-                gen_helper_cmpxchg8b_unlocked(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+                gen_helper_cmpxchg8b_unlocked(tcg_ctx, cpu_env, s->A0);
             }
             set_cc_op(s, CC_OP_EFLAGS);
             break;
@@ -5877,14 +4152,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             }
             if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
                 gen_io_start(tcg_ctx);
+                s->base.is_jmp = DISAS_TOO_MANY;
             }
-            gen_helper_rdrand(tcg_ctx, s->T0, tcg_ctx->cpu_env);
+            gen_helper_rdrand(tcg_ctx, s->T0, cpu_env);
             rm = (modrm & 7) | REX_B(s);
             gen_op_mov_reg_v(s, dflag, rm, s->T0);
             set_cc_op(s, CC_OP_EFLAGS);
-            if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-                gen_jmp(s, s->pc - s->cs_base);
-            }
             break;
 
         default:
@@ -5894,25 +4167,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
 
         /**************************/
         /* push/pop */
-    case 0x50: /* push */
-    case 0x51: /* push */
-    case 0x52: /* push */
-    case 0x53: /* push */
-    case 0x54: /* push */
-    case 0x55: /* push */
-    case 0x56: /* push */
-    case 0x57: /* push */
+    case 0x50: case 0x51: case 0x52: case 0x53: case 0x54: case 0x55: case 0x56: case 0x57: /* push */
         gen_op_mov_v_reg(s, MO_32, s->T0, (b & 7) | REX_B(s));
         gen_push_v(s, s->T0);
         break;
-    case 0x58: /* pop */
-    case 0x59: /* pop */
-    case 0x5a: /* pop */
-    case 0x5b: /* pop */
-    case 0x5c: /* pop */
-    case 0x5d: /* pop */
-    case 0x5e: /* pop */
-    case 0x5f: /* pop */
+    case 0x58: case 0x59: case 0x5a: case 0x5b: case 0x5c: case 0x5d: case 0x5e: case 0x5f: /* pop */
         ot = gen_pop_T0(s);
         /* NOTE: order is important for pop %sp */
         gen_pop_update(s, ot);
@@ -5989,26 +4248,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         ot = gen_pop_T0(s);
         gen_movl_seg_T0(s, reg);
         gen_pop_update(s, ot);
-        /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp.  */
-        if (s->base.is_jmp) {
-            gen_jmp_im(s, s->pc - s->cs_base);
-            if (reg == R_SS) {
-                s->tf = 0;
-                gen_eob_inhibit_irq(s, true);
-            } else {
-                gen_eob(s);
-            }
-        }
         break;
     case 0x1a1: /* pop fs */
     case 0x1a9: /* pop gs */
         ot = gen_pop_T0(s);
         gen_movl_seg_T0(s, (b >> 3) & 7);
         gen_pop_update(s, ot);
-        if (s->base.is_jmp) {
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
-        }
         break;
 
         /**************************/
@@ -6017,7 +4262,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x89: /* mov Gv, Ev */
         ot = mo_b_d(b, dflag);
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
 
         /* generate a generic store */
         gen_ldst_modrm(env, s, modrm, ot, reg, 1);
@@ -6027,15 +4272,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         ot = mo_b_d(b, dflag);
         modrm = x86_ldub_code(env, s);
         mod = (modrm >> 6) & 3;
-        reg = ((modrm >> 3) & 7) | rex_r;
         if (mod != 3) {
-            if (reg != 0)
-                goto illegal_op;
             s->rip_offset = insn_const_size(ot);
             gen_lea_modrm(env, s, modrm);
-        } else {
-            if (reg != 0 && reg != 7)
-                goto illegal_op;
         }
         val = insn_get(env, s, ot);
         tcg_gen_movi_tl(tcg_ctx, s->T0, val);
@@ -6049,7 +4288,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x8b: /* mov Ev, Gv */
         ot = mo_b_d(b, dflag);
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_reg_v(s, ot, reg, s->T0);
@@ -6060,17 +4299,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         if (reg >= 6 || reg == R_CS)
             goto illegal_op;
         gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-        gen_movl_seg_T0(s, reg);
-        /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp.  */
-        if (s->base.is_jmp) {
-            gen_jmp_im(s, s->pc - s->cs_base);
-            if (reg == R_SS) {
-                s->tf = 0;
-                gen_eob_inhibit_irq(s, true);
-            } else {
-                gen_eob(s);
-            }
-        }
+        gen_movl_seg_T0(s, reg);
         break;
     case 0x8c: /* mov Gv, seg */
         modrm = x86_ldub_code(env, s);
@@ -6099,13 +4328,13 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             s_ot = b & 8 ? MO_SIGN | ot : ot;
 
             modrm = x86_ldub_code(env, s);
-            reg = ((modrm >> 3) & 7) | rex_r;
+            reg = ((modrm >> 3) & 7) | REX_R(s);
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
 
             if (mod == 3) {
                 if (s_ot == MO_SB && byte_reg_is_xH(s, rm)) {
-                    tcg_gen_sextract_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[rm - 4], 8, 8);
+                    tcg_gen_sextract_tl(tcg_ctx, s->T0, cpu_regs[rm - 4], 8, 8);
                 } else {
                     gen_op_mov_v_reg(s, ot, s->T0, rm);
                     switch (s_ot) {
@@ -6138,10 +4367,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         {
             AddressParts a = gen_lea_modrm_0(env, s, modrm);
-            TCGv ea = gen_lea_modrm_1(s, a);
+            TCGv ea = gen_lea_modrm_1(s, a, false);
             gen_lea_v_seg(s, s->aflag, ea, -1, -1);
             gen_op_mov_reg_v(s, dflag, reg, s->A0);
         }
@@ -6155,16 +4384,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             target_ulong offset_addr;
 
             ot = mo_b_d(b, dflag);
-            switch (s->aflag) {
-#ifdef TARGET_X86_64
-            case MO_64:
-                offset_addr = x86_ldq_code(env, s);
-                break;
-#endif
-            default:
-                offset_addr = insn_get(env, s, s->aflag);
-                break;
-            }
+            offset_addr = insn_get_addr(env, s, s->aflag);
             tcg_gen_movi_tl(tcg_ctx, s->A0, offset_addr);
             gen_add_A0_ds_seg(s);
             if ((b & 2) == 0) {
@@ -6177,34 +4397,20 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         }
         break;
     case 0xd7: /* xlat */
-        tcg_gen_mov_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[R_EBX]);
-        tcg_gen_ext8u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[R_EAX]);
+        tcg_gen_mov_tl(tcg_ctx, s->A0, cpu_regs[R_EBX]);
+        tcg_gen_ext8u_tl(tcg_ctx, s->T0, cpu_regs[R_EAX]);
         tcg_gen_add_tl(tcg_ctx, s->A0, s->A0, s->T0);
         gen_extu(tcg_ctx, s->aflag, s->A0);
         gen_add_A0_ds_seg(s);
         gen_op_ld_v(s, MO_8, s->T0, s->A0);
         gen_op_mov_reg_v(s, MO_8, R_EAX, s->T0);
         break;
-    case 0xb0: /* mov R, Ib */
-    case 0xb1: /* mov R, Ib */
-    case 0xb2: /* mov R, Ib */
-    case 0xb3: /* mov R, Ib */
-    case 0xb4: /* mov R, Ib */
-    case 0xb5: /* mov R, Ib */
-    case 0xb6: /* mov R, Ib */
-    case 0xb7: /* mov R, Ib */
+    case 0xb0: case 0xb1: case 0xb2: case 0xb3: case 0xb4: case 0xb5: case 0xb6: case 0xb7: /* mov R, Ib */
         val = insn_get(env, s, MO_8);
         tcg_gen_movi_tl(tcg_ctx, s->T0, val);
         gen_op_mov_reg_v(s, MO_8, (b & 7) | REX_B(s), s->T0);
         break;
-    case 0xb8: /* mov R, Iv */
-    case 0xb9: /* mov R, Iv */
-    case 0xba: /* mov R, Iv */
-    case 0xbb: /* mov R, Iv */
-    case 0xbc: /* mov R, Iv */
-    case 0xbd: /* mov R, Iv */
-    case 0xbe: /* mov R, Iv */
-    case 0xbf: /* mov R, Iv */
+    case 0xb8: case 0xb9: case 0xba: case 0xbb: case 0xbc: case 0xbd: case 0xbe: case 0xbf: /* mov R, Iv */
 #ifdef TARGET_X86_64
         if (dflag == MO_64) {
             uint64_t tmp;
@@ -6224,13 +4430,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         }
         break;
 
-    case 0x91: /* xchg R, EAX */
-    case 0x92: /* xchg R, EAX */
-    case 0x93: /* xchg R, EAX */
-    case 0x94: /* xchg R, EAX */
-    case 0x95: /* xchg R, EAX */
-    case 0x96: /* xchg R, EAX */
-    case 0x97: /* xchg R, EAX */
+    case 0x91: case 0x92: case 0x93: case 0x94: case 0x95: case 0x96: case 0x97: /* xchg R, EAX */
     do_xchg_reg_eax:
         ot = dflag;
         reg = (b & 7) | REX_B(s);
@@ -6240,7 +4440,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x87: /* xchg Ev, Gv */
         ot = mo_b_d(b, dflag);
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
             rm = (modrm & 7) | REX_B(s);
@@ -6277,7 +4477,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     do_lxx:
         ot = dflag != MO_16 ? MO_32 : MO_16;
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
@@ -6289,10 +4489,6 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         gen_movl_seg_T0(s, op);
         /* then put the data */
         gen_op_mov_reg_v(s, ot, reg, s->T1);
-        if (s->base.is_jmp) {
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
-        }
         break;
 
         /************************/
@@ -6301,7 +4497,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xc1:
         /* shift Ev,Ib */
         shift = 2;
-    grp2_label:
+    grp2:
         {
             ot = mo_b_d(b, dflag);
             modrm = x86_ldub_code(env, s);
@@ -6333,12 +4529,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xd1:
         /* shift Ev,1 */
         shift = 1;
-        goto grp2_label;
+        goto grp2;
     case 0xd2:
     case 0xd3:
         /* shift Ev,cl */
         shift = 0;
-        goto grp2_label;
+        goto grp2;
 
     case 0x1a4: /* shld imm */
         op = 0;
@@ -6360,7 +4556,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         modrm = x86_ldub_code(env, s);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         if (mod != 3) {
             if (shift) {
                 s->rip_offset = 1;
@@ -6377,27 +4573,20 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_shiftd_rm_T1(s, ot, opreg, op, imm);
             tcg_temp_free(tcg_ctx, imm);
         } else {
-            gen_shiftd_rm_T1(s, ot, opreg, op, tcg_ctx->cpu_regs[R_ECX]);
+            gen_shiftd_rm_T1(s, ot, opreg, op, cpu_regs[R_ECX]);
         }
         break;
 
         /************************/
         /* floats */
-    case 0xd8:
-    case 0xd9:
-    case 0xda:
-    case 0xdb:
-    case 0xdc:
-    case 0xdd:
-    case 0xde:
-    case 0xdf:
+    case 0xd8: case 0xd9: case 0xda: case 0xdb: case 0xdc: case 0xdd: case 0xde: case 0xdf:
         {
             bool update_fip = true;
 
             if (s->flags & (HF_EM_MASK | HF_TS_MASK)) {
                 /* if CR0.EM or CR0.TS are set, generate an FPU exception */
                 /* XXX: what to do if illegal op ? */
-                gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+                gen_exception(s, EXCP07_PREX);
                 break;
             }
             modrm = x86_ldub_code(env, s);
@@ -6407,234 +4596,199 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             if (mod != 3) {
                 /* memory op */
                 AddressParts a = gen_lea_modrm_0(env, s, modrm);
-                TCGv ea = gen_lea_modrm_1(s, a);
+                TCGv ea = gen_lea_modrm_1(s, a, false);
                 TCGv last_addr = tcg_temp_new(tcg_ctx);
                 bool update_fdp = true;
 
                 tcg_gen_mov_tl(tcg_ctx, last_addr, ea);
                 gen_lea_v_seg(s, s->aflag, ea, a.def_seg, s->override);
-                switch(op) {
-                case 0x00: /* fxxxs */
-                case 0x01: /* fxxxs */
-                case 0x02: /* fxxxs */
-                case 0x03: /* fxxxs */
-                case 0x04: /* fxxxs */
-                case 0x05: /* fxxxs */
-                case 0x06: /* fxxxs */
-                case 0x07: /* fxxxs */
-
-                case 0x10: /* fixxxl */
-                case 0x11: /* fixxxl */
-                case 0x12: /* fixxxl */
-                case 0x13: /* fixxxl */
-                case 0x14: /* fixxxl */
-                case 0x15: /* fixxxl */
-                case 0x16: /* fixxxl */
-                case 0x17: /* fixxxl */
-
-                case 0x20: /* fxxxl */
-                case 0x21: /* fxxxl */
-                case 0x22: /* fxxxl */
-                case 0x23: /* fxxxl */
-                case 0x24: /* fxxxl */
-                case 0x25: /* fxxxl */
-                case 0x26: /* fxxxl */
-                case 0x27: /* fxxxl */
-
-                case 0x30: /* fixxx */
-                case 0x31: /* fixxx */
-                case 0x32: /* fixxx */
-                case 0x33: /* fixxx */
-                case 0x34: /* fixxx */
-                case 0x35: /* fixxx */
-                case 0x36: /* fixxx */
-                case 0x37: /* fixxx */
+
+                switch (op) {
+                case 0x0: case 0x1: case 0x2: case 0x3: case 0x4: case 0x5: case 0x6: case 0x7: /* fxxxs */
+                case 0x10: case 0x11: case 0x12: case 0x13: case 0x14: case 0x15: case 0x16: case 0x17: /* fixxxl */
+                case 0x20: case 0x21: case 0x22: case 0x23: case 0x24: case 0x25: case 0x26: case 0x27: /* fxxxl */
+                case 0x30: case 0x31: case 0x32: case 0x33: case 0x34: case 0x35: case 0x36: case 0x37: /* fixxx */
                     {
                         int op1;
                         op1 = op & 7;
 
-                        switch(op >> 4) {
+                        switch (op >> 4) {
                         case 0:
                             tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUL);
-                            gen_helper_flds_FT0(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                            gen_helper_flds_FT0(tcg_ctx, cpu_env, s->tmp2_i32);
                             break;
                         case 1:
                             tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUL);
-                            gen_helper_fildl_FT0(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                            gen_helper_fildl_FT0(tcg_ctx, cpu_env, s->tmp2_i32);
                             break;
                         case 2:
                             tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0,
-                                                s->mem_index, MO_LEQ);
-                            gen_helper_fldl_FT0(tcg_ctx, tcg_ctx->cpu_env, s->tmp1_i64);
+                                                s->mem_index, MO_LEUQ);
+                            gen_helper_fldl_FT0(tcg_ctx, cpu_env, s->tmp1_i64);
                             break;
                         case 3:
                         default:
                             tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LESW);
-                            gen_helper_fildl_FT0(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                            gen_helper_fildl_FT0(tcg_ctx, cpu_env, s->tmp2_i32);
                             break;
                         }
 
                         gen_helper_fp_arith_ST0_FT0(tcg_ctx, op1);
                         if (op1 == 3) {
                             /* fcomp needs pop */
-                            gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpop(tcg_ctx, cpu_env);
                         }
                     }
                     break;
                 case 0x08: /* flds */
                 case 0x0a: /* fsts */
                 case 0x0b: /* fstps */
-
-                case 0x18: /* fildl, fisttpl, fistl, fistpl */
-                case 0x19: /* fildl, fisttpl, fistl, fistpl */
-                case 0x1a: /* fildl, fisttpl, fistl, fistpl */
-                case 0x1b: /* fildl, fisttpl, fistl, fistpl */
-
-                case 0x28: /* fldl, fisttpll, fstl, fstpl */
-                case 0x29: /* fldl, fisttpll, fstl, fstpl */
-                case 0x2a: /* fldl, fisttpll, fstl, fstpl */
-                case 0x2b: /* fldl, fisttpll, fstl, fstpl */
-
-                case 0x38: /* filds, fisttps, fists, fistps */
-                case 0x39: /* filds, fisttps, fists, fistps */
-                case 0x3a: /* filds, fisttps, fists, fistps */
-                case 0x3b: /* filds, fisttps, fists, fistps */
-                    switch(op & 7) {
+                case 0x18: case 0x19: case 0x1a: case 0x1b: /* fildl, fisttpl, fistl, fistpl */
+                case 0x28: case 0x29: case 0x2a: case 0x2b: /* fldl, fisttpll, fstl, fstpl */
+                case 0x38: case 0x39: case 0x3a: case 0x3b: /* filds, fisttps, fists, fistps */
+                    switch (op & 7) {
                     case 0:
-                        switch(op >> 4) {
+                        switch (op >> 4) {
                         case 0:
                             tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUL);
-                            gen_helper_flds_ST0(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                            gen_helper_flds_ST0(tcg_ctx, cpu_env, s->tmp2_i32);
                             break;
                         case 1:
                             tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUL);
-                            gen_helper_fildl_ST0(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                            gen_helper_fildl_ST0(tcg_ctx, cpu_env, s->tmp2_i32);
                             break;
                         case 2:
                             tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0,
-                                                s->mem_index, MO_LEQ);
-                            gen_helper_fldl_ST0(tcg_ctx, tcg_ctx->cpu_env, s->tmp1_i64);
+                                                s->mem_index, MO_LEUQ);
+                            gen_helper_fldl_ST0(tcg_ctx, cpu_env, s->tmp1_i64);
                             break;
                         case 3:
                         default:
                             tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LESW);
-                            gen_helper_fildl_ST0(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                            gen_helper_fildl_ST0(tcg_ctx, cpu_env, s->tmp2_i32);
                             break;
                         }
                         break;
                     case 1:
                         /* XXX: the corresponding CPUID bit must be tested ! */
-                        switch(op >> 4) {
+                        switch (op >> 4) {
                         case 1:
-                            gen_helper_fisttl_ST0(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                            gen_helper_fisttl_ST0(tcg_ctx, s->tmp2_i32, cpu_env);
                             tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUL);
                             break;
                         case 2:
-                            gen_helper_fisttll_ST0(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env);
+                            gen_helper_fisttll_ST0(tcg_ctx, s->tmp1_i64, cpu_env);
                             tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0,
-                                                s->mem_index, MO_LEQ);
+                                                s->mem_index, MO_LEUQ);
                             break;
                         case 3:
                         default:
-                            gen_helper_fistt_ST0(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                            gen_helper_fistt_ST0(tcg_ctx, s->tmp2_i32, cpu_env);
                             tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUW);
                             break;
                         }
-                        gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fpop(tcg_ctx, cpu_env);
                         break;
                     default:
-                        switch(op >> 4) {
+                        switch (op >> 4) {
                         case 0:
-                            gen_helper_fsts_ST0(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                            gen_helper_fsts_ST0(tcg_ctx, s->tmp2_i32, cpu_env);
                             tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUL);
                             break;
                         case 1:
-                            gen_helper_fistl_ST0(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                            gen_helper_fistl_ST0(tcg_ctx, s->tmp2_i32, cpu_env);
                             tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUL);
                             break;
                         case 2:
-                            gen_helper_fstl_ST0(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env);
+                            gen_helper_fstl_ST0(tcg_ctx, s->tmp1_i64, cpu_env);
                             tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0,
-                                                s->mem_index, MO_LEQ);
+                                                s->mem_index, MO_LEUQ);
                             break;
                         case 3:
                         default:
-                            gen_helper_fist_ST0(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                            gen_helper_fist_ST0(tcg_ctx, s->tmp2_i32, cpu_env);
                             tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                                 s->mem_index, MO_LEUW);
                             break;
                         }
-                        if ((op & 7) == 3)
-                            gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                        if ((op & 7) == 3) {
+                            gen_helper_fpop(tcg_ctx, cpu_env);
+                        }
                         break;
                     }
                     break;
                 case 0x0c: /* fldenv mem */
-                    gen_helper_fldenv(tcg_ctx, tcg_ctx->cpu_env, s->A0, tcg_const_i32(tcg_ctx, dflag - 1));
+                    gen_helper_fldenv(tcg_ctx, cpu_env, s->A0,
+                                      tcg_const_i32(tcg_ctx, dflag - 1));
                     update_fip = update_fdp = false;
                     break;
                 case 0x0d: /* fldcw mem */
                     tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                         s->mem_index, MO_LEUW);
-                    gen_helper_fldcw(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                    gen_helper_fldcw(tcg_ctx, cpu_env, s->tmp2_i32);
                     update_fip = update_fdp = false;
                     break;
                 case 0x0e: /* fnstenv mem */
-                    gen_helper_fstenv(tcg_ctx, tcg_ctx->cpu_env, s->A0, tcg_const_i32(tcg_ctx, dflag - 1));
+                    gen_helper_fstenv(tcg_ctx, cpu_env, s->A0,
+                                      tcg_const_i32(tcg_ctx, dflag - 1));
                     update_fip = update_fdp = false;
                     break;
                 case 0x0f: /* fnstcw mem */
-                    gen_helper_fnstcw(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                    gen_helper_fnstcw(tcg_ctx, s->tmp2_i32, cpu_env);
                     tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                         s->mem_index, MO_LEUW);
                     update_fip = update_fdp = false;
                     break;
                 case 0x1d: /* fldt mem */
-                    gen_helper_fldt_ST0(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+                    gen_helper_fldt_ST0(tcg_ctx, cpu_env, s->A0);
                     break;
                 case 0x1f: /* fstpt mem */
-                    gen_helper_fstt_ST0(tcg_ctx, tcg_ctx->cpu_env, s->A0);
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fstt_ST0(tcg_ctx, cpu_env, s->A0);
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     break;
                 case 0x2c: /* frstor mem */
-                    gen_helper_frstor(tcg_ctx, tcg_ctx->cpu_env, s->A0, tcg_const_i32(tcg_ctx, dflag - 1));
+                    gen_helper_frstor(tcg_ctx, cpu_env, s->A0,
+                                      tcg_const_i32(tcg_ctx, dflag - 1));
                     update_fip = update_fdp = false;
                     break;
                 case 0x2e: /* fnsave mem */
-                    gen_helper_fsave(tcg_ctx, tcg_ctx->cpu_env, s->A0, tcg_const_i32(tcg_ctx, dflag - 1));
+                    gen_helper_fsave(tcg_ctx, cpu_env, s->A0,
+                                     tcg_const_i32(tcg_ctx, dflag - 1));
                     update_fip = update_fdp = false;
                     break;
                 case 0x2f: /* fnstsw mem */
-                    gen_helper_fnstsw(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                    gen_helper_fnstsw(tcg_ctx, s->tmp2_i32, cpu_env);
                     tcg_gen_qemu_st_i32(tcg_ctx, s->tmp2_i32, s->A0,
                                         s->mem_index, MO_LEUW);
                     update_fip = update_fdp = false;
                     break;
                 case 0x3c: /* fbld */
-                    gen_helper_fbld_ST0(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+                    gen_helper_fbld_ST0(tcg_ctx, cpu_env, s->A0);
                     break;
                 case 0x3e: /* fbstp */
-                    gen_helper_fbst_ST0(tcg_ctx, tcg_ctx->cpu_env, s->A0);
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fbst_ST0(tcg_ctx, cpu_env, s->A0);
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     break;
                 case 0x3d: /* fildll */
-                    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEQ);
-                    gen_helper_fildll_ST0(tcg_ctx, tcg_ctx->cpu_env, s->tmp1_i64);
+                    tcg_gen_qemu_ld_i64(tcg_ctx, s->tmp1_i64, s->A0,
+                                        s->mem_index, MO_LEUQ);
+                    gen_helper_fildll_ST0(tcg_ctx, cpu_env, s->tmp1_i64);
                     break;
                 case 0x3f: /* fistpll */
-                    gen_helper_fistll_ST0(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env);
-                    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0, s->mem_index, MO_LEQ);
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fistll_ST0(tcg_ctx, s->tmp1_i64, cpu_env);
+                    tcg_gen_qemu_st_i64(tcg_ctx, s->tmp1_i64, s->A0,
+                                        s->mem_index, MO_LEUQ);
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     break;
                 default:
                     goto unknown_op;
@@ -6643,35 +4797,35 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 if (update_fdp) {
                     int last_seg = s->override >= 0 ? s->override : a.def_seg;
 
-                    tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                    offsetof(CPUX86State,
+                    tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, cpu_env,
+                                   offsetof(CPUX86State,
                                             segs[last_seg].selector));
-                    tcg_gen_st16_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                        offsetof(CPUX86State, fpds));
-                    tcg_gen_st_tl(tcg_ctx, last_addr, tcg_ctx->cpu_env,
-                                    offsetof(CPUX86State, fpdp));
+                    tcg_gen_st16_i32(tcg_ctx, s->tmp2_i32, cpu_env,
+                                     offsetof(CPUX86State, fpds));
+                    tcg_gen_st_tl(tcg_ctx, last_addr, cpu_env,
+                                  offsetof(CPUX86State, fpdp));
                 }
                 tcg_temp_free(tcg_ctx, last_addr);
             } else {
                 /* register float ops */
                 opreg = rm;
 
-                switch(op) {
+                switch (op) {
                 case 0x08: /* fld sti */
-                    gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                    gen_helper_fmov_ST0_STN(tcg_ctx, tcg_ctx->cpu_env,
+                    gen_helper_fpush(tcg_ctx, cpu_env);
+                    gen_helper_fmov_ST0_STN(tcg_ctx, cpu_env,
                                             tcg_const_i32(tcg_ctx, (opreg + 1) & 7));
                     break;
                 case 0x09: /* fxchg sti */
                 case 0x29: /* fxchg4 sti, undocumented op */
                 case 0x39: /* fxchg7 sti, undocumented op */
-                    gen_helper_fxchg_ST0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fxchg_ST0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
                     break;
                 case 0x0a: /* grp d9/2 */
-                    switch(rm) {
+                    switch (rm) {
                     case 0: /* fnop */
                         /* check exceptions (FreeBSD FPU probe) */
-                        gen_helper_fwait(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fwait(tcg_ctx, cpu_env);
                         update_fip = false;
                         break;
                     default:
@@ -6679,19 +4833,19 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     }
                     break;
                 case 0x0c: /* grp d9/4 */
-                    switch(rm) {
+                    switch (rm) {
                     case 0: /* fchs */
-                        gen_helper_fchs_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fchs_ST0(tcg_ctx, cpu_env);
                         break;
                     case 1: /* fabs */
-                        gen_helper_fabs_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fabs_ST0(tcg_ctx, cpu_env);
                         break;
                     case 4: /* ftst */
-                        gen_helper_fldz_FT0(tcg_ctx, tcg_ctx->cpu_env);
-                        gen_helper_fcom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fldz_FT0(tcg_ctx, cpu_env);
+                        gen_helper_fcom_ST0_FT0(tcg_ctx, cpu_env);
                         break;
                     case 5: /* fxam */
-                        gen_helper_fxam_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fxam_ST0(tcg_ctx, cpu_env);
                         break;
                     default:
                         goto unknown_op;
@@ -6699,34 +4853,34 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     break;
                 case 0x0d: /* grp d9/5 */
                     {
-                        switch(rm) {
+                        switch (rm) {
                         case 0:
-                            gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                            gen_helper_fld1_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpush(tcg_ctx, cpu_env);
+                            gen_helper_fld1_ST0(tcg_ctx, cpu_env);
                             break;
                         case 1:
-                            gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                            gen_helper_fldl2t_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpush(tcg_ctx, cpu_env);
+                            gen_helper_fldl2t_ST0(tcg_ctx, cpu_env);
                             break;
                         case 2:
-                            gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                            gen_helper_fldl2e_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpush(tcg_ctx, cpu_env);
+                            gen_helper_fldl2e_ST0(tcg_ctx, cpu_env);
                             break;
                         case 3:
-                            gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                            gen_helper_fldpi_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpush(tcg_ctx, cpu_env);
+                            gen_helper_fldpi_ST0(tcg_ctx, cpu_env);
                             break;
                         case 4:
-                            gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                            gen_helper_fldlg2_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpush(tcg_ctx, cpu_env);
+                            gen_helper_fldlg2_ST0(tcg_ctx, cpu_env);
                             break;
                         case 5:
-                            gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                            gen_helper_fldln2_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpush(tcg_ctx, cpu_env);
+                            gen_helper_fldln2_ST0(tcg_ctx, cpu_env);
                             break;
                         case 6:
-                            gen_helper_fpush(tcg_ctx, tcg_ctx->cpu_env);
-                            gen_helper_fldz_ST0(tcg_ctx, tcg_ctx->cpu_env);
+                            gen_helper_fpush(tcg_ctx, cpu_env);
+                            gen_helper_fldz_ST0(tcg_ctx, cpu_env);
                             break;
                         default:
                             goto unknown_op;
@@ -6734,130 +4888,118 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     }
                     break;
                 case 0x0e: /* grp d9/6 */
-                    switch(rm) {
+                    switch (rm) {
                     case 0: /* f2xm1 */
-                        gen_helper_f2xm1(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_f2xm1(tcg_ctx, cpu_env);
                         break;
                     case 1: /* fyl2x */
-                        gen_helper_fyl2x(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fyl2x(tcg_ctx, cpu_env);
                         break;
                     case 2: /* fptan */
-                        gen_helper_fptan(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fptan(tcg_ctx, cpu_env);
                         break;
                     case 3: /* fpatan */
-                        gen_helper_fpatan(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fpatan(tcg_ctx, cpu_env);
                         break;
                     case 4: /* fxtract */
-                        gen_helper_fxtract(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fxtract(tcg_ctx, cpu_env);
                         break;
                     case 5: /* fprem1 */
-                        gen_helper_fprem1(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fprem1(tcg_ctx, cpu_env);
                         break;
                     case 6: /* fdecstp */
-                        gen_helper_fdecstp(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fdecstp(tcg_ctx, cpu_env);
                         break;
                     default:
                     case 7: /* fincstp */
-                        gen_helper_fincstp(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fincstp(tcg_ctx, cpu_env);
                         break;
                     }
                     break;
                 case 0x0f: /* grp d9/7 */
-                    switch(rm) {
+                    switch (rm) {
                     case 0: /* fprem */
-                        gen_helper_fprem(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fprem(tcg_ctx, cpu_env);
                         break;
                     case 1: /* fyl2xp1 */
-                        gen_helper_fyl2xp1(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fyl2xp1(tcg_ctx, cpu_env);
                         break;
                     case 2: /* fsqrt */
-                        gen_helper_fsqrt(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fsqrt(tcg_ctx, cpu_env);
                         break;
                     case 3: /* fsincos */
-                        gen_helper_fsincos(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fsincos(tcg_ctx, cpu_env);
                         break;
                     case 5: /* fscale */
-                        gen_helper_fscale(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fscale(tcg_ctx, cpu_env);
                         break;
                     case 4: /* frndint */
-                        gen_helper_frndint(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_frndint(tcg_ctx, cpu_env);
                         break;
                     case 6: /* fsin */
-                        gen_helper_fsin(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fsin(tcg_ctx, cpu_env);
                         break;
                     default:
                     case 7: /* fcos */
-                        gen_helper_fcos(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fcos(tcg_ctx, cpu_env);
                         break;
                     }
                     break;
-                case 0x00: case 0x01:
-                case 0x04: /* fxxx st, sti */
-                case 0x05: /* fxxx st, sti */
-                case 0x06: /* fxxx st, sti */
-                case 0x07: /* fxxx st, sti */
-
-                case 0x20: case 0x21:
-                case 0x24: /* fxxx sti, st */
-                case 0x25: /* fxxx sti, st */
-                case 0x26: /* fxxx sti, st */
-                case 0x27: /* fxxx sti, st */
-
-                case 0x30: case 0x31:
-                case 0x34: /* fxxxp sti, st */
-                case 0x35: /* fxxxp sti, st */
-                case 0x36: /* fxxxp sti, st */
-                case 0x37: /* fxxxp sti, st */
+                case 0x00: case 0x01: case 0x4: case 0x5: case 0x6: case 0x7: /* fxxx st, sti */
+                case 0x20: case 0x21: case 0x24: case 0x25: case 0x26: case 0x27: /* fxxx sti, st */
+                case 0x30: case 0x31: case 0x34: case 0x35: case 0x36: case 0x37: /* fxxxp sti, st */
                     {
                         int op1;
 
                         op1 = op & 7;
                         if (op >= 0x20) {
                             gen_helper_fp_arith_STN_ST0(tcg_ctx, op1, opreg);
-                            if (op >= 0x30)
-                                gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                            if (op >= 0x30) {
+                                gen_helper_fpop(tcg_ctx, cpu_env);
+                            }
                         } else {
-                            gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                            gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env,
+                                                    tcg_const_i32(tcg_ctx, opreg));
                             gen_helper_fp_arith_ST0_FT0(tcg_ctx, op1);
                         }
                     }
                     break;
                 case 0x02: /* fcom */
                 case 0x22: /* fcom2, undocumented op */
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fcom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fcom_ST0_FT0(tcg_ctx, cpu_env);
                     break;
                 case 0x03: /* fcomp */
                 case 0x23: /* fcomp3, undocumented op */
                 case 0x32: /* fcomp5, undocumented op */
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fcom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fcom_ST0_FT0(tcg_ctx, cpu_env);
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     break;
                 case 0x15: /* da/5 */
-                    switch(rm) {
+                    switch (rm) {
                     case 1: /* fucompp */
-                        gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, 1));
-                        gen_helper_fucom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
-                        gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
-                        gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, 1));
+                        gen_helper_fucom_ST0_FT0(tcg_ctx, cpu_env);
+                        gen_helper_fpop(tcg_ctx, cpu_env);
+                        gen_helper_fpop(tcg_ctx, cpu_env);
                         break;
                     default:
                         goto unknown_op;
                     }
                     break;
                 case 0x1c:
-                    switch(rm) {
+                    switch (rm) {
                     case 0: /* feni (287 only, just do nop here) */
                         break;
                     case 1: /* fdisi (287 only, just do nop here) */
                         break;
                     case 2: /* fclex */
-                        gen_helper_fclex(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fclex(tcg_ctx, cpu_env);
                         update_fip = false;
                         break;
                     case 3: /* fninit */
-                        gen_helper_fninit(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fninit(tcg_ctx, cpu_env);
                         update_fip = false;
                         break;
                     case 4: /* fsetpm (287 only, just do nop here) */
@@ -6871,8 +5013,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                         goto illegal_op;
                     }
                     gen_update_cc_op(s);
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fucomi_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fucomi_ST0_FT0(tcg_ctx, cpu_env);
                     set_cc_op(s, CC_OP_EFLAGS);
                     break;
                 case 0x1e: /* fcomi */
@@ -6880,52 +5022,52 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                         goto illegal_op;
                     }
                     gen_update_cc_op(s);
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fcomi_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fcomi_ST0_FT0(tcg_ctx, cpu_env);
                     set_cc_op(s, CC_OP_EFLAGS);
                     break;
                 case 0x28: /* ffree sti */
-                    gen_helper_ffree_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_ffree_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
                     break;
                 case 0x2a: /* fst sti */
-                    gen_helper_fmov_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fmov_STN_ST0(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
                     break;
                 case 0x2b: /* fstp sti */
                 case 0x0b: /* fstp1 sti, undocumented op */
                 case 0x3a: /* fstp8 sti, undocumented op */
                 case 0x3b: /* fstp9 sti, undocumented op */
-                    gen_helper_fmov_STN_ST0(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_STN_ST0(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     break;
                 case 0x2c: /* fucom st(i) */
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fucom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fucom_ST0_FT0(tcg_ctx, cpu_env);
                     break;
                 case 0x2d: /* fucomp st(i) */
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fucom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fucom_ST0_FT0(tcg_ctx, cpu_env);
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     break;
                 case 0x33: /* de/3 */
-                    switch(rm) {
+                    switch (rm) {
                     case 1: /* fcompp */
-                        gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, 1));
-                        gen_helper_fcom_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
-                        gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
-                        gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                        gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, 1));
+                        gen_helper_fcom_ST0_FT0(tcg_ctx, cpu_env);
+                        gen_helper_fpop(tcg_ctx, cpu_env);
+                        gen_helper_fpop(tcg_ctx, cpu_env);
                         break;
                     default:
                         goto unknown_op;
                     }
                     break;
                 case 0x38: /* ffreep sti, undocumented op */
-                    gen_helper_ffree_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_ffree_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     break;
                 case 0x3c: /* df/4 */
-                    switch(rm) {
+                    switch (rm) {
                     case 0:
-                        gen_helper_fnstsw(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env);
+                        gen_helper_fnstsw(tcg_ctx, s->tmp2_i32, cpu_env);
                         tcg_gen_extu_i32_tl(tcg_ctx, s->T0, s->tmp2_i32);
                         gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0);
                         break;
@@ -6938,9 +5080,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                         goto illegal_op;
                     }
                     gen_update_cc_op(s);
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fucomi_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fucomi_ST0_FT0(tcg_ctx, cpu_env);
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     set_cc_op(s, CC_OP_EFLAGS);
                     break;
                 case 0x3e: /* fcomip */
@@ -6948,20 +5090,13 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                         goto illegal_op;
                     }
                     gen_update_cc_op(s);
-                    gen_helper_fmov_FT0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
-                    gen_helper_fcomi_ST0_FT0(tcg_ctx, tcg_ctx->cpu_env);
-                    gen_helper_fpop(tcg_ctx, tcg_ctx->cpu_env);
+                    gen_helper_fmov_FT0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                    gen_helper_fcomi_ST0_FT0(tcg_ctx, cpu_env);
+                    gen_helper_fpop(tcg_ctx, cpu_env);
                     set_cc_op(s, CC_OP_EFLAGS);
                     break;
-                case 0x10: /* fcmovxx */
-                case 0x11: /* fcmovxx */
-                case 0x12: /* fcmovxx */
-                case 0x13: /* fcmovxx */
-
-                case 0x18:
-                case 0x19:
-                case 0x1a:
-                case 0x1b:
+                case 0x10: case 0x11: case 0x12: case 0x13: /* fcmovxx */
+                case 0x18: case 0x19: case 0x1a: case 0x1b:
                     {
                         int op1;
                         TCGLabel *l1;
@@ -6978,7 +5113,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                         op1 = fcmov_cc[op & 3] | (((op >> 3) & 1) ^ 1);
                         l1 = gen_new_label(tcg_ctx);
                         gen_jcc1_noeob(s, op1, l1);
-                        gen_helper_fmov_ST0_STN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, opreg));
+                        gen_helper_fmov_ST0_STN(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, opreg));
                         gen_set_label(tcg_ctx, l1);
                     }
                     break;
@@ -6988,12 +5123,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             }
 
             if (update_fip) {
-                tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                offsetof(CPUX86State, segs[R_CS].selector));
-                tcg_gen_st16_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env,
-                                    offsetof(CPUX86State, fpcs));
-                tcg_gen_st_tl(tcg_ctx, tcg_const_tl(tcg_ctx, pc_start - s->cs_base),
-                                tcg_ctx->cpu_env, offsetof(CPUX86State, fpip));
+                tcg_gen_ld_i32(tcg_ctx, s->tmp2_i32, cpu_env,
+                               offsetof(CPUX86State, segs[R_CS].selector));
+                tcg_gen_st16_i32(tcg_ctx, s->tmp2_i32, cpu_env,
+                                 offsetof(CPUX86State, fpcs));
+                tcg_gen_st_tl(tcg_ctx, eip_cur_tl(s),
+                              cpu_env, offsetof(CPUX86State, fpip));
             }
         }
         break;
@@ -7004,7 +5139,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xa5:
         ot = mo_b_d(b, dflag);
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
-            gen_repz_movs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
+            gen_repz_movs(s, ot);
         } else {
             gen_movs(s, ot);
         }
@@ -7014,7 +5149,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xab:
         ot = mo_b_d(b, dflag);
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
-            gen_repz_stos(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
+            gen_repz_stos(s, ot);
         } else {
             gen_stos(s, ot);
         }
@@ -7023,7 +5158,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xad:
         ot = mo_b_d(b, dflag);
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
-            gen_repz_lods(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
+            gen_repz_lods(s, ot);
         } else {
             gen_lods(s, ot);
         }
@@ -7032,9 +5167,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xaf:
         ot = mo_b_d(b, dflag);
         if (prefixes & PREFIX_REPNZ) {
-            gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1);
+            gen_repz_scas(s, ot, 1);
         } else if (prefixes & PREFIX_REPZ) {
-            gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0);
+            gen_repz_scas(s, ot, 0);
         } else {
             gen_scas(s, ot);
         }
@@ -7044,9 +5179,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xa7:
         ot = mo_b_d(b, dflag);
         if (prefixes & PREFIX_REPNZ) {
-            gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1);
+            gen_repz_cmps(s, ot, 1);
         } else if (prefixes & PREFIX_REPZ) {
-            gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0);
+            gen_repz_cmps(s, ot, 0);
         } else {
             gen_cmps(s, ot);
         }
@@ -7054,31 +5189,38 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x6c: /* insS */
     case 0x6d:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[R_EDX]);
-        gen_check_io(s, ot, pc_start - s->cs_base, 
-                     SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes) | 4);
+        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_EDX]);
+        tcg_gen_ext16u_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32);
+        if (!gen_check_io(s, ot, s->tmp2_i32,
+                          SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) {
+            break;
+        }
+        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
+            gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
+        }
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
-            gen_repz_ins(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
+            gen_repz_ins(s, ot);
         } else {
             gen_ins(s, ot);
-            if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-                gen_jmp(s, s->pc - s->cs_base);
-            }
         }
         break;
     case 0x6e: /* outsS */
     case 0x6f:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[R_EDX]);
-        gen_check_io(s, ot, pc_start - s->cs_base,
-                     svm_is_rep(prefixes) | 4);
+        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_EDX]);
+        tcg_gen_ext16u_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32);
+        if (!gen_check_io(s, ot, s->tmp2_i32, SVM_IOIO_STR_MASK)) {
+            break;
+        }
+        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
+            gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
+        }
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
-            gen_repz_outs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
+            gen_repz_outs(s, ot);
         } else {
             gen_outs(s, ot);
-            if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-                gen_jmp(s, s->pc - s->cs_base);
-            }
         }
         break;
 
@@ -7089,75 +5231,67 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xe5:
         ot = mo_b_d32(b, dflag);
         val = x86_ldub_code(env, s);
-        tcg_gen_movi_tl(tcg_ctx, s->T0, val);
-        gen_check_io(s, ot, pc_start - s->cs_base,
-                     SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
+        tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, val);
+        if (!gen_check_io(s, ot, s->tmp2_i32, SVM_IOIO_TYPE_MASK)) {
+            break;
+        }
         if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
             gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
         }
-        tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, val);
         gen_helper_in_func(tcg_ctx, ot, s->T1, s->tmp2_i32);
         gen_op_mov_reg_v(s, ot, R_EAX, s->T1);
         gen_bpt_io(s, s->tmp2_i32, ot);
-        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-            gen_jmp(s, s->pc - s->cs_base);
-        }
         break;
     case 0xe6:
     case 0xe7:
         ot = mo_b_d32(b, dflag);
         val = x86_ldub_code(env, s);
-        tcg_gen_movi_tl(tcg_ctx, s->T0, val);
-        gen_check_io(s, ot, pc_start - s->cs_base,
-                     svm_is_rep(prefixes));
-        gen_op_mov_v_reg(s, ot, s->T1, R_EAX);
-
+        tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, val);
+        if (!gen_check_io(s, ot, s->tmp2_i32, 0)) {
+            break;
+        }
         if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
             gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
         }
-        tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, val);
+        gen_op_mov_v_reg(s, ot, s->T1, R_EAX);
         tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, s->T1);
         gen_helper_out_func(tcg_ctx, ot, s->tmp2_i32, s->tmp3_i32);
         gen_bpt_io(s, s->tmp2_i32, ot);
-        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-            gen_jmp(s, s->pc - s->cs_base);
-        }
         break;
     case 0xec:
     case 0xed:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[R_EDX]);
-        gen_check_io(s, ot, pc_start - s->cs_base,
-                     SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
+        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_EDX]);
+        tcg_gen_ext16u_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32);
+        if (!gen_check_io(s, ot, s->tmp2_i32, SVM_IOIO_TYPE_MASK)) {
+            break;
+        }
         if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
             gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
         }
-        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
         gen_helper_in_func(tcg_ctx, ot, s->T1, s->tmp2_i32);
         gen_op_mov_reg_v(s, ot, R_EAX, s->T1);
         gen_bpt_io(s, s->tmp2_i32, ot);
-        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-            gen_jmp(s, s->pc - s->cs_base);
-        }
         break;
     case 0xee:
     case 0xef:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[R_EDX]);
-        gen_check_io(s, ot, pc_start - s->cs_base,
-                     svm_is_rep(prefixes));
-        gen_op_mov_v_reg(s, ot, s->T1, R_EAX);
-
+        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_EDX]);
+        tcg_gen_ext16u_i32(tcg_ctx, s->tmp2_i32, s->tmp2_i32);
+        if (!gen_check_io(s, ot, s->tmp2_i32, 0)) {
+            break;
+        }
         if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
             gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
         }
-        tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
+        gen_op_mov_v_reg(s, ot, s->T1, R_EAX);
         tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp3_i32, s->T1);
         gen_helper_out_func(tcg_ctx, ot, s->tmp2_i32, s->tmp3_i32);
         gen_bpt_io(s, s->tmp2_i32, ot);
-        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-            gen_jmp(s, s->pc - s->cs_base);
-        }
         break;
 
         /************************/
@@ -7167,25 +5301,25 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         ot = gen_pop_T0(s);
         gen_stack_update(s, val + (1 << ot));
         /* Note that gen_pop_T0 uses a zero-extending load.  */
-        gen_op_jmp_v(tcg_ctx, s->T0);
+        gen_op_jmp_v(s, s->T0);
         gen_bnd_jmp(s);
-        gen_jr(s, s->T0);
+        s->base.is_jmp = DISAS_JUMP;
         break;
     case 0xc3: /* ret */
         ot = gen_pop_T0(s);
         gen_pop_update(s, ot);
         /* Note that gen_pop_T0 uses a zero-extending load.  */
-        gen_op_jmp_v(tcg_ctx, s->T0);
+        gen_op_jmp_v(s, s->T0);
         gen_bnd_jmp(s);
-        gen_jr(s, s->T0);
+        s->base.is_jmp = DISAS_JUMP;
         break;
     case 0xca: /* lret im */
         val = x86_lduw_code(env, s);
     do_lret:
-        if (s->pe && !s->vm86) {
+        if (PE(s) && !VM86(s)) {
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_lret_protected(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, dflag - 1),
+            gen_update_eip_cur(s);
+            gen_helper_lret_protected(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, dflag - 1),
                                       tcg_const_i32(tcg_ctx, val));
         } else {
             gen_stack_A0(s);
@@ -7193,7 +5327,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_op_ld_v(s, dflag, s->T0, s->A0);
             /* NOTE: keeping EIP updated is not a problem in case of
                exception */
-            gen_op_jmp_v(tcg_ctx, s->T0);
+            gen_op_jmp_v(s, s->T0);
             /* pop selector */
             gen_add_A0_im(s, 1 << dflag);
             gen_op_ld_v(s, dflag, s->T0, s->A0);
@@ -7201,49 +5335,34 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             /* add stack offset */
             gen_stack_update(s, val + (2 << dflag));
         }
-        gen_eob(s);
+        s->base.is_jmp = DISAS_EOB_ONLY;
         break;
     case 0xcb: /* lret */
         val = 0;
         goto do_lret;
     case 0xcf: /* iret */
-        gen_svm_check_intercept(s, pc_start, SVM_EXIT_IRET);
-        if (!s->pe) {
-            /* real mode */
-            gen_helper_iret_real(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, dflag - 1));
-            set_cc_op(s, CC_OP_EFLAGS);
-        } else if (s->vm86) {
-            if (s->iopl != 3) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            } else {
-                gen_helper_iret_real(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, dflag - 1));
-                set_cc_op(s, CC_OP_EFLAGS);
+        gen_svm_check_intercept(s, SVM_EXIT_IRET);
+        if (!PE(s) || VM86(s)) {
+            /* real mode or vm86 mode */
+            if (!check_vm86_iopl(s)) {
+                break;
             }
+            gen_helper_iret_real(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, dflag - 1));
         } else {
-            gen_helper_iret_protected(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, dflag - 1),
-                                      tcg_const_i32(tcg_ctx, s->pc - s->cs_base));
-            set_cc_op(s, CC_OP_EFLAGS);
+            gen_helper_iret_protected(tcg_ctx, cpu_env, tcg_constant_i32(tcg_ctx, dflag - 1),
+                                      eip_next_i32(s));
         }
-        gen_eob(s);
+        set_cc_op(s, CC_OP_EFLAGS);
+        s->base.is_jmp = DISAS_EOB_ONLY;
         break;
     case 0xe8: /* call im */
         {
-            if (dflag != MO_16) {
-                tval = (int32_t)insn_get(env, s, MO_32);
-            } else {
-                tval = (int16_t)insn_get(env, s, MO_16);
-            }
-            next_eip = s->pc - s->cs_base;
-            tval += next_eip;
-            if (dflag == MO_16) {
-                tval &= 0xffff;
-            } else if (!CODE64(s)) {
-                tval &= 0xffffffff;
-            }
-            tcg_gen_movi_tl(tcg_ctx, s->T0, next_eip);
-            gen_push_v(s, s->T0);
+            int diff = (dflag != MO_16
+                        ? (int32_t)insn_get(env, s, MO_32)
+                        : (int16_t)insn_get(env, s, MO_16));
+            gen_push_v(s, eip_next_tl(s));
             gen_bnd_jmp(s);
-            gen_jmp(s, tval);
+            gen_jmp_rel(s, dflag, diff, 0);
         }
         break;
     case 0x9a: /* lcall im */
@@ -7261,19 +5380,13 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         }
         goto do_lcall;
     case 0xe9: /* jmp im */
-        if (dflag != MO_16) {
-            tval = (int32_t)insn_get(env, s, MO_32);
-        } else {
-            tval = (int16_t)insn_get(env, s, MO_16);
-        }
-        tval += s->pc - s->cs_base;
-        if (dflag == MO_16) {
-            tval &= 0xffff;
-        } else if (!CODE64(s)) {
-            tval &= 0xffffffff;
+        {
+            int diff = (dflag != MO_16
+                        ? (int32_t)insn_get(env, s, MO_32)
+                        : (int16_t)insn_get(env, s, MO_16));
+            gen_bnd_jmp(s);
+            gen_jmp_rel(s, dflag, diff, 0);
         }
-        gen_bnd_jmp(s);
-        gen_jmp(s, tval);
         break;
     case 0xea: /* ljmp im */
         {
@@ -7290,150 +5403,82 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         }
         goto do_ljmp;
     case 0xeb: /* jmp Jb */
-        tval = (int8_t)insn_get(env, s, MO_8);
-        tval += s->pc - s->cs_base;
-        if (dflag == MO_16) {
-            tval &= 0xffff;
-        }
-        gen_jmp(s, tval);
-        break;
-    case 0x70: /* jcc Jb */
-    case 0x71: /* jcc Jb */
-    case 0x72: /* jcc Jb */
-    case 0x73: /* jcc Jb */
-    case 0x74: /* jcc Jb */
-    case 0x75: /* jcc Jb */
-    case 0x76: /* jcc Jb */
-    case 0x77: /* jcc Jb */
-    case 0x78: /* jcc Jb */
-    case 0x79: /* jcc Jb */
-    case 0x7a: /* jcc Jb */
-    case 0x7b: /* jcc Jb */
-    case 0x7c: /* jcc Jb */
-    case 0x7d: /* jcc Jb */
-    case 0x7e: /* jcc Jb */
-    case 0x7f: /* jcc Jb */
-        tval = (int8_t)insn_get(env, s, MO_8);
-        goto do_jcc;
-    case 0x180: /* jcc Jv */
-    case 0x181: /* jcc Jv */
-    case 0x182: /* jcc Jv */
-    case 0x183: /* jcc Jv */
-    case 0x184: /* jcc Jv */
-    case 0x185: /* jcc Jv */
-    case 0x186: /* jcc Jv */
-    case 0x187: /* jcc Jv */
-    case 0x188: /* jcc Jv */
-    case 0x189: /* jcc Jv */
-    case 0x18a: /* jcc Jv */
-    case 0x18b: /* jcc Jv */
-    case 0x18c: /* jcc Jv */
-    case 0x18d: /* jcc Jv */
-    case 0x18e: /* jcc Jv */
-    case 0x18f: /* jcc Jv */
-        if (dflag != MO_16) {
-            tval = (int32_t)insn_get(env, s, MO_32);
-        } else {
-            tval = (int16_t)insn_get(env, s, MO_16);
+        {
+            int diff = (int8_t)insn_get(env, s, MO_8);
+            gen_jmp_rel(s, dflag, diff, 0);
         }
-    do_jcc:
-        next_eip = s->pc - s->cs_base;
-        tval += next_eip;
-        if (dflag == MO_16) {
-            tval &= 0xffff;
+        break;
+    case 0x70: case 0x71: case 0x72: case 0x73: case 0x74: case 0x75: case 0x76: case 0x77: case 0x78: case 0x79: case 0x7a: case 0x7b: case 0x7c: case 0x7d: case 0x7e: case 0x7f: /* jcc Jb */
+        {
+            int diff = (int8_t)insn_get(env, s, MO_8);
+            gen_bnd_jmp(s);
+            gen_jcc(s, b, diff);
         }
-        gen_bnd_jmp(s);
-        gen_jcc(s, b, tval, next_eip);
-        break;
-
-    case 0x190: /* setcc Gv */
-    case 0x191: /* setcc Gv */
-    case 0x192: /* setcc Gv */
-    case 0x193: /* setcc Gv */
-    case 0x194: /* setcc Gv */
-    case 0x195: /* setcc Gv */
-    case 0x196: /* setcc Gv */
-    case 0x197: /* setcc Gv */
-    case 0x198: /* setcc Gv */
-    case 0x199: /* setcc Gv */
-    case 0x19a: /* setcc Gv */
-    case 0x19b: /* setcc Gv */
-    case 0x19c: /* setcc Gv */
-    case 0x19d: /* setcc Gv */
-    case 0x19e: /* setcc Gv */
-    case 0x19f: /* setcc Gv */
+        break;
+    case 0x180: case 0x181: case 0x182: case 0x183: case 0x184: case 0x185: case 0x186: case 0x187: case 0x188: case 0x189: case 0x18a: case 0x18b: case 0x18c: case 0x18d: case 0x18e: case 0x18f: /* jcc Jv */
+        {
+            int diff = (dflag != MO_16
+                        ? (int32_t)insn_get(env, s, MO_32)
+                        : (int16_t)insn_get(env, s, MO_16));
+            gen_bnd_jmp(s);
+            gen_jcc(s, b, diff);
+        }
+        break;
+
+    case 0x190: case 0x191: case 0x192: case 0x193: case 0x194: case 0x195: case 0x196: case 0x197: case 0x198: case 0x199: case 0x19a: case 0x19b: case 0x19c: case 0x19d: case 0x19e: case 0x19f: /* setcc Gv */
         modrm = x86_ldub_code(env, s);
         gen_setcc1(s, b, s->T0);
         gen_ldst_modrm(env, s, modrm, MO_8, OR_TMP0, 1);
         break;
-    case 0x140: /* cmov Gv, Ev */
-    case 0x141: /* cmov Gv, Ev */
-    case 0x142: /* cmov Gv, Ev */
-    case 0x143: /* cmov Gv, Ev */
-    case 0x144: /* cmov Gv, Ev */
-    case 0x145: /* cmov Gv, Ev */
-    case 0x146: /* cmov Gv, Ev */
-    case 0x147: /* cmov Gv, Ev */
-    case 0x148: /* cmov Gv, Ev */
-    case 0x149: /* cmov Gv, Ev */
-    case 0x14a: /* cmov Gv, Ev */
-    case 0x14b: /* cmov Gv, Ev */
-    case 0x14c: /* cmov Gv, Ev */
-    case 0x14d: /* cmov Gv, Ev */
-    case 0x14e: /* cmov Gv, Ev */
-    case 0x14f: /* cmov Gv, Ev */
+    case 0x140: case 0x141: case 0x142: case 0x143: case 0x144: case 0x145: case 0x146: case 0x147: case 0x148: case 0x149: case 0x14a: case 0x14b: case 0x14c: case 0x14d: case 0x14e: case 0x14f: /* cmov Gv, Ev */
         if (!(s->cpuid_features & CPUID_CMOV)) {
             goto illegal_op;
         }
         ot = dflag;
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         gen_cmovcc1(env, s, ot, b, modrm, reg);
         break;
 
         /************************/
         /* flags */
     case 0x9c: /* pushf */
-        gen_svm_check_intercept(s, pc_start, SVM_EXIT_PUSHF);
-        if (s->vm86 && s->iopl != 3) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
+        gen_svm_check_intercept(s, SVM_EXIT_PUSHF);
+        if (check_vm86_iopl(s)) {
             gen_update_cc_op(s);
-            gen_helper_read_eflags(tcg_ctx, s->T0, tcg_ctx->cpu_env);
+            gen_helper_read_eflags(tcg_ctx, s->T0, cpu_env);
             gen_push_v(s, s->T0);
         }
         break;
     case 0x9d: /* popf */
-        gen_svm_check_intercept(s, pc_start, SVM_EXIT_POPF);
-        if (s->vm86 && s->iopl != 3) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
+        gen_svm_check_intercept(s, SVM_EXIT_POPF);
+        if (check_vm86_iopl(s)) {
             ot = gen_pop_T0(s);
-            if (s->cpl == 0) {
+            if (CPL(s) == 0) {
                 if (dflag != MO_16) {
-                    gen_helper_write_eflags(tcg_ctx, tcg_ctx->cpu_env, s->T0,
+                    gen_helper_write_eflags(tcg_ctx, cpu_env, s->T0,
                                             tcg_const_i32(tcg_ctx, (TF_MASK | AC_MASK |
                                                            ID_MASK | NT_MASK |
                                                            IF_MASK |
                                                            IOPL_MASK)));
                 } else {
-                    gen_helper_write_eflags(tcg_ctx, tcg_ctx->cpu_env, s->T0,
+                    gen_helper_write_eflags(tcg_ctx, cpu_env, s->T0,
                                             tcg_const_i32(tcg_ctx, (TF_MASK | AC_MASK |
                                                            ID_MASK | NT_MASK |
                                                            IF_MASK | IOPL_MASK)
                                                           & 0xffff));
                 }
             } else {
-                if (s->cpl <= s->iopl) {
+                if (CPL(s) <= IOPL(s)) {
                     if (dflag != MO_16) {
-                        gen_helper_write_eflags(tcg_ctx, tcg_ctx->cpu_env, s->T0,
+                        gen_helper_write_eflags(tcg_ctx, cpu_env, s->T0,
                                                 tcg_const_i32(tcg_ctx, (TF_MASK |
                                                                AC_MASK |
                                                                ID_MASK |
                                                                NT_MASK |
                                                                IF_MASK)));
                     } else {
-                        gen_helper_write_eflags(tcg_ctx, tcg_ctx->cpu_env, s->T0,
+                        gen_helper_write_eflags(tcg_ctx, cpu_env, s->T0,
                                                 tcg_const_i32(tcg_ctx, (TF_MASK |
                                                                AC_MASK |
                                                                ID_MASK |
@@ -7443,11 +5488,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     }
                 } else {
                     if (dflag != MO_16) {
-                        gen_helper_write_eflags(tcg_ctx, tcg_ctx->cpu_env, s->T0,
+                        gen_helper_write_eflags(tcg_ctx, cpu_env, s->T0,
                                            tcg_const_i32(tcg_ctx, (TF_MASK | AC_MASK |
                                                           ID_MASK | NT_MASK)));
                     } else {
-                        gen_helper_write_eflags(tcg_ctx, tcg_ctx->cpu_env, s->T0,
+                        gen_helper_write_eflags(tcg_ctx, cpu_env, s->T0,
                                            tcg_const_i32(tcg_ctx, (TF_MASK | AC_MASK |
                                                           ID_MASK | NT_MASK)
                                                          & 0xffff));
@@ -7457,46 +5502,45 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_pop_update(s, ot);
             set_cc_op(s, CC_OP_EFLAGS);
             /* abort translation because TF/AC flag may change */
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            s->base.is_jmp = DISAS_EOB_NEXT;
         }
         break;
     case 0x9e: /* sahf */
         if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
             goto illegal_op;
-        gen_op_mov_v_reg(s, MO_8, s->T0, R_AH);
+        tcg_gen_shri_tl(tcg_ctx, s->T0, cpu_regs[R_EAX], 8);
         gen_compute_eflags(s);
-        tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, CC_O);
+        tcg_gen_andi_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, CC_O);
         tcg_gen_andi_tl(tcg_ctx, s->T0, s->T0, CC_S | CC_Z | CC_A | CC_P | CC_C);
-        tcg_gen_or_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, s->T0);
+        tcg_gen_or_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, s->T0);
         break;
     case 0x9f: /* lahf */
         if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
             goto illegal_op;
         gen_compute_eflags(s);
         /* Note: gen_compute_eflags() only gives the condition codes */
-        tcg_gen_ori_tl(tcg_ctx, s->T0, tcg_ctx->cpu_cc_src, 0x02);
-        gen_op_mov_reg_v(s, MO_8, R_AH, s->T0);
+        tcg_gen_ori_tl(tcg_ctx, s->T0, cpu_cc_src, 0x02);
+        tcg_gen_deposit_tl(tcg_ctx, cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8);
         break;
     case 0xf5: /* cmc */
         gen_compute_eflags(s);
-        tcg_gen_xori_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, CC_C);
+        tcg_gen_xori_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, CC_C);
         break;
     case 0xf8: /* clc */
         gen_compute_eflags(s);
-        tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, ~CC_C);
+        tcg_gen_andi_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, ~CC_C);
         break;
     case 0xf9: /* stc */
         gen_compute_eflags(s);
-        tcg_gen_ori_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, CC_C);
+        tcg_gen_ori_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, CC_C);
         break;
     case 0xfc: /* cld */
         tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, 1);
-        tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, offsetof(CPUX86State, df));
+        tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, cpu_env, offsetof(CPUX86State, df));
         break;
     case 0xfd: /* std */
         tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, -1);
-        tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_env, offsetof(CPUX86State, df));
+        tcg_gen_st_i32(tcg_ctx, s->tmp2_i32, cpu_env, offsetof(CPUX86State, df));
         break;
 
         /************************/
@@ -7537,7 +5581,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     do_btx:
         ot = dflag;
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         gen_op_mov_v_reg(s, MO_32, s->T1, reg);
@@ -7547,7 +5591,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_exts(tcg_ctx, ot, s->T1);
             tcg_gen_sari_tl(tcg_ctx, s->tmp0, s->T1, 3 + ot);
             tcg_gen_shli_tl(tcg_ctx, s->tmp0, s->tmp0, ot);
-            tcg_gen_add_tl(tcg_ctx, s->A0, gen_lea_modrm_1(s, a), s->tmp0);
+            tcg_gen_add_tl(tcg_ctx, s->A0, gen_lea_modrm_1(s, a, false), s->tmp0);
             gen_lea_v_seg(s, s->aflag, s->A0, a.def_seg, s->override);
             if (!(s->prefix & PREFIX_LOCK)) {
                 gen_op_ld_v(s, ot, s->T0, s->A0);
@@ -7612,71 +5656,28 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
            C is the result of the test, Z is unchanged, and the others
            are all undefined.  */
         switch (s->cc_op) {
-        case CC_OP_MULB:
-        case CC_OP_MULW:
-        case CC_OP_MULL:
-        case CC_OP_MULQ:
-
-        case CC_OP_ADDB:
-        case CC_OP_ADDW:
-        case CC_OP_ADDL:
-        case CC_OP_ADDQ:
-
-        case CC_OP_ADCB:
-        case CC_OP_ADCW:
-        case CC_OP_ADCL:
-        case CC_OP_ADCQ:
-
-        case CC_OP_SUBB:
-        case CC_OP_SUBW:
-        case CC_OP_SUBL:
-        case CC_OP_SUBQ:
-
-        case CC_OP_SBBB:
-        case CC_OP_SBBW:
-        case CC_OP_SBBL:
-        case CC_OP_SBBQ:
-
-        case CC_OP_LOGICB:
-        case CC_OP_LOGICW:
-        case CC_OP_LOGICL:
-        case CC_OP_LOGICQ:
-
-        case CC_OP_INCB:
-        case CC_OP_INCW:
-        case CC_OP_INCL:
-        case CC_OP_INCQ:
-
-        case CC_OP_DECB:
-        case CC_OP_DECW:
-        case CC_OP_DECL:
-        case CC_OP_DECQ:
-
-        case CC_OP_SHLB:
-        case CC_OP_SHLW:
-        case CC_OP_SHLL:
-        case CC_OP_SHLQ:
-
-        case CC_OP_SARB:
-        case CC_OP_SARW:
-        case CC_OP_SARL:
-        case CC_OP_SARQ:
-
-        case CC_OP_BMILGB:
-        case CC_OP_BMILGW:
-        case CC_OP_BMILGL:
-        case CC_OP_BMILGQ:
+        case CC_OP_MULB: case CC_OP_MULW: case CC_OP_MULL: case CC_OP_MULQ:
+        case CC_OP_ADDB: case CC_OP_ADDW: case CC_OP_ADDL: case CC_OP_ADDQ:
+        case CC_OP_ADCB: case CC_OP_ADCW: case CC_OP_ADCL: case CC_OP_ADCQ:
+        case CC_OP_SUBB: case CC_OP_SUBW: case CC_OP_SUBL: case CC_OP_SUBQ:
+        case CC_OP_SBBB: case CC_OP_SBBW: case CC_OP_SBBL: case CC_OP_SBBQ:
+        case CC_OP_LOGICB: case CC_OP_LOGICW: case CC_OP_LOGICL: case CC_OP_LOGICQ:
+        case CC_OP_INCB: case CC_OP_INCW: case CC_OP_INCL: case CC_OP_INCQ:
+        case CC_OP_DECB: case CC_OP_DECW: case CC_OP_DECL: case CC_OP_DECQ:
+        case CC_OP_SHLB: case CC_OP_SHLW: case CC_OP_SHLL: case CC_OP_SHLQ:
+        case CC_OP_SARB: case CC_OP_SARW: case CC_OP_SARL: case CC_OP_SARQ:
+        case CC_OP_BMILGB: case CC_OP_BMILGW: case CC_OP_BMILGL: case CC_OP_BMILGQ:
             /* Z was going to be computed from the non-zero status of CC_DST.
                We can get that same Z value (and the new C value) by leaving
                CC_DST alone, setting CC_SRC, and using a CC_OP_SAR of the
                same width.  */
-            tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->tmp4);
+            tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->tmp4);
             set_cc_op(s, ((s->cc_op - CC_OP_MULB) & 3) + CC_OP_SARB);
             break;
         default:
             /* Otherwise, generate EFLAGS and replace the C bit.  */
             gen_compute_eflags(s);
-            tcg_gen_deposit_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, s->tmp4,
+            tcg_gen_deposit_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, s->tmp4,
                                ctz32(CC_C), 1);
             break;
         }
@@ -7685,7 +5686,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0x1bd: /* bsr / lzcnt */
         ot = dflag;
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_extu(tcg_ctx, ot, s->T0);
 
@@ -7696,7 +5697,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 : s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI1)) {
             int size = 8 << ot;
             /* For lzcnt/tzcnt, C bit is defined related to the input. */
-            tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0);
+            tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T0);
             if (b & 1) {
                 /* For lzcnt, reduce the target_ulong result by the
                    number of zeros that we expect to find at the top.  */
@@ -7712,7 +5713,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         } else {
             /* For bsr/bsf, only the Z bit is defined and it is related
                to the input and not the result.  */
-            tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_dst, s->T0);
+            tcg_gen_mov_tl(tcg_ctx, cpu_cc_dst, s->T0);
             set_cc_op(s, CC_OP_LOGICB + ot);
 
             /* ??? The manual says that the output is undefined when the
@@ -7722,11 +5723,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             if (b & 1) {
                 /* For bsr, return the bit index of the first 1 bit,
                    not the count of leading zeros.  */
-                tcg_gen_xori_tl(tcg_ctx, s->T1, tcg_ctx->cpu_regs[reg], TARGET_LONG_BITS - 1);
+                tcg_gen_xori_tl(tcg_ctx, s->T1, cpu_regs[reg], TARGET_LONG_BITS - 1);
                 tcg_gen_clz_tl(tcg_ctx, s->T0, s->T0, s->T1);
                 tcg_gen_xori_tl(tcg_ctx, s->T0, s->T0, TARGET_LONG_BITS - 1);
             } else {
-                tcg_gen_ctz_tl(tcg_ctx, s->T0, s->T0, tcg_ctx->cpu_regs[reg]);
+                tcg_gen_ctz_tl(tcg_ctx, s->T0, s->T0, cpu_regs[reg]);
             }
         }
         gen_op_mov_reg_v(s, ot, reg, s->T0);
@@ -7737,28 +5738,28 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         if (CODE64(s))
             goto illegal_op;
         gen_update_cc_op(s);
-        gen_helper_daa(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_daa(tcg_ctx, cpu_env);
         set_cc_op(s, CC_OP_EFLAGS);
         break;
     case 0x2f: /* das */
         if (CODE64(s))
             goto illegal_op;
         gen_update_cc_op(s);
-        gen_helper_das(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_das(tcg_ctx, cpu_env);
         set_cc_op(s, CC_OP_EFLAGS);
         break;
     case 0x37: /* aaa */
         if (CODE64(s))
             goto illegal_op;
         gen_update_cc_op(s);
-        gen_helper_aaa(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_aaa(tcg_ctx, cpu_env);
         set_cc_op(s, CC_OP_EFLAGS);
         break;
     case 0x3f: /* aas */
         if (CODE64(s))
             goto illegal_op;
         gen_update_cc_op(s);
-        gen_helper_aas(tcg_ctx, tcg_ctx->cpu_env);
+        gen_helper_aas(tcg_ctx, cpu_env);
         set_cc_op(s, CC_OP_EFLAGS);
         break;
     case 0xd4: /* aam */
@@ -7766,9 +5767,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             goto illegal_op;
         val = x86_ldub_code(env, s);
         if (val == 0) {
-            gen_exception(s, EXCP00_DIVZ, pc_start - s->cs_base);
+            gen_exception(s, EXCP00_DIVZ);
         } else {
-            gen_helper_aam(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, val));
+            gen_helper_aam(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, val));
             set_cc_op(s, CC_OP_LOGICB);
         }
         break;
@@ -7776,7 +5777,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         if (CODE64(s))
             goto illegal_op;
         val = x86_ldub_code(env, s);
-        gen_helper_aad(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, val));
+        gen_helper_aad(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, val));
         set_cc_op(s, CC_OP_LOGICB);
         break;
         /************************/
@@ -7792,66 +5793,52 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         }
         if (prefixes & PREFIX_REPZ) {
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_pause(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->pc - pc_start));
+            gen_update_eip_cur(s);
+            gen_helper_pause(tcg_ctx, cpu_env, cur_insn_len_i32(s));
             s->base.is_jmp = DISAS_NORETURN;
         }
         break;
     case 0x9b: /* fwait */
         if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) ==
             (HF_MP_MASK | HF_TS_MASK)) {
-            gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+            gen_exception(s, EXCP07_PREX);
         } else {
-            gen_helper_fwait(tcg_ctx, tcg_ctx->cpu_env);
+            gen_helper_fwait(tcg_ctx, cpu_env);
         }
         break;
     case 0xcc: /* int3 */
-        gen_interrupt(s, EXCP03_INT3, pc_start - s->cs_base, s->pc - s->cs_base);
+        gen_interrupt(s, EXCP03_INT3);
         break;
     case 0xcd: /* int N */
         val = x86_ldub_code(env, s);
-        if (s->vm86 && s->iopl != 3) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
-            gen_interrupt(s, val, pc_start - s->cs_base, s->pc - s->cs_base);
+        if (check_vm86_iopl(s)) {
+            gen_interrupt(s, val);
         }
         break;
     case 0xce: /* into */
         if (CODE64(s))
             goto illegal_op;
         gen_update_cc_op(s);
-        gen_jmp_im(s, pc_start - s->cs_base);
-        gen_helper_into(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->pc - pc_start));
+        gen_update_eip_cur(s);
+        gen_helper_into(tcg_ctx, cpu_env, cur_insn_len_i32(s));
         break;
 #ifdef WANT_ICEBP
     case 0xf1: /* icebp (undocumented, exits to external debugger) */
-        gen_svm_check_intercept(s, pc_start, SVM_EXIT_ICEBP);
-        gen_debug(s, pc_start - s->cs_base);
+        gen_svm_check_intercept(s, SVM_EXIT_ICEBP);
+        gen_debug(s, s->base.pc_next - s->cs_base);
         break;
 #endif
     case 0xfa: /* cli */
-        if (!s->vm86) {
-            if (s->cpl <= s->iopl) {
-                gen_helper_cli(tcg_ctx, tcg_ctx->cpu_env);
-            } else {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            }
-        } else {
-            if (s->iopl == 3) {
-                gen_helper_cli(tcg_ctx, tcg_ctx->cpu_env);
-            } else {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            }
+        if (check_iopl(s)) {
+            gen_reset_eflags(s, IF_MASK);
         }
         break;
     case 0xfb: /* sti */
-        if (s->vm86 ? s->iopl == 3 : s->cpl <= s->iopl) {
-            gen_helper_sti(tcg_ctx, tcg_ctx->cpu_env);
+        if (check_iopl(s)) {
+            gen_set_eflags(s, IF_MASK);
             /* interruptions are enabled only the first insn after sti */
-            gen_jmp_im(s, s->pc - s->cs_base);
+            gen_update_eip_next(s);
             gen_eob_inhibit_irq(s, true);
-        } else {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         }
         break;
     case 0x62: /* bound */
@@ -7867,19 +5854,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         gen_lea_modrm(env, s, modrm);
         tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
         if (ot == MO_16) {
-            gen_helper_boundw(tcg_ctx, tcg_ctx->cpu_env, s->A0, s->tmp2_i32);
+            gen_helper_boundw(tcg_ctx, cpu_env, s->A0, s->tmp2_i32);
         } else {
-            gen_helper_boundl(tcg_ctx, tcg_ctx->cpu_env, s->A0, s->tmp2_i32);
+            gen_helper_boundl(tcg_ctx, cpu_env, s->A0, s->tmp2_i32);
         }
         break;
-    case 0x1c8: /* bswap reg */
-    case 0x1c9: /* bswap reg */
-    case 0x1ca: /* bswap reg */
-    case 0x1cb: /* bswap reg */
-    case 0x1cc: /* bswap reg */
-    case 0x1cd: /* bswap reg */
-    case 0x1ce: /* bswap reg */
-    case 0x1cf: /* bswap reg */
+    case 0x1c8: case 0x1c9: case 0x1ca: case 0x1cb: case 0x1cc: case 0x1cd: case 0x1ce: case 0x1cf: /* bswap reg */
         reg = (b & 7) | REX_B(s);
 #ifdef TARGET_X86_64
         if (dflag == MO_64) {
@@ -7893,8 +5873,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             tcg_gen_ext32u_tl(tcg_ctx, s->T0, s->T0);
             tcg_gen_bswap32_tl(tcg_ctx, s->T0, s->T0);
             gen_op_mov_reg_v(s, MO_32, reg, s->T0);
-        }
-        else {
+        } else {
             tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
             gen_op_mov_reg_v(s, MO_16, reg, s->T0);
         }
@@ -7911,118 +5890,105 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
     case 0xe2: /* loop */
     case 0xe3: /* jecxz */
         {
-            TCGLabel *l1, *l2, *l3;
-
-            tval = (int8_t)insn_get(env, s, MO_8);
-            next_eip = s->pc - s->cs_base;
-            tval += next_eip;
-            if (dflag == MO_16) {
-                tval &= 0xffff;
-            }
+            TCGLabel *l1, *l2;
+            int diff = (int8_t)insn_get(env, s, MO_8);
 
             l1 = gen_new_label(tcg_ctx);
             l2 = gen_new_label(tcg_ctx);
-            l3 = gen_new_label(tcg_ctx);
+            gen_update_cc_op(s);
             b &= 3;
             switch(b) {
             case 0: /* loopnz */
             case 1: /* loopz */
                 gen_op_add_reg_im(s, s->aflag, R_ECX, -1);
-                gen_op_jz_ecx(s, s->aflag, l3);
+                gen_op_jz_ecx(s, l2);
                 gen_jcc1(s, (JCC_Z << 1) | (b ^ 1), l1);
                 break;
             case 2: /* loop */
                 gen_op_add_reg_im(s, s->aflag, R_ECX, -1);
-                gen_op_jnz_ecx(s, s->aflag, l1);
+                gen_op_jnz_ecx(s, l1);
                 break;
             default:
             case 3: /* jcxz */
-                gen_op_jz_ecx(s, s->aflag, l1);
+                gen_op_jz_ecx(s, l1);
                 break;
             }
 
-            gen_set_label(tcg_ctx, l3);
-            gen_jmp_im(s, next_eip);
-            tcg_gen_br(tcg_ctx, l2);
+            gen_set_label(tcg_ctx, l2);
+            gen_jmp_rel_csize(s, 0, 1);
 
             gen_set_label(tcg_ctx, l1);
-            gen_jmp_im(s, tval);
-            gen_set_label(tcg_ctx, l2);
-            gen_eob(s);
+            gen_jmp_rel(s, dflag, diff, 0);
         }
         break;
     case 0x130: /* wrmsr */
     case 0x132: /* rdmsr */
-        if (s->cpl != 0) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
+        if (check_cpl0(s)) {
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
+            gen_update_eip_cur(s);
             if (b & 2) {
-                gen_helper_rdmsr(tcg_ctx, tcg_ctx->cpu_env);
+                gen_helper_rdmsr(tcg_ctx, cpu_env);
             } else {
-                gen_helper_wrmsr(tcg_ctx, tcg_ctx->cpu_env);
+                gen_helper_wrmsr(tcg_ctx, cpu_env);
+                s->base.is_jmp = DISAS_EOB_NEXT;
             }
         }
         break;
     case 0x131: /* rdtsc */
         gen_update_cc_op(s);
-        gen_jmp_im(s, pc_start - s->cs_base);
+        gen_update_eip_cur(s);
         if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
             gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
         }
-        gen_helper_rdtsc(tcg_ctx, tcg_ctx->cpu_env);
-        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-            gen_jmp(s, s->pc - s->cs_base);
-        }
+        gen_helper_rdtsc(tcg_ctx, cpu_env);
         break;
     case 0x133: /* rdpmc */
         gen_update_cc_op(s);
-        gen_jmp_im(s, pc_start - s->cs_base);
-        gen_helper_rdpmc(tcg_ctx, tcg_ctx->cpu_env);
+        gen_update_eip_cur(s);
+        gen_helper_rdpmc(tcg_ctx, cpu_env);
+        s->base.is_jmp = DISAS_NORETURN;
         break;
     case 0x134: /* sysenter */
         /* For Intel SYSENTER is valid on 64-bit */
         if (CODE64(s) && env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
             goto illegal_op;
-        if (!s->pe) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+        if (!PE(s)) {
+            gen_exception_gpf(s);
         } else {
-            TCGv_i32 addend = tcg_const_i32(tcg_ctx, s->pc - pc_start);
-            gen_helper_sysenter(tcg_ctx, tcg_ctx->cpu_env, addend);
-            gen_eob(s);
-            tcg_temp_free_i32(tcg_ctx, addend);
+            gen_helper_sysenter(tcg_ctx, cpu_env, cur_insn_len_i32(s));
+            s->base.is_jmp = DISAS_EOB_ONLY;
         }
         break;
     case 0x135: /* sysexit */
         /* For Intel SYSEXIT is valid on 64-bit */
         if (CODE64(s) && env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
             goto illegal_op;
-        if (!s->pe) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+        if (!PE(s)) {
+            gen_exception_gpf(s);
         } else {
-            gen_helper_sysexit(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, dflag - 1));
-            gen_eob(s);
+            gen_helper_sysexit(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, dflag - 1));
+            s->base.is_jmp = DISAS_EOB_ONLY;
         }
         break;
 #ifdef TARGET_X86_64
     case 0x105: /* syscall */
         /* XXX: is it usable in real mode ? */
         gen_update_cc_op(s);
-        gen_jmp_im(s, pc_start - s->cs_base);
-        gen_helper_syscall(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->pc - pc_start));
+        gen_update_eip_cur(s);
+        gen_helper_syscall(tcg_ctx, cpu_env, cur_insn_len_i32(s));
         /* TF handling for the syscall insn is different. The TF bit is  checked
            after the syscall insn completes. This allows #DB to not be
            generated after one has entered CPL0 if TF is set in FMASK.  */
         gen_eob_worker(s, false, true);
         break;
     case 0x107: /* sysret */
-        if (!s->pe) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+        if (!PE(s)) {
+            gen_exception_gpf(s);
         } else {
-            gen_helper_sysret(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, dflag - 1));
+            gen_helper_sysret(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, dflag - 1));
             /* condition codes are modified only in long mode */
-            if (s->lma) {
+            if (LMA(s)) {
                 set_cc_op(s, CC_OP_EFLAGS);
             }
             /* TF handling for the sysret insn is different. The TF bit is
@@ -8035,16 +6001,14 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
 #endif
     case 0x1a2: /* cpuid */
         gen_update_cc_op(s);
-        gen_jmp_im(s, pc_start - s->cs_base);
-        gen_helper_cpuid(tcg_ctx, tcg_ctx->cpu_env);
+        gen_update_eip_cur(s);
+        gen_helper_cpuid(tcg_ctx, cpu_env);
         break;
     case 0xf4: /* hlt */
-        if (s->cpl != 0) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
+        if (check_cpl0(s)) {
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_hlt(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->pc - pc_start));
+            gen_update_eip_cur(s);
+            gen_helper_hlt(tcg_ctx, cpu_env, cur_insn_len_i32(s));
             s->base.is_jmp = DISAS_NORETURN;
         }
         break;
@@ -8054,57 +6018,59 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         op = (modrm >> 3) & 7;
         switch(op) {
         case 0: /* sldt */
-            if (!s->pe || s->vm86)
+            if (!PE(s) || VM86(s))
                 goto illegal_op;
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_READ);
-            tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
+            if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
+                break;
+            }
+            gen_svm_check_intercept(s, SVM_EXIT_LDTR_READ);
+            tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env,
                              offsetof(CPUX86State, ldt.selector));
             ot = mod == 3 ? dflag : MO_16;
             gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
         case 2: /* lldt */
-            if (!s->pe || s->vm86)
+            if (!PE(s) || VM86(s))
                 goto illegal_op;
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            } else {
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
+            if (check_cpl0(s)) {
+                gen_svm_check_intercept(s, SVM_EXIT_LDTR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
                 tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                gen_helper_lldt(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                gen_helper_lldt(tcg_ctx, cpu_env, s->tmp2_i32);
             }
             break;
         case 1: /* str */
-            if (!s->pe || s->vm86)
+            if (!PE(s) || VM86(s))
                 goto illegal_op;
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_READ);
-            tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
+            if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
+                break;
+            }
+            gen_svm_check_intercept(s, SVM_EXIT_TR_READ);
+            tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env,
                              offsetof(CPUX86State, tr.selector));
             ot = mod == 3 ? dflag : MO_16;
             gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
         case 3: /* ltr */
-            if (!s->pe || s->vm86)
+            if (!PE(s) || VM86(s))
                 goto illegal_op;
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            } else {
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
+            if (check_cpl0(s)) {
+                gen_svm_check_intercept(s, SVM_EXIT_TR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
                 tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, s->T0);
-                gen_helper_ltr(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                gen_helper_ltr(tcg_ctx, cpu_env, s->tmp2_i32);
             }
             break;
         case 4: /* verr */
         case 5: /* verw */
-            if (!s->pe || s->vm86)
+            if (!PE(s) || VM86(s))
                 goto illegal_op;
             gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
             gen_update_cc_op(s);
             if (op == 4) {
-                gen_helper_verr(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_verr(tcg_ctx, cpu_env, s->T0);
             } else {
-                gen_helper_verw(tcg_ctx, tcg_ctx->cpu_env, s->T0);
+                gen_helper_verw(tcg_ctx, cpu_env, s->T0);
             }
             set_cc_op(s, CC_OP_EFLAGS);
             break;
@@ -8117,13 +6083,16 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         modrm = x86_ldub_code(env, s);
         switch (modrm) {
         CASE_MODRM_MEM_OP(0): /* sgdt */
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ);
+            if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
+                break;
+            }
+            gen_svm_check_intercept(s, SVM_EXIT_GDTR_READ);
             gen_lea_modrm(env, s, modrm);
             tcg_gen_ld32u_tl(tcg_ctx, s->T0,
-                             tcg_ctx->cpu_env, offsetof(CPUX86State, gdt.limit));
+                             cpu_env, offsetof(CPUX86State, gdt.limit));
             gen_op_st_v(s, MO_16, s->T0, s->A0);
             gen_add_A0_im(s, 2);
-            tcg_gen_ld_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, gdt.base));
+            tcg_gen_ld_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, gdt.base));
             if (dflag == MO_16) {
                 tcg_gen_andi_tl(tcg_ctx, s->T0, s->T0, 0xffffff);
             }
@@ -8131,54 +6100,55 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             break;
 
         case 0xc8: /* monitor */
-            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || s->cpl != 0) {
+            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || CPL(s) != 0) {
                 goto illegal_op;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            tcg_gen_mov_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[R_EAX]);
+            gen_update_eip_cur(s);
+            tcg_gen_mov_tl(tcg_ctx, s->A0, cpu_regs[R_EAX]);
             gen_extu(tcg_ctx, s->aflag, s->A0);
             gen_add_A0_ds_seg(s);
-            gen_helper_monitor(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+            gen_helper_monitor(tcg_ctx, cpu_env, s->A0);
             break;
 
         case 0xc9: /* mwait */
-            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || s->cpl != 0) {
+            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || CPL(s) != 0) {
                 goto illegal_op;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_mwait(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->pc - pc_start));
-            gen_eob(s);
+            gen_update_eip_cur(s);
+            gen_helper_mwait(tcg_ctx, cpu_env, cur_insn_len_i32(s));
+            s->base.is_jmp = DISAS_NORETURN;
             break;
 
         case 0xca: /* clac */
             if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
-                || s->cpl != 0) {
+                || CPL(s) != 0) {
                 goto illegal_op;
             }
-            gen_helper_clac(tcg_ctx, tcg_ctx->cpu_env);
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            gen_reset_eflags(s, AC_MASK);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         case 0xcb: /* stac */
             if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
-                || s->cpl != 0) {
+                || CPL(s) != 0) {
                 goto illegal_op;
             }
-            gen_helper_stac(tcg_ctx, tcg_ctx->cpu_env);
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            gen_set_eflags(s, AC_MASK);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         CASE_MODRM_MEM_OP(1): /* sidt */
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
+            if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
+                break;
+            }
+            gen_svm_check_intercept(s, SVM_EXIT_IDTR_READ);
             gen_lea_modrm(env, s, modrm);
-            tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, idt.limit));
+            tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, idt.limit));
             gen_op_st_v(s, MO_16, s->T0, s->A0);
             gen_add_A0_im(s, 2);
-            tcg_gen_ld_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, idt.base));
+            tcg_gen_ld_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, idt.base));
             if (dflag == MO_16) {
                 tcg_gen_andi_tl(tcg_ctx, s->T0, s->T0, 0xffffff);
             }
@@ -8191,9 +6161,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                                  | PREFIX_REPZ | PREFIX_REPNZ))) {
                 goto illegal_op;
             }
-            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[R_ECX]);
-            gen_helper_xgetbv(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, s->tmp2_i32);
-            tcg_gen_extr_i64_tl(tcg_ctx, tcg_ctx->cpu_regs[R_EAX], tcg_ctx->cpu_regs[R_EDX], s->tmp1_i64);
+            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_xgetbv(tcg_ctx, s->tmp1_i64, cpu_env, s->tmp2_i32);
+            tcg_gen_extr_i64_tl(tcg_ctx, cpu_regs[R_EAX], cpu_regs[R_EDX], s->tmp1_i64);
             break;
 
         case 0xd1: /* xsetbv */
@@ -8202,129 +6172,121 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                                  | PREFIX_REPZ | PREFIX_REPNZ))) {
                 goto illegal_op;
             }
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
-            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_regs[R_EAX],
-                                  tcg_ctx->cpu_regs[R_EDX]);
-            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[R_ECX]);
-            gen_helper_xsetbv(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32, s->tmp1_i64);
+            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_xsetbv(tcg_ctx, cpu_env, s->tmp2_i32, s->tmp1_i64);
             /* End TB because translation flags may change.  */
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         case 0xd8: /* VMRUN */
-            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+            if (!SVME(s) || !PE(s)) {
                 goto illegal_op;
             }
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_vmrun(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->aflag - 1),
-                             tcg_const_i32(tcg_ctx, s->pc - pc_start));
+            gen_update_eip_cur(s);
+            gen_helper_vmrun(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, s->aflag - 1),
+                             cur_insn_len_i32(s));
             tcg_gen_exit_tb(tcg_ctx, NULL, 0);
             s->base.is_jmp = DISAS_NORETURN;
             break;
 
         case 0xd9: /* VMMCALL */
-            if (!(s->flags & HF_SVME_MASK)) {
+            if (!SVME(s)) {
                 goto illegal_op;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_vmmcall(tcg_ctx, tcg_ctx->cpu_env);
+            gen_update_eip_cur(s);
+            gen_helper_vmmcall(tcg_ctx, cpu_env);
             break;
 
         case 0xda: /* VMLOAD */
-            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+            if (!SVME(s) || !PE(s)) {
                 goto illegal_op;
             }
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_vmload(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->aflag - 1));
+            gen_update_eip_cur(s);
+            gen_helper_vmload(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, s->aflag - 1));
             break;
 
         case 0xdb: /* VMSAVE */
-            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+            if (!SVME(s) || !PE(s)) {
                 goto illegal_op;
             }
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_vmsave(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->aflag - 1));
+            gen_update_eip_cur(s);
+            gen_helper_vmsave(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, s->aflag - 1));
             break;
 
         case 0xdc: /* STGI */
-            if ((!(s->flags & HF_SVME_MASK)
-                   && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
-                || !s->pe) {
+            if ((!SVME(s) && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
+                || !PE(s)) {
                 goto illegal_op;
             }
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
             gen_update_cc_op(s);
-            gen_helper_stgi(tcg_ctx, tcg_ctx->cpu_env);
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            gen_helper_stgi(tcg_ctx, cpu_env);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         case 0xdd: /* CLGI */
-            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+            if (!SVME(s) || !PE(s)) {
                 goto illegal_op;
             }
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_clgi(tcg_ctx, tcg_ctx->cpu_env);
+            gen_update_eip_cur(s);
+            gen_helper_clgi(tcg_ctx, cpu_env);
             break;
 
         case 0xde: /* SKINIT */
-            if ((!(s->flags & HF_SVME_MASK)
-                 && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
-                || !s->pe) {
+            if ((!SVME(s) && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
+                || !PE(s)) {
                 goto illegal_op;
             }
-            gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_skinit(tcg_ctx, tcg_ctx->cpu_env);
-            break;
+            gen_svm_check_intercept(s, SVM_EXIT_SKINIT);
+            /* If not intercepted, not implemented -- raise #UD. */
+            goto illegal_op;
 
         case 0xdf: /* INVLPGA */
-            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+            if (!SVME(s) || !PE(s)) {
                 goto illegal_op;
             }
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
-            gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
-            gen_helper_invlpga(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, s->aflag - 1));
+            gen_svm_check_intercept(s, SVM_EXIT_INVLPGA);
+            if (s->aflag == MO_64) {
+                tcg_gen_mov_tl(tcg_ctx, s->A0, cpu_regs[R_EAX]);
+            } else {
+                tcg_gen_ext32u_tl(tcg_ctx, s->A0, cpu_regs[R_EAX]);
+            }
+            gen_helper_flush_page(tcg_ctx, cpu_env, s->A0);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         CASE_MODRM_MEM_OP(2): /* lgdt */
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_WRITE);
+            gen_svm_check_intercept(s, SVM_EXIT_GDTR_WRITE);
             gen_lea_modrm(env, s, modrm);
             gen_op_ld_v(s, MO_16, s->T1, s->A0);
             gen_add_A0_im(s, 2);
@@ -8332,16 +6294,15 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             if (dflag == MO_16) {
                 tcg_gen_andi_tl(tcg_ctx, s->T0, s->T0, 0xffffff);
             }
-            tcg_gen_st_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, gdt.base));
-            tcg_gen_st32_tl(tcg_ctx, s->T1, tcg_ctx->cpu_env, offsetof(CPUX86State, gdt.limit));
+            tcg_gen_st_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, gdt.base));
+            tcg_gen_st32_tl(tcg_ctx, s->T1, cpu_env, offsetof(CPUX86State, gdt.limit));
             break;
 
         CASE_MODRM_MEM_OP(3): /* lidt */
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_WRITE);
+            gen_svm_check_intercept(s, SVM_EXIT_IDTR_WRITE);
             gen_lea_modrm(env, s, modrm);
             gen_op_ld_v(s, MO_16, s->T1, s->A0);
             gen_add_A0_im(s, 2);
@@ -8349,73 +6310,81 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             if (dflag == MO_16) {
                 tcg_gen_andi_tl(tcg_ctx, s->T0, s->T0, 0xffffff);
             }
-            tcg_gen_st_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, idt.base));
-            tcg_gen_st32_tl(tcg_ctx, s->T1, tcg_ctx->cpu_env, offsetof(CPUX86State, idt.limit));
+            tcg_gen_st_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, idt.base));
+            tcg_gen_st32_tl(tcg_ctx, s->T1, cpu_env, offsetof(CPUX86State, idt.limit));
             break;
 
         CASE_MODRM_OP(4): /* smsw */
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
-            tcg_gen_ld_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, cr[0]));
-            if (CODE64(s)) {
-                mod = (modrm >> 6) & 3;
-                ot = (mod != 3 ? MO_16 : s->dflag);
-            } else {
-                ot = MO_16;
+            if (s->flags & HF_UMIP_MASK && !check_cpl0(s)) {
+                break;
             }
+            gen_svm_check_intercept(s, SVM_EXIT_READ_CR0);
+            tcg_gen_ld_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, cr[0]));
+            /*
+             * In 32-bit mode, the higher 16 bits of the destination
+             * register are undefined.  In practice CR0[31:0] is stored
+             * just like in 64-bit mode.
+             */
+            mod = (modrm >> 6) & 3;
+            ot = (mod != 3 ? MO_16 : s->dflag);
             gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
         case 0xee: /* rdpkru */
-            if (prefixes & PREFIX_LOCK) {
+            if (s->prefix & (PREFIX_LOCK | PREFIX_DATA
+                             | PREFIX_REPZ | PREFIX_REPNZ)) {
                 goto illegal_op;
             }
-            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[R_ECX]);
-            gen_helper_rdpkru(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_env, s->tmp2_i32);
-            tcg_gen_extr_i64_tl(tcg_ctx, tcg_ctx->cpu_regs[R_EAX], tcg_ctx->cpu_regs[R_EDX], s->tmp1_i64);
+            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_rdpkru(tcg_ctx, s->tmp1_i64, cpu_env, s->tmp2_i32);
+            tcg_gen_extr_i64_tl(tcg_ctx, cpu_regs[R_EAX], cpu_regs[R_EDX], s->tmp1_i64);
             break;
         case 0xef: /* wrpkru */
-            if (prefixes & PREFIX_LOCK) {
+            if (s->prefix & (PREFIX_LOCK | PREFIX_DATA
+                             | PREFIX_REPZ | PREFIX_REPNZ)) {
                 goto illegal_op;
             }
-            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_regs[R_EAX],
-                                  tcg_ctx->cpu_regs[R_EDX]);
-            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, tcg_ctx->cpu_regs[R_ECX]);
-            gen_helper_wrpkru(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32, s->tmp1_i64);
+            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            tcg_gen_trunc_tl_i32(tcg_ctx, s->tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_wrpkru(tcg_ctx, cpu_env, s->tmp2_i32, s->tmp1_i64);
             break;
+
         CASE_MODRM_OP(6): /* lmsw */
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
+            gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0);
             gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-            gen_helper_lmsw(tcg_ctx, tcg_ctx->cpu_env, s->T0);
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            /*
+             * Only the 4 lower bits of CR0 are modified.
+             * PE cannot be set to zero if already set to one.
+             */
+            tcg_gen_ld_tl(tcg_ctx, s->T1, cpu_env, offsetof(CPUX86State, cr[0]));
+            tcg_gen_andi_tl(tcg_ctx, s->T0, s->T0, 0xf);
+            tcg_gen_andi_tl(tcg_ctx, s->T1, s->T1, ~0xe);
+            tcg_gen_or_tl(tcg_ctx, s->T0, s->T0, s->T1);
+            gen_helper_write_crN(tcg_ctx, cpu_env, tcg_constant_i32(tcg_ctx, 0), s->T0);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         CASE_MODRM_MEM_OP(7): /* invlpg */
-            if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+            if (!check_cpl0(s)) {
                 break;
             }
-            gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
+            gen_svm_check_intercept(s, SVM_EXIT_INVLPG);
             gen_lea_modrm(env, s, modrm);
-            gen_helper_invlpg(tcg_ctx, tcg_ctx->cpu_env, s->A0);
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            gen_helper_flush_page(tcg_ctx, cpu_env, s->A0);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         case 0xf8: /* swapgs */
 #ifdef TARGET_X86_64
             if (CODE64(s)) {
-                if (s->cpl != 0) {
-                    gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                } else {
-                    tcg_gen_mov_tl(tcg_ctx, s->T0, tcg_ctx->cpu_seg_base[R_GS]);
-                    tcg_gen_ld_tl(tcg_ctx, tcg_ctx->cpu_seg_base[R_GS], tcg_ctx->cpu_env,
+                if (check_cpl0(s)) {
+                    tcg_gen_mov_tl(tcg_ctx, s->T0, cpu_seg_base[R_GS]);
+                    tcg_gen_ld_tl(tcg_ctx, cpu_seg_base[R_GS], cpu_env,
                                   offsetof(CPUX86State, kernelgsbase));
-                    tcg_gen_st_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env,
+                    tcg_gen_st_tl(tcg_ctx, s->T0, cpu_env,
                                   offsetof(CPUX86State, kernelgsbase));
                 }
                 break;
@@ -8428,14 +6397,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             gen_update_cc_op(s);
-            gen_jmp_im(s, pc_start - s->cs_base);
+            gen_update_eip_cur(s);
             if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
                 gen_io_start(tcg_ctx);
+                s->base.is_jmp = DISAS_TOO_MANY;
             }
-            gen_helper_rdtscp(tcg_ctx, tcg_ctx->cpu_env);
-            if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-                gen_jmp(s, s->pc - s->cs_base);
-            }
+            gen_helper_rdtscp(tcg_ctx, cpu_env);
             break;
 
         default:
@@ -8445,10 +6412,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
 
     case 0x108: /* invd */
     case 0x109: /* wbinvd */
-        if (s->cpl != 0) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
-            gen_svm_check_intercept(s, pc_start, (b & 2) ? SVM_EXIT_INVD : SVM_EXIT_WBINVD);
+        if (check_cpl0(s)) {
+            gen_svm_check_intercept(s, (b & 2) ? SVM_EXIT_INVD : SVM_EXIT_WBINVD);
             /* nothing to do */
         }
         break;
@@ -8460,7 +6425,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             d_ot = dflag;
 
             modrm = x86_ldub_code(env, s);
-            reg = ((modrm >> 3) & 7) | rex_r;
+            reg = ((modrm >> 3) & 7) | REX_R(s);
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
 
@@ -8482,7 +6447,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             TCGLabel *label1;
             TCGv t0, t1, t2, a0;
 
-            if (!s->pe || s->vm86)
+            if (!PE(s) || VM86(s))
                 goto illegal_op;
             t0 = tcg_temp_local_new(tcg_ctx);
             t1 = tcg_temp_local_new(tcg_ctx);
@@ -8518,8 +6483,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 gen_op_mov_reg_v(s, ot, rm, t0);
             }
             gen_compute_eflags(s);
-            tcg_gen_andi_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, ~CC_Z);
-            tcg_gen_or_tl(tcg_ctx, tcg_ctx->cpu_cc_src, tcg_ctx->cpu_cc_src, t2);
+            tcg_gen_andi_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, ~CC_Z);
+            tcg_gen_or_tl(tcg_ctx, cpu_cc_src, cpu_cc_src, t2);
             tcg_temp_free(tcg_ctx, t0);
             tcg_temp_free(tcg_ctx, t1);
             tcg_temp_free(tcg_ctx, t2);
@@ -8530,20 +6495,20 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         {
             TCGLabel *label1;
             TCGv t0;
-            if (!s->pe || s->vm86)
+            if (!PE(s) || VM86(s))
                 goto illegal_op;
             ot = dflag != MO_16 ? MO_32 : MO_16;
             modrm = x86_ldub_code(env, s);
-            reg = ((modrm >> 3) & 7) | rex_r;
+            reg = ((modrm >> 3) & 7) | REX_R(s);
             gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
             t0 = tcg_temp_local_new(tcg_ctx);
             gen_update_cc_op(s);
             if (b == 0x102) {
-                gen_helper_lar(tcg_ctx, t0, tcg_ctx->cpu_env, s->T0);
+                gen_helper_lar(tcg_ctx, t0, cpu_env, s->T0);
             } else {
-                gen_helper_lsl(tcg_ctx, t0, tcg_ctx->cpu_env, s->T0);
+                gen_helper_lsl(tcg_ctx, t0, cpu_env, s->T0);
             }
-            tcg_gen_andi_tl(tcg_ctx, s->tmp0, tcg_ctx->cpu_cc_src, CC_Z);
+            tcg_gen_andi_tl(tcg_ctx, s->tmp0, cpu_cc_src, CC_Z);
             label1 = gen_new_label(tcg_ctx);
             tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, s->tmp0, 0, label1);
             gen_op_mov_reg_v(s, ot, reg, t0);
@@ -8575,7 +6540,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         modrm = x86_ldub_code(env, s);
         if (s->flags & HF_MPX_EN_MASK) {
             mod = (modrm >> 6) & 3;
-            reg = ((modrm >> 3) & 7) | rex_r;
+            reg = ((modrm >> 3) & 7) | REX_R(s);
             if (prefixes & PREFIX_REPZ) {
                 /* bndcl */
                 if (reg >= 4
@@ -8583,7 +6548,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     || s->aflag == MO_16) {
                     goto illegal_op;
                 }
-                gen_bndck(env, s, modrm, TCG_COND_LTU, tcg_ctx->cpu_bndl[reg]);
+                gen_bndck(env, s, modrm, TCG_COND_LTU, cpu_bndl[reg]);
             } else if (prefixes & PREFIX_REPNZ) {
                 /* bndcu */
                 if (reg >= 4
@@ -8592,7 +6557,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     goto illegal_op;
                 }
                 TCGv_i64 notu = tcg_temp_new_i64(tcg_ctx);
-                tcg_gen_not_i64(tcg_ctx, notu, tcg_ctx->cpu_bndu[reg]);
+                tcg_gen_not_i64(tcg_ctx, notu, cpu_bndu[reg]);
                 gen_bndck(env, s, modrm, TCG_COND_GTU, notu);
                 tcg_temp_free_i64(tcg_ctx, notu);
             } else if (prefixes & PREFIX_DATA) {
@@ -8606,22 +6571,22 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                         goto illegal_op;
                     }
                     if (s->flags & HF_MPX_IU_MASK) {
-                        tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], tcg_ctx->cpu_bndl[reg2]);
-                        tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], tcg_ctx->cpu_bndu[reg2]);
+                        tcg_gen_mov_i64(tcg_ctx, cpu_bndl[reg], cpu_bndl[reg2]);
+                        tcg_gen_mov_i64(tcg_ctx, cpu_bndu[reg], cpu_bndu[reg2]);
                     }
                 } else {
                     gen_lea_modrm(env, s, modrm);
                     if (CODE64(s)) {
-                        tcg_gen_qemu_ld_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], s->A0,
-                                            s->mem_index, MO_LEQ);
+                        tcg_gen_qemu_ld_i64(tcg_ctx, cpu_bndl[reg], s->A0,
+                                            s->mem_index, MO_LEUQ);
                         tcg_gen_addi_tl(tcg_ctx, s->A0, s->A0, 8);
-                        tcg_gen_qemu_ld_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], s->A0,
-                                            s->mem_index, MO_LEQ);
+                        tcg_gen_qemu_ld_i64(tcg_ctx, cpu_bndu[reg], s->A0,
+                                            s->mem_index, MO_LEUQ);
                     } else {
-                        tcg_gen_qemu_ld_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], s->A0,
+                        tcg_gen_qemu_ld_i64(tcg_ctx, cpu_bndl[reg], s->A0,
                                             s->mem_index, MO_LEUL);
                         tcg_gen_addi_tl(tcg_ctx, s->A0, s->A0, 4);
-                        tcg_gen_qemu_ld_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], s->A0,
+                        tcg_gen_qemu_ld_i64(tcg_ctx, cpu_bndu[reg], s->A0,
                                             s->mem_index, MO_LEUL);
                     }
                     /* bnd registers are now in-use */
@@ -8637,24 +6602,24 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     goto illegal_op;
                 }
                 if (a.base >= 0) {
-                    tcg_gen_addi_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[a.base], a.disp);
+                    tcg_gen_addi_tl(tcg_ctx, s->A0, cpu_regs[a.base], a.disp);
                 } else {
                     tcg_gen_movi_tl(tcg_ctx, s->A0, 0);
                 }
                 gen_lea_v_seg(s, s->aflag, s->A0, a.def_seg, s->override);
                 if (a.index >= 0) {
-                    tcg_gen_mov_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[a.index]);
+                    tcg_gen_mov_tl(tcg_ctx, s->T0, cpu_regs[a.index]);
                 } else {
                     tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
                 }
                 if (CODE64(s)) {
-                    gen_helper_bndldx64(tcg_ctx, tcg_ctx->cpu_bndl[reg], tcg_ctx->cpu_env, s->A0, s->T0);
-                    tcg_gen_ld_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], tcg_ctx->cpu_env,
+                    gen_helper_bndldx64(tcg_ctx, cpu_bndl[reg], cpu_env, s->A0, s->T0);
+                    tcg_gen_ld_i64(tcg_ctx, cpu_bndu[reg], cpu_env,
                                    offsetof(CPUX86State, mmx_t0.MMX_Q(0)));
                 } else {
-                    gen_helper_bndldx32(tcg_ctx, tcg_ctx->cpu_bndu[reg], tcg_ctx->cpu_env, s->A0, s->T0);
-                    tcg_gen_ext32u_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], tcg_ctx->cpu_bndu[reg]);
-                    tcg_gen_shri_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], tcg_ctx->cpu_bndu[reg], 32);
+                    gen_helper_bndldx32(tcg_ctx, cpu_bndu[reg], cpu_env, s->A0, s->T0);
+                    tcg_gen_ext32u_i64(tcg_ctx, cpu_bndl[reg], cpu_bndu[reg]);
+                    tcg_gen_shri_i64(tcg_ctx, cpu_bndu[reg], cpu_bndu[reg], 32);
                 }
                 gen_set_hflag(s, HF_MPX_IU_MASK);
             }
@@ -8665,7 +6630,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         modrm = x86_ldub_code(env, s);
         if (s->flags & HF_MPX_EN_MASK) {
             mod = (modrm >> 6) & 3;
-            reg = ((modrm >> 3) & 7) | rex_r;
+            reg = ((modrm >> 3) & 7) | REX_R(s);
             if (mod != 3 && (prefixes & PREFIX_REPZ)) {
                 /* bndmk */
                 if (reg >= 4
@@ -8675,22 +6640,22 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 }
                 AddressParts a = gen_lea_modrm_0(env, s, modrm);
                 if (a.base >= 0) {
-                    tcg_gen_extu_tl_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], tcg_ctx->cpu_regs[a.base]);
+                    tcg_gen_extu_tl_i64(tcg_ctx, cpu_bndl[reg], cpu_regs[a.base]);
                     if (!CODE64(s)) {
-                        tcg_gen_ext32u_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], tcg_ctx->cpu_bndl[reg]);
+                        tcg_gen_ext32u_i64(tcg_ctx, cpu_bndl[reg], cpu_bndl[reg]);
                     }
                 } else if (a.base == -1) {
                     /* no base register has lower bound of 0 */
-                    tcg_gen_movi_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], 0);
+                    tcg_gen_movi_i64(tcg_ctx, cpu_bndl[reg], 0);
                 } else {
                     /* rip-relative generates #ud */
                     goto illegal_op;
                 }
-                tcg_gen_not_tl(tcg_ctx, s->A0, gen_lea_modrm_1(s, a));
+                tcg_gen_not_tl(tcg_ctx, s->A0, gen_lea_modrm_1(s, a, false));
                 if (!CODE64(s)) {
                     tcg_gen_ext32u_tl(tcg_ctx, s->A0, s->A0);
                 }
-                tcg_gen_extu_tl_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], s->A0);
+                tcg_gen_extu_tl_i64(tcg_ctx, cpu_bndu[reg], s->A0);
                 /* bnd registers are now in-use */
                 gen_set_hflag(s, HF_MPX_IU_MASK);
                 break;
@@ -8701,7 +6666,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     || s->aflag == MO_16) {
                     goto illegal_op;
                 }
-                gen_bndck(env, s, modrm, TCG_COND_GTU, tcg_ctx->cpu_bndu[reg]);
+                gen_bndck(env, s, modrm, TCG_COND_GTU, cpu_bndu[reg]);
             } else if (prefixes & PREFIX_DATA) {
                 /* bndmov -- to reg/mem */
                 if (reg >= 4 || s->aflag == MO_16) {
@@ -8713,22 +6678,22 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                         goto illegal_op;
                     }
                     if (s->flags & HF_MPX_IU_MASK) {
-                        tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg2], tcg_ctx->cpu_bndl[reg]);
-                        tcg_gen_mov_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg2], tcg_ctx->cpu_bndu[reg]);
+                        tcg_gen_mov_i64(tcg_ctx, cpu_bndl[reg2], cpu_bndl[reg]);
+                        tcg_gen_mov_i64(tcg_ctx, cpu_bndu[reg2], cpu_bndu[reg]);
                     }
                 } else {
                     gen_lea_modrm(env, s, modrm);
                     if (CODE64(s)) {
-                        tcg_gen_qemu_st_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], s->A0,
-                                            s->mem_index, MO_LEQ);
+                        tcg_gen_qemu_st_i64(tcg_ctx, cpu_bndl[reg], s->A0,
+                                            s->mem_index, MO_LEUQ);
                         tcg_gen_addi_tl(tcg_ctx, s->A0, s->A0, 8);
-                        tcg_gen_qemu_st_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], s->A0,
-                                            s->mem_index, MO_LEQ);
+                        tcg_gen_qemu_st_i64(tcg_ctx, cpu_bndu[reg], s->A0,
+                                            s->mem_index, MO_LEUQ);
                     } else {
-                        tcg_gen_qemu_st_i64(tcg_ctx, tcg_ctx->cpu_bndl[reg], s->A0,
+                        tcg_gen_qemu_st_i64(tcg_ctx, cpu_bndl[reg], s->A0,
                                             s->mem_index, MO_LEUL);
                         tcg_gen_addi_tl(tcg_ctx, s->A0, s->A0, 4);
-                        tcg_gen_qemu_st_i64(tcg_ctx, tcg_ctx->cpu_bndu[reg], s->A0,
+                        tcg_gen_qemu_st_i64(tcg_ctx, cpu_bndu[reg], s->A0,
                                             s->mem_index, MO_LEUL);
                     }
                 }
@@ -8742,94 +6707,82 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     goto illegal_op;
                 }
                 if (a.base >= 0) {
-                    tcg_gen_addi_tl(tcg_ctx, s->A0, tcg_ctx->cpu_regs[a.base], a.disp);
+                    tcg_gen_addi_tl(tcg_ctx, s->A0, cpu_regs[a.base], a.disp);
                 } else {
                     tcg_gen_movi_tl(tcg_ctx, s->A0, 0);
                 }
                 gen_lea_v_seg(s, s->aflag, s->A0, a.def_seg, s->override);
                 if (a.index >= 0) {
-                    tcg_gen_mov_tl(tcg_ctx, s->T0, tcg_ctx->cpu_regs[a.index]);
+                    tcg_gen_mov_tl(tcg_ctx, s->T0, cpu_regs[a.index]);
                 } else {
                     tcg_gen_movi_tl(tcg_ctx, s->T0, 0);
                 }
                 if (CODE64(s)) {
-                    gen_helper_bndstx64(tcg_ctx, tcg_ctx->cpu_env, s->A0, s->T0,
-                                        tcg_ctx->cpu_bndl[reg], tcg_ctx->cpu_bndu[reg]);
+                    gen_helper_bndstx64(tcg_ctx, cpu_env, s->A0, s->T0,
+                                        cpu_bndl[reg], cpu_bndu[reg]);
                 } else {
-                    gen_helper_bndstx32(tcg_ctx, tcg_ctx->cpu_env, s->A0, s->T0,
-                                        tcg_ctx->cpu_bndl[reg], tcg_ctx->cpu_bndu[reg]);
+                    gen_helper_bndstx32(tcg_ctx, cpu_env, s->A0, s->T0,
+                                        cpu_bndl[reg], cpu_bndu[reg]);
                 }
             }
         }
         gen_nop_modrm(env, s, modrm);
         break;
-    case 0x119:
-    case 0x11c: /* nop (multi byte) */
-    case 0x11d: /* nop (multi byte) */
-    case 0x11e: /* nop (multi byte) */
-    case 0x11f: /* nop (multi byte) */
+    case 0x119: case 0x11c: case 0x11d: case 0x11e: case 0x11f: /* nop (multi byte) */
         modrm = x86_ldub_code(env, s);
         gen_nop_modrm(env, s, modrm);
         break;
+
     case 0x120: /* mov reg, crN */
     case 0x122: /* mov crN, reg */
-        if (s->cpl != 0) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
-            modrm = x86_ldub_code(env, s);
-            /* Ignore the mod bits (assume (modrm&0xc0)==0xc0).
-             * AMD documentation (24594.pdf) and testing of
-             * intel 386 and 486 processors all show that the mod bits
-             * are assumed to be 1's, regardless of actual values.
-             */
-            rm = (modrm & 7) | REX_B(s);
-            reg = ((modrm >> 3) & 7) | rex_r;
-            if (CODE64(s))
-                ot = MO_64;
-            else
-                ot = MO_32;
-            if ((prefixes & PREFIX_LOCK) && (reg == 0) &&
+        if (!check_cpl0(s)) {
+            break;
+        }
+        modrm = x86_ldub_code(env, s);
+        /*
+         * Ignore the mod bits (assume (modrm&0xc0)==0xc0).
+         * AMD documentation (24594.pdf) and testing of Intel 386 and 486
+         * processors all show that the mod bits are assumed to be 1's,
+         * regardless of actual values.
+         */
+        rm = (modrm & 7) | REX_B(s);
+        reg = ((modrm >> 3) & 7) | REX_R(s);
+        switch (reg) {
+        case 0:
+            if ((prefixes & PREFIX_LOCK) &&
                 (s->cpuid_ext3_features & CPUID_EXT3_CR8LEG)) {
                 reg = 8;
             }
-            switch(reg) {
-            case 0:
-            case 2:
-            case 3:
-            case 4:
-            case 8:
-                gen_update_cc_op(s);
-                gen_jmp_im(s, pc_start - s->cs_base);
-                if (b & 2) {
-                    if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-                        gen_io_start(tcg_ctx);
-                    }
-                    gen_op_mov_v_reg(s, ot, s->T0, rm);
-                    gen_helper_write_crN(tcg_ctx, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, reg),
-                                         s->T0);
-                    gen_jmp_im(s, s->pc - s->cs_base);
-                    gen_eob(s);
-                } else {
-                    if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-                        gen_io_start(tcg_ctx);
-                    }
-                    gen_helper_read_crN(tcg_ctx, s->T0, tcg_ctx->cpu_env, tcg_const_i32(tcg_ctx, reg));
-                    gen_op_mov_reg_v(s, ot, rm, s->T0);
-                    if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
-                        gen_io_end(tcg_ctx);
-                    }
-                }
-                break;
-            default:
-                goto unknown_op;
-            }
+            break;
+        case 2:
+        case 3:
+        case 4:
+        case 8:
+            break;
+        default:
+            goto unknown_op;
+        }
+        ot  = (CODE64(s) ? MO_64 : MO_32);
+
+        if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) {
+            gen_io_start(tcg_ctx);
+            s->base.is_jmp = DISAS_TOO_MANY;
+        }
+        if (b & 2) {
+            gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0 + reg);
+            gen_op_mov_v_reg(s, ot, s->T0, rm);
+            gen_helper_write_crN(tcg_ctx, cpu_env, tcg_constant_i32(tcg_ctx, reg), s->T0);
+            s->base.is_jmp = DISAS_EOB_NEXT;
+        } else {
+            gen_svm_check_intercept(s, SVM_EXIT_READ_CR0 + reg);
+            gen_helper_read_crN(tcg_ctx, s->T0, cpu_env, tcg_constant_i32(tcg_ctx, reg));
+            gen_op_mov_reg_v(s, ot, rm, s->T0);
         }
         break;
+
     case 0x121: /* mov reg, drN */
     case 0x123: /* mov drN, reg */
-        if (s->cpl != 0) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
+        if (check_cpl0(s)) {
             modrm = x86_ldub_code(env, s);
             /* Ignore the mod bits (assume (modrm&0xc0)==0xc0).
              * AMD documentation (24594.pdf) and testing of
@@ -8837,7 +6790,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
              * are assumed to be 1's, regardless of actual values.
              */
             rm = (modrm & 7) | REX_B(s);
-            reg = ((modrm >> 3) & 7) | rex_r;
+            reg = ((modrm >> 3) & 7) | REX_R(s);
             if (CODE64(s))
                 ot = MO_64;
             else
@@ -8846,29 +6799,25 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             if (b & 2) {
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_DR0 + reg);
+                gen_svm_check_intercept(s, SVM_EXIT_WRITE_DR0 + reg);
                 gen_op_mov_v_reg(s, ot, s->T0, rm);
                 tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, reg);
-                gen_helper_set_dr(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32, s->T0);
-                gen_jmp_im(s, s->pc - s->cs_base);
-                gen_eob(s);
+                gen_helper_set_dr(tcg_ctx, cpu_env, s->tmp2_i32, s->T0);
+                s->base.is_jmp = DISAS_EOB_NEXT;
             } else {
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_DR0 + reg);
+                gen_svm_check_intercept(s, SVM_EXIT_READ_DR0 + reg);
                 tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, reg);
-                gen_helper_get_dr(tcg_ctx, s->T0, tcg_ctx->cpu_env, s->tmp2_i32);
+                gen_helper_get_dr(tcg_ctx, s->T0, cpu_env, s->tmp2_i32);
                 gen_op_mov_reg_v(s, ot, rm, s->T0);
             }
         }
         break;
     case 0x106: /* clts */
-        if (s->cpl != 0) {
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-        } else {
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
-            gen_helper_clts(tcg_ctx, tcg_ctx->cpu_env);
+        if (check_cpl0(s)) {
+            gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0);
+            gen_helper_clts(tcg_ctx, cpu_env);
             /* abort block because static cpu state changed */
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            s->base.is_jmp = DISAS_EOB_NEXT;
         }
         break;
     /* MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4 support */
@@ -8880,7 +6829,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        reg = ((modrm >> 3) & 7) | rex_r;
+        reg = ((modrm >> 3) & 7) | REX_R(s);
         /* generate a generic store */
         gen_ldst_modrm(env, s, modrm, ot, reg, 1);
         break;
@@ -8893,11 +6842,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) {
-                gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+                gen_exception(s, EXCP07_PREX);
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_helper_fxsave(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+            gen_helper_fxsave(tcg_ctx, cpu_env, s->A0);
             break;
 
         CASE_MODRM_MEM_OP(1): /* fxrstor */
@@ -8906,11 +6855,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) {
-                gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+                gen_exception(s, EXCP07_PREX);
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_helper_fxrstor(tcg_ctx, tcg_ctx->cpu_env, s->A0);
+            gen_helper_fxrstor(tcg_ctx, cpu_env, s->A0);
             break;
 
         CASE_MODRM_MEM_OP(2): /* ldmxcsr */
@@ -8918,12 +6867,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             if (s->flags & HF_TS_MASK) {
-                gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+                gen_exception(s, EXCP07_PREX);
                 break;
             }
             gen_lea_modrm(env, s, modrm);
             tcg_gen_qemu_ld_i32(tcg_ctx, s->tmp2_i32, s->A0, s->mem_index, MO_LEUL);
-            gen_helper_ldmxcsr(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+            gen_helper_ldmxcsr(tcg_ctx, cpu_env, s->tmp2_i32);
             break;
 
         CASE_MODRM_MEM_OP(3): /* stmxcsr */
@@ -8931,11 +6880,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             if (s->flags & HF_TS_MASK) {
-                gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+                gen_exception(s, EXCP07_PREX);
                 break;
             }
+            gen_helper_update_mxcsr(tcg_ctx, cpu_env);
             gen_lea_modrm(env, s, modrm);
-            tcg_gen_ld32u_tl(tcg_ctx, s->T0, tcg_ctx->cpu_env, offsetof(CPUX86State, mxcsr));
+            tcg_gen_ld32u_tl(tcg_ctx, s->T0, cpu_env, offsetof(CPUX86State, mxcsr));
             gen_op_st_v(s, MO_32, s->T0, s->A0);
             break;
 
@@ -8946,9 +6896,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             gen_lea_modrm(env, s, modrm);
-            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_regs[R_EAX],
-                                  tcg_ctx->cpu_regs[R_EDX]);
-            gen_helper_xsave(tcg_ctx, tcg_ctx->cpu_env, s->A0, s->tmp1_i64);
+            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            gen_helper_xsave(tcg_ctx, cpu_env, s->A0, s->tmp1_i64);
             break;
 
         CASE_MODRM_MEM_OP(5): /* xrstor */
@@ -8958,14 +6908,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 goto illegal_op;
             }
             gen_lea_modrm(env, s, modrm);
-            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_regs[R_EAX],
-                                  tcg_ctx->cpu_regs[R_EDX]);
-            gen_helper_xrstor(tcg_ctx, tcg_ctx->cpu_env, s->A0, s->tmp1_i64);
+            tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            gen_helper_xrstor(tcg_ctx, cpu_env, s->A0, s->tmp1_i64);
             /* XRSTOR is how MPX is enabled, which changes how
                we translate.  Thus we need to end the TB.  */
-            gen_update_cc_op(s);
-            gen_jmp_im(s, s->pc - s->cs_base);
-            gen_eob(s);
+            s->base.is_jmp = DISAS_EOB_NEXT;
             break;
 
         CASE_MODRM_MEM_OP(6): /* xsaveopt / clwb */
@@ -8986,9 +6934,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                     goto illegal_op;
                 }
                 gen_lea_modrm(env, s, modrm);
-                tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, tcg_ctx->cpu_regs[R_EAX],
-                                      tcg_ctx->cpu_regs[R_EDX]);
-                gen_helper_xsaveopt(tcg_ctx, tcg_ctx->cpu_env, s->A0, s->tmp1_i64);
+                tcg_gen_concat_tl_i64(tcg_ctx, s->tmp1_i64, cpu_regs[R_EAX],
+                                      cpu_regs[R_EDX]);
+                gen_helper_xsaveopt(tcg_ctx, cpu_env, s->A0, s->tmp1_i64);
             }
             break;
 
@@ -9011,41 +6959,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             gen_nop_modrm(env, s, modrm);
             break;
 
-        case 0xc0: /* rdfsbase (f3 0f ae /0) */
-        case 0xc1: /* rdfsbase (f3 0f ae /0) */
-        case 0xc2: /* rdfsbase (f3 0f ae /0) */
-        case 0xc3: /* rdfsbase (f3 0f ae /0) */
-        case 0xc4: /* rdfsbase (f3 0f ae /0) */
-        case 0xc5: /* rdfsbase (f3 0f ae /0) */
-        case 0xc6: /* rdfsbase (f3 0f ae /0) */
-        case 0xc7: /* rdfsbase (f3 0f ae /0) */
-
-        case 0xc8: /* rdgsbase (f3 0f ae /1) */
-        case 0xc9: /* rdgsbase (f3 0f ae /1) */
-        case 0xca: /* rdgsbase (f3 0f ae /1) */
-        case 0xcb: /* rdgsbase (f3 0f ae /1) */
-        case 0xcc: /* rdgsbase (f3 0f ae /1) */
-        case 0xcd: /* rdgsbase (f3 0f ae /1) */
-        case 0xce: /* rdgsbase (f3 0f ae /1) */
-        case 0xcf: /* rdgsbase (f3 0f ae /1) */
-
-        case 0xd0: /* wrfsbase (f3 0f ae /2) */
-        case 0xd1: /* wrfsbase (f3 0f ae /2) */
-        case 0xd2: /* wrfsbase (f3 0f ae /2) */
-        case 0xd3: /* wrfsbase (f3 0f ae /2) */
-        case 0xd4: /* wrfsbase (f3 0f ae /2) */
-        case 0xd5: /* wrfsbase (f3 0f ae /2) */
-        case 0xd6: /* wrfsbase (f3 0f ae /2) */
-        case 0xd7: /* wrfsbase (f3 0f ae /2) */
-
-        case 0xd8: /* wrgsbase (f3 0f ae /3) */
-        case 0xd9: /* wrgsbase (f3 0f ae /3) */
-        case 0xda: /* wrgsbase (f3 0f ae /3) */
-        case 0xdb: /* wrgsbase (f3 0f ae /3) */
-        case 0xdc: /* wrgsbase (f3 0f ae /3) */
-        case 0xdd: /* wrgsbase (f3 0f ae /3) */
-        case 0xde: /* wrgsbase (f3 0f ae /3) */
-        case 0xdf: /* wrgsbase (f3 0f ae /3) */
+        case 0xc0: case 0xc1: case 0xc2: case 0xc3: case 0xc4: case 0xc5: case 0xc6: case 0xc7: /* rdfsbase (f3 0f ae /0) */
+        case 0xc8: case 0xc9: case 0xca: case 0xcb: case 0xcc: case 0xcd: case 0xce: case 0xcf: /* rdgsbase (f3 0f ae /1) */
+        case 0xd0: case 0xd1: case 0xd2: case 0xd3: case 0xd4: case 0xd5: case 0xd6: case 0xd7: /* wrfsbase (f3 0f ae /2) */
+        case 0xd8: case 0xd9: case 0xda: case 0xdb: case 0xdc: case 0xdd: case 0xde: case 0xdf: /* wrgsbase (f3 0f ae /3) */
             if (CODE64(s)
                 && (prefixes & PREFIX_REPZ)
                 && !(prefixes & PREFIX_LOCK)
@@ -9054,10 +6971,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
 
                 /* Preserve hflags bits by testing CR4 at runtime.  */
                 tcg_gen_movi_i32(tcg_ctx, s->tmp2_i32, CR4_FSGSBASE_MASK);
-                gen_helper_cr4_testbit(tcg_ctx, tcg_ctx->cpu_env, s->tmp2_i32);
+                gen_helper_cr4_testbit(tcg_ctx, cpu_env, s->tmp2_i32);
 
-                base = tcg_ctx->cpu_seg_base[modrm & 8 ? R_GS : R_FS];
-                treg = tcg_ctx->cpu_regs[(modrm & 7) | REX_B(s)];
+                base = cpu_seg_base[modrm & 8 ? R_GS : R_FS];
+                treg = cpu_regs[(modrm & 7) | REX_B(s)];
 
                 if (modrm & 0x10) {
                     /* wr*base */
@@ -9086,41 +7003,21 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
                 break;
             }
             /* fallthru */
-        case 0xf9: /* sfence */
-        case 0xfa: /* sfence */
-        case 0xfb: /* sfence */
-        case 0xfc: /* sfence */
-        case 0xfd: /* sfence */
-        case 0xfe: /* sfence */
-        case 0xff: /* sfence */
+        case 0xf9: case 0xfa: case 0xfb: case 0xfc: case 0xfd: case 0xfe: case 0xff: /* sfence */
             if (!(s->cpuid_features & CPUID_SSE)
                 || (prefixes & PREFIX_LOCK)) {
                 goto illegal_op;
             }
             tcg_gen_mb(tcg_ctx, TCG_MO_ST_ST | TCG_BAR_SC);
             break;
-        case 0xe8: /* lfence */
-        case 0xe9: /* lfence */
-        case 0xea: /* lfence */
-        case 0xeb: /* lfence */
-        case 0xec: /* lfence */
-        case 0xed: /* lfence */
-        case 0xee: /* lfence */
-        case 0xef: /* lfence */
+        case 0xe8: case 0xe9: case 0xea: case 0xeb: case 0xec: case 0xed: case 0xee: case 0xef: /* lfence */
             if (!(s->cpuid_features & CPUID_SSE)
                 || (prefixes & PREFIX_LOCK)) {
                 goto illegal_op;
             }
             tcg_gen_mb(tcg_ctx, TCG_MO_LD_LD | TCG_BAR_SC);
             break;
-        case 0xf0: /* mfence */
-        case 0xf1: /* mfence */
-        case 0xf2: /* mfence */
-        case 0xf3: /* mfence */
-        case 0xf4: /* mfence */
-        case 0xf5: /* mfence */
-        case 0xf6: /* mfence */
-        case 0xf7: /* mfence */
+        case 0xf0: case 0xf1: case 0xf2: case 0xf3: case 0xf4: case 0xf5: case 0xf6: case 0xf7: /* mfence */
             if (!(s->cpuid_features & CPUID_SSE2)
                 || (prefixes & PREFIX_LOCK)) {
                 goto illegal_op;
@@ -9141,13 +7038,18 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         gen_nop_modrm(env, s, modrm);
         break;
     case 0x1aa: /* rsm */
-        gen_svm_check_intercept(s, pc_start, SVM_EXIT_RSM);
+        gen_svm_check_intercept(s, SVM_EXIT_RSM);
         if (!(s->flags & HF_SMM_MASK))
             goto illegal_op;
+#ifdef CONFIG_USER_ONLY
+        /* we should not be in SMM mode */
+        g_assert_not_reached();
+#else
         gen_update_cc_op(s);
-        gen_jmp_im(s, s->pc - s->cs_base);
-        gen_helper_rsm(tcg_ctx, tcg_ctx->cpu_env);
-        gen_eob(s);
+        gen_update_eip_next(s);
+        gen_helper_rsm(tcg_ctx, cpu_env);
+#endif /* CONFIG_USER_ONLY */
+        s->base.is_jmp = DISAS_EOB_ONLY;
         break;
     case 0x1b8: /* SSE4.2 popcnt */
         if ((prefixes & (PREFIX_REPZ | PREFIX_LOCK | PREFIX_REPNZ)) !=
@@ -9157,101 +7059,45 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
             goto illegal_op;
 
         modrm = x86_ldub_code(env, s);
-        reg = ((modrm >> 3) & 7) | rex_r;
-
-        if (s->prefix & PREFIX_DATA) {
-            ot = MO_16;
-        } else {
-            ot = mo_64_32(dflag);
-        }
+        reg = ((modrm >> 3) & 7) | REX_R(s);
 
+        ot = dflag;
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_extu(tcg_ctx, ot, s->T0);
-        tcg_gen_mov_tl(tcg_ctx, tcg_ctx->cpu_cc_src, s->T0);
+        tcg_gen_mov_tl(tcg_ctx, cpu_cc_src, s->T0);
         tcg_gen_ctpop_tl(tcg_ctx, s->T0, s->T0);
         gen_op_mov_reg_v(s, ot, reg, s->T0);
 
         set_cc_op(s, CC_OP_POPCNT);
         break;
-    case 0x10e:
-    case 0x10f:
-        /* 3DNow! instructions, ignore prefixes */
-        s->prefix &= ~(PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA);
-        /* fall through */
-    case 0x110:
-    case 0x111:
-    case 0x112:
-    case 0x113:
-    case 0x114:
-    case 0x115:
-    case 0x116:
-    case 0x117:
-
-    case 0x128:
-    case 0x129:
-    case 0x12a:
-    case 0x12b:
-    case 0x12c:
-    case 0x12d:
-    case 0x12e:
-    case 0x12f:
-
-    case 0x138:
-    case 0x139:
-    case 0x13a:
-
-    // case 0x150 ... 0x179:
-
-    case 0x17c:
-    case 0x17d:
-    case 0x17e:
-    case 0x17f:
+    case 0x10e: case 0x10f: case 0x110: case 0x111: case 0x112: case 0x113: case 0x114: case 0x115: case 0x116: case 0x117:
+    case 0x128: case 0x129: case 0x12a: case 0x12b: case 0x12c: case 0x12d: case 0x12e: case 0x12f:
+    case 0x138: case 0x139: case 0x13a:
+    case 0x150: case 0x151: case 0x152: case 0x153: case 0x154: case 0x155: case 0x156: case 0x157: case 0x158: case 0x159: case 0x15a: case 0x15b: case 0x15c: case 0x15d: case 0x15e: case 0x15f: case 0x160: case 0x161: case 0x162: case 0x163: case 0x164: case 0x165: case 0x166: case 0x167: case 0x168: case 0x169: case 0x16a: case 0x16b: case 0x16c: case 0x16d: case 0x16e: case 0x16f: case 0x170: case 0x171: case 0x172: case 0x173: case 0x174: case 0x175: case 0x176: case 0x177: case 0x178: case 0x179:
+    case 0x17c: case 0x17d: case 0x17e: case 0x17f:
     case 0x1c2:
-    case 0x1c4:
-    case 0x1c5:
-    case 0x1c6:
-    // case 0x1d0 ... 0x1fe:
-        gen_sse(env, s, b, pc_start, rex_r);
+    case 0x1c4: case 0x1c5: case 0x1c6:
+    case 0x1d0: case 0x1d1: case 0x1d2: case 0x1d3: case 0x1d4: case 0x1d5: case 0x1d6: case 0x1d7: case 0x1d8: case 0x1d9: case 0x1da: case 0x1db: case 0x1dc: case 0x1dd: case 0x1de: case 0x1df: case 0x1e0: case 0x1e1: case 0x1e2: case 0x1e3: case 0x1e4: case 0x1e5: case 0x1e6: case 0x1e7: case 0x1e8: case 0x1e9: case 0x1ea: case 0x1eb: case 0x1ec: case 0x1ed: case 0x1ee: case 0x1ef: case 0x1f0: case 0x1f1: case 0x1f2: case 0x1f3: case 0x1f4: case 0x1f5: case 0x1f6: case 0x1f7: case 0x1f8: case 0x1f9: case 0x1fa: case 0x1fb: case 0x1fc: case 0x1fd: case 0x1fe:
+        disas_insn_new(s, cpu, b);
         break;
     default:
-        if (b >= 0x150 && b <= 0x179) {
-            gen_sse(env, s, b, pc_start, rex_r);
-            break;
-        }
-
-        if (b >= 0x1d0 && b <= 0x1fe) {
-            gen_sse(env, s, b, pc_start, rex_r);
-            break;
-        }
-
         goto unknown_op;
     }
-
     if (insn_hook) {
-        // Unicorn: patch the callback to have the proper instruction size.
         if (prev_op) {
-            // As explained further up in the function where prev_op is
-            // assigned, we move forward in the tail queue, so we're modifying the
-            // move instruction generated by gen_uc_tracecode() that contains
-            // the instruction size to assign the proper size (replacing 0xF1F1F1F1).
             tcg_op = QTAILQ_NEXT(prev_op, link);
         } else {
-            // this instruction is the first emulated code ever,
-            // so the operand is the first operand
             tcg_op = QTAILQ_FIRST(&tcg_ctx->ops);
         }
         tcg_op->args[1] = s->pc - pc_start;
     }
-
-    return s->pc;
-
+    return true;
  illegal_op:
     gen_illegal_opcode(s);
-    return s->pc;
-
+    return true;
  unknown_op:
     gen_unknown_opcode(env, s);
-    return s->pc;
+    return true;
 }
 
 void tcg_x86_init(struct uc_struct *uc)
@@ -9283,6 +7129,13 @@ void tcg_x86_init(struct uc_struct *uc)
         [R_EDI] = "edi",
         [R_EBP] = "ebp",
         [R_ESP] = "esp",
+#endif
+    };
+    static const char eip_name[] = {
+#ifdef TARGET_X86_64
+        "rip"
+#else
+        "eip"
 #endif
     };
     static const char seg_base_names[6][8] = {
@@ -9302,35 +7155,36 @@ void tcg_x86_init(struct uc_struct *uc)
     int i;
     TCGContext *tcg_ctx = uc->tcg_ctx;
 
-    tcg_ctx->cpu_cc_op = tcg_global_mem_new_i32(tcg_ctx, tcg_ctx->cpu_env,
+    cpu_cc_op = tcg_global_mem_new_i32(tcg_ctx, cpu_env,
                                        offsetof(CPUX86State, cc_op), "cc_op");
-    tcg_ctx->cpu_cc_dst = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env, offsetof(CPUX86State, cc_dst),
+    cpu_cc_dst = tcg_global_mem_new(tcg_ctx, cpu_env, offsetof(CPUX86State, cc_dst),
                                     "cc_dst");
-    tcg_ctx->cpu_cc_src = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env, offsetof(CPUX86State, cc_src),
+    cpu_cc_src = tcg_global_mem_new(tcg_ctx, cpu_env, offsetof(CPUX86State, cc_src),
                                     "cc_src");
-    tcg_ctx->cpu_cc_src2 = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env, offsetof(CPUX86State, cc_src2),
+    cpu_cc_src2 = tcg_global_mem_new(tcg_ctx, cpu_env, offsetof(CPUX86State, cc_src2),
                                      "cc_src2");
+    cpu_eip = tcg_global_mem_new(tcg_ctx, cpu_env, offsetof(CPUX86State, eip), eip_name);
 
     for (i = 0; i < CPU_NB_REGS; ++i) {
-        tcg_ctx->cpu_regs[i] = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env,
+        cpu_regs[i] = tcg_global_mem_new(tcg_ctx, cpu_env,
                                          offsetof(CPUX86State, regs[i]),
                                          reg_names[i]);
     }
 
     for (i = 0; i < 6; ++i) {
-        tcg_ctx->cpu_seg_base[i]
-            = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env,
+        cpu_seg_base[i]
+            = tcg_global_mem_new(tcg_ctx, cpu_env,
                                  offsetof(CPUX86State, segs[i].base),
                                  seg_base_names[i]);
     }
 
     for (i = 0; i < 4; ++i) {
-        tcg_ctx->cpu_bndl[i]
-            = tcg_global_mem_new_i64(tcg_ctx, tcg_ctx->cpu_env,
+        cpu_bndl[i]
+            = tcg_global_mem_new_i64(tcg_ctx, cpu_env,
                                      offsetof(CPUX86State, bnd_regs[i].lb),
                                      bnd_regl_names[i]);
-        tcg_ctx->cpu_bndu[i]
-            = tcg_global_mem_new_i64(tcg_ctx, tcg_ctx->cpu_env,
+        cpu_bndu[i]
+            = tcg_global_mem_new_i64(tcg_ctx, cpu_env,
                                      offsetof(CPUX86State, bnd_regs[i].ub),
                                      bnd_regu_names[i]);
     }
@@ -9342,55 +7196,54 @@ static void i386_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cpu)
     TCGContext *tcg_ctx = cpu->uc->tcg_ctx;
     CPUX86State *env = cpu->env_ptr;
     uint32_t flags = dc->base.tb->flags;
-    target_ulong cs_base = dc->base.tb->cs_base;
+    uint32_t cflags = tb_cflags(dc->base.tb);
+    int cpl = (flags >> HF_CPL_SHIFT) & 3;
+    int iopl = (flags >> IOPL_SHIFT) & 3;
 
-    // unicorn setup
     dc->uc = cpu->uc;
-    dc->pe = (flags >> HF_PE_SHIFT) & 1;
-    dc->code32 = (flags >> HF_CS32_SHIFT) & 1;
-    dc->ss32 = (flags >> HF_SS32_SHIFT) & 1;
-    dc->addseg = (flags >> HF_ADDSEG_SHIFT) & 1;
-    dc->f_st = 0;
-    dc->vm86 = (flags >> VM_SHIFT) & 1;
-    dc->cpl = (flags >> HF_CPL_SHIFT) & 3;
-    dc->iopl = (flags >> IOPL_SHIFT) & 3;
-    dc->tf = (flags >> TF_SHIFT) & 1;
+    dc->cs_base = dc->base.tb->cs_base;
+    dc->pc_save = dc->base.pc_next;
+    dc->flags = flags;
+#ifndef CONFIG_USER_ONLY
+    dc->cpl = cpl;
+    dc->iopl = iopl;
+#endif
+
+    /* We make some simplifying assumptions; validate they're correct. */
+    g_assert(PE(dc) == ((flags & HF_PE_MASK) != 0));
+    g_assert(CPL(dc) == cpl);
+    g_assert(IOPL(dc) == iopl);
+    g_assert(VM86(dc) == ((flags & HF_VM_MASK) != 0));
+    g_assert(CODE32(dc) == ((flags & HF_CS32_MASK) != 0));
+    g_assert(CODE64(dc) == ((flags & HF_CS64_MASK) != 0));
+    g_assert(SS32(dc) == ((flags & HF_SS32_MASK) != 0));
+    g_assert(LMA(dc) == ((flags & HF_LMA_MASK) != 0));
+    g_assert(ADDSEG(dc) == ((flags & HF_ADDSEG_MASK) != 0));
+    g_assert(SVME(dc) == ((flags & HF_SVME_MASK) != 0));
+    g_assert(GUEST(dc) == ((flags & HF_GUEST_MASK) != 0));
+
     dc->cc_op = CC_OP_DYNAMIC;
     dc->cc_op_dirty = false;
-    dc->cs_base = cs_base;
     dc->popl_esp_hack = 0;
     /* select memory access functions */
     dc->mem_index = 0;
+#ifdef CONFIG_SOFTMMU
     dc->mem_index = cpu_mmu_index(env, false);
+#endif
     dc->cpuid_features = env->features[FEAT_1_EDX];
     dc->cpuid_ext_features = env->features[FEAT_1_ECX];
     dc->cpuid_ext2_features = env->features[FEAT_8000_0001_EDX];
     dc->cpuid_ext3_features = env->features[FEAT_8000_0001_ECX];
     dc->cpuid_7_0_ebx_features = env->features[FEAT_7_0_EBX];
+    dc->cpuid_7_0_ecx_features = env->features[FEAT_7_0_ECX];
     dc->cpuid_xsave_features = env->features[FEAT_XSAVE];
-#ifdef TARGET_X86_64
-    dc->lma = (flags >> HF_LMA_SHIFT) & 1;
-    dc->code64 = (flags >> HF_CS64_SHIFT) & 1;
-#endif
-    dc->flags = flags;
-    dc->jmp_opt = !(dc->tf || dc->base.singlestep_enabled ||
-                    (flags & HF_INHIBIT_IRQ_MASK));
-    /* Do not optimize repz jumps at all in icount mode, because
-       rep movsS instructions are execured with different paths
-       in !repz_opt and repz_opt modes. The first one was used
-       always except single step mode. And this setting
-       disables jumps optimization and control paths become
-       equivalent in run and single step modes.
-       Now there will be no jump optimization for repz in
-       record/replay modes and there will always be an
-       additional step for ecx=0 when icount is enabled.
+    dc->jmp_opt = !(flags & (HF_RF_MASK | HF_TF_MASK | HF_INHIBIT_IRQ_MASK));
+    /*
+     * If jmp_opt, we want to handle each string instruction individually.
+     * For icount also disable repz optimization so that each iteration
+     * is accounted separately.
      */
-    dc->repz_opt = !dc->jmp_opt && !(tb_cflags(dc->base.tb) & CF_USE_ICOUNT);
-#if 0
-    /* check addseg logic */
-    if (!dc->addseg && (dc->vm86 || !dc->pe || !dc->code32))
-        printf("ERROR addseg\n");
-#endif
+    dc->repz_opt = !dc->jmp_opt && !(cflags & CF_USE_ICOUNT);
 
     dc->T0 = tcg_temp_new(tcg_ctx);
     dc->T1 = tcg_temp_new(tcg_ctx);
@@ -9401,8 +7254,6 @@ static void i386_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cpu)
     dc->tmp2_i32 = tcg_temp_new_i32(tcg_ctx);
     dc->tmp3_i32 = tcg_temp_new_i32(tcg_ctx);
     dc->tmp4 = tcg_temp_new(tcg_ctx);
-    dc->ptr0 = tcg_temp_new_ptr(tcg_ctx);
-    dc->ptr1 = tcg_temp_new_ptr(tcg_ctx);
     dc->cc_srcT = tcg_temp_local_new(tcg_ctx);
 }
 
@@ -9416,6 +7267,7 @@ static void i386_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu)
     TCGContext *tcg_ctx = dc->uc->tcg_ctx;
 
     dc->prev_pc = dc->base.pc_next - dc->cs_base;
+    dc->prev_insn_end = tcg_last_op(tcg_ctx);
     tcg_gen_insn_start(tcg_ctx, dc->base.pc_next, dc->cc_op);
 }
 
@@ -9424,70 +7276,93 @@ static bool i386_tr_breakpoint_check(DisasContextBase *dcbase, CPUState *cpu,
 {
     DisasContext *dc = container_of(dcbase, DisasContext, base);
 
-    /* If RF is set, suppress an internally generated breakpoint.  */
+    /* If RF is set, suppress an internally generated breakpoint. */
     int flags = dc->base.tb->flags & HF_RF_MASK ? BP_GDB : BP_ANY;
+
     if (bp->flags & flags) {
         gen_debug(dc, dc->base.pc_next - dc->cs_base);
         dc->base.is_jmp = DISAS_NORETURN;
-        /* The address covered by the breakpoint must be included in
-           [tb->pc, tb->pc + tb->size) in order to for it to be
-           properly cleared -- thus we increment the PC here so that
-           the generic logic setting tb->size later does the right thing.  */
-        dc->base.pc_next += 1;
+        dc->base.pc_next++;
         return true;
-    } else {
-        return false;
     }
+
+    return false;
 }
 
 static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
 {
     DisasContext *dc = container_of(dcbase, DisasContext, base);
-    target_ulong pc_next;
-
-    pc_next = disas_insn(dc, cpu);
-
-    if (dc->tf || (dc->base.tb->flags & HF_INHIBIT_IRQ_MASK)) {
-        /* if single step mode, we generate only one instruction and
-           generate an exception */
-        /* if irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear
-           the flag and abort the translation to give the irqs a
-           chance to happen */
-        dc->base.is_jmp = DISAS_TOO_MANY;
-    } else if ((tb_cflags(dc->base.tb) & CF_USE_ICOUNT)
-               && ((pc_next & TARGET_PAGE_MASK)
-                   != ((pc_next + TARGET_MAX_INSN_SIZE - 1)
-                       & TARGET_PAGE_MASK)
-                   || (pc_next & ~TARGET_PAGE_MASK) == 0)) {
-        /* Do not cross the boundary of the pages in icount mode,
-           it can cause an exception. Do it only when boundary is
-           crossed by the first instruction in the block.
-           If current instruction already crossed the bound - it's ok,
-           because an exception hasn't stopped this code.
-         */
-        dc->base.is_jmp = DISAS_TOO_MANY;
-    } else if ((pc_next - dc->base.pc_first) >= (TARGET_PAGE_SIZE - 32)) {
-        dc->base.is_jmp = DISAS_TOO_MANY;
+
+#ifdef TARGET_VSYSCALL_PAGE
+    /*
+     * Detect entry into the vsyscall page and invoke the syscall.
+     */
+    if ((dc->base.pc_next & TARGET_PAGE_MASK) == TARGET_VSYSCALL_PAGE) {
+        gen_exception(dc, EXCP_VSYSCALL);
+        dc->base.pc_next = dc->pc + 1;
+        return;
     }
+#endif
 
-    dc->base.pc_next = pc_next;
+    if (disas_insn(dc, cpu)) {
+        target_ulong pc_next = dc->pc;
+        dc->base.pc_next = pc_next;
+
+        if (dc->base.is_jmp == DISAS_NEXT) {
+            if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) {
+                /*
+                 * If single step mode, we generate only one instruction and
+                 * generate an exception.
+                 * If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear
+                 * the flag and abort the translation to give the irqs a
+                 * chance to happen.
+                 */
+                dc->base.is_jmp = DISAS_EOB_NEXT;
+            } else if (uc_addr_is_exit(dc->uc, pc_next)) {
+                dc->base.is_jmp = DISAS_EOB_NEXT;
+            } else if (!is_same_page(&dc->base, pc_next)) {
+                dc->base.is_jmp = DISAS_TOO_MANY;
+            }
+        }
+    }
 }
 
 static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu)
 {
     DisasContext *dc = container_of(dcbase, DisasContext, base);
 
-    if (dc->base.is_jmp == DISAS_TOO_MANY) {
-        gen_jmp_im(dc, dc->base.pc_next - dc->cs_base);
+    switch (dc->base.is_jmp) {
+    case DISAS_NORETURN:
+        break;
+    case DISAS_TOO_MANY:
+        gen_update_cc_op(dc);
+        gen_jmp_rel_csize(dc, 0, 0);
+        break;
+    case DISAS_EOB_NEXT:
+        gen_update_cc_op(dc);
+        gen_update_eip_cur(dc);
+        /* fall through */
+    case DISAS_EOB_ONLY:
         gen_eob(dc);
+        break;
+    case DISAS_EOB_INHIBIT_IRQ:
+        gen_update_cc_op(dc);
+        gen_update_eip_cur(dc);
+        gen_eob_inhibit_irq(dc, true);
+        break;
+    case DISAS_JUMP:
+        gen_jr(dc);
+        break;
+    default:
+        g_assert_not_reached();
     }
 }
 
-static void i386_sync_pc(DisasContextBase *db, CPUState *cpu)
+static void i386_sync_pc(DisasContextBase *dcbase, CPUState *cpu)
 {
-    DisasContext *dc = container_of(db, DisasContext, base);
+    DisasContext *dc = container_of(dcbase, DisasContext, base);
 
-    gen_jmp_im(dc, dc->base.pc_next - dc->cs_base);
+    gen_update_eip_cur(dc);
 }
 
 static const TranslatorOps i386_tr_ops = {
@@ -9513,7 +7388,22 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb,
                           target_ulong *data)
 {
     int cc_op = data[1];
-    env->eip = data[0] - tb->cs_base;
+    target_ulong new_pc;
+
+    if (TARGET_TB_PCREL) {
+        target_ulong pc = env->eip + tb->cs_base;
+
+        new_pc = (pc & TARGET_PAGE_MASK) | data[0];
+    } else {
+        new_pc = data[0];
+    }
+
+    if (tb->flags & HF_CS64_MASK) {
+        env->eip = new_pc;
+    } else {
+        env->eip = (uint32_t)(new_pc - tb->cs_base);
+    }
+
     if (cc_op != CC_OP_DYNAMIC) {
         env->cc_op = cc_op;
     }
diff --git a/qemu/target/i386/unicorn.c b/qemu/target/i386/unicorn.c
index 96d8159449..83e72dd16f 100644
--- a/qemu/target/i386/unicorn.c
+++ b/qemu/target/i386/unicorn.c
@@ -300,6 +300,17 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
         *(uint16_t *)value = fptag;
         return ret;
     }
+    case UC_X86_REG_K0:
+    case UC_X86_REG_K1:
+    case UC_X86_REG_K2:
+    case UC_X86_REG_K3:
+    case UC_X86_REG_K4:
+    case UC_X86_REG_K5:
+    case UC_X86_REG_K6:
+    case UC_X86_REG_K7:
+        CHECK_REG_TYPE(uint64_t);
+        *(uint64_t *)value = env->opmask_regs[regid - UC_X86_REG_K0];
+        return ret;
     case UC_X86_REG_XMM0:
     case UC_X86_REG_XMM1:
     case UC_X86_REG_XMM2:
@@ -1155,6 +1166,17 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
 
         return ret;
     }
+    case UC_X86_REG_K0:
+    case UC_X86_REG_K1:
+    case UC_X86_REG_K2:
+    case UC_X86_REG_K3:
+    case UC_X86_REG_K4:
+    case UC_X86_REG_K5:
+    case UC_X86_REG_K6:
+    case UC_X86_REG_K7:
+        CHECK_REG_TYPE(uint64_t);
+        env->opmask_regs[regid - UC_X86_REG_K0] = *(uint64_t *)value;
+        return ret;
     case UC_X86_REG_XMM0:
     case UC_X86_REG_XMM1:
     case UC_X86_REG_XMM2:
@@ -1494,6 +1516,7 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             CHECK_REG_TYPE(uint64_t);
             env->xcr0 = *(uint64_t *)value;
             cpu_sync_bndcs_hflags(env);
+            cpu_sync_avx_hflag(env);
             break;
         }
         break;
@@ -2019,6 +2042,7 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             CHECK_REG_TYPE(uint64_t);
             env->xcr0 = *(uint64_t *)value;
             cpu_sync_bndcs_hflags(env);
+            cpu_sync_avx_hflag(env);
             break;
         }
         break;
diff --git a/qemu/target/i386/xsave_helper.c b/qemu/target/i386/xsave_helper.c
index 818115e7d2..2431733df0 100644
--- a/qemu/target/i386/xsave_helper.c
+++ b/qemu/target/i386/xsave_helper.c
@@ -6,14 +6,24 @@
 
 #include "cpu.h"
 
-void x86_cpu_xsave_all_areas(X86CPU *cpu, X86XSaveArea *buf)
+void x86_cpu_xsave_all_areas(X86CPU *cpu, void *buf, uint32_t buflen)
 {
     CPUX86State *env = &cpu->env;
-    X86XSaveArea *xsave = buf;
+    const ExtSaveArea *e, *f;
+    uint8_t *raw = buf;
+    int i;
 
+    X86LegacyXSaveArea *legacy;
+    X86XSaveHeader *header;
     uint16_t cwd, swd, twd;
-    int i;
-    memset(xsave, 0, sizeof(X86XSaveArea));
+
+    memset(buf, 0, buflen);
+
+    e = &x86_ext_save_areas[XSTATE_FP_BIT];
+
+    legacy = (X86LegacyXSaveArea *)(raw + e->offset);
+    header = (X86XSaveHeader *)(raw + e->offset + sizeof(*legacy));
+
     twd = 0;
     swd = env->fpus & ~(7 << 11);
     swd |= (env->fpstt & 7) << 11;
@@ -21,92 +31,251 @@ void x86_cpu_xsave_all_areas(X86CPU *cpu, X86XSaveArea *buf)
     for (i = 0; i < 8; ++i) {
         twd |= (!env->fptags[i]) << i;
     }
-    xsave->legacy.fcw = cwd;
-    xsave->legacy.fsw = swd;
-    xsave->legacy.ftw = twd;
-    xsave->legacy.fpop = env->fpop;
-    xsave->legacy.fpip = env->fpip;
-    xsave->legacy.fpdp = env->fpdp;
-    memcpy(&xsave->legacy.fpregs, env->fpregs,
-            sizeof env->fpregs);
-    xsave->legacy.mxcsr = env->mxcsr;
-    xsave->header.xstate_bv = env->xstate_bv;
-    memcpy(&xsave->bndreg_state.bnd_regs, env->bnd_regs,
-            sizeof env->bnd_regs);
-    xsave->bndcsr_state.bndcsr = env->bndcs_regs;
-    memcpy(&xsave->opmask_state.opmask_regs, env->opmask_regs,
-            sizeof env->opmask_regs);
+    legacy->fcw = cwd;
+    legacy->fsw = swd;
+    legacy->ftw = twd;
+    legacy->fpop = env->fpop;
+    legacy->fpip = env->fpip;
+    legacy->fpdp = env->fpdp;
+    memcpy(&legacy->fpregs, env->fpregs,
+           sizeof(env->fpregs));
+    legacy->mxcsr = env->mxcsr;
 
     for (i = 0; i < CPU_NB_REGS; i++) {
-        uint8_t *xmm = xsave->legacy.xmm_regs[i];
-        uint8_t *ymmh = xsave->avx_state.ymmh[i];
-        uint8_t *zmmh = xsave->zmm_hi256_state.zmm_hi256[i];
+        uint8_t *xmm = legacy->xmm_regs[i];
+
         stq_p(xmm,     env->xmm_regs[i].ZMM_Q(0));
-        stq_p(xmm+8,   env->xmm_regs[i].ZMM_Q(1));
-        stq_p(ymmh,    env->xmm_regs[i].ZMM_Q(2));
-        stq_p(ymmh+8,  env->xmm_regs[i].ZMM_Q(3));
-        stq_p(zmmh,    env->xmm_regs[i].ZMM_Q(4));
-        stq_p(zmmh+8,  env->xmm_regs[i].ZMM_Q(5));
-        stq_p(zmmh+16, env->xmm_regs[i].ZMM_Q(6));
-        stq_p(zmmh+24, env->xmm_regs[i].ZMM_Q(7));
+        stq_p(xmm + 8, env->xmm_regs[i].ZMM_Q(1));
+    }
+
+    header->xstate_bv = env->xstate_bv;
+
+    e = &x86_ext_save_areas[XSTATE_YMM_BIT];
+    if (e->size && e->offset) {
+        XSaveAVX *avx;
+
+        avx = (XSaveAVX *)(raw + e->offset);
+
+        for (i = 0; i < CPU_NB_REGS; i++) {
+            uint8_t *ymmh = avx->ymmh[i];
+
+            stq_p(ymmh,     env->xmm_regs[i].ZMM_Q(2));
+            stq_p(ymmh + 8, env->xmm_regs[i].ZMM_Q(3));
+        }
+    }
+
+    e = &x86_ext_save_areas[XSTATE_BNDREGS_BIT];
+    if (e->size && e->offset) {
+        XSaveBNDREG *bndreg;
+        XSaveBNDCSR *bndcsr;
+
+        f = &x86_ext_save_areas[XSTATE_BNDCSR_BIT];
+        assert(f->size);
+        assert(f->offset);
+
+        bndreg = (XSaveBNDREG *)(raw + e->offset);
+        bndcsr = (XSaveBNDCSR *)(raw + f->offset);
+
+        memcpy(&bndreg->bnd_regs, env->bnd_regs,
+               sizeof(env->bnd_regs));
+        bndcsr->bndcsr = env->bndcs_regs;
     }
 
+    e = &x86_ext_save_areas[XSTATE_OPMASK_BIT];
+    if (e->size && e->offset) {
+        XSaveOpmask *opmask;
+        XSaveZMM_Hi256 *zmm_hi256;
 #ifdef TARGET_X86_64
-    memcpy(&xsave->hi16_zmm_state.hi16_zmm, &env->xmm_regs[16],
-            16 * sizeof env->xmm_regs[16]);
-    memcpy(&xsave->pkru_state, &env->pkru, sizeof env->pkru);
+        XSaveHi16_ZMM *hi16_zmm;
 #endif
 
+        f = &x86_ext_save_areas[XSTATE_ZMM_Hi256_BIT];
+        assert(f->size);
+        assert(f->offset);
+
+        opmask = (XSaveOpmask *)(raw + e->offset);
+        zmm_hi256 = (XSaveZMM_Hi256 *)(raw + f->offset);
+
+        memcpy(&opmask->opmask_regs, env->opmask_regs,
+               sizeof(env->opmask_regs));
+
+        for (i = 0; i < CPU_NB_REGS; i++) {
+            uint8_t *zmmh = zmm_hi256->zmm_hi256[i];
+
+            stq_p(zmmh,      env->xmm_regs[i].ZMM_Q(4));
+            stq_p(zmmh + 8,  env->xmm_regs[i].ZMM_Q(5));
+            stq_p(zmmh + 16, env->xmm_regs[i].ZMM_Q(6));
+            stq_p(zmmh + 24, env->xmm_regs[i].ZMM_Q(7));
+        }
+
+#ifdef TARGET_X86_64
+        f = &x86_ext_save_areas[XSTATE_Hi16_ZMM_BIT];
+        assert(f->size);
+        assert(f->offset);
+
+        hi16_zmm = (XSaveHi16_ZMM *)(raw + f->offset);
+
+        memcpy(&hi16_zmm->hi16_zmm, &env->xmm_regs[16],
+               16 * sizeof(env->xmm_regs[16]));
+#endif
+    }
+
+#ifdef TARGET_X86_64
+    e = &x86_ext_save_areas[XSTATE_PKRU_BIT];
+    if (e->size && e->offset) {
+        XSavePKRU *pkru = (XSavePKRU *)(raw + e->offset);
+
+        memcpy(pkru, &env->pkru, sizeof(env->pkru));
+    }
+
+    e = &x86_ext_save_areas[XSTATE_XTILE_CFG_BIT];
+    if (e->size && e->offset) {
+        XSaveXTILECFG *tilecfg = (XSaveXTILECFG *)(raw + e->offset);
+
+        memcpy(tilecfg, &env->xtilecfg, sizeof(env->xtilecfg));
+    }
+
+    e = &x86_ext_save_areas[XSTATE_XTILE_DATA_BIT];
+    if (e->size && e->offset && buflen >= e->size + e->offset) {
+        XSaveXTILEDATA *tiledata = (XSaveXTILEDATA *)(raw + e->offset);
+
+        memcpy(tiledata, &env->xtiledata, sizeof(env->xtiledata));
+    }
+#endif
 }
 
-void x86_cpu_xrstor_all_areas(X86CPU *cpu, const X86XSaveArea *buf)
+void x86_cpu_xrstor_all_areas(X86CPU *cpu, const void *buf, uint32_t buflen)
 {
-
     CPUX86State *env = &cpu->env;
-    const X86XSaveArea *xsave = buf;
-
+    const ExtSaveArea *e, *f, *g;
+    const uint8_t *raw = buf;
     int i;
+
+    const X86LegacyXSaveArea *legacy;
+    const X86XSaveHeader *header;
     uint16_t cwd, swd, twd;
-    cwd = xsave->legacy.fcw;
-    swd = xsave->legacy.fsw;
-    twd = xsave->legacy.ftw;
-    env->fpop = xsave->legacy.fpop;
+
+    e = &x86_ext_save_areas[XSTATE_FP_BIT];
+
+    legacy = (const X86LegacyXSaveArea *)(raw + e->offset);
+    header = (const X86XSaveHeader *)(raw + e->offset + sizeof(*legacy));
+
+    cwd = legacy->fcw;
+    swd = legacy->fsw;
+    twd = legacy->ftw;
+    env->fpop = legacy->fpop;
     env->fpstt = (swd >> 11) & 7;
     env->fpus = swd;
     env->fpuc = cwd;
     for (i = 0; i < 8; ++i) {
         env->fptags[i] = !((twd >> i) & 1);
     }
-    env->fpip = xsave->legacy.fpip;
-    env->fpdp = xsave->legacy.fpdp;
-    env->mxcsr = xsave->legacy.mxcsr;
-    memcpy(env->fpregs, &xsave->legacy.fpregs,
-            sizeof env->fpregs);
-    env->xstate_bv = xsave->header.xstate_bv;
-    memcpy(env->bnd_regs, &xsave->bndreg_state.bnd_regs,
-            sizeof env->bnd_regs);
-    env->bndcs_regs = xsave->bndcsr_state.bndcsr;
-    memcpy(env->opmask_regs, &xsave->opmask_state.opmask_regs,
-            sizeof env->opmask_regs);
+    env->fpip = legacy->fpip;
+    env->fpdp = legacy->fpdp;
+    env->mxcsr = legacy->mxcsr;
+    memcpy(env->fpregs, &legacy->fpregs,
+           sizeof(env->fpregs));
 
     for (i = 0; i < CPU_NB_REGS; i++) {
-        const uint8_t *xmm = xsave->legacy.xmm_regs[i];
-        const uint8_t *ymmh = xsave->avx_state.ymmh[i];
-        const uint8_t *zmmh = xsave->zmm_hi256_state.zmm_hi256[i];
+        const uint8_t *xmm = legacy->xmm_regs[i];
+
         env->xmm_regs[i].ZMM_Q(0) = ldq_p(xmm);
-        env->xmm_regs[i].ZMM_Q(1) = ldq_p(xmm+8);
-        env->xmm_regs[i].ZMM_Q(2) = ldq_p(ymmh);
-        env->xmm_regs[i].ZMM_Q(3) = ldq_p(ymmh+8);
-        env->xmm_regs[i].ZMM_Q(4) = ldq_p(zmmh);
-        env->xmm_regs[i].ZMM_Q(5) = ldq_p(zmmh+8);
-        env->xmm_regs[i].ZMM_Q(6) = ldq_p(zmmh+16);
-        env->xmm_regs[i].ZMM_Q(7) = ldq_p(zmmh+24);
+        env->xmm_regs[i].ZMM_Q(1) = ldq_p(xmm + 8);
+    }
+
+    env->xstate_bv = header->xstate_bv;
+
+    e = &x86_ext_save_areas[XSTATE_YMM_BIT];
+    if (e->size && e->offset) {
+        const XSaveAVX *avx;
+
+        avx = (const XSaveAVX *)(raw + e->offset);
+        for (i = 0; i < CPU_NB_REGS; i++) {
+            const uint8_t *ymmh = avx->ymmh[i];
+
+            env->xmm_regs[i].ZMM_Q(2) = ldq_p(ymmh);
+            env->xmm_regs[i].ZMM_Q(3) = ldq_p(ymmh + 8);
+        }
+    }
+
+    e = &x86_ext_save_areas[XSTATE_BNDREGS_BIT];
+    if (e->size && e->offset) {
+        const XSaveBNDREG *bndreg;
+        const XSaveBNDCSR *bndcsr;
+
+        f = &x86_ext_save_areas[XSTATE_BNDCSR_BIT];
+        assert(f->size);
+        assert(f->offset);
+
+        bndreg = (const XSaveBNDREG *)(raw + e->offset);
+        bndcsr = (const XSaveBNDCSR *)(raw + f->offset);
+
+        memcpy(env->bnd_regs, &bndreg->bnd_regs,
+               sizeof(env->bnd_regs));
+        env->bndcs_regs = bndcsr->bndcsr;
     }
 
+    e = &x86_ext_save_areas[XSTATE_OPMASK_BIT];
+    if (e->size && e->offset) {
+        const XSaveOpmask *opmask;
+        const XSaveZMM_Hi256 *zmm_hi256;
 #ifdef TARGET_X86_64
-    memcpy(&env->xmm_regs[16], &xsave->hi16_zmm_state.hi16_zmm,
-           16 * sizeof env->xmm_regs[16]);
-    memcpy(&env->pkru, &xsave->pkru_state, sizeof env->pkru);
+        const XSaveHi16_ZMM *hi16_zmm;
 #endif
 
+        f = &x86_ext_save_areas[XSTATE_ZMM_Hi256_BIT];
+        assert(f->size);
+        assert(f->offset);
+
+        g = &x86_ext_save_areas[XSTATE_Hi16_ZMM_BIT];
+        assert(g->size);
+        assert(g->offset);
+
+        opmask = (const XSaveOpmask *)(raw + e->offset);
+        zmm_hi256 = (const XSaveZMM_Hi256 *)(raw + f->offset);
+#ifdef TARGET_X86_64
+        hi16_zmm = (const XSaveHi16_ZMM *)(raw + g->offset);
+#endif
+
+        memcpy(env->opmask_regs, &opmask->opmask_regs,
+               sizeof(env->opmask_regs));
+
+        for (i = 0; i < CPU_NB_REGS; i++) {
+            const uint8_t *zmmh = zmm_hi256->zmm_hi256[i];
+
+            env->xmm_regs[i].ZMM_Q(4) = ldq_p(zmmh);
+            env->xmm_regs[i].ZMM_Q(5) = ldq_p(zmmh + 8);
+            env->xmm_regs[i].ZMM_Q(6) = ldq_p(zmmh + 16);
+            env->xmm_regs[i].ZMM_Q(7) = ldq_p(zmmh + 24);
+        }
+
+#ifdef TARGET_X86_64
+        memcpy(&env->xmm_regs[16], &hi16_zmm->hi16_zmm,
+               16 * sizeof(env->xmm_regs[16]));
+#endif
+    }
+
+#ifdef TARGET_X86_64
+    e = &x86_ext_save_areas[XSTATE_PKRU_BIT];
+    if (e->size && e->offset) {
+        const XSavePKRU *pkru;
+
+        pkru = (const XSavePKRU *)(raw + e->offset);
+        memcpy(&env->pkru, pkru, sizeof(env->pkru));
+    }
+
+    e = &x86_ext_save_areas[XSTATE_XTILE_CFG_BIT];
+    if (e->size && e->offset) {
+        const XSaveXTILECFG *tilecfg = (const XSaveXTILECFG *)(raw + e->offset);
+
+        memcpy(&env->xtilecfg, tilecfg, sizeof(env->xtilecfg));
+    }
+
+    e = &x86_ext_save_areas[XSTATE_XTILE_DATA_BIT];
+    if (e->size && e->offset && buflen >= e->size + e->offset) {
+        const XSaveXTILEDATA *tiledata = (const XSaveXTILEDATA *)(raw + e->offset);
+
+        memcpy(&env->xtiledata, tiledata, sizeof(env->xtiledata));
+    }
+#endif
 }
diff --git a/qemu/target/m68k/cpu.c b/qemu/target/m68k/cpu.c
index 6b636b80eb..e9707a28be 100644
--- a/qemu/target/m68k/cpu.c
+++ b/qemu/target/m68k/cpu.c
@@ -37,7 +37,7 @@ static bool m68k_cpu_has_work(CPUState *cs)
 
 static void m68k_set_feature(CPUM68KState *env, int feature)
 {
-    env->features |= (1u << feature);
+    env->features |= (1ull << feature);
 }
 
 static void m68k_cpu_reset(CPUState *dev)
@@ -71,6 +71,7 @@ static void m5206_cpu_initfn(CPUState *obj)
     CPUM68KState *env = &cpu->env;
 
     m68k_set_feature(env, M68K_FEATURE_CF_ISA_A);
+    m68k_set_feature(env, M68K_FEATURE_MOVEFROMSR_PRIV);
 }
 
 static void m68000_cpu_initfn(CPUState *obj)
@@ -84,12 +85,27 @@ static void m68000_cpu_initfn(CPUState *obj)
     m68k_set_feature(env, M68K_FEATURE_MOVEP);
 }
 
+static void m68010_cpu_initfn(CPUState *obj)
+{
+    M68kCPU *cpu = M68K_CPU(obj);
+    CPUM68KState *env = &cpu->env;
+
+    m68000_cpu_initfn(obj);
+    m68k_set_feature(env, M68K_FEATURE_M68010);
+    m68k_set_feature(env, M68K_FEATURE_BKPT);
+    m68k_set_feature(env, M68K_FEATURE_RTD);
+    m68k_set_feature(env, M68K_FEATURE_MOVEC);
+    m68k_set_feature(env, M68K_FEATURE_MOVEFROMSR_PRIV);
+}
+
 /* common features for 68020, 68030 and 68040 */
 static void m680x0_cpu_common(CPUM68KState *env)
 {
     m68k_set_feature(env, M68K_FEATURE_M68000);
     m68k_set_feature(env, M68K_FEATURE_USP);
+    m68k_set_feature(env, M68K_FEATURE_MSP);
     m68k_set_feature(env, M68K_FEATURE_WORD_INDEX);
+    m68k_set_feature(env, M68K_FEATURE_UNALIGNED_DATA);
     m68k_set_feature(env, M68K_FEATURE_QUAD_MULDIV);
     m68k_set_feature(env, M68K_FEATURE_BRAL);
     m68k_set_feature(env, M68K_FEATURE_BCCL);
@@ -103,6 +119,9 @@ static void m680x0_cpu_common(CPUM68KState *env)
     m68k_set_feature(env, M68K_FEATURE_RTD);
     m68k_set_feature(env, M68K_FEATURE_CHK2);
     m68k_set_feature(env, M68K_FEATURE_MOVEP);
+    m68k_set_feature(env, M68K_FEATURE_MOVEC);
+    m68k_set_feature(env, M68K_FEATURE_TRAPCC);
+    m68k_set_feature(env, M68K_FEATURE_MOVEFROMSR_PRIV);
 }
 
 static void m68020_cpu_initfn(CPUState *obj)
@@ -139,7 +158,9 @@ static void m68060_cpu_initfn(CPUState *obj)
 
     m68k_set_feature(env, M68K_FEATURE_M68000);
     m68k_set_feature(env, M68K_FEATURE_USP);
+    m68k_set_feature(env, M68K_FEATURE_MSP);
     m68k_set_feature(env, M68K_FEATURE_WORD_INDEX);
+    m68k_set_feature(env, M68K_FEATURE_UNALIGNED_DATA);
     m68k_set_feature(env, M68K_FEATURE_BRAL);
     m68k_set_feature(env, M68K_FEATURE_BCCL);
     m68k_set_feature(env, M68K_FEATURE_BITFIELD);
@@ -152,6 +173,9 @@ static void m68060_cpu_initfn(CPUState *obj)
     m68k_set_feature(env, M68K_FEATURE_RTD);
     m68k_set_feature(env, M68K_FEATURE_CHK2);
     m68k_set_feature(env, M68K_FEATURE_M68060);
+    m68k_set_feature(env, M68K_FEATURE_MOVEC);
+    m68k_set_feature(env, M68K_FEATURE_TRAPCC);
+    m68k_set_feature(env, M68K_FEATURE_MOVEFROMSR_PRIV);
 }
 
 static void m5208_cpu_initfn(CPUState *obj)
@@ -164,6 +188,7 @@ static void m5208_cpu_initfn(CPUState *obj)
     m68k_set_feature(env, M68K_FEATURE_BRAL);
     m68k_set_feature(env, M68K_FEATURE_CF_EMAC);
     m68k_set_feature(env, M68K_FEATURE_USP);
+    m68k_set_feature(env, M68K_FEATURE_MOVEFROMSR_PRIV);
 }
 
 static void cfv4e_cpu_initfn(CPUState *obj)
@@ -177,6 +202,7 @@ static void cfv4e_cpu_initfn(CPUState *obj)
     m68k_set_feature(env, M68K_FEATURE_CF_FPU);
     m68k_set_feature(env, M68K_FEATURE_CF_EMAC);
     m68k_set_feature(env, M68K_FEATURE_USP);
+    m68k_set_feature(env, M68K_FEATURE_MOVEFROMSR_PRIV);
 }
 
 static void any_cpu_initfn(CPUState *obj)
@@ -198,6 +224,8 @@ static void any_cpu_initfn(CPUState *obj)
     m68k_set_feature(env, M68K_FEATURE_USP);
     m68k_set_feature(env, M68K_FEATURE_EXT_FULL);
     m68k_set_feature(env, M68K_FEATURE_WORD_INDEX);
+    m68k_set_feature(env, M68K_FEATURE_MOVEC);
+    m68k_set_feature(env, M68K_FEATURE_TRAPCC);
 }
 
 static void m68k_cpu_realizefn(CPUState *dev)
@@ -248,15 +276,16 @@ struct M68kCPUInfo {
 };
 
 static struct M68kCPUInfo m68k_cpus_type_infos[] = {
+    { "m5206", m5206_cpu_initfn },
     { "m68000", m68000_cpu_initfn },
     { "m68020", m68020_cpu_initfn },
     { "m68030", m68030_cpu_initfn },
     { "m68040", m68040_cpu_initfn },
     { "m68060", m68060_cpu_initfn },
-    { "m5206", m5206_cpu_initfn },
     { "m5208", m5208_cpu_initfn },
     { "cfv4e", cfv4e_cpu_initfn },
     { "any", any_cpu_initfn },
+    { "m68010", m68010_cpu_initfn },
 };
 
 M68kCPU *cpu_m68k_init(struct uc_struct *uc)
diff --git a/qemu/target/m68k/cpu.h b/qemu/target/m68k/cpu.h
index cc2c50e463..008c2f3743 100644
--- a/qemu/target/m68k/cpu.h
+++ b/qemu/target/m68k/cpu.h
@@ -145,7 +145,7 @@ typedef struct CPUM68KState {
     int end_reset_fields;
 
     /* Fields from here on are preserved across CPU reset. */
-    uint32_t features;
+    uint64_t features;
 
     // translate opcode
     void* opcode_table[65536];
@@ -465,6 +465,7 @@ void do_m68k_semihosting(CPUM68KState *env, int nr);
 
 enum m68k_features {
     M68K_FEATURE_M68000,
+    M68K_FEATURE_M68010,
     M68K_FEATURE_M68020,
     M68K_FEATURE_M68030,
     M68K_FEATURE_M68040,
@@ -478,6 +479,7 @@ enum m68k_features {
     M68K_FEATURE_CF_EMAC,
     M68K_FEATURE_CF_EMAC_B, /* Revision B EMAC (dual accumulate).  */
     M68K_FEATURE_USP, /* User Stack Pointer.  (ISA A+, B or C).  */
+    M68K_FEATURE_MSP, /* Master Stack Pointer.  (68020+).  */
     M68K_FEATURE_EXT_FULL, /* 68020+ full extension word.  */
     M68K_FEATURE_WORD_INDEX, /* word sized address index registers.  */
     M68K_FEATURE_SCALED_INDEX, /* scaled address index registers.  */
@@ -491,11 +493,15 @@ enum m68k_features {
     M68K_FEATURE_RTD,
     M68K_FEATURE_CHK2,
     M68K_FEATURE_MOVEP,
+    M68K_FEATURE_MOVEC,
+    M68K_FEATURE_UNALIGNED_DATA, /* Unaligned data accesses.  */
+    M68K_FEATURE_TRAPCC,
+    M68K_FEATURE_MOVEFROMSR_PRIV,
 };
 
 static inline int m68k_feature(CPUM68KState *env, int feature)
 {
-    return (env->features & (1u << feature)) != 0;
+    return (env->features & (1ull << feature)) != 0;
 }
 
 void m68k_cpu_list(void);
diff --git a/qemu/target/m68k/fpu_helper.c b/qemu/target/m68k/fpu_helper.c
index 3f544a0572..22211daea9 100644
--- a/qemu/target/m68k/fpu_helper.c
+++ b/qemu/target/m68k/fpu_helper.c
@@ -94,13 +94,13 @@ static void m68k_restore_precision_mode(CPUM68KState *env)
 {
     switch (env->fpcr & FPCR_PREC_MASK) {
     case FPCR_PREC_X: /* extended */
-        set_floatx80_rounding_precision(80, &env->fp_status);
+        set_floatx80_rounding_precision(floatx80_precision_x, &env->fp_status);
         break;
     case FPCR_PREC_S: /* single */
-        set_floatx80_rounding_precision(32, &env->fp_status);
+        set_floatx80_rounding_precision(floatx80_precision_s, &env->fp_status);
         break;
     case FPCR_PREC_D: /* double */
-        set_floatx80_rounding_precision(64, &env->fp_status);
+        set_floatx80_rounding_precision(floatx80_precision_d, &env->fp_status);
         break;
     case FPCR_PREC_U: /* undefined */
     default:
@@ -111,9 +111,9 @@ static void m68k_restore_precision_mode(CPUM68KState *env)
 static void cf_restore_precision_mode(CPUM68KState *env)
 {
     if (env->fpcr & FPCR_PREC_S) { /* single */
-        set_floatx80_rounding_precision(32, &env->fp_status);
+        set_floatx80_rounding_precision(floatx80_precision_s, &env->fp_status);
     } else { /* double */
-        set_floatx80_rounding_precision(64, &env->fp_status);
+        set_floatx80_rounding_precision(floatx80_precision_d, &env->fp_status);
     }
 }
 
@@ -135,10 +135,8 @@ static void restore_rounding_mode(CPUM68KState *env)
     }
 }
 
-void cpu_m68k_set_fpcr(CPUM68KState *env, uint32_t val)
+void cpu_m68k_restore_fp_status(CPUM68KState *env)
 {
-    env->fpcr = val & 0xffff;
-
     if (m68k_feature(env, M68K_FEATURE_CF_FPU)) {
         cf_restore_precision_mode(env);
     } else {
@@ -147,9 +145,15 @@ void cpu_m68k_set_fpcr(CPUM68KState *env, uint32_t val)
     restore_rounding_mode(env);
 }
 
+void cpu_m68k_set_fpcr(CPUM68KState *env, uint32_t val)
+{
+    env->fpcr = val & 0xffff;
+    cpu_m68k_restore_fp_status(env);
+}
+
 void HELPER(fitrunc)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    int rounding_mode = get_float_rounding_mode(&env->fp_status);
+    FloatRoundMode rounding_mode = get_float_rounding_mode(&env->fp_status);
     set_float_rounding_mode(float_round_to_zero, &env->fp_status);
     res->d = floatx80_round_to_int(val->d, &env->fp_status);
     set_float_rounding_mode(rounding_mode, &env->fp_status);
@@ -162,8 +166,8 @@ void HELPER(set_fpcr)(CPUM68KState *env, uint32_t val)
 
 #define PREC_BEGIN(prec)                                        \
     do {                                                        \
-        int old;                                                \
-        old = get_floatx80_rounding_precision(&env->fp_status); \
+        FloatX80RoundPrec old =                                 \
+            get_floatx80_rounding_precision(&env->fp_status);   \
         set_floatx80_rounding_precision(prec, &env->fp_status)  \
 
 #define PREC_END()                                              \
@@ -172,14 +176,14 @@ void HELPER(set_fpcr)(CPUM68KState *env, uint32_t val)
 
 void HELPER(fsround)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_round(val->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fdround)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_round(val->d, &env->fp_status);
     PREC_END();
 }
@@ -191,14 +195,14 @@ void HELPER(fsqrt)(CPUM68KState *env, FPReg *res, FPReg *val)
 
 void HELPER(fssqrt)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_sqrt(val->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fdsqrt)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_sqrt(val->d, &env->fp_status);
     PREC_END();
 }
@@ -210,14 +214,14 @@ void HELPER(fabs)(CPUM68KState *env, FPReg *res, FPReg *val)
 
 void HELPER(fsabs)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_round(floatx80_abs(val->d), &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fdabs)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_round(floatx80_abs(val->d), &env->fp_status);
     PREC_END();
 }
@@ -229,14 +233,14 @@ void HELPER(fneg)(CPUM68KState *env, FPReg *res, FPReg *val)
 
 void HELPER(fsneg)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_round(floatx80_chs(val->d), &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fdneg)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_round(floatx80_chs(val->d), &env->fp_status);
     PREC_END();
 }
@@ -248,14 +252,14 @@ void HELPER(fadd)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 
 void HELPER(fsadd)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_add(val0->d, val1->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fdadd)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_add(val0->d, val1->d, &env->fp_status);
     PREC_END();
 }
@@ -267,14 +271,14 @@ void HELPER(fsub)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 
 void HELPER(fssub)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_sub(val1->d, val0->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fdsub)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_sub(val1->d, val0->d, &env->fp_status);
     PREC_END();
 }
@@ -286,24 +290,24 @@ void HELPER(fmul)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 
 void HELPER(fsmul)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_mul(val0->d, val1->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fdmul)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_mul(val0->d, val1->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fsglmul)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    int rounding_mode = get_float_rounding_mode(&env->fp_status);
+    FloatRoundMode rounding_mode = get_float_rounding_mode(&env->fp_status);
     floatx80 a, b;
 
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     set_float_rounding_mode(float_round_to_zero, &env->fp_status);
     a = floatx80_round(val0->d, &env->fp_status);
     b = floatx80_round(val1->d, &env->fp_status);
@@ -319,24 +323,24 @@ void HELPER(fdiv)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 
 void HELPER(fsdiv)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     res->d = floatx80_div(val1->d, val0->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fddiv)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    PREC_BEGIN(64);
+    PREC_BEGIN(floatx80_precision_d);
     res->d = floatx80_div(val1->d, val0->d, &env->fp_status);
     PREC_END();
 }
 
 void HELPER(fsgldiv)(CPUM68KState *env, FPReg *res, FPReg *val0, FPReg *val1)
 {
-    int rounding_mode = get_float_rounding_mode(&env->fp_status);
+    FloatRoundMode rounding_mode = get_float_rounding_mode(&env->fp_status);
     floatx80 a, b;
 
-    PREC_BEGIN(32);
+    PREC_BEGIN(floatx80_precision_s);
     set_float_rounding_mode(float_round_to_zero, &env->fp_status);
     a = floatx80_round(val1->d, &env->fp_status);
     b = floatx80_round(val0->d, &env->fp_status);
@@ -642,6 +646,11 @@ void HELPER(fatanh)(CPUM68KState *env, FPReg *res, FPReg *val)
     res->d = floatx80_atanh(val->d, &env->fp_status);
 }
 
+void HELPER(fetoxm1)(CPUM68KState *env, FPReg *res, FPReg *val)
+{
+    res->d = floatx80_etoxm1(val->d, &env->fp_status);
+}
+
 void HELPER(ftanh)(CPUM68KState *env, FPReg *res, FPReg *val)
 {
     res->d = floatx80_tanh(val->d, &env->fp_status);
diff --git a/qemu/target/m68k/helper.c b/qemu/target/m68k/helper.c
index 98463eb0c4..e773246ae6 100644
--- a/qemu/target/m68k/helper.c
+++ b/qemu/target/m68k/helper.c
@@ -26,6 +26,14 @@
 
 #define SIGNBIT (1u << 31)
 
+static void raise_exception_ra(CPUM68KState *env, int tt, uintptr_t raddr)
+{
+    CPUState *cs = env_cpu(env);
+
+    cs->exception_index = tt;
+    cpu_loop_exit_restore(cs, raddr);
+}
+
 void HELPER(cf_movec_to)(CPUM68KState *env, uint32_t reg, uint32_t val)
 {
     switch (reg) {
@@ -53,13 +61,15 @@ void HELPER(cf_movec_to)(CPUM68KState *env, uint32_t reg, uint32_t val)
 void HELPER(m68k_movec_to)(CPUM68KState *env, uint32_t reg, uint32_t val)
 {
     switch (reg) {
-    /* MC680[1234]0 */
+    /* MC680[12346]0 */
     case M68K_CR_SFC:
         env->sfc = val & 7;
         return;
+    /* MC680[12346]0 */
     case M68K_CR_DFC:
         env->dfc = val & 7;
         return;
+    /* MC680[12346]0 */
     case M68K_CR_VBR:
         env->vbr = val;
         return;
@@ -73,90 +83,194 @@ void HELPER(m68k_movec_to)(CPUM68KState *env, uint32_t reg, uint32_t val)
             env->cacr = val & 0x80008000;
         } else if (m68k_feature(env, M68K_FEATURE_M68060)) {
             env->cacr = val & 0xf8e0e000;
+        } else {
+            break;
         }
         m68k_switch_sp(env);
         return;
-    /* MC680[34]0 */
+    /* MC680[46]0 */
     case M68K_CR_TC:
-        env->mmu.tcr = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040) ||
+            m68k_feature(env, M68K_FEATURE_M68060)) {
+            env->mmu.tcr = val;
+            return;
+        }
+        break;
+    /* MC68040 */
     case M68K_CR_MMUSR:
-        env->mmu.mmusr = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            env->mmu.mmusr = val;
+            return;
+        }
+        break;
+    /* MC680[46]0 */
     case M68K_CR_SRP:
-        env->mmu.srp = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040) ||
+            m68k_feature(env, M68K_FEATURE_M68060)) {
+            env->mmu.srp = val;
+            return;
+        }
+        break;
+    /* MC680[46]0 */
     case M68K_CR_URP:
-        env->mmu.urp = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040) ||
+            m68k_feature(env, M68K_FEATURE_M68060)) {
+            env->mmu.urp = val;
+            return;
+        }
+        break;
+    /* MC680[12346]0 */
     case M68K_CR_USP:
         env->sp[M68K_USP] = val;
         return;
+    /* MC680[234]0 */
     case M68K_CR_MSP:
-        env->sp[M68K_SSP] = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68020) ||
+            m68k_feature(env, M68K_FEATURE_M68030) ||
+            m68k_feature(env, M68K_FEATURE_M68040)) {
+            env->sp[M68K_SSP] = val;
+            return;
+        }
+        break;
+    /* MC680[234]0 */
     case M68K_CR_ISP:
-        env->sp[M68K_ISP] = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68020) ||
+            m68k_feature(env, M68K_FEATURE_M68030) ||
+            m68k_feature(env, M68K_FEATURE_M68040)) {
+            env->sp[M68K_ISP] = val;
+            return;
+        }
+        break;
     /* MC68040/MC68LC040 */
     case M68K_CR_ITT0:
-        env->mmu.ttr[M68K_ITTR0] = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            env->mmu.ttr[M68K_ITTR0] = val;
+            return;
+        }
+        break;
     case M68K_CR_ITT1:
-         env->mmu.ttr[M68K_ITTR1] = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            env->mmu.ttr[M68K_ITTR1] = val;
+            return;
+        }
+        break;
     case M68K_CR_DTT0:
-        env->mmu.ttr[M68K_DTTR0] = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            env->mmu.ttr[M68K_DTTR0] = val;
+            return;
+        }
+        break;
     case M68K_CR_DTT1:
-        env->mmu.ttr[M68K_DTTR1] = val;
-        return;
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            env->mmu.ttr[M68K_DTTR1] = val;
+            return;
+        }
+        break;
+    case M68K_CR_CAAR:
+        cpu_abort(env_cpu(env),
+                  "Unimplemented control register write 0x%x = 0x%x\n",
+                  reg, val);
     }
-    cpu_abort(env_cpu(env),
-              "Unimplemented control register write 0x%x = 0x%x\n",
-              reg, val);
+
+    raise_exception_ra(env, EXCP_ILLEGAL, 0);
+    return;
 }
 
 uint32_t HELPER(m68k_movec_from)(CPUM68KState *env, uint32_t reg)
 {
     switch (reg) {
-    /* MC680[1234]0 */
+    /* MC680[12346]0 */
     case M68K_CR_SFC:
         return env->sfc;
+    /* MC680[12346]0 */
     case M68K_CR_DFC:
         return env->dfc;
+    /* MC680[12346]0 */
     case M68K_CR_VBR:
         return env->vbr;
-    /* MC680[234]0 */
+    /* MC680[2346]0 */
     case M68K_CR_CACR:
-        return env->cacr;
-    /* MC680[34]0 */
+        if (m68k_feature(env, M68K_FEATURE_M68020) ||
+            m68k_feature(env, M68K_FEATURE_M68030) ||
+            m68k_feature(env, M68K_FEATURE_M68040) ||
+            m68k_feature(env, M68K_FEATURE_M68060)) {
+            return env->cacr;
+        }
+        break;
+    /* MC680[46]0 */
     case M68K_CR_TC:
-        return env->mmu.tcr;
+        if (m68k_feature(env, M68K_FEATURE_M68040) ||
+            m68k_feature(env, M68K_FEATURE_M68060)) {
+            return env->mmu.tcr;
+        }
+        break;
+    /* MC68040 */
     case M68K_CR_MMUSR:
-        return env->mmu.mmusr;
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            return env->mmu.mmusr;
+        }
+        break;
+    /* MC680[46]0 */
     case M68K_CR_SRP:
-        return env->mmu.srp;
+        if (m68k_feature(env, M68K_FEATURE_M68040) ||
+            m68k_feature(env, M68K_FEATURE_M68060)) {
+            return env->mmu.srp;
+        }
+        break;
+    /* MC68040/MC68LC040 */
+    case M68K_CR_URP:
+        if (m68k_feature(env, M68K_FEATURE_M68040) ||
+            m68k_feature(env, M68K_FEATURE_M68060)) {
+            return env->mmu.urp;
+        }
+        break;
+    /* MC680[46]0 */
     case M68K_CR_USP:
         return env->sp[M68K_USP];
+    /* MC680[234]0 */
     case M68K_CR_MSP:
-        return env->sp[M68K_SSP];
+        if (m68k_feature(env, M68K_FEATURE_M68020) ||
+            m68k_feature(env, M68K_FEATURE_M68030) ||
+            m68k_feature(env, M68K_FEATURE_M68040)) {
+            return env->sp[M68K_SSP];
+        }
+        break;
+    /* MC680[234]0 */
     case M68K_CR_ISP:
-        return env->sp[M68K_ISP];
+        if (m68k_feature(env, M68K_FEATURE_M68020) ||
+            m68k_feature(env, M68K_FEATURE_M68030) ||
+            m68k_feature(env, M68K_FEATURE_M68040)) {
+            return env->sp[M68K_ISP];
+        }
+        break;
     /* MC68040/MC68LC040 */
-    case M68K_CR_URP:
-        return env->mmu.urp;
     case M68K_CR_ITT0:
-        return env->mmu.ttr[M68K_ITTR0];
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            return env->mmu.ttr[M68K_ITTR0];
+        }
+        break;
     case M68K_CR_ITT1:
-        return env->mmu.ttr[M68K_ITTR1];
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            return env->mmu.ttr[M68K_ITTR1];
+        }
+        break;
     case M68K_CR_DTT0:
-        return env->mmu.ttr[M68K_DTTR0];
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            return env->mmu.ttr[M68K_DTTR0];
+        }
+        break;
     case M68K_CR_DTT1:
-        return env->mmu.ttr[M68K_DTTR1];
+        if (m68k_feature(env, M68K_FEATURE_M68040)) {
+            return env->mmu.ttr[M68K_DTTR1];
+        }
+        break;
+    case M68K_CR_CAAR:
+        cpu_abort(env_cpu(env), "Unimplemented control register read 0x%x\n",
+                  reg);
     }
-    cpu_abort(env_cpu(env), "Unimplemented control register read 0x%x\n",
-              reg);
+
+    raise_exception_ra(env, EXCP_ILLEGAL, 0);
+    return 0;
 }
 
 void HELPER(set_macsr)(CPUM68KState *env, uint32_t val)
@@ -200,7 +314,8 @@ void m68k_switch_sp(CPUM68KState *env)
     env->sp[env->current_sp] = env->aregs[7];
     if (m68k_feature(env, M68K_FEATURE_M68000)) {
         if (env->sr & SR_S) {
-            if (env->sr & SR_M) {
+            if (!m68k_feature(env, M68K_FEATURE_MSP) ||
+                env->sr & SR_M) {
                 new_sp = M68K_SSP;
             } else {
                 new_sp = M68K_ISP;
diff --git a/qemu/target/m68k/helper.h b/qemu/target/m68k/helper.h
index 413f88dc65..fea07f143b 100644
--- a/qemu/target/m68k/helper.h
+++ b/qemu/target/m68k/helper.h
@@ -76,6 +76,7 @@ DEF_HELPER_3(flogn, void, env, fp, fp)
 DEF_HELPER_3(flog10, void, env, fp, fp)
 DEF_HELPER_3(flog2, void, env, fp, fp)
 DEF_HELPER_3(fetox, void, env, fp, fp)
+DEF_HELPER_3(fetoxm1, void, env, fp, fp)
 DEF_HELPER_3(ftwotox, void, env, fp, fp)
 DEF_HELPER_3(ftentox, void, env, fp, fp)
 DEF_HELPER_3(ftan, void, env, fp, fp)
diff --git a/qemu/target/m68k/op_helper.c b/qemu/target/m68k/op_helper.c
index 91c504f0e3..5d67a34dee 100644
--- a/qemu/target/m68k/op_helper.c
+++ b/qemu/target/m68k/op_helper.c
@@ -224,7 +224,9 @@ static void m68k_interrupt_all(CPUM68KState *env, int is_hw)
     cpu_m68k_set_sr(env, sr);
     sp = env->aregs[7];
 
-    sp &= ~1;
+    if (!m68k_feature(env, M68K_FEATURE_UNALIGNED_DATA)) {
+        sp &= ~1;
+    }
     if (cs->exception_index == EXCP_ACCESS) {
         if (env->mmu.fault) {
             cpu_abort(cs, "DOUBLE MMU FAULT\n");
@@ -280,10 +282,11 @@ static void m68k_interrupt_all(CPUM68KState *env, int is_hw)
         env->mmu.fault = false;
     } else if (cs->exception_index == EXCP_ADDRESS) {
         do_stack_frame(env, &sp, 2, oldsr, 0, retaddr);
+    } else if (cs->exception_index == EXCP_TRAPCC) {
+        do_stack_frame(env, &sp, 2, oldsr, env->mmu.ar, retaddr);
     } else if (cs->exception_index == EXCP_ILLEGAL ||
                cs->exception_index == EXCP_DIV0 ||
                cs->exception_index == EXCP_CHK ||
-               cs->exception_index == EXCP_TRAPCC ||
                cs->exception_index == EXCP_TRACE) {
         /* FIXME: addr is not only env->pc */
         do_stack_frame(env, &sp, 2, oldsr, env->pc, retaddr);
@@ -294,7 +297,10 @@ static void m68k_interrupt_all(CPUM68KState *env, int is_hw)
         oldsr = sr;
         env->aregs[7] = sp;
         cpu_m68k_set_sr(env, sr &= ~SR_M);
-        sp = env->aregs[7] & ~1;
+        sp = env->aregs[7];
+        if (!m68k_feature(env, M68K_FEATURE_UNALIGNED_DATA)) {
+            sp &= ~1;
+        }
         do_stack_frame(env, &sp, 1, oldsr, 0, retaddr);
     } else {
         do_stack_frame(env, &sp, 0, oldsr, 0, retaddr);
diff --git a/qemu/target/m68k/softfloat.c b/qemu/target/m68k/softfloat.c
index 24c313ed69..02dcc03d15 100644
--- a/qemu/target/m68k/softfloat.c
+++ b/qemu/target/m68k/softfloat.c
@@ -42,89 +42,6 @@ static floatx80 propagateFloatx80NaNOneArg(floatx80 a, float_status *status)
     return a;
 }
 
-/*
- * Returns the modulo remainder of the extended double-precision floating-point
- * value `a' with respect to the corresponding value `b'.
- */
-
-floatx80 floatx80_mod(floatx80 a, floatx80 b, float_status *status)
-{
-    flag aSign, zSign;
-    int32_t aExp, bExp, expDiff;
-    uint64_t aSig0, aSig1, bSig;
-    uint64_t qTemp, term0, term1;
-
-    aSig0 = extractFloatx80Frac(a);
-    aExp = extractFloatx80Exp(a);
-    aSign = extractFloatx80Sign(a);
-    bSig = extractFloatx80Frac(b);
-    bExp = extractFloatx80Exp(b);
-
-    if (aExp == 0x7FFF) {
-        if ((uint64_t) (aSig0 << 1)
-            || ((bExp == 0x7FFF) && (uint64_t) (bSig << 1))) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        goto invalid;
-    }
-    if (bExp == 0x7FFF) {
-        if ((uint64_t) (bSig << 1)) {
-            return propagateFloatx80NaN(a, b, status);
-        }
-        return a;
-    }
-    if (bExp == 0) {
-        if (bSig == 0) {
-        invalid:
-            float_raise(float_flag_invalid, status);
-            return floatx80_default_nan(status);
-        }
-        normalizeFloatx80Subnormal(bSig, &bExp, &bSig);
-    }
-    if (aExp == 0) {
-        if ((uint64_t) (aSig0 << 1) == 0) {
-            return a;
-        }
-        normalizeFloatx80Subnormal(aSig0, &aExp, &aSig0);
-    }
-    bSig |= UINT64_C(0x8000000000000000);
-    zSign = aSign;
-    expDiff = aExp - bExp;
-    aSig1 = 0;
-    if (expDiff < 0) {
-        return a;
-    }
-    qTemp = (bSig <= aSig0);
-    if (qTemp) {
-        aSig0 -= bSig;
-    }
-    expDiff -= 64;
-    while (0 < expDiff) {
-        qTemp = estimateDiv128To64(aSig0, aSig1, bSig);
-        qTemp = (2 < qTemp) ? qTemp - 2 : 0;
-        mul64To128(bSig, qTemp, &term0, &term1);
-        sub128(aSig0, aSig1, term0, term1, &aSig0, &aSig1);
-        shortShift128Left(aSig0, aSig1, 62, &aSig0, &aSig1);
-        expDiff -= 62;
-    }
-    expDiff += 64;
-    if (0 < expDiff) {
-        qTemp = estimateDiv128To64(aSig0, aSig1, bSig);
-        qTemp = (2 < qTemp) ? qTemp - 2 : 0;
-        qTemp >>= 64 - expDiff;
-        mul64To128(bSig, qTemp << (64 - expDiff), &term0, &term1);
-        sub128(aSig0, aSig1, term0, term1, &aSig0, &aSig1);
-        shortShift128Left(0, bSig, 64 - expDiff, &term0, &term1);
-        while (le128(term0, term1, aSig0, aSig1)) {
-            ++qTemp;
-            sub128(aSig0, aSig1, term0, term1, &aSig0, &aSig1);
-        }
-    }
-    return
-        normalizeRoundAndPackFloatx80(
-            80, zSign, bExp + expDiff, aSig0, aSig1, status);
-}
-
 /*
  * Returns the mantissa of the extended double-precision floating-point
  * value `a'.
@@ -132,7 +49,7 @@ floatx80 floatx80_mod(floatx80 a, floatx80 b, float_status *status)
 
 floatx80 floatx80_getman(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
@@ -166,7 +83,7 @@ floatx80 floatx80_getman(floatx80 a, float_status *status)
 
 floatx80 floatx80_getexp(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
@@ -202,7 +119,7 @@ floatx80 floatx80_getexp(floatx80 a, float_status *status)
 
 floatx80 floatx80_scale(floatx80 a, floatx80 b, float_status *status)
 {
-    flag aSign, bSign;
+    bool aSign, bSign;
     int32_t aExp, bExp, shiftCount;
     uint64_t aSig, bSig;
 
@@ -258,7 +175,7 @@ floatx80 floatx80_scale(floatx80 a, floatx80 b, float_status *status)
 
 floatx80 floatx80_move(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
@@ -306,11 +223,12 @@ static int32_t floatx80_make_compact(int32_t aExp, uint64_t aSig)
 
 floatx80 floatx80_lognp1(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig, fSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, j, k;
     floatx80 fp0, fp1, fp2, fp3, f, logof2, klog2, saveu;
@@ -353,7 +271,7 @@ floatx80 floatx80_lognp1(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
@@ -505,11 +423,12 @@ floatx80 floatx80_lognp1(floatx80 a, float_status *status)
 
 floatx80 floatx80_logn(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig, fSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, j, k, adjk;
     floatx80 fp0, fp1, fp2, fp3, f, logof2, klog2, saveu;
@@ -552,7 +471,7 @@ floatx80 floatx80_logn(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
@@ -673,11 +592,12 @@ floatx80 floatx80_logn(floatx80 a, float_status *status)
 
 floatx80 floatx80_log10(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     floatx80 fp0, fp1;
 
@@ -709,7 +629,7 @@ floatx80 floatx80_log10(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     fp0 = floatx80_logn(a, status);
     fp1 = packFloatx80(0, 0x3FFD, UINT64_C(0xDE5BD8A937287195)); /* INV_L10 */
@@ -730,11 +650,12 @@ floatx80 floatx80_log10(floatx80 a, float_status *status)
 
 floatx80 floatx80_log2(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     floatx80 fp0, fp1;
 
@@ -769,7 +690,7 @@ floatx80 floatx80_log2(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     if (aSig == one_sig) { /* X is 2^k */
         status->float_rounding_mode = user_rnd_mode;
@@ -797,15 +718,16 @@ floatx80 floatx80_log2(floatx80 a, float_status *status)
 
 floatx80 floatx80_etox(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, n, j, k, m, m1;
     floatx80 fp0, fp1, fp2, fp3, l2, scale, adjscale;
-    flag adjflag;
+    bool adjflag;
 
     aSig = extractFloatx80Frac(a);
     aExp = extractFloatx80Exp(a);
@@ -829,7 +751,7 @@ floatx80 floatx80_etox(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     adjflag = 0;
 
@@ -981,11 +903,12 @@ floatx80 floatx80_etox(floatx80 a, float_status *status)
 
 floatx80 floatx80_twotox(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, n, j, l, m, m1;
     floatx80 fp0, fp1, fp2, fp3, adjfact, fact1, fact2;
@@ -1012,7 +935,7 @@ floatx80 floatx80_twotox(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     fp0 = a;
 
@@ -1131,11 +1054,12 @@ floatx80 floatx80_twotox(floatx80 a, float_status *status)
 
 floatx80 floatx80_tentox(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, n, j, l, m, m1;
     floatx80 fp0, fp1, fp2, fp3, adjfact, fact1, fact2;
@@ -1162,7 +1086,7 @@ floatx80 floatx80_tentox(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     fp0 = a;
 
@@ -1286,16 +1210,17 @@ floatx80 floatx80_tentox(floatx80 a, float_status *status)
 
 floatx80 floatx80_tan(floatx80 a, float_status *status)
 {
-    flag aSign, xSign;
+    bool aSign, xSign;
     int32_t aExp, xExp;
     uint64_t aSig, xSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, l, n, j;
     floatx80 fp0, fp1, fp2, fp3, fp4, fp5, invtwopi, twopi1, twopi2;
     float32 twoto63;
-    flag endflag;
+    bool endflag;
 
     aSig = extractFloatx80Frac(a);
     aExp = extractFloatx80Exp(a);
@@ -1316,7 +1241,7 @@ floatx80 floatx80_tan(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
@@ -1344,10 +1269,10 @@ floatx80 floatx80_tan(floatx80 a, float_status *status)
             xExp -= 0x3FFF;
             if (xExp <= 28) {
                 l = 0;
-                endflag = 1;
+                endflag = true;
             } else {
                 l = xExp - 27;
-                endflag = 0;
+                endflag = false;
             }
             invtwopi = packFloatx80(0, 0x3FFE - l,
                                     UINT64_C(0xA2F9836E4E44152A)); /* INVTWOPI */
@@ -1372,7 +1297,7 @@ floatx80 floatx80_tan(floatx80 a, float_status *status)
             fp1 = floatx80_sub(fp1, fp4, status); /* FP1 is a := r - p */
             fp0 = floatx80_add(fp0, fp1, status); /* FP0 is R := A+a */
 
-            if (endflag > 0) {
+            if (endflag) {
                 n = floatx80_to_int32(fp2, status);
                 goto tancont;
             }
@@ -1496,16 +1421,17 @@ floatx80 floatx80_tan(floatx80 a, float_status *status)
 
 floatx80 floatx80_sin(floatx80 a, float_status *status)
 {
-    flag aSign, xSign;
+    bool aSign, xSign;
     int32_t aExp, xExp;
     uint64_t aSig, xSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, l, n, j;
     floatx80 fp0, fp1, fp2, fp3, fp4, fp5, x, invtwopi, twopi1, twopi2;
     float32 posneg1, twoto63;
-    flag endflag;
+    bool endflag;
 
     aSig = extractFloatx80Frac(a);
     aExp = extractFloatx80Exp(a);
@@ -1526,7 +1452,7 @@ floatx80 floatx80_sin(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
@@ -1554,10 +1480,10 @@ floatx80 floatx80_sin(floatx80 a, float_status *status)
             xExp -= 0x3FFF;
             if (xExp <= 28) {
                 l = 0;
-                endflag = 1;
+                endflag = true;
             } else {
                 l = xExp - 27;
-                endflag = 0;
+                endflag = false;
             }
             invtwopi = packFloatx80(0, 0x3FFE - l,
                                     UINT64_C(0xA2F9836E4E44152A)); /* INVTWOPI */
@@ -1582,7 +1508,7 @@ floatx80 floatx80_sin(floatx80 a, float_status *status)
             fp1 = floatx80_sub(fp1, fp4, status); /* FP1 is a := r - p */
             fp0 = floatx80_add(fp0, fp1, status); /* FP0 is R := A+a */
 
-            if (endflag > 0) {
+            if (endflag) {
                 n = floatx80_to_int32(fp2, status);
                 goto sincont;
             }
@@ -1735,16 +1661,17 @@ floatx80 floatx80_sin(floatx80 a, float_status *status)
 
 floatx80 floatx80_cos(floatx80 a, float_status *status)
 {
-    flag aSign, xSign;
+    bool aSign, xSign;
     int32_t aExp, xExp;
     uint64_t aSig, xSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, l, n, j;
     floatx80 fp0, fp1, fp2, fp3, fp4, fp5, x, invtwopi, twopi1, twopi2;
     float32 posneg1, twoto63;
-    flag endflag;
+    bool endflag;
 
     aSig = extractFloatx80Frac(a);
     aExp = extractFloatx80Exp(a);
@@ -1765,7 +1692,7 @@ floatx80 floatx80_cos(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
@@ -1793,10 +1720,10 @@ floatx80 floatx80_cos(floatx80 a, float_status *status)
             xExp -= 0x3FFF;
             if (xExp <= 28) {
                 l = 0;
-                endflag = 1;
+                endflag = true;
             } else {
                 l = xExp - 27;
-                endflag = 0;
+                endflag = false;
             }
             invtwopi = packFloatx80(0, 0x3FFE - l,
                                     UINT64_C(0xA2F9836E4E44152A)); /* INVTWOPI */
@@ -1821,7 +1748,7 @@ floatx80 floatx80_cos(floatx80 a, float_status *status)
             fp1 = floatx80_sub(fp1, fp4, status); /* FP1 is a := r - p */
             fp0 = floatx80_add(fp0, fp1, status); /* FP0 is R := A+a */
 
-            if (endflag > 0) {
+            if (endflag) {
                 n = floatx80_to_int32(fp2, status);
                 goto sincont;
             }
@@ -1972,11 +1899,12 @@ floatx80 floatx80_cos(floatx80 a, float_status *status)
 
 floatx80 floatx80_atan(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, tbl_index;
     floatx80 fp0, fp1, fp2, fp3, xsave;
@@ -2003,7 +1931,7 @@ floatx80 floatx80_atan(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     if (compact < 0x3FFB8000 || compact > 0x4002FFFF) {
         /* |X| >= 16 or |X| < 1/16 */
@@ -2169,11 +2097,12 @@ floatx80 floatx80_atan(floatx80 a, float_status *status)
 
 floatx80 floatx80_asin(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact;
     floatx80 fp0, fp1, fp2, one;
@@ -2207,7 +2136,7 @@ floatx80 floatx80_asin(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     one = packFloatx80(0, one_exp, one_sig);
     fp0 = a;
@@ -2234,11 +2163,12 @@ floatx80 floatx80_asin(floatx80 a, float_status *status)
 
 floatx80 floatx80_acos(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact;
     floatx80 fp0, fp1, one;
@@ -2276,7 +2206,7 @@ floatx80 floatx80_acos(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     one = packFloatx80(0, one_exp, one_sig);
     fp0 = a;
@@ -2303,11 +2233,12 @@ floatx80 floatx80_acos(floatx80 a, float_status *status)
 
 floatx80 floatx80_atanh(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact;
     floatx80 fp0, fp1, fp2, one;
@@ -2340,7 +2271,7 @@ floatx80 floatx80_atanh(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     one = packFloatx80(0, one_exp, one_sig);
     fp2 = packFloatx80(aSign, 0x3FFE, one_sig); /* SIGN(X) * (1/2) */
@@ -2368,11 +2299,12 @@ floatx80 floatx80_atanh(floatx80 a, float_status *status)
 
 floatx80 floatx80_etoxm1(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact, n, j, m, m1;
     floatx80 fp0, fp1, fp2, fp3, l2, sc, onebysc;
@@ -2399,7 +2331,7 @@ floatx80 floatx80_etoxm1(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     if (aExp >= 0x3FFD) { /* |X| >= 1/4 */
         compact = floatx80_make_compact(aExp, aSig);
@@ -2620,11 +2552,12 @@ floatx80 floatx80_etoxm1(floatx80 a, float_status *status)
 
 floatx80 floatx80_tanh(floatx80 a, float_status *status)
 {
-    flag aSign, vSign;
+    bool aSign, vSign;
     int32_t aExp, vExp;
     uint64_t aSig, vSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact;
     floatx80 fp0, fp1;
@@ -2648,7 +2581,7 @@ floatx80 floatx80_tanh(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
@@ -2735,11 +2668,12 @@ floatx80 floatx80_tanh(floatx80 a, float_status *status)
 
 floatx80 floatx80_sinh(floatx80 a, float_status *status)
 {
-    flag aSign;
+    bool aSign;
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact;
     floatx80 fp0, fp1, fp2;
@@ -2764,7 +2698,7 @@ floatx80 floatx80_sinh(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
@@ -2827,7 +2761,8 @@ floatx80 floatx80_cosh(floatx80 a, float_status *status)
     int32_t aExp;
     uint64_t aSig;
 
-    int8_t user_rnd_mode, user_rnd_prec;
+    FloatRoundMode user_rnd_mode;
+    FloatX80RoundPrec user_rnd_prec;
 
     int32_t compact;
     floatx80 fp0, fp1;
@@ -2850,7 +2785,7 @@ floatx80 floatx80_cosh(floatx80 a, float_status *status)
     user_rnd_mode = status->float_rounding_mode;
     user_rnd_prec = status->floatx80_rounding_precision;
     status->float_rounding_mode = float_round_nearest_even;
-    status->floatx80_rounding_precision = 80;
+    status->floatx80_rounding_precision = floatx80_precision_x;
 
     compact = floatx80_make_compact(aExp, aSig);
 
diff --git a/qemu/target/m68k/softfloat.h b/qemu/target/m68k/softfloat.h
index 365ef6ac7a..4bb9567134 100644
--- a/qemu/target/m68k/softfloat.h
+++ b/qemu/target/m68k/softfloat.h
@@ -23,7 +23,6 @@
 #define TARGET_M68K_SOFTFLOAT_H
 #include "fpu/softfloat.h"
 
-floatx80 floatx80_mod(floatx80 a, floatx80 b, float_status *status);
 floatx80 floatx80_getman(floatx80 a, float_status *status);
 floatx80 floatx80_getexp(floatx80 a, float_status *status);
 floatx80 floatx80_scale(floatx80 a, floatx80 b, float_status *status);
diff --git a/qemu/target/m68k/translate.c b/qemu/target/m68k/translate.c
index 5b9a74ce2b..833720162d 100644
--- a/qemu/target/m68k/translate.c
+++ b/qemu/target/m68k/translate.c
@@ -309,6 +309,26 @@ static void gen_raise_exception(TCGContext *tcg_ctx, int nr)
     tcg_temp_free_i32(tcg_ctx, tmp);
 }
 
+static void gen_raise_exception_format2(DisasContext *s, int nr,
+                                        target_ulong this_pc)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 addr;
+
+    /*
+     * Pass the address of the insn to the exception handler,
+     * for recording in the Format 2 (6-word) stack frame.
+     * Re-use mmu.ar for the purpose, since that's only valid
+     * after tlb_fill.
+     */
+    addr = tcg_const_i32(tcg_ctx, this_pc);
+    tcg_gen_st_i32(tcg_ctx, addr, tcg_ctx->cpu_env,
+                   offsetof(CPUM68KState, mmu.ar));
+    tcg_temp_free_i32(tcg_ctx, addr);
+    gen_raise_exception(tcg_ctx, nr);
+    s->base.is_jmp = DISAS_NORETURN;
+}
+
 static void gen_exception(DisasContext *s, uint32_t dest, int nr)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -3043,6 +3063,26 @@ DISAS_INSN(rtd)
     gen_jmp(s, tmp);
 }
 
+DISAS_INSN(rtr)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv tmp;
+    TCGv ccr;
+    TCGv sp;
+
+    sp = tcg_temp_new(tcg_ctx);
+    ccr = gen_load(s, OS_WORD, QREG_SP, 0, IS_USER(s));
+    tcg_gen_addi_i32(tcg_ctx, sp, QREG_SP, 2);
+    tmp = gen_load(s, OS_LONG, sp, 0, IS_USER(s));
+    tcg_gen_addi_i32(tcg_ctx, QREG_SP, sp, 4);
+    tcg_temp_free(tcg_ctx, sp);
+
+    gen_set_sr(s, ccr, true);
+    tcg_temp_free(tcg_ctx, ccr);
+
+    gen_jmp(s, tmp);
+}
+
 DISAS_INSN(rts)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -4719,7 +4759,7 @@ DISAS_INSN(move_from_sr)
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     TCGv sr;
 
-    if (IS_USER(s) && !m68k_feature(env, M68K_FEATURE_M68000)) {
+    if (IS_USER(s) && m68k_feature(env, M68K_FEATURE_MOVEFROMSR_PRIV)) {
         gen_exception(s, s->base.pc_next, EXCP_PRIVILEGE);
         return;
     }
@@ -4988,6 +5028,61 @@ DISAS_INSN(trap)
     gen_exception(s, s->base.pc_next, EXCP_TRAP0 + (insn & 0xf));
 }
 
+static void do_trapcc(DisasContext *s, DisasCompare *c)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    if (c->tcond != TCG_COND_NEVER) {
+        TCGLabel *over = NULL;
+
+        update_cc_op(s);
+
+        if (c->tcond != TCG_COND_ALWAYS) {
+            over = gen_new_label(tcg_ctx);
+            tcg_gen_brcond_i32(tcg_ctx, tcg_invert_cond(c->tcond),
+                               c->v1, c->v2, over);
+        }
+
+        tcg_gen_movi_i32(tcg_ctx, QREG_PC, s->pc);
+        gen_raise_exception_format2(s, EXCP_TRAPCC, s->base.pc_next);
+
+        if (over != NULL) {
+            gen_set_label(tcg_ctx, over);
+            s->base.is_jmp = DISAS_NEXT;
+        }
+    }
+    free_cond(tcg_ctx, c);
+}
+
+DISAS_INSN(trapcc)
+{
+    DisasCompare c;
+
+    switch (extract32(insn, 0, 3)) {
+    case 2:
+        (void)read_im16(env, s);
+        break;
+    case 3:
+        (void)read_im32(env, s);
+        break;
+    case 4:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    gen_cc_cond(&c, s, extract32(insn, 8, 4));
+    do_trapcc(s, &c);
+}
+
+DISAS_INSN(trapv)
+{
+    DisasCompare c;
+
+    gen_cc_cond(&c, s, 9);
+    do_trapcc(s, &c);
+}
+
 static void gen_load_fcr(DisasContext *s, TCGv res, int reg)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -5289,6 +5384,9 @@ DISAS_INSN(fpu)
     case 0x06: /* flognp1 */
         gen_helper_flognp1(tcg_ctx, tcg_ctx->cpu_env, cpu_dest, cpu_src);
         break;
+    case 0x08: /* fetoxm1 */
+        gen_helper_fetoxm1(tcg_ctx, tcg_ctx->cpu_env, cpu_dest, cpu_src);
+        break;
     case 0x09: /* ftanh */
         gen_helper_ftanh(tcg_ctx, tcg_ctx->cpu_env, cpu_dest, cpu_src);
         break;
@@ -5630,6 +5728,32 @@ DISAS_INSN(fscc)
     tcg_temp_free(tcg_ctx, tmp);
 }
 
+DISAS_INSN(ftrapcc)
+{
+    DisasCompare c;
+    uint16_t ext;
+    int cond;
+
+    ext = read_im16(env, s);
+    cond = ext & 0x3f;
+
+    switch (extract32(insn, 0, 3)) {
+    case 2:
+        (void)read_im16(env, s);
+        break;
+    case 3:
+        (void)read_im32(env, s);
+        break;
+    case 4:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    gen_fcc_cond(&c, s, cond);
+    do_trapcc(s, &c);
+}
+
 DISAS_INSN(frestore)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -5936,8 +6060,8 @@ DISAS_INSN(macsr_to_ccr)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     TCGv tmp = tcg_temp_new(tcg_ctx);
-    tcg_gen_andi_i32(tcg_ctx, tmp, QREG_MACSR, 0xf);
-    gen_helper_set_sr(tcg_ctx, tcg_ctx->cpu_env, tmp);
+    tcg_gen_andi_i32(tcg_ctx, tmp, QREG_MACSR, CCF_N | CCF_Z | CCF_V);
+    gen_helper_set_ccr(tcg_ctx, tcg_ctx->cpu_env, tmp);
     tcg_temp_free(tcg_ctx, tmp);
     set_cc_op(s, CC_OP_FLAGS);
 }
@@ -6142,11 +6266,13 @@ void register_m68k_insns (CPUM68KState *env)
     INSN(reset,     4e70, ffff, M68000);
     BASE(stop,      4e72, ffff);
     BASE(rte,       4e73, ffff);
+    INSN(m68k_movec, 4e7a, fffe, MOVEC);
     INSN(cf_movec,  4e7b, ffff, CF_ISA_A);
-    INSN(m68k_movec, 4e7a, fffe, M68000);
     BASE(nop,       4e71, ffff);
     INSN(rtd,       4e74, ffff, RTD);
     BASE(rts,       4e75, ffff);
+    INSN(trapv,     4e76, ffff, M68000);
+    INSN(rtr,       4e77, ffff, M68000);
     BASE(jump,      4e80, ffc0);
     BASE(jump,      4ec0, ffc0);
     INSN(addsubq,   5000, f080, M68000);
@@ -6155,6 +6281,10 @@ void register_m68k_insns (CPUM68KState *env)
     INSN(scc,       50c0, f0c0, M68000);   /* Scc.B <EA> */
     INSN(dbcc,      50c8, f0f8, M68000);
     INSN(tpf,       51f8, fff8, CF_ISA_A);
+    INSN(trapcc,    50fa, f0fe, TRAPCC);   /* opmode 010, 011 */
+    INSN(trapcc,    50fc, f0ff, TRAPCC);   /* opmode 100 */
+    INSN(trapcc,    51fa, fffe, CF_ISA_A); /* TPF opmode 010, 011 */
+    INSN(trapcc,    51fc, ffff, CF_ISA_A); /* TPF opmode 100 */
 
     /* Branch instructions.  */
     BASE(branch,    6000, f000);
@@ -6252,6 +6382,8 @@ void register_m68k_insns (CPUM68KState *env)
     INSN(fbcc,      f280, ffc0, CF_FPU);
     INSN(fpu,       f200, ffc0, FPU);
     INSN(fscc,      f240, ffc0, FPU);
+    INSN(ftrapcc,   f27a, fffe, FPU);       /* opmode 010, 011 */
+    INSN(ftrapcc,   f27c, ffff, FPU);       /* opmode 100 */
     INSN(fbcc,      f280, ff80, FPU);
     INSN(frestore,  f340, ffc0, CF_FPU);
     INSN(fsave,     f300, ffc0, CF_FPU);
diff --git a/qemu/target/mips/cp0_helper.c b/qemu/target/mips/cp0_helper.c
index e3600c26d7..65d8c3ef35 100644
--- a/qemu/target/mips/cp0_helper.c
+++ b/qemu/target/mips/cp0_helper.c
@@ -347,8 +347,7 @@ target_ulong helper_mftc0_tcschefback(CPUMIPSState *env)
 
 target_ulong helper_mfc0_count(CPUMIPSState *env)
 {
-    // return (int32_t)cpu_mips_get_count(env);
-    return 0;
+    return (int32_t)cpu_mips_get_count(env);
 }
 
 target_ulong helper_mfc0_saar(CPUMIPSState *env)
@@ -875,15 +874,21 @@ void helper_mtc0_memorymapid(CPUMIPSState *env, target_ulong arg1)
     }
 }
 
-void update_pagemask(CPUMIPSState *env, target_ulong arg1, int32_t *pagemask)
+static uint32_t compute_pagemask(uint32_t val)
 {
-    uint64_t mask = arg1 >> (TARGET_PAGE_BITS + 1);
-    if (!(env->insn_flags & ISA_MIPS32R6) || (arg1 == ~0) ||
-        (mask == 0x0000 || mask == 0x0003 || mask == 0x000F ||
-         mask == 0x003F || mask == 0x00FF || mask == 0x03FF ||
-         mask == 0x0FFF || mask == 0x3FFF || mask == 0xFFFF)) {
-        env->CP0_PageMask = arg1 & (0x1FFFFFFF & (TARGET_PAGE_MASK << 1));
+    uint32_t mask = extract32(val, 13, 16);
+    int maskbits = cto32(mask);
+
+    if ((mask >> maskbits) == 0 && (maskbits % 2) == 0) {
+        return mask << 13;
     }
+
+    return 0;
+}
+
+void update_pagemask(CPUMIPSState *env, target_ulong arg1, int32_t *pagemask)
+{
+    *pagemask = compute_pagemask(arg1);
 }
 
 void helper_mtc0_pagemask(CPUMIPSState *env, target_ulong arg1)
@@ -1068,7 +1073,7 @@ void helper_mtc0_hwrena(CPUMIPSState *env, target_ulong arg1)
 
 void helper_mtc0_count(CPUMIPSState *env, target_ulong arg1)
 {
-    //cpu_mips_store_count(env, arg1);
+    cpu_mips_store_count(env, arg1);
 }
 
 void helper_mtc0_saari(CPUMIPSState *env, target_ulong arg1)
@@ -1157,7 +1162,7 @@ void helper_mttc0_entryhi(CPUMIPSState *env, target_ulong arg1)
 
 void helper_mtc0_compare(CPUMIPSState *env, target_ulong arg1)
 {
-    // cpu_mips_store_compare(env, arg1);
+    cpu_mips_store_compare(env, arg1);
 }
 
 void helper_mtc0_status(CPUMIPSState *env, target_ulong arg1)
diff --git a/qemu/target/mips/cp0_timer.c b/qemu/target/mips/cp0_timer.c
index 223e070b6b..2c28bea3c5 100644
--- a/qemu/target/mips/cp0_timer.c
+++ b/qemu/target/mips/cp0_timer.c
@@ -54,6 +54,24 @@ uint32_t cpu_mips_get_random(CPUMIPSState *env)
     return idx;
 }
 
+uint32_t cpu_mips_get_count(CPUMIPSState *env)
+{
+    return env->CP0_Count;
+}
+
+void cpu_mips_store_count(CPUMIPSState *env, uint32_t count)
+{
+    env->CP0_Count = count;
+}
+
+void cpu_mips_store_compare(CPUMIPSState *env, uint32_t value)
+{
+    env->CP0_Compare = value;
+    if (env->insn_flags & ISA_MIPS32R2) {
+        env->CP0_Cause &= ~(1 << CP0Ca_TI);
+    }
+}
+
 #if 0
 /* MIPS R4K timer */
 static void cpu_mips_timer_update(CPUMIPSState *env)
diff --git a/qemu/target/mips/cpu.c b/qemu/target/mips/cpu.c
index ac2a6145db..f1fe63d39e 100644
--- a/qemu/target/mips/cpu.c
+++ b/qemu/target/mips/cpu.c
@@ -172,7 +172,13 @@ MIPSCPU *cpu_mips_init(struct uc_struct *uc)
     }
 #else
     if (uc->cpu_model == INT_MAX) {
-        uc->cpu_model = UC_CPU_MIPS32_74KF; // 74kf
+        if (uc->mode & UC_MODE_MIPS32R6) {
+            uc->cpu_model = UC_CPU_MIPS32_MIPS32R6_GENERIC;
+        } else if (uc->mode & UC_MODE_MICRO) {
+            uc->cpu_model = UC_CPU_MIPS32_M14K;
+        } else {
+            uc->cpu_model = UC_CPU_MIPS32_74KF; // 74kf
+        }
     } else if (uc->cpu_model >= mips_defs_number) {
         free(cpu);
         return NULL;
@@ -207,6 +213,9 @@ MIPSCPU *cpu_mips_init(struct uc_struct *uc)
     }
 
     mips_cpu_realizefn(cs);
+    if (uc->mode & UC_MODE_MICRO) {
+        env->hflags |= MIPS_HFLAG_M16;
+    }
 
     // init address space
     cpu_address_space_init(cs, 0, cs->memory);
diff --git a/qemu/target/mips/cpu.h b/qemu/target/mips/cpu.h
index 95f6bf5077..53cde9c2fa 100644
--- a/qemu/target/mips/cpu.h
+++ b/qemu/target/mips/cpu.h
@@ -940,7 +940,33 @@ struct CPUMIPSState {
 #define CP0C5_UFR          2
 #define CP0C5_NFExists     0
     int32_t CP0_Config6;
+#define CP0C6_BPPASS       31
+#define CP0C6_KPOS         24
+#define CP0C6_KE           23
+#define CP0C6_VTLBONLY     22
+#define CP0C6_LASX         21
+#define CP0C6_SSEN         20
+#define CP0C6_DISDRTIME    19
+#define CP0C6_PIXNUEN      18
+#define CP0C6_SCRAND       17
+#define CP0C6_LLEXCEN      16
+#define CP0C6_DISVC        15
+#define CP0C6_VCLRU        14
+#define CP0C6_DCLRU        13
+#define CP0C6_PIXUEN       12
+#define CP0C6_DISBLKLYEN   11
+#define CP0C6_UMEMUALEN    10
+#define CP0C6_SFBEN        8
+#define CP0C6_FLTINT       7
+#define CP0C6_VLTINT       6
+#define CP0C6_DISBTB       5
+#define CP0C6_STPREFCTL    2
+#define CP0C6_INSTPREF     1
+#define CP0C6_DATAPREF     0
     int32_t CP0_Config7;
+#define CP0C7_NAPCGEN      2
+#define CP0C7_UNIMUEN      1
+#define CP0C7_VFPUCGEN     0
     uint64_t CP0_LLAddr;
     uint64_t CP0_MAAR[MIPS_MAAR_MAX];
     int32_t CP0_MAARI;
diff --git a/qemu/target/mips/mips-defs.h b/qemu/target/mips/mips-defs.h
index a831bb4384..6a86c193ae 100644
--- a/qemu/target/mips/mips-defs.h
+++ b/qemu/target/mips/mips-defs.h
@@ -53,11 +53,15 @@
 #define INSN_LOONGSON2F   0x0002000000000000ULL
 #define INSN_VR54XX       0x0004000000000000ULL
 #define INSN_R5900        0x0008000000000000ULL
+#define INSN_OCTEON       0x0010000000000000ULL
+#define INSN_LOONGSON3A   0x0020000000000000ULL
 /*
  *   bits 56-63: vendor-specific ASEs
  */
 #define ASE_MMI           0x0100000000000000ULL
 #define ASE_MXU           0x0200000000000000ULL
+#define ASE_LMMI          0x0400000000000000ULL
+#define ASE_LEXT          0x0800000000000000ULL
 
 /* MIPS CPU defines. */
 #define CPU_MIPS1       (ISA_MIPS1)
@@ -67,8 +71,7 @@
 #define CPU_VR54XX      (CPU_MIPS4 | INSN_VR54XX)
 #define CPU_R5900       (CPU_MIPS3 | INSN_R5900)
 #define CPU_LOONGSON2E  (CPU_MIPS3 | INSN_LOONGSON2E)
-#define CPU_LOONGSON2F  (CPU_MIPS3 | INSN_LOONGSON2F)
-
+#define CPU_LOONGSON2F  (CPU_MIPS3 | INSN_LOONGSON2F | ASE_LMMI)
 #define CPU_MIPS5       (CPU_MIPS4 | ISA_MIPS5)
 
 /* MIPS Technologies "Release 1" */
@@ -78,6 +81,9 @@
 /* MIPS Technologies "Release 2" */
 #define CPU_MIPS32R2    (CPU_MIPS32 | ISA_MIPS32R2)
 #define CPU_MIPS64R2    (CPU_MIPS64 | CPU_MIPS32R2 | ISA_MIPS64R2)
+#define CPU_OCTEON      (CPU_MIPS64R2 | INSN_OCTEON)
+#define CPU_LOONGSON3A  (CPU_MIPS64R2 | INSN_LOONGSON3A | ASE_LMMI | \
+                         ASE_LEXT)
 
 /* MIPS Technologies "Release 3" */
 #define CPU_MIPS32R3    (CPU_MIPS32R2 | ISA_MIPS32R3)
diff --git a/qemu/target/mips/op_helper.c b/qemu/target/mips/op_helper.c
index 9802b9cebd..a3e8156c0b 100644
--- a/qemu/target/mips/op_helper.c
+++ b/qemu/target/mips/op_helper.c
@@ -1014,8 +1014,7 @@ target_ulong helper_rdhwr_synci_step(CPUMIPSState *env)
 target_ulong helper_rdhwr_cc(CPUMIPSState *env)
 {
     check_hwrena(env, 2, GETPC());
-    // return (int32_t)cpu_mips_get_count(env);
-    return 0;
+    return (int32_t)cpu_mips_get_count(env);
 }
 
 target_ulong helper_rdhwr_ccres(CPUMIPSState *env)
diff --git a/qemu/target/mips/translate.c b/qemu/target/mips/translate.c
index 3fab57b251..66d785c1b2 100644
--- a/qemu/target/mips/translate.c
+++ b/qemu/target/mips/translate.c
@@ -454,6 +454,50 @@ enum {
     R6_OPC_SCD         = 0x27 | OPC_SPECIAL3,
 };
 
+/* Loongson EXT load/store quad word opcodes */
+#define MASK_LOONGSON_GSLSQ(op)           (MASK_OP_MAJOR(op) | (op & 0x8020))
+
+enum {
+    OPC_GSLQ        = 0x0020 | OPC_LWC2,
+    OPC_GSLQC1      = 0x8020 | OPC_LWC2,
+    OPC_GSSHFL      = OPC_LWC2,
+    OPC_GSSQ        = 0x0020 | OPC_SWC2,
+    OPC_GSSQC1      = 0x8020 | OPC_SWC2,
+    OPC_GSSHFS      = OPC_SWC2,
+};
+
+/* Loongson EXT shifted load/store opcodes */
+#define MASK_LOONGSON_GSSHFLS(op)         (MASK_OP_MAJOR(op) | (op & 0xc03f))
+
+enum {
+    OPC_GSLWLC1     = 0x4 | OPC_GSSHFL,
+    OPC_GSLWRC1     = 0x5 | OPC_GSSHFL,
+    OPC_GSLDLC1     = 0x6 | OPC_GSSHFL,
+    OPC_GSLDRC1     = 0x7 | OPC_GSSHFL,
+    OPC_GSSWLC1     = 0x4 | OPC_GSSHFS,
+    OPC_GSSWRC1     = 0x5 | OPC_GSSHFS,
+    OPC_GSSDLC1     = 0x6 | OPC_GSSHFS,
+    OPC_GSSDRC1     = 0x7 | OPC_GSSHFS,
+};
+
+/* Loongson EXT LDC2/SDC2 opcodes */
+#define MASK_LOONGSON_LSDC2(op)           (MASK_OP_MAJOR(op) | (op & 0x7))
+
+enum {
+    OPC_GSLBX      = 0x0 | OPC_LDC2,
+    OPC_GSLHX      = 0x1 | OPC_LDC2,
+    OPC_GSLWX      = 0x2 | OPC_LDC2,
+    OPC_GSLDX      = 0x3 | OPC_LDC2,
+    OPC_GSLWXC1    = 0x6 | OPC_LDC2,
+    OPC_GSLDXC1    = 0x7 | OPC_LDC2,
+    OPC_GSSBX      = 0x0 | OPC_SDC2,
+    OPC_GSSHX      = 0x1 | OPC_SDC2,
+    OPC_GSSWX      = 0x2 | OPC_SDC2,
+    OPC_GSSDX      = 0x3 | OPC_SDC2,
+    OPC_GSSWXC1    = 0x6 | OPC_SDC2,
+    OPC_GSSDXC1    = 0x7 | OPC_SDC2,
+};
+
 /* BSHFL opcodes */
 #define MASK_BSHFL(op)              (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
 
@@ -3384,7 +3428,8 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
     TCGv t0, t1, t2;
     int mem_idx = ctx->mem_idx;
 
-    if (rt == 0 && ctx->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
+    if (rt == 0 && ctx->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F |
+                                      INSN_LOONGSON3A)) {
         /*
          * Loongson CPU uses a load to zero register for prefetch.
          * We emulate it as a NOP. On other CPU we must perform the
@@ -5898,6 +5943,402 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
     tcg_temp_free_i64(tcg_ctx, t1);
 }
 
+/* Loongson EXT LWC2/SWC2 */
+static void gen_loongson_lswc2(DisasContext *ctx, int rt, int rs)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int rt1 = ctx->opcode & 0x1f;
+    int lsq_offset = sextract32(ctx->opcode, 6, 9) << 4;
+    int shf_offset = sextract32(ctx->opcode, 6, 8);
+    uint32_t opc = MASK_LOONGSON_GSLSQ(ctx->opcode);
+    TCGv t0, t1, t2;
+    TCGv_i32 fp0;
+
+    switch (opc) {
+#if defined(TARGET_MIPS64)
+    case OPC_GSLQ:
+    case OPC_GSLQC1:
+    case OPC_GSSQ:
+    case OPC_GSSQC1:
+        break;
+#endif
+    case OPC_GSSHFL:
+    case OPC_GSSHFS:
+        break;
+    default:
+        MIPS_INVAL("loongson_lswc2");
+        generate_exception_end(ctx, EXCP_RI);
+        return;
+    }
+
+    t0 = tcg_temp_new(tcg_ctx);
+
+    switch (opc) {
+#if defined(TARGET_MIPS64)
+    case OPC_GSLQ:
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset);
+        tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset + 8);
+        tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_store_gpr(tcg_ctx, t1, rt);
+        gen_store_gpr(tcg_ctx, t0, rt1);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+    case OPC_GSLQC1:
+        check_cp1_enabled(ctx);
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset);
+        tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset + 8);
+        tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_store_fpr64(ctx, t1, rt);
+        gen_store_fpr64(ctx, t0, rt1);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+    case OPC_GSSQ:
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset + 8);
+        gen_load_gpr(tcg_ctx, t1, rt1);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+    case OPC_GSSQC1:
+        check_cp1_enabled(ctx);
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset);
+        gen_load_fpr64(ctx, t1, rt);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_base_offset_addr(ctx, t0, rs, lsq_offset + 8);
+        gen_load_fpr64(ctx, t1, rt1);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+#endif
+    case OPC_GSSHFL:
+        switch (MASK_LOONGSON_GSSHFLS(ctx->opcode)) {
+        case OPC_GSLWLC1:
+            check_cp1_enabled(ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            t1 = tcg_temp_new(tcg_ctx);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
+            tcg_gen_andi_tl(tcg_ctx, t1, t0, 3);
+#ifndef TARGET_WORDS_BIGENDIAN
+            tcg_gen_xori_tl(tcg_ctx, t1, t1, 3);
+#endif
+            tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
+            tcg_gen_andi_tl(tcg_ctx, t0, t0, ~3);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUL);
+            tcg_gen_shl_tl(tcg_ctx, t0, t0, t1);
+            t2 = tcg_const_tl(tcg_ctx, -1);
+            tcg_gen_shl_tl(tcg_ctx, t2, t2, t1);
+            fp0 = tcg_temp_new_i32(tcg_ctx);
+            gen_load_fpr32(ctx, fp0, rt);
+            tcg_gen_ext_i32_tl(tcg_ctx, t1, fp0);
+            tcg_gen_andc_tl(tcg_ctx, t1, t1, t2);
+            tcg_temp_free(tcg_ctx, t2);
+            tcg_gen_or_tl(tcg_ctx, t0, t0, t1);
+            tcg_temp_free(tcg_ctx, t1);
+#if defined(TARGET_MIPS64)
+            tcg_gen_extrl_i64_i32(tcg_ctx, fp0, t0);
+#else
+            tcg_gen_ext32s_tl(tcg_ctx, fp0, t0);
+#endif
+            gen_store_fpr32(ctx, fp0, rt);
+            tcg_temp_free_i32(tcg_ctx, fp0);
+            break;
+        case OPC_GSLWRC1:
+            check_cp1_enabled(ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            t1 = tcg_temp_new(tcg_ctx);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
+            tcg_gen_andi_tl(tcg_ctx, t1, t0, 3);
+#ifdef TARGET_WORDS_BIGENDIAN
+            tcg_gen_xori_tl(tcg_ctx, t1, t1, 3);
+#endif
+            tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
+            tcg_gen_andi_tl(tcg_ctx, t0, t0, ~3);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUL);
+            tcg_gen_shr_tl(tcg_ctx, t0, t0, t1);
+            tcg_gen_xori_tl(tcg_ctx, t1, t1, 31);
+            t2 = tcg_const_tl(tcg_ctx, 0xfffffffeull);
+            tcg_gen_shl_tl(tcg_ctx, t2, t2, t1);
+            fp0 = tcg_temp_new_i32(tcg_ctx);
+            gen_load_fpr32(ctx, fp0, rt);
+            tcg_gen_ext_i32_tl(tcg_ctx, t1, fp0);
+            tcg_gen_and_tl(tcg_ctx, t1, t1, t2);
+            tcg_temp_free(tcg_ctx, t2);
+            tcg_gen_or_tl(tcg_ctx, t0, t0, t1);
+            tcg_temp_free(tcg_ctx, t1);
+#if defined(TARGET_MIPS64)
+            tcg_gen_extrl_i64_i32(tcg_ctx, fp0, t0);
+#else
+            tcg_gen_ext32s_tl(tcg_ctx, fp0, t0);
+#endif
+            gen_store_fpr32(ctx, fp0, rt);
+            tcg_temp_free_i32(tcg_ctx, fp0);
+            break;
+#if defined(TARGET_MIPS64)
+        case OPC_GSLDLC1:
+            check_cp1_enabled(ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            t1 = tcg_temp_new(tcg_ctx);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
+            tcg_gen_andi_tl(tcg_ctx, t1, t0, 7);
+#ifndef TARGET_WORDS_BIGENDIAN
+            tcg_gen_xori_tl(tcg_ctx, t1, t1, 7);
+#endif
+            tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
+            tcg_gen_andi_tl(tcg_ctx, t0, t0, ~7);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ);
+            tcg_gen_shl_tl(tcg_ctx, t0, t0, t1);
+            t2 = tcg_const_tl(tcg_ctx, -1);
+            tcg_gen_shl_tl(tcg_ctx, t2, t2, t1);
+            gen_load_fpr64(ctx, t1, rt);
+            tcg_gen_andc_tl(tcg_ctx, t1, t1, t2);
+            tcg_temp_free(tcg_ctx, t2);
+            tcg_gen_or_tl(tcg_ctx, t0, t0, t1);
+            tcg_temp_free(tcg_ctx, t1);
+            gen_store_fpr64(ctx, t0, rt);
+            break;
+        case OPC_GSLDRC1:
+            check_cp1_enabled(ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            t1 = tcg_temp_new(tcg_ctx);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
+            tcg_gen_andi_tl(tcg_ctx, t1, t0, 7);
+#ifdef TARGET_WORDS_BIGENDIAN
+            tcg_gen_xori_tl(tcg_ctx, t1, t1, 7);
+#endif
+            tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
+            tcg_gen_andi_tl(tcg_ctx, t0, t0, ~7);
+            tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ);
+            tcg_gen_shr_tl(tcg_ctx, t0, t0, t1);
+            tcg_gen_xori_tl(tcg_ctx, t1, t1, 63);
+            t2 = tcg_const_tl(tcg_ctx, 0xfffffffffffffffeull);
+            tcg_gen_shl_tl(tcg_ctx, t2, t2, t1);
+            gen_load_fpr64(ctx, t1, rt);
+            tcg_gen_and_tl(tcg_ctx, t1, t1, t2);
+            tcg_temp_free(tcg_ctx, t2);
+            tcg_gen_or_tl(tcg_ctx, t0, t0, t1);
+            tcg_temp_free(tcg_ctx, t1);
+            gen_store_fpr64(ctx, t0, rt);
+            break;
+#endif
+        default:
+            MIPS_INVAL("loongson_gsshfl");
+            generate_exception_end(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    case OPC_GSSHFS:
+        switch (MASK_LOONGSON_GSSHFLS(ctx->opcode)) {
+        case OPC_GSSWLC1:
+            check_cp1_enabled(ctx);
+            t1 = tcg_temp_new(tcg_ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            fp0 = tcg_temp_new_i32(tcg_ctx);
+            gen_load_fpr32(ctx, fp0, rt);
+            tcg_gen_ext_i32_tl(tcg_ctx, t1, fp0);
+            gen_helper_0e2i(swl, t1, t0, ctx->mem_idx);
+            tcg_temp_free_i32(tcg_ctx, fp0);
+            tcg_temp_free(tcg_ctx, t1);
+            break;
+        case OPC_GSSWRC1:
+            check_cp1_enabled(ctx);
+            t1 = tcg_temp_new(tcg_ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            fp0 = tcg_temp_new_i32(tcg_ctx);
+            gen_load_fpr32(ctx, fp0, rt);
+            tcg_gen_ext_i32_tl(tcg_ctx, t1, fp0);
+            gen_helper_0e2i(swr, t1, t0, ctx->mem_idx);
+            tcg_temp_free_i32(tcg_ctx, fp0);
+            tcg_temp_free(tcg_ctx, t1);
+            break;
+#if defined(TARGET_MIPS64)
+        case OPC_GSSDLC1:
+            check_cp1_enabled(ctx);
+            t1 = tcg_temp_new(tcg_ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            gen_load_fpr64(ctx, t1, rt);
+            gen_helper_0e2i(sdl, t1, t0, ctx->mem_idx);
+            tcg_temp_free(tcg_ctx, t1);
+            break;
+        case OPC_GSSDRC1:
+            check_cp1_enabled(ctx);
+            t1 = tcg_temp_new(tcg_ctx);
+            gen_base_offset_addr(ctx, t0, rs, shf_offset);
+            gen_load_fpr64(ctx, t1, rt);
+            gen_helper_0e2i(sdr, t1, t0, ctx->mem_idx);
+            tcg_temp_free(tcg_ctx, t1);
+            break;
+#endif
+        default:
+            MIPS_INVAL("loongson_gsshfs");
+            generate_exception_end(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    tcg_temp_free(tcg_ctx, t0);
+}
+
+/* Loongson EXT LDC2/SDC2 */
+static void gen_loongson_lsdc2(DisasContext *ctx, int rt, int rs, int rd)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int offset = sextract32(ctx->opcode, 3, 8);
+    uint32_t opc = MASK_LOONGSON_LSDC2(ctx->opcode);
+    TCGv t0, t1;
+    TCGv_i32 fp0;
+
+    switch (opc) {
+    case OPC_GSLBX:
+    case OPC_GSLHX:
+    case OPC_GSLWX:
+#if defined(TARGET_MIPS64)
+    case OPC_GSLDX:
+#endif
+        if (rt == 0) {
+            return;
+        }
+        break;
+    case OPC_GSSBX:
+    case OPC_GSSHX:
+    case OPC_GSSWX:
+#if defined(TARGET_MIPS64)
+    case OPC_GSSDX:
+#endif
+        break;
+    case OPC_GSLWXC1:
+#if defined(TARGET_MIPS64)
+    case OPC_GSLDXC1:
+#endif
+        check_cp1_enabled(ctx);
+        if (rt == 0) {
+            return;
+        }
+        break;
+    case OPC_GSSWXC1:
+#if defined(TARGET_MIPS64)
+    case OPC_GSSDXC1:
+#endif
+        check_cp1_enabled(ctx);
+        break;
+    default:
+        MIPS_INVAL("loongson_lsdc2");
+        generate_exception_end(ctx, EXCP_RI);
+        return;
+    }
+
+    t0 = tcg_temp_new(tcg_ctx);
+    gen_base_offset_addr(ctx, t0, rs, offset);
+    if (rd != 0) {
+        gen_op_addr_add(ctx, t0, tcg_ctx->cpu_gpr[rd], t0);
+    }
+
+    switch (opc) {
+    case OPC_GSLBX:
+        tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_SB);
+        gen_store_gpr(tcg_ctx, t0, rt);
+        break;
+    case OPC_GSLHX:
+        tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TESW |
+                           ctx->default_tcg_memop_mask);
+        gen_store_gpr(tcg_ctx, t0, rt);
+        break;
+    case OPC_GSLWX:
+        tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TESL |
+                           ctx->default_tcg_memop_mask);
+        gen_store_gpr(tcg_ctx, t0, rt);
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_GSLDX:
+        tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_store_gpr(tcg_ctx, t0, rt);
+        break;
+#endif
+    case OPC_GSLWXC1:
+        fp0 = tcg_temp_new_i32(tcg_ctx);
+        tcg_gen_qemu_ld_i32(tcg_ctx, fp0, t0, ctx->mem_idx, MO_TESL |
+                            ctx->default_tcg_memop_mask);
+        gen_store_fpr32(ctx, fp0, rt);
+        tcg_temp_free_i32(tcg_ctx, fp0);
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_GSLDXC1:
+        tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        gen_store_fpr64(ctx, t0, rt);
+        break;
+#endif
+    case OPC_GSSBX:
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_8);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+    case OPC_GSSHX:
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUW |
+                           ctx->default_tcg_memop_mask);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+    case OPC_GSSWX:
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUL |
+                           ctx->default_tcg_memop_mask);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_GSSDX:
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_qemu_st_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                           ctx->default_tcg_memop_mask);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+#endif
+    case OPC_GSSWXC1:
+        fp0 = tcg_temp_new_i32(tcg_ctx);
+        gen_load_fpr32(ctx, fp0, rt);
+        tcg_gen_qemu_st_i32(tcg_ctx, fp0, t0, ctx->mem_idx, MO_TEUL |
+                            ctx->default_tcg_memop_mask);
+        tcg_temp_free_i32(tcg_ctx, fp0);
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_GSSDXC1:
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_fpr64(ctx, t1, rt);
+        tcg_gen_qemu_st_i64(tcg_ctx, t1, t0, ctx->mem_idx, MO_TEUQ |
+                            ctx->default_tcg_memop_mask);
+        tcg_temp_free(tcg_ctx, t1);
+        break;
+#endif
+    default:
+        g_assert_not_reached();
+    }
+
+    tcg_temp_free(tcg_ctx, t0);
+}
+
 /* Traps */
 static void gen_trap(DisasContext *ctx, uint32_t opc,
                      int rs, int rt, int16_t imm)
@@ -27207,7 +27648,7 @@ static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
     case OPC_MULTU_G_2F:
     case OPC_MOD_G_2F:
     case OPC_MODU_G_2F:
-        check_insn(ctx, INSN_LOONGSON2F);
+        check_insn(ctx, INSN_LOONGSON2F | ASE_LEXT);
         gen_loongson_integer(ctx, op1, rd, rs, rt);
         break;
     case OPC_CLO:
@@ -27240,7 +27681,7 @@ static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
     case OPC_DDIVU_G_2F:
     case OPC_DMOD_G_2F:
     case OPC_DMODU_G_2F:
-        check_insn(ctx, INSN_LOONGSON2F);
+        check_insn(ctx, INSN_LOONGSON2F | ASE_LEXT);
         gen_loongson_integer(ctx, op1, rd, rs, rt);
         break;
 #endif
@@ -27251,6 +27692,187 @@ static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
     }
 }
 
+#if defined(TARGET_MIPS64)
+static bool decode_opc_octeon_special2(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int rs = (ctx->opcode >> 21) & 0x1f;
+    int rt = (ctx->opcode >> 16) & 0x1f;
+    int rd = (ctx->opcode >> 11) & 0x1f;
+    int sa = (ctx->opcode >> 6) & 0x1f;
+    int func = ctx->opcode & 0x3f;
+    TCGv t0;
+    TCGv t1;
+
+    switch (func) {
+    case 0x03: /* DMUL */
+        if (sa != 0) {
+            generate_exception_end(ctx, EXCP_RI);
+            return true;
+        }
+        if (rt == 0) {
+            return true;
+        }
+        t0 = tcg_temp_new(tcg_ctx);
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t0, rs);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_mul_tl(tcg_ctx, t0, t0, t1);
+        gen_store_gpr(tcg_ctx, t0, rd);
+        tcg_temp_free(tcg_ctx, t0);
+        tcg_temp_free(tcg_ctx, t1);
+        return true;
+    case 0x28: /* BADDU */
+        if (sa != 0) {
+            generate_exception_end(ctx, EXCP_RI);
+            return true;
+        }
+        if (rt == 0) {
+            return true;
+        }
+        t0 = tcg_temp_new(tcg_ctx);
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t0, rs);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_add_tl(tcg_ctx, t0, t0, t1);
+        tcg_gen_andi_tl(tcg_ctx, t0, t0, 0xff);
+        gen_store_gpr(tcg_ctx, t0, rd);
+        tcg_temp_free(tcg_ctx, t0);
+        tcg_temp_free(tcg_ctx, t1);
+        return true;
+    case 0x2a: /* SEQ */
+    case 0x2b: /* SNE */
+        if (sa != 0) {
+            generate_exception_end(ctx, EXCP_RI);
+            return true;
+        }
+        if (rd == 0) {
+            return true;
+        }
+        t0 = tcg_temp_new(tcg_ctx);
+        t1 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t0, rs);
+        gen_load_gpr(tcg_ctx, t1, rt);
+        tcg_gen_setcond_tl(tcg_ctx, func == 0x2b ? TCG_COND_NE : TCG_COND_EQ,
+                           tcg_ctx->cpu_gpr[rd], t1, t0);
+        tcg_temp_free(tcg_ctx, t0);
+        tcg_temp_free(tcg_ctx, t1);
+        return true;
+    case 0x2c: /* POP */
+    case 0x2d: /* DPOP */
+        if (rt != 0 || sa != 0) {
+            generate_exception_end(ctx, EXCP_RI);
+            return true;
+        }
+        if (rd == 0) {
+            return true;
+        }
+        t0 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t0, rs);
+        if (func == 0x2c) {
+            tcg_gen_andi_tl(tcg_ctx, t0, t0, 0xffffffff);
+        }
+        tcg_gen_ctpop_tl(tcg_ctx, t0, t0);
+        gen_store_gpr(tcg_ctx, t0, rd);
+        tcg_temp_free(tcg_ctx, t0);
+        return true;
+    case 0x2e: /* SEQI */
+    case 0x2f: /* SNEI */
+        if (rt == 0) {
+            return true;
+        }
+        t0 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t0, rs);
+        tcg_gen_setcondi_tl(tcg_ctx,
+                            func == 0x2f ? TCG_COND_NE : TCG_COND_EQ,
+                            tcg_ctx->cpu_gpr[rt], t0,
+                            sextract32(ctx->opcode, 6, 10));
+        tcg_temp_free(tcg_ctx, t0);
+        return true;
+    default:
+        break;
+    }
+
+    switch (func & 0x3e) {
+    case 0x32: /* CINS/CINS32 */
+    case 0x3a: /* EXTS/EXTS32 */
+    {
+        int p = ((ctx->opcode & 1) << 5) | sa;
+        int len = rd + 1;
+
+        if (rt == 0) {
+            return true;
+        }
+        if (p + len > 64) {
+            generate_exception_end(ctx, EXCP_RI);
+            return true;
+        }
+        t0 = tcg_temp_new(tcg_ctx);
+        gen_load_gpr(tcg_ctx, t0, rs);
+        if ((func & 0x3e) == 0x32) {
+            tcg_gen_deposit_z_tl(tcg_ctx, t0, t0, p, len);
+        } else {
+            tcg_gen_sextract_tl(tcg_ctx, t0, t0, p, len);
+        }
+        gen_store_gpr(tcg_ctx, t0, rt);
+        tcg_temp_free(tcg_ctx, t0);
+        return true;
+    }
+    default:
+        break;
+    }
+
+    return false;
+}
+
+static bool decode_opc_octeon_bbit(DisasContext *ctx, uint32_t op,
+                                   int rs, int rt, int16_t imm)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int p;
+    bool set;
+    TCGv mask;
+    TCGv t0;
+
+    if (!(ctx->insn_flags & INSN_OCTEON)) {
+        return false;
+    }
+
+    switch (op) {
+    case OPC_BC:
+    case OPC_BALC:
+    case OPC_BEQZC:
+    case OPC_BNEZC:
+        break;
+    default:
+        return false;
+    }
+
+    if (ctx->hflags & MIPS_HFLAG_BMASK) {
+        generate_exception_end(ctx, EXCP_RI);
+        return true;
+    }
+
+    set = extract32(ctx->opcode, 29, 1);
+    p = (extract32(ctx->opcode, 28, 1) << 5) | rt;
+    mask = tcg_const_tl(tcg_ctx, 1ULL << p);
+    t0 = tcg_temp_new(tcg_ctx);
+    gen_load_gpr(tcg_ctx, t0, rs);
+    if (set) {
+        tcg_gen_and_tl(tcg_ctx, tcg_ctx->bcond, mask, t0);
+    } else {
+        tcg_gen_andc_tl(tcg_ctx, tcg_ctx->bcond, mask, t0);
+    }
+
+    ctx->hflags |= MIPS_HFLAG_BC | MIPS_HFLAG_BDS32;
+    ctx->btarget = addr_add(ctx, ctx->base.pc_next + 4, (int32_t)imm * 4);
+
+    tcg_temp_free(tcg_ctx, t0);
+    tcg_temp_free(tcg_ctx, mask);
+    return true;
+}
+#endif
+
 static void decode_opc_special3_r6(CPUMIPSState *env, DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -30133,6 +30755,10 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
         break;
     case OPC_SPECIAL2:
 #if defined(TARGET_MIPS64)
+        if ((ctx->insn_flags & INSN_OCTEON) &&
+            decode_opc_octeon_special2(ctx)) {
+            break;
+        }
         if ((ctx->insn_flags & INSN_R5900) && (ctx->insn_flags & ASE_MMI)) {
             decode_mmi(env, ctx);
 #else
@@ -30655,10 +31281,17 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     /* Compact branches [R6] and COP2 [non-R6] */
     case OPC_BC: /* OPC_LWC2 */
     case OPC_BALC: /* OPC_SWC2 */
+#if defined(TARGET_MIPS64)
+        if (decode_opc_octeon_bbit(ctx, op, rs, rt, imm)) {
+            break;
+        }
+#endif
         if (ctx->insn_flags & ISA_MIPS32R6) {
             /* OPC_BC, OPC_BALC */
             gen_compute_compact_branch(ctx, op, 0, 0,
                                        sextract32(ctx->opcode << 2, 0, 28));
+        } else if (ctx->insn_flags & ASE_LEXT) {
+            gen_loongson_lswc2(ctx, rt, rs);
         } else {
             /* OPC_LWC2, OPC_SWC2 */
             /* COP2: Not implemented. */
@@ -30667,6 +31300,11 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
         break;
     case OPC_BEQZC: /* OPC_JIC, OPC_LDC2 */
     case OPC_BNEZC: /* OPC_JIALC, OPC_SDC2 */
+#if defined(TARGET_MIPS64)
+        if (decode_opc_octeon_bbit(ctx, op, rs, rt, imm)) {
+            break;
+        }
+#endif
         if (ctx->insn_flags & ISA_MIPS32R6) {
             if (rs != 0) {
                 /* OPC_BEQZC, OPC_BNEZC */
@@ -30676,14 +31314,16 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                 /* OPC_JIC, OPC_JIALC */
                 gen_compute_compact_branch(ctx, op, 0, rt, imm);
             }
+        } else if (ctx->insn_flags & ASE_LEXT) {
+            gen_loongson_lsdc2(ctx, rt, rs, rd);
         } else {
-            /* OPC_LWC2, OPC_SWC2 */
+            /* OPC_LDC2, OPC_SDC2 */
             /* COP2: Not implemented. */
             generate_exception_err(ctx, EXCP_CpU, 2);
         }
         break;
     case OPC_CP2:
-        check_insn(ctx, INSN_LOONGSON2F);
+        check_insn(ctx, ASE_LMMI);
         /* Note that these instructions use different fields.  */
         gen_loongson_multimedia(ctx, sa, rd, rt);
         break;
@@ -30883,8 +31523,9 @@ static void mips_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs)
     ctx->gi = (env->CP0_Config5 >> CP0C5_GI) & 3;
     restore_cpu_state(env, ctx);
     ctx->mem_idx = hflags_mmu_index(ctx->hflags);
-    ctx->default_tcg_memop_mask = (ctx->insn_flags & ISA_MIPS32R6) ?
-                                  MO_UNALN : MO_ALIGN;
+    ctx->default_tcg_memop_mask =
+        (ctx->insn_flags & (ISA_MIPS32R6 | INSN_LOONGSON3A)) ? MO_UNALN :
+                                                               MO_ALIGN;
 
     LOG_DISAS("\ntb %p idx %d hflags %04x\n", ctx->base.tb, ctx->mem_idx,
               ctx->hflags);
@@ -31303,6 +31944,10 @@ void cpu_state_reset(CPUMIPSState *env)
     env->CP0_EntryHi_ASID_mask = (env->CP0_Config5 & (1 << CP0C5_MI)) ?
             0x0 : (env->CP0_Config4 & (1 << CP0C4_AE)) ? 0x3ff : 0xff;
     env->CP0_Status = (1 << CP0St_BEV) | (1 << CP0St_ERL);
+    if (env->insn_flags & INSN_LOONGSON2F) {
+        env->CP0_Status |= (1 << CP0St_KX) | (1 << CP0St_SX) |
+                            (1 << CP0St_UX);
+    }
     /*
      * Vectored interrupts not implemented, timer on int 7,
      * no performance counters.
@@ -31321,7 +31966,7 @@ void cpu_state_reset(CPUMIPSState *env)
     /* Count register increments in debug mode, EJTAG version 1 */
     env->CP0_Debug = (1 << CP0DB_CNT) | (0x1 << CP0DB_VER);
 
-    // cpu_mips_store_count(env, 1);
+    cpu_mips_store_count(env, 1);
 
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
         int i;
diff --git a/qemu/target/mips/translate_init.inc.c b/qemu/target/mips/translate_init.inc.c
index 3e395c7e6a..426c909b72 100644
--- a/qemu/target/mips/translate_init.inc.c
+++ b/qemu/target/mips/translate_init.inc.c
@@ -830,6 +830,121 @@ const mips_def_t mips_defs[] =
         .insn_flags = CPU_MIPS64R2 | ASE_DSP | ASE_DSP_R2,
         .mmu_type = MMU_TYPE_R4000,
     },
+    {
+        .name = "Octeon68XX",
+        .CP0_PRid = 0x000D9100,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+                       (0x2 << CP0C0_AT) | (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (0x3F << CP0C1_MMU) |
+                       (1 << CP0C1_IS) | (4 << CP0C1_IL) |
+                       (1 << CP0C1_IA) | (1 << CP0C1_DS) |
+                       (4 << CP0C1_DL) | (1 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_WR) |
+                       (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_LPA),
+        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) |
+                       (0x3c << CP0C4_KScrExist) |
+                       (1U << CP0C4_MMUExtDef) |
+                       (3U << CP0C4_MMUSizeExt),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 4,
+        .CP0_PageGrain = (1 << CP0PG_ELPA),
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x12F8FFFF,
+        .SEGBITS = 42,
+        .PABITS = 49,
+        .insn_flags = CPU_OCTEON,
+        .mmu_type = MMU_TYPE_R4000,
+    },
+    {
+        .name = "Loongson-3A1000",
+        .CP0_PRid = 0x6305,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+                       (0x2 << CP0C0_AT) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) |
+                       (63 << CP0C1_MMU) | (3 << CP0C1_IS) |
+                       (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (3 << CP0C1_DS) | (4 << CP0C1_DL) |
+                       (3 << CP0C1_DA) | (1 << CP0C1_PC) |
+                       (1 << CP0C1_WR) | (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2 | (7 << CP0C2_SS) |
+                       (4 << CP0C2_SL) | (3 << CP0C2_SA),
+        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_LPA),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x74D8FFFF,
+        .CP0_PageGrain = (1 << CP0PG_ELPA),
+        .CP0_PageGrain_rw_bitmask = (1 << CP0PG_ELPA),
+        .CP1_fcr0 = (0x5 << FCR0_PRID) | (0x1 << FCR0_REV) |
+                    (0x1 << FCR0_F64) | (0x1 << FCR0_PS) |
+                    (0x1 << FCR0_L) | (0x1 << FCR0_W) |
+                    (0x1 << FCR0_D) | (0x1 << FCR0_S),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
+        .SEGBITS = 48,
+        .PABITS = 48,
+        .insn_flags = CPU_LOONGSON3A,
+        .mmu_type = MMU_TYPE_R4000,
+    },
+    {
+        .name = "Loongson-3A4000",
+        .CP0_PRid = 0x14C000,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+                       (0x2 << CP0C0_AT) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) |
+                       (63 << CP0C1_MMU) | (2 << CP0C1_IS) |
+                       (5 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (5 << CP0C1_DL) |
+                       (3 << CP0C1_DA) | (1 << CP0C1_PC) |
+                       (1 << CP0C1_WR) | (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2 | (5 << CP0C2_SS) |
+                       (5 << CP0C2_SL) | (15 << CP0C2_SA),
+        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) |
+                       (1 << CP0C3_MSAP) | (1 << CP0C3_BP) |
+                       (1 << CP0C3_BI) | (1 << CP0C3_ULRI) |
+                       (1 << CP0C3_RXI) | (1 << CP0C3_LPA) |
+                       (1 << CP0C3_VInt),
+        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) |
+                       (2 << CP0C4_IE) | (1 << CP0C4_AE) |
+                       (0x1c << CP0C4_KScrExist),
+        .CP0_Config4_rw_bitmask = 0,
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_CRCP) |
+                       (1 << CP0C5_NFExists),
+        .CP0_Config5_rw_bitmask = (1 << CP0C5_K) | (1 << CP0C5_CV) |
+                                  (1 << CP0C5_MSAEn) |
+                                  (1 << CP0C5_UFE) |
+                                  (1 << CP0C5_FRE) |
+                                  (1 << CP0C5_SBRI),
+        .CP0_Config6 = (1 << CP0C6_VCLRU) | (1 << CP0C6_DCLRU) |
+                       (1 << CP0C6_SFBEN) | (1 << CP0C6_VLTINT) |
+                       (1 << CP0C6_INSTPREF) | (1 << CP0C6_DATAPREF),
+        .CP0_Config7 = 0,
+        .CP0_LLAddr_rw_bitmask = 1,
+        .SYNCI_Step = 16,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x7DDBFFFF,
+        .CP0_PageGrain = (1 << CP0PG_ELPA),
+        .CP0_PageGrain_rw_bitmask = (1U << CP0PG_RIE) |
+                                    (1 << CP0PG_XIE) |
+                                    (1 << CP0PG_ELPA) |
+                                    (1 << CP0PG_IEC),
+        .CP1_fcr0 = (0x5 << FCR0_PRID) | (0x1 << FCR0_REV) |
+                    (0x1 << FCR0_F64) | (0x1 << FCR0_PS) |
+                    (0x1 << FCR0_L) | (0x1 << FCR0_W) |
+                    (0x1 << FCR0_D) | (0x1 << FCR0_S),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
+        .MSAIR = (0x01 << MSAIR_ProcID) | (0x40 << MSAIR_Rev),
+        .SEGBITS = 48,
+        .PABITS = 48,
+        .insn_flags = CPU_LOONGSON3A,
+        .mmu_type = MMU_TYPE_R4000,
+    },
 
 #endif
 };
diff --git a/qemu/target/mips/unicorn.c b/qemu/target/mips/unicorn.c
index 7785265e58..e5b4dced6d 100644
--- a/qemu/target/mips/unicorn.c
+++ b/qemu/target/mips/unicorn.c
@@ -9,6 +9,8 @@
 #include "unicorn.h"
 #include "internal.h"
 
+#include <string.h>
+
 #ifdef TARGET_MIPS64
 typedef uint64_t mipsreg_t;
 #else
@@ -19,12 +21,13 @@ MIPSCPU *cpu_mips_init(struct uc_struct *uc);
 
 static void mips_set_pc(struct uc_struct *uc, uint64_t address)
 {
-    ((CPUMIPSState *)uc->cpu->env_ptr)->active_tc.PC =
-        address & ~(uint64_t)1ULL;
-    if (address & 1) {
-        ((CPUMIPSState *)uc->cpu->env_ptr)->hflags |= MIPS_HFLAG_M16;
+    CPUMIPSState *env = (CPUMIPSState *)uc->cpu->env_ptr;
+
+    env->active_tc.PC = address & ~(uint64_t)1ULL;
+    if ((address & 1) || (uc->mode & UC_MODE_MICRO)) {
+        env->hflags |= MIPS_HFLAG_M16;
     } else {
-        ((CPUMIPSState *)uc->cpu->env_ptr)->hflags &= ~(MIPS_HFLAG_M16);
+        env->hflags &= ~(MIPS_HFLAG_M16);
     }
 }
 
@@ -76,6 +79,10 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
     if (regid >= UC_MIPS_REG_0 && regid <= UC_MIPS_REG_31) {
         CHECK_REG_TYPE(mipsreg_t);
         *(mipsreg_t *)value = env->active_tc.gpr[regid - UC_MIPS_REG_0];
+    } else if (regid >= UC_MIPS_REG_W0 && regid <= UC_MIPS_REG_W31) {
+        CHECK_REG_TYPE(uint64_t[2]);
+        memcpy(value, &env->active_fpu.fpr[regid - UC_MIPS_REG_W0].wr,
+               sizeof(uint64_t[2]));
     } else {
         switch (regid) {
         default:
@@ -164,6 +171,10 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
     if (regid >= UC_MIPS_REG_0 && regid <= UC_MIPS_REG_31) {
         CHECK_REG_TYPE(mipsreg_t);
         env->active_tc.gpr[regid - UC_MIPS_REG_0] = *(mipsreg_t *)value;
+    } else if (regid >= UC_MIPS_REG_W0 && regid <= UC_MIPS_REG_W31) {
+        CHECK_REG_TYPE(uint64_t[2]);
+        memcpy(&env->active_fpu.fpr[regid - UC_MIPS_REG_W0].wr, value,
+               sizeof(uint64_t[2]));
     } else {
         switch (regid) {
         default:
@@ -178,8 +189,8 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             break;
         case UC_MIPS_REG_PC:
             CHECK_REG_TYPE(mipsreg_t);
-            env->active_tc.PC = *(mipsreg_t *)value & ~1ULL;
-            if ((*(uint32_t *)value & 1)) {
+            env->active_tc.PC = *(mipsreg_t *)value & ~(mipsreg_t)1;
+            if ((*(mipsreg_t *)value & 1) || (mode & UC_MODE_MICRO)) {
                 env->hflags |= MIPS_HFLAG_M16;
             } else {
                 env->hflags &= ~(MIPS_HFLAG_M16);
diff --git a/qemu/target/ppc/cpu.h b/qemu/target/ppc/cpu.h
index 191050fa12..ee98a161a3 100644
--- a/qemu/target/ppc/cpu.h
+++ b/qemu/target/ppc/cpu.h
@@ -148,6 +148,7 @@ enum {
     POWERPC_EXCP_ALIGN_PROT    = 0x04,  /* Access cross protection boundary  */
     POWERPC_EXCP_ALIGN_BAT     = 0x05,  /* Access cross a BAT/seg boundary   */
     POWERPC_EXCP_ALIGN_CACHE   = 0x06,  /* Impossible dcbz access            */
+    POWERPC_EXCP_ALIGN_INSN    = 0x07,  /* Pref. insn x-ing 64-byte boundary */
     /* Exception subtypes for POWERPC_EXCP_PROGRAM                           */
     /* FP exceptions                                                         */
     POWERPC_EXCP_FP            = 0x10,
@@ -220,6 +221,7 @@ typedef union _ppc_vsr_t {
     int16_t s16[8];
     int32_t s32[4];
     int64_t s64[2];
+    float16 f16[8];
     float32 f32[4];
     float64 f64[2];
     float128 f128;
@@ -231,6 +233,7 @@ typedef union _ppc_vsr_t {
 
 typedef ppc_vsr_t ppc_avr_t;
 typedef ppc_vsr_t ppc_fprp_t;
+typedef ppc_vsr_t ppc_acc_t;
 
 /* Software TLB cache */
 typedef struct ppc6xx_tlb_t ppc6xx_tlb_t;
@@ -1011,9 +1014,9 @@ struct CPUPPCState {
     /* Vector status and control register, minus VSCR_SAT */
     uint32_t vscr;
     /* VSX registers (including FP and AVR) */
-    ppc_vsr_t vsr[64] QEMU_ALIGNED(16);
+    QEMU_ALIGN(16, ppc_vsr_t vsr[64]);
     /* Non-zero if and only if VSCR_SAT should be set */
-    ppc_vsr_t vscr_sat QEMU_ALIGNED(16);
+    QEMU_ALIGN(16, ppc_vsr_t vscr_sat);
     /* SPE registers */
     uint64_t spe_acc;
     uint32_t spe_fscr;
@@ -1140,7 +1143,7 @@ struct PowerPCCPU {
     /*< public >*/
 
     CPUNegativeOffsetState neg;
-    CPUPPCState env;
+    QEMU_ALIGN(16, CPUPPCState env);
 
     int vcpu_id;
     uint32_t compat_pvr;
@@ -1530,6 +1533,8 @@ typedef PowerPCCPU ArchCPU;
 #define SPR_BOOKE_GIVOR14     (0x1BD)
 #define SPR_TIR               (0x1BE)
 #define SPR_PTCR              (0x1D0)
+#define SPR_HASHKEYR          (0x1D4)
+#define SPR_HASHPKEYR         (0x1D5)
 #define SPR_BOOKE_SPEFSCR     (0x200)
 #define SPR_Exxx_BBEAR        (0x201)
 #define SPR_Exxx_BBTAR        (0x202)
@@ -2186,6 +2191,10 @@ enum {
     PPC2_PM_ISA206     = 0x0000000000040000ULL,
     /* POWER ISA 3.0                                                         */
     PPC2_ISA300        = 0x0000000000080000ULL,
+    /* POWER ISA 3.1                                                         */
+    PPC2_ISA310        = 0x0000000000100000ULL,
+    /* ISA 2.06 BCD assist instructions                                      */
+    PPC2_BCDA_ISA206   = 0x0000000000400000ULL,
 
 #define PPC_TCG_INSNS2 (PPC2_BOOKE206 | PPC2_VSX | PPC2_PRCNTL | PPC2_DBRX | \
                         PPC2_ISA205 | PPC2_VSX207 | PPC2_PERM_ISA206 | \
@@ -2194,7 +2203,7 @@ enum {
                         PPC2_BCTAR_ISA207 | PPC2_LSQ_ISA207 | \
                         PPC2_ALTIVEC_207 | PPC2_ISA207S | PPC2_DFP | \
                         PPC2_FP_CVT_S64 | PPC2_TM | PPC2_PM_ISA206 | \
-                        PPC2_ISA300)
+                        PPC2_ISA300 | PPC2_ISA310 | PPC2_BCDA_ISA206)
 };
 
 /*****************************************************************************/
@@ -2573,6 +2582,9 @@ static inline bool lsw_reg_in_range(int start, int nregs, int rx)
 #define VsrSW(i) s32[i]
 #define VsrD(i) u64[i]
 #define VsrSD(i) s64[i]
+#define VsrHF(i) f16[i]
+#define VsrSF(i) f32[i]
+#define VsrDF(i) f64[i]
 #else
 #define VsrB(i) u8[15 - (i)]
 #define VsrSB(i) s8[15 - (i)]
@@ -2582,6 +2594,9 @@ static inline bool lsw_reg_in_range(int start, int nregs, int rx)
 #define VsrSW(i) s32[3 - (i)]
 #define VsrD(i) u64[1 - (i)]
 #define VsrSD(i) s64[1 - (i)]
+#define VsrHF(i) f16[7 - (i)]
+#define VsrSF(i) f32[3 - (i)]
+#define VsrDF(i) f64[1 - (i)]
 #endif
 
 static inline int vsr64_offset(int i, bool high)
@@ -2594,6 +2609,11 @@ static inline int vsr_full_offset(int i)
     return offsetof(CPUPPCState, vsr[i].u64[0]);
 }
 
+static inline int acc_full_offset(int i)
+{
+    return vsr_full_offset(i * 4);
+}
+
 static inline int fpr_offset(int i)
 {
     return vsr64_offset(i, true);
diff --git a/qemu/target/ppc/dfp_helper.c b/qemu/target/ppc/dfp_helper.c
index a025ed362e..2e318d5e5a 100644
--- a/qemu/target/ppc/dfp_helper.c
+++ b/qemu/target/ppc/dfp_helper.c
@@ -51,6 +51,11 @@ static void set_dfp128(ppc_fprp_t *dfp, ppc_vsr_t *src)
     dfp[1].VsrD(0) = src->VsrD(1);
 }
 
+static void set_dfp128_to_avr(ppc_avr_t *dst, ppc_vsr_t *src)
+{
+    *dst = *src;
+}
+
 struct PPC_DFP {
     CPUPPCState *env;
     ppc_vsr_t vt, va, vb;
@@ -970,6 +975,18 @@ static void CFFIX_PPs(struct PPC_DFP *dfp)
 DFP_HELPER_CFFIX(dcffix, 64)
 DFP_HELPER_CFFIX(dcffixq, 128)
 
+void helper_DCFFIXQQ(CPUPPCState *env, ppc_fprp_t *t, ppc_avr_t *b)
+{
+    struct PPC_DFP dfp;
+
+    dfp_prepare_decimal128(&dfp, NULL, NULL, env);
+    decNumberFromInt128(&dfp.t, (uint64_t)b->VsrD(1), (int64_t)b->VsrD(0));
+    dfp_finalize_decimal128(&dfp);
+    CFFIX_PPs(&dfp);
+
+    set_dfp128(t, &dfp.vt);
+}
+
 #define DFP_HELPER_CTFIX(op, size)                                            \
 void helper_##op(CPUPPCState *env, ppc_fprp_t *t, ppc_fprp_t *b)              \
 {                                                                             \
@@ -1008,6 +1025,55 @@ void helper_##op(CPUPPCState *env, ppc_fprp_t *t, ppc_fprp_t *b)              \
 DFP_HELPER_CTFIX(dctfix, 64)
 DFP_HELPER_CTFIX(dctfixq, 128)
 
+void helper_DCTFIXQQ(CPUPPCState *env, ppc_avr_t *t, ppc_fprp_t *b)
+{
+    struct PPC_DFP dfp;
+
+    dfp_prepare_decimal128(&dfp, 0, b, env);
+
+    if (unlikely(decNumberIsSpecial(&dfp.b))) {
+        uint64_t invalid_flags = FP_VX | FP_VXCVI;
+
+        if (decNumberIsInfinite(&dfp.b)) {
+            if (decNumberIsNegative(&dfp.b)) {
+                dfp.vt.VsrD(0) = INT64_MIN;
+                dfp.vt.VsrD(1) = 0;
+            } else {
+                dfp.vt.VsrD(0) = INT64_MAX;
+                dfp.vt.VsrD(1) = UINT64_MAX;
+            }
+        } else {
+            dfp.vt.VsrD(0) = INT64_MIN;
+            dfp.vt.VsrD(1) = 0;
+            if (decNumberIsSNaN(&dfp.b)) {
+                invalid_flags |= FP_VXSNAN;
+            }
+        }
+        dfp_set_FPSCR_flag(&dfp, invalid_flags, FP_VE);
+    } else if (unlikely(decNumberIsZero(&dfp.b))) {
+        dfp.vt.VsrD(0) = 0;
+        dfp.vt.VsrD(1) = 0;
+    } else {
+        decNumberToIntegralExact(&dfp.b, &dfp.b, &dfp.context);
+        decNumberIntegralToInt128(&dfp.b, &dfp.context, &dfp.vt.VsrD(1),
+                                  &dfp.vt.VsrD(0));
+        if (decContextTestStatus(&dfp.context, DEC_Invalid_operation)) {
+            if (decNumberIsNegative(&dfp.b)) {
+                dfp.vt.VsrD(0) = INT64_MIN;
+                dfp.vt.VsrD(1) = 0;
+            } else {
+                dfp.vt.VsrD(0) = INT64_MAX;
+                dfp.vt.VsrD(1) = UINT64_MAX;
+            }
+            dfp_set_FPSCR_flag(&dfp, FP_VX | FP_VXCVI, FP_VE);
+        } else {
+            dfp_check_for_XX(&dfp);
+        }
+    }
+
+    set_dfp128_to_avr(t, &dfp.vt);
+}
+
 static inline void dfp_set_bcd_digit_64(ppc_vsr_t *t, uint8_t digit,
                                         unsigned n)
 {
@@ -1329,3 +1395,63 @@ DFP_HELPER_SHIFT(dscli, 64, 1)
 DFP_HELPER_SHIFT(dscliq, 128, 1)
 DFP_HELPER_SHIFT(dscri, 64, 0)
 DFP_HELPER_SHIFT(dscriq, 128, 0)
+
+target_ulong helper_CDTBCD(target_ulong s)
+{
+    uint64_t res = 0;
+    uint32_t dec32;
+    uint32_t declets;
+    uint8_t bcd[6];
+    int i;
+    int w;
+    int sh;
+    decNumber a;
+
+    for (w = 1; w >= 0; w--) {
+        res <<= 32;
+        declets = extract64(s, 32 * w, 20);
+        if (declets) {
+            dec32 = (0x225ULL << 20) | declets;
+            decimal32ToNumber((decimal32 *)&dec32, &a);
+            decNumberGetBCD(&a, bcd);
+            for (i = 0; i < a.digits; i++) {
+                sh = 4 * (a.digits - 1 - i);
+                res |= (uint64_t)bcd[i] << sh;
+            }
+        }
+    }
+
+    return res;
+}
+
+target_ulong helper_CBCDTD(target_ulong s)
+{
+    uint64_t res = 0;
+    uint32_t dec32;
+    uint8_t bcd[6];
+    int i;
+    int offs;
+    int w;
+    decNumber a;
+    decContext context;
+
+    decContextDefault(&context, DEC_INIT_DECIMAL32);
+
+    for (w = 1; w >= 0; w--) {
+        res <<= 32;
+        decNumberZero(&a);
+        for (i = 5; i >= 0; i--) {
+            offs = 4 * (5 - i) + 32 * w;
+            bcd[i] = extract64(s, offs, 4);
+            if (bcd[i] > 9) {
+                bcd[i] &= 9;
+            }
+        }
+
+        decNumberSetBCD(&a, bcd, 6);
+        decimal32FromNumber((decimal32 *)&dec32, &a, &context);
+        res |= dec32 & 0xfffff;
+    }
+
+    return res;
+}
diff --git a/qemu/target/ppc/excp_helper.c b/qemu/target/ppc/excp_helper.c
index 298b7730a1..7aa761a807 100644
--- a/qemu/target/ppc/excp_helper.c
+++ b/qemu/target/ppc/excp_helper.c
@@ -1193,6 +1193,95 @@ void helper_td(CPUPPCState *env, target_ulong arg1, target_ulong arg2,
 }
 #endif
 
+#if defined(TARGET_PPC64)
+static uint32_t helper_SIMON_LIKE_32_64(uint32_t x, uint64_t key,
+                                        uint32_t lane)
+{
+    const uint16_t c = 0xfffc;
+    const uint64_t z0 = 0xfa2561cdf44ac398ULL;
+    uint16_t z = 0, temp;
+    uint16_t k[32], eff_k[32], xleft[33], xright[33], fxleft[32];
+    int i;
+
+    for (i = 3; i >= 0; i--) {
+        k[i] = key & 0xffff;
+        key >>= 16;
+    }
+    xleft[0] = x & 0xffff;
+    xright[0] = (x >> 16) & 0xffff;
+
+    for (i = 0; i < 28; i++) {
+        z = (z0 >> (63 - i)) & 1;
+        temp = ror16(k[i + 3], 3) ^ k[i + 1];
+        k[i + 4] = c ^ z ^ k[i] ^ temp ^ ror16(temp, 1);
+    }
+
+    for (i = 0; i < 8; i++) {
+        eff_k[4 * i + 0] = k[4 * i + ((0 + lane) % 4)];
+        eff_k[4 * i + 1] = k[4 * i + ((1 + lane) % 4)];
+        eff_k[4 * i + 2] = k[4 * i + ((2 + lane) % 4)];
+        eff_k[4 * i + 3] = k[4 * i + ((3 + lane) % 4)];
+    }
+
+    for (i = 0; i < 32; i++) {
+        fxleft[i] = (rol16(xleft[i], 1) & rol16(xleft[i], 8)) ^
+                    rol16(xleft[i], 2);
+        xleft[i + 1] = xright[i] ^ fxleft[i] ^ eff_k[i];
+        xright[i + 1] = xleft[i];
+    }
+
+    return (((uint32_t)xright[32]) << 16) | xleft[32];
+}
+
+static uint64_t hash_digest(uint64_t ra, uint64_t rb, uint64_t key)
+{
+    uint64_t stage0_h = 0, stage0_l = 0;
+    uint64_t stage1_h, stage1_l;
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        stage0_h |= ror64(rb & 0xff, 8 * (2 * i + 1));
+        stage0_h |= ((ra >> 32) & 0xff) << (8 * 2 * i);
+        stage0_l |= ror64((rb >> 32) & 0xff, 8 * (2 * i + 1));
+        stage0_l |= (ra & 0xff) << (8 * 2 * i);
+        rb >>= 8;
+        ra >>= 8;
+    }
+
+    stage1_h = (uint64_t)helper_SIMON_LIKE_32_64(stage0_h >> 32,
+                                                 key, 0) << 32;
+    stage1_h |= helper_SIMON_LIKE_32_64(stage0_h, key, 1);
+    stage1_l = (uint64_t)helper_SIMON_LIKE_32_64(stage0_l >> 32,
+                                                 key, 2) << 32;
+    stage1_l |= helper_SIMON_LIKE_32_64(stage0_l, key, 3);
+
+    return stage1_h ^ stage1_l;
+}
+
+#define HELPER_HASH(op, key, store)                                           \
+void helper_##op(CPUPPCState *env, target_ulong ea, target_ulong ra,          \
+                 target_ulong rb)                                             \
+{                                                                             \
+    uint64_t calculated_hash = hash_digest(ra, rb, key);                      \
+    uint64_t loaded_hash;                                                     \
+                                                                              \
+    if (store) {                                                              \
+        cpu_stq_data_ra(env, ea, calculated_hash, GETPC());                   \
+    } else {                                                                  \
+        loaded_hash = cpu_ldq_data_ra(env, ea, GETPC());                      \
+        if (loaded_hash != calculated_hash) {                                 \
+            raise_exception_err_ra(env, POWERPC_EXCP_PROGRAM,                 \
+                                   POWERPC_EXCP_TRAP, GETPC());               \
+        }                                                                     \
+    }                                                                         \
+}
+
+HELPER_HASH(HASHST, env->spr[SPR_HASHKEYR], true)
+HELPER_HASH(HASHCHK, env->spr[SPR_HASHKEYR], false)
+HELPER_HASH(HASHSTP, env->spr[SPR_HASHPKEYR], true)
+HELPER_HASH(HASHCHKP, env->spr[SPR_HASHPKEYR], false)
+#endif
+
 /*****************************************************************************/
 /* PowerPC 601 specific instructions (POWER bridge) */
 
diff --git a/qemu/target/ppc/fpu_helper.c b/qemu/target/ppc/fpu_helper.c
index c6630e5ffc..25acf38d04 100644
--- a/qemu/target/ppc/fpu_helper.c
+++ b/qemu/target/ppc/fpu_helper.c
@@ -36,6 +36,15 @@ static inline float128 float128_snan_to_qnan(float128 x)
 #define float32_snan_to_qnan(x) ((x) | 0x00400000)
 #define float16_snan_to_qnan(x) ((x) | 0x0200)
 
+static inline float32 bfp32_neg(float32 x)
+{
+    if (likely(!float32_is_any_nan(x))) {
+        x = float32_chs(x);
+    }
+
+    return x;
+}
+
 static inline bool fp_exceptions_enabled(CPUPPCState *env)
 {
     return (env->msr & ((1U << MSR_FE0) | (1U << MSR_FE1))) != 0;
@@ -710,6 +719,18 @@ static void float_invalid_op_mul(CPUPPCState *env, bool set_fprc,
     }
 }
 
+static void float_invalid_op_madd(CPUPPCState *env, int flags,
+                                  bool set_fpcc, uintptr_t retaddr)
+{
+    if (flags & float_flag_invalid_imz) {
+        float_invalid_op_vximz(env, set_fpcc, retaddr);
+    } else if (flags & float_flag_invalid_isi) {
+        float_invalid_op_vxisi(env, set_fpcc, retaddr);
+    } else if (flags & float_flag_invalid_snan) {
+        float_invalid_op_vxsnan(env, retaddr);
+    }
+}
+
 /* fmul - fmul. */
 float64 helper_fmul(CPUPPCState *env, float64 arg1, float64 arg2)
 {
@@ -2354,6 +2375,40 @@ VSX_MADD(xvmsubsp, 4, float32, VsrW(i), MSUB_FLGS, 0, 0)
 VSX_MADD(xvnmaddsp, 4, float32, VsrW(i), NMADD_FLGS, 0, 0)
 VSX_MADD(xvnmsubsp, 4, float32, VsrW(i), NMSUB_FLGS, 0, 0)
 
+#define VSX_MADDQ(op, maddflgs, ro)                                            \
+void helper_##op(CPUPPCState *env, ppc_vsr_t *xt, ppc_vsr_t *s1, ppc_vsr_t *s2,\
+                 ppc_vsr_t *s3)                                                \
+{                                                                              \
+    ppc_vsr_t t = *xt;                                                         \
+    float_status tstat;                                                        \
+                                                                               \
+    helper_reset_fpstatus(env);                                                \
+    tstat = env->fp_status;                                                    \
+    set_float_exception_flags(0, &tstat);                                      \
+    if (ro) {                                                                  \
+        tstat.float_rounding_mode = float_round_to_odd;                        \
+    }                                                                          \
+    t.f128 = float128_muladd(s1->f128, s3->f128, s2->f128, maddflgs, &tstat);  \
+    env->fp_status.float_exception_flags |= tstat.float_exception_flags;       \
+                                                                               \
+    if (unlikely(tstat.float_exception_flags & float_flag_invalid)) {          \
+        float_invalid_op_madd(env, tstat.float_exception_flags, false, GETPC());\
+    }                                                                          \
+                                                                               \
+    helper_compute_fprf_float128(env, t.f128);                                 \
+    *xt = t;                                                                   \
+    do_float_check_status(env, GETPC());                                       \
+}
+
+VSX_MADDQ(XSMADDQP, MADD_FLGS, 0)
+VSX_MADDQ(XSMADDQPO, MADD_FLGS, 1)
+VSX_MADDQ(XSMSUBQP, MSUB_FLGS, 0)
+VSX_MADDQ(XSMSUBQPO, MSUB_FLGS, 1)
+VSX_MADDQ(XSNMADDQP, NMADD_FLGS, 0)
+VSX_MADDQ(XSNMADDQPO, NMADD_FLGS, 1)
+VSX_MADDQ(XSNMSUBQP, NMSUB_FLGS, 0)
+VSX_MADDQ(XSNMSUBQPO, NMSUB_FLGS, 0)
+
 /*
  * VSX_SCALAR_CMP_DP - VSX scalar floating point compare double precision
  *   op    - instruction mnemonic
@@ -2405,6 +2460,44 @@ VSX_SCALAR_CMP_DP(xscmpgedp, le, 1, 1)
 VSX_SCALAR_CMP_DP(xscmpgtdp, lt, 1, 1)
 VSX_SCALAR_CMP_DP(xscmpnedp, eq, 0, 0)
 
+#define VSX_SCALAR_CMP_QP_VECTOR(op, cmp, svxvc)                              \
+void helper_##op(CPUPPCState *env, ppc_vsr_t *xt,                             \
+                 ppc_vsr_t *xa, ppc_vsr_t *xb)                                \
+{                                                                             \
+    ppc_vsr_t t = { 0 };                                                      \
+    bool r;                                                                   \
+    int flags;                                                                \
+    bool vxvc = false;                                                        \
+                                                                              \
+    helper_reset_fpstatus(env);                                               \
+                                                                              \
+    if (svxvc) {                                                              \
+        r = float128_##cmp(xb->f128, xa->f128, &env->fp_status);              \
+    } else {                                                                  \
+        r = float128_##cmp##_quiet(xb->f128, xa->f128, &env->fp_status);      \
+    }                                                                         \
+                                                                              \
+    flags = get_float_exception_flags(&env->fp_status);                       \
+    if (unlikely(flags & float_flag_invalid)) {                               \
+        vxvc = svxvc;                                                         \
+        if (flags & float_flag_invalid_snan) {                                \
+            float_invalid_op_vxsnan(env, GETPC());                            \
+            vxvc &= !(env->fpscr & FP_VE);                                    \
+        }                                                                     \
+        if (vxvc) {                                                           \
+            float_invalid_op_vxvc(env, 0, GETPC());                           \
+        }                                                                     \
+    }                                                                         \
+                                                                              \
+    memset(&t.f128, r ? 0xff : 0, sizeof(t.f128));                            \
+    *xt = t;                                                                  \
+    do_float_check_status(env, GETPC());                                      \
+}
+
+VSX_SCALAR_CMP_QP_VECTOR(XSCMPEQQP, eq, 0)
+VSX_SCALAR_CMP_QP_VECTOR(XSCMPGEQP, le, 1)
+VSX_SCALAR_CMP_QP_VECTOR(XSCMPGTQP, lt, 1)
+
 void helper_xscmpexpdp(CPUPPCState *env, uint32_t opcode,
                        ppc_vsr_t *xa, ppc_vsr_t *xb)
 {
@@ -2628,6 +2721,37 @@ void helper_##name(CPUPPCState *env, uint32_t opcode,                         \
 VSX_MAX_MINC(xsmaxcdp, 1);
 VSX_MAX_MINC(xsmincdp, 0);
 
+#define VSX_MAX_MINC_QP(name, max)                                            \
+void helper_##name(CPUPPCState *env,                                          \
+                   ppc_vsr_t *xt, ppc_vsr_t *xa, ppc_vsr_t *xb)               \
+{                                                                             \
+    ppc_vsr_t t = { 0 };                                                      \
+    bool first;                                                               \
+                                                                              \
+    helper_reset_fpstatus(env);                                               \
+                                                                              \
+    if (max) {                                                                \
+        first = float128_le_quiet(xb->f128, xa->f128, &env->fp_status);       \
+    } else {                                                                  \
+        first = float128_lt_quiet(xa->f128, xb->f128, &env->fp_status);       \
+    }                                                                         \
+                                                                              \
+    if (first) {                                                              \
+        t.f128 = xa->f128;                                                    \
+    } else {                                                                  \
+        t.f128 = xb->f128;                                                    \
+        if (env->fp_status.float_exception_flags & float_flag_invalid_snan) { \
+            float_invalid_op_vxsnan(env, GETPC());                            \
+        }                                                                     \
+    }                                                                         \
+                                                                              \
+    *xt = t;                                                                  \
+    do_float_check_status(env, GETPC());                                      \
+}
+
+VSX_MAX_MINC_QP(XSMAXCQP, true)
+VSX_MAX_MINC_QP(XSMINCQP, false)
+
 #define VSX_MAX_MINJ(name, max)                                               \
 void helper_##name(CPUPPCState *env, uint32_t opcode,                         \
                    ppc_vsr_t *xt, ppc_vsr_t *xa, ppc_vsr_t *xb)               \
@@ -2850,6 +2974,39 @@ VSX_CVT_FP_TO_FP_HP(xscvhpdp, 1, float16, float64, VsrH(3), VsrD(0), 1)
 VSX_CVT_FP_TO_FP_HP(xvcvsphp, 4, float32, float16, VsrW(i), VsrH(2 * i  + 1), 0)
 VSX_CVT_FP_TO_FP_HP(xvcvhpsp, 4, float16, float32, VsrH(2 * i + 1), VsrW(i), 0)
 
+void helper_XVCVSPBF16(CPUPPCState *env, ppc_vsr_t *xt, ppc_vsr_t *xb)
+{
+    ppc_vsr_t t = { 0 };
+    int i, status;
+
+    helper_reset_fpstatus(env);
+
+    for (i = 0; i < 4; i++) {
+        t.VsrH(2 * i + 1) = float32_to_bfloat16(xb->VsrW(i),
+                                                &env->fp_status);
+    }
+
+    status = get_float_exception_flags(&env->fp_status);
+    if (unlikely(status & float_flag_invalid_snan)) {
+        float_invalid_op_vxsnan(env, GETPC());
+    }
+
+    *xt = t;
+    do_float_check_status(env, GETPC());
+}
+
+void helper_XVCVBF16SPN(ppc_vsr_t *xt, ppc_vsr_t *xb)
+{
+    ppc_vsr_t t = { 0 };
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        t.VsrW(i) = xb->VsrW(i) << 16;
+    }
+
+    *xt = t;
+}
+
 /*
  * xscvqpdp isn't using VSX_CVT_FP_TO_FP() because xscvqpdpo will be
  * added to this later.
@@ -3009,6 +3166,28 @@ VSX_CVT_FP_TO_INT_VECTOR(xscvqpswz, float128, int32, f128, VsrD(0),          \
 VSX_CVT_FP_TO_INT_VECTOR(xscvqpudz, float128, uint64, f128, VsrD(0), 0x0ULL)
 VSX_CVT_FP_TO_INT_VECTOR(xscvqpuwz, float128, uint32, f128, VsrD(0), 0x0ULL)
 
+#define VSX_CVT_FP_TO_INT128(op, tp, rnan)                                    \
+void helper_##op(CPUPPCState *env, ppc_vsr_t *xt, ppc_vsr_t *xb)              \
+{                                                                             \
+    ppc_vsr_t t;                                                              \
+    int flags;                                                                \
+                                                                              \
+    helper_reset_fpstatus(env);                                               \
+    t.s128 = float128_to_##tp##_round_to_zero(xb->f128, &env->fp_status);     \
+    flags = get_float_exception_flags(&env->fp_status);                       \
+    if (unlikely(flags & float_flag_invalid)) {                               \
+        float_invalid_cvt(env, 0, GETPC(), float128_classify(xb->f128));      \
+        t.VsrD(0) = rnan;                                                     \
+        t.VsrD(1) = -(t.VsrD(0) & 1);                                         \
+    }                                                                         \
+                                                                              \
+    *xt = t;                                                                  \
+    do_float_check_status(env, GETPC());                                      \
+}
+
+VSX_CVT_FP_TO_INT128(XSCVQPUQZ, uint128, 0)
+VSX_CVT_FP_TO_INT128(XSCVQPSQZ, int128, 0x8000000000000000ULL)
+
 /*
  * VSX_CVT_INT_TO_FP - VSX integer to floating point conversion
  *   op    - instruction mnemonic
@@ -3077,6 +3256,18 @@ void helper_##op(CPUPPCState *env, uint32_t opcode,                     \
 VSX_CVT_INT_TO_FP_VECTOR(xscvsdqp, int64, float128, VsrD(0), f128)
 VSX_CVT_INT_TO_FP_VECTOR(xscvudqp, uint64, float128, VsrD(0), f128)
 
+#define VSX_CVT_INT128_TO_FP(op, tp)                            \
+void helper_##op(CPUPPCState *env, ppc_vsr_t *xt, ppc_vsr_t *xb)\
+{                                                               \
+    helper_reset_fpstatus(env);                                 \
+    xt->f128 = tp##_to_float128(xb->s128, &env->fp_status);     \
+    helper_compute_fprf_float128(env, xt->f128);                \
+    do_float_check_status(env, GETPC());                        \
+}
+
+VSX_CVT_INT128_TO_FP(XSCVUQQP, uint128)
+VSX_CVT_INT128_TO_FP(XSCVSQQP, int128)
+
 /*
  * For "use current rounding mode", define a value that will not be
  * one of the existing rounding model enums.
@@ -3459,3 +3650,326 @@ void helper_xssubqp(CPUPPCState *env, uint32_t opcode,
     *xt = t;
     do_float_check_status(env, GETPC());
 }
+
+static inline void vsxger_excp(CPUPPCState *env, uintptr_t retaddr)
+{
+    target_ulong enable = env->fpscr & (FP_ENABLES | FP_FI | FP_FR);
+    int status;
+
+    env->fpscr &= ~(FP_ENABLES | FP_FI | FP_FR);
+    status = get_float_exception_flags(&env->fp_status);
+    if (unlikely(status & float_flag_invalid)) {
+        if (status & float_flag_invalid_snan) {
+            float_invalid_op_vxsnan(env, retaddr);
+        }
+        if (status & float_flag_invalid_imz) {
+            float_invalid_op_vximz(env, false, retaddr);
+        }
+        if (status & float_flag_invalid_isi) {
+            float_invalid_op_vxisi(env, false, retaddr);
+        }
+    }
+    env->fpscr |= enable;
+    do_float_check_status(env, retaddr);
+}
+
+typedef float64 extract_f16(float16, float_status *);
+
+static float64 extract_hf16(float16 in, float_status *fp_status)
+{
+    return float16_to_float64(in, true, fp_status);
+}
+
+static float64 extract_bf16(bfloat16 in, float_status *fp_status)
+{
+    return bfloat16_to_float64(in, fp_status);
+}
+
+static void vsxger16(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                     ppc_acc_t *at, uint32_t mask, bool acc, bool neg_mul,
+                     bool neg_acc, extract_f16 extract)
+{
+    float_status *excp_ptr = &env->fp_status;
+    uint8_t pmsk = ger_mask_pmsk(mask);
+    uint8_t xmsk = ger_mask_xmsk(mask);
+    uint8_t ymsk = ger_mask_ymsk(mask);
+    float32 r;
+    float32 aux_acc;
+    float64 psum;
+    float64 va;
+    float64 vb;
+    float64 vc;
+    float64 vd;
+    int i;
+    int j;
+    int xmsk_bit;
+    int ymsk_bit;
+
+    helper_reset_fpstatus(env);
+    for (i = 0, xmsk_bit = 1 << 3; i < 4; i++, xmsk_bit >>= 1) {
+        for (j = 0, ymsk_bit = 1 << 3; j < 4; j++, ymsk_bit >>= 1) {
+            if ((xmsk_bit & xmsk) && (ymsk_bit & ymsk)) {
+                va = !(pmsk & 2) ? float64_zero :
+                     extract(a->VsrHF(2 * i), excp_ptr);
+                vb = !(pmsk & 2) ? float64_zero :
+                     extract(b->VsrHF(2 * j), excp_ptr);
+                vc = !(pmsk & 1) ? float64_zero :
+                     extract(a->VsrHF(2 * i + 1), excp_ptr);
+                vd = !(pmsk & 1) ? float64_zero :
+                     extract(b->VsrHF(2 * j + 1), excp_ptr);
+                psum = float64_mul(va, vb, excp_ptr);
+                psum = float64r32_muladd(vc, vd, psum, 0, excp_ptr);
+                r = float64_to_float32(psum, excp_ptr);
+                if (acc) {
+                    aux_acc = at[i].VsrSF(j);
+                    if (neg_mul) {
+                        r = bfp32_neg(r);
+                    }
+                    if (neg_acc) {
+                        aux_acc = bfp32_neg(aux_acc);
+                    }
+                    r = float32_add(r, aux_acc, excp_ptr);
+                }
+                at[i].VsrSF(j) = r;
+            } else {
+                at[i].VsrSF(j) = float32_zero;
+            }
+        }
+    }
+    vsxger_excp(env, GETPC());
+}
+
+typedef void vsxger_zero(ppc_vsr_t *at, int, int);
+typedef void vsxger_muladd_f(ppc_vsr_t *, ppc_vsr_t *, ppc_vsr_t *, int, int,
+                             int, float_status *);
+
+static void vsxger_muladd32(ppc_vsr_t *at, ppc_vsr_t *a, ppc_vsr_t *b, int i,
+                            int j, int flags, float_status *s)
+{
+    at[i].VsrSF(j) = float32_muladd(a->VsrSF(i), b->VsrSF(j),
+                                    at[i].VsrSF(j), flags, s);
+}
+
+static void vsxger_mul32(ppc_vsr_t *at, ppc_vsr_t *a, ppc_vsr_t *b, int i,
+                         int j, int flags, float_status *s)
+{
+    at[i].VsrSF(j) = float32_mul(a->VsrSF(i), b->VsrSF(j), s);
+}
+
+static void vsxger_zero32(ppc_vsr_t *at, int i, int j)
+{
+    at[i].VsrSF(j) = float32_zero;
+}
+
+static void vsxger_muladd64(ppc_vsr_t *at, ppc_vsr_t *a, ppc_vsr_t *b, int i,
+                            int j, int flags, float_status *s)
+{
+    if (j >= 2) {
+        j -= 2;
+        at[i].VsrDF(j) = float64_muladd(a[i / 2].VsrDF(i % 2),
+                                        b->VsrDF(j), at[i].VsrDF(j),
+                                        flags, s);
+    }
+}
+
+static void vsxger_mul64(ppc_vsr_t *at, ppc_vsr_t *a, ppc_vsr_t *b, int i,
+                         int j, int flags, float_status *s)
+{
+    if (j >= 2) {
+        j -= 2;
+        at[i].VsrDF(j) = float64_mul(a[i / 2].VsrDF(i % 2),
+                                     b->VsrDF(j), s);
+    }
+}
+
+static void vsxger_zero64(ppc_vsr_t *at, int i, int j)
+{
+    if (j >= 2) {
+        j -= 2;
+        at[i].VsrDF(j) = float64_zero;
+    }
+}
+
+static void vsxger(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                   ppc_acc_t *at, uint32_t mask, bool acc, bool neg_mul,
+                   bool neg_acc, vsxger_muladd_f mul,
+                   vsxger_muladd_f muladd, vsxger_zero zero)
+{
+    float_status *excp_ptr = &env->fp_status;
+    uint8_t xmsk = mask & 0x0f;
+    uint8_t ymsk = (mask >> 4) & 0x0f;
+    int op_flags;
+    int i;
+    int j;
+    int xmsk_bit;
+    int ymsk_bit;
+
+    op_flags = (neg_acc ^ neg_mul) ? float_muladd_negate_c : 0;
+    op_flags |= neg_mul ? float_muladd_negate_result : 0;
+
+    helper_reset_fpstatus(env);
+    for (i = 0, xmsk_bit = 1 << 3; i < 4; i++, xmsk_bit >>= 1) {
+        for (j = 0, ymsk_bit = 1 << 3; j < 4; j++, ymsk_bit >>= 1) {
+            if ((xmsk_bit & xmsk) && (ymsk_bit & ymsk)) {
+                if (acc) {
+                    muladd(at, a, b, i, j, op_flags, excp_ptr);
+                } else {
+                    mul(at, a, b, i, j, op_flags, excp_ptr);
+                }
+            } else {
+                zero(at, i, j);
+            }
+        }
+    }
+    vsxger_excp(env, GETPC());
+}
+
+QEMU_FLATTEN
+void helper_XVBF16GER2(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, false, false, false, extract_bf16);
+}
+
+QEMU_FLATTEN
+void helper_XVBF16GER2PP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                         ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, false, false, extract_bf16);
+}
+
+QEMU_FLATTEN
+void helper_XVBF16GER2PN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                         ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, false, true, extract_bf16);
+}
+
+QEMU_FLATTEN
+void helper_XVBF16GER2NP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                         ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, true, false, extract_bf16);
+}
+
+QEMU_FLATTEN
+void helper_XVBF16GER2NN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                         ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, true, true, extract_bf16);
+}
+
+QEMU_FLATTEN
+void helper_XVF16GER2(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                     ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, false, false, false, extract_hf16);
+}
+
+QEMU_FLATTEN
+void helper_XVF16GER2PP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                        ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, false, false, extract_hf16);
+}
+
+QEMU_FLATTEN
+void helper_XVF16GER2PN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                        ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, false, true, extract_hf16);
+}
+
+QEMU_FLATTEN
+void helper_XVF16GER2NP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                        ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, true, false, extract_hf16);
+}
+
+QEMU_FLATTEN
+void helper_XVF16GER2NN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                        ppc_acc_t *at, uint32_t mask)
+{
+    vsxger16(env, a, b, at, mask, true, true, true, extract_hf16);
+}
+
+QEMU_FLATTEN
+void helper_XVF32GER(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                     ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, false, false, false, vsxger_mul32,
+           vsxger_muladd32, vsxger_zero32);
+}
+
+QEMU_FLATTEN
+void helper_XVF32GERPP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, false, false, vsxger_mul32,
+           vsxger_muladd32, vsxger_zero32);
+}
+
+QEMU_FLATTEN
+void helper_XVF32GERPN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, false, true, vsxger_mul32,
+           vsxger_muladd32, vsxger_zero32);
+}
+
+QEMU_FLATTEN
+void helper_XVF32GERNP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, true, false, vsxger_mul32,
+           vsxger_muladd32, vsxger_zero32);
+}
+
+QEMU_FLATTEN
+void helper_XVF32GERNN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, true, true, vsxger_mul32,
+           vsxger_muladd32, vsxger_zero32);
+}
+
+QEMU_FLATTEN
+void helper_XVF64GER(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                     ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, false, false, false, vsxger_mul64,
+           vsxger_muladd64, vsxger_zero64);
+}
+
+QEMU_FLATTEN
+void helper_XVF64GERPP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, false, false, vsxger_mul64,
+           vsxger_muladd64, vsxger_zero64);
+}
+
+QEMU_FLATTEN
+void helper_XVF64GERPN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, false, true, vsxger_mul64,
+           vsxger_muladd64, vsxger_zero64);
+}
+
+QEMU_FLATTEN
+void helper_XVF64GERNP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, true, false, vsxger_mul64,
+           vsxger_muladd64, vsxger_zero64);
+}
+
+QEMU_FLATTEN
+void helper_XVF64GERNN(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    vsxger(env, a, b, at, mask, true, true, true, vsxger_mul64,
+           vsxger_muladd64, vsxger_zero64);
+}
diff --git a/qemu/target/ppc/helper.h b/qemu/target/ppc/helper.h
index b1c4343908..c62d9d26fc 100644
--- a/qemu/target/ppc/helper.h
+++ b/qemu/target/ppc/helper.h
@@ -45,11 +45,20 @@ DEF_HELPER_4(divwe, tl, env, tl, tl, i32)
 
 DEF_HELPER_FLAGS_1(popcntb, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_FLAGS_2(cmpb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_1(CDTBCD, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(CBCDTD, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_3(sraw, tl, env, tl, tl)
 #if defined(TARGET_PPC64)
 DEF_HELPER_FLAGS_2(cmpeqb, TCG_CALL_NO_RWG_SE, i32, tl, tl)
 DEF_HELPER_FLAGS_1(popcntw, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_FLAGS_2(bpermd, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(CFUGED, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(PDEPD, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(PEXTD, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_4(HASHST, void, env, tl, tl, tl)
+DEF_HELPER_4(HASHCHK, void, env, tl, tl, tl)
+DEF_HELPER_4(HASHSTP, void, env, tl, tl, tl)
+DEF_HELPER_4(HASHCHKP, void, env, tl, tl, tl)
 DEF_HELPER_3(srad, tl, env, tl, tl)
 DEF_HELPER_0(darn32, tl)
 DEF_HELPER_0(darn64, tl)
@@ -114,6 +123,10 @@ DEF_HELPER_FLAGS_1(ftsqrt, TCG_CALL_NO_RWG_SE, i32, i64)
 #define dh_ctype_vsr ppc_vsr_t *
 #define dh_is_signed_vsr dh_is_signed_ptr
 
+#define dh_alias_acc ptr
+#define dh_ctype_acc ppc_acc_t *
+#define dh_is_signed_acc dh_is_signed_ptr
+
 DEF_HELPER_3(vavgub, void, avr, avr, avr)
 DEF_HELPER_3(vavguh, void, avr, avr, avr)
 DEF_HELPER_3(vavguw, void, avr, avr, avr)
@@ -186,6 +199,18 @@ DEF_HELPER_3(vmuloub, void, avr, avr, avr)
 DEF_HELPER_3(vmulouh, void, avr, avr, avr)
 DEF_HELPER_3(vmulouw, void, avr, avr, avr)
 DEF_HELPER_3(vmuluwm, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VDIVSQ, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VDIVUQ, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VDIVESD, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VDIVEUD, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VDIVESQ, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VDIVEUQ, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VMODSQ, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_3(VMODUQ, TCG_CALL_NO_RWG, void, avr, avr, avr)
+DEF_HELPER_FLAGS_2(VSTRIBL, TCG_CALL_NO_RWG, i32, avr, avr)
+DEF_HELPER_FLAGS_2(VSTRIBR, TCG_CALL_NO_RWG, i32, avr, avr)
+DEF_HELPER_FLAGS_2(VSTRIHL, TCG_CALL_NO_RWG, i32, avr, avr)
+DEF_HELPER_FLAGS_2(VSTRIHR, TCG_CALL_NO_RWG, i32, avr, avr)
 DEF_HELPER_3(vslo, void, avr, avr, avr)
 DEF_HELPER_3(vsro, void, avr, avr, avr)
 DEF_HELPER_3(vsrv, void, avr, avr, avr)
@@ -228,6 +253,10 @@ DEF_HELPER_3(vinsertb, void, avr, avr, i32)
 DEF_HELPER_3(vinserth, void, avr, avr, i32)
 DEF_HELPER_3(vinsertw, void, avr, avr, i32)
 DEF_HELPER_3(vinsertd, void, avr, avr, i32)
+DEF_HELPER_4(VINSBLX, void, env, avr, i64, tl)
+DEF_HELPER_4(VINSHLX, void, env, avr, i64, tl)
+DEF_HELPER_4(VINSWLX, void, env, avr, i64, tl)
+DEF_HELPER_4(VINSDLX, void, env, avr, i64, tl)
 DEF_HELPER_2(vextsb2w, void, avr, avr)
 DEF_HELPER_2(vextsh2w, void, avr, avr)
 DEF_HELPER_2(vextsb2d, void, avr, avr)
@@ -334,6 +363,10 @@ DEF_HELPER_2(vextuwlx, tl, tl, avr)
 DEF_HELPER_2(vextubrx, tl, tl, avr)
 DEF_HELPER_2(vextuhrx, tl, tl, avr)
 DEF_HELPER_2(vextuwrx, tl, tl, avr)
+DEF_HELPER_5(VEXTDUBVLX, void, env, avr, avr, avr, tl)
+DEF_HELPER_5(VEXTDUHVLX, void, env, avr, avr, avr, tl)
+DEF_HELPER_5(VEXTDUWVLX, void, env, avr, avr, avr, tl)
+DEF_HELPER_5(VEXTDDVLX, void, env, avr, avr, avr, tl)
 
 DEF_HELPER_2(vsbox, void, avr, avr)
 DEF_HELPER_3(vcipher, void, avr, avr, avr)
@@ -376,10 +409,21 @@ DEF_HELPER_5(xsmadddp, void, env, vsr, vsr, vsr, vsr)
 DEF_HELPER_5(xsmsubdp, void, env, vsr, vsr, vsr, vsr)
 DEF_HELPER_5(xsnmadddp, void, env, vsr, vsr, vsr, vsr)
 DEF_HELPER_5(xsnmsubdp, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSMADDQP, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSMADDQPO, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSMSUBQP, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSMSUBQPO, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSNMADDQP, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSNMADDQPO, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSNMSUBQP, void, env, vsr, vsr, vsr, vsr)
+DEF_HELPER_5(XSNMSUBQPO, void, env, vsr, vsr, vsr, vsr)
 DEF_HELPER_4(xscmpeqdp, void, env, vsr, vsr, vsr)
 DEF_HELPER_4(xscmpgtdp, void, env, vsr, vsr, vsr)
 DEF_HELPER_4(xscmpgedp, void, env, vsr, vsr, vsr)
 DEF_HELPER_4(xscmpnedp, void, env, vsr, vsr, vsr)
+DEF_HELPER_4(XSCMPEQQP, void, env, vsr, vsr, vsr)
+DEF_HELPER_4(XSCMPGTQP, void, env, vsr, vsr, vsr)
+DEF_HELPER_4(XSCMPGEQP, void, env, vsr, vsr, vsr)
 DEF_HELPER_4(xscmpexpdp, void, env, i32, vsr, vsr)
 DEF_HELPER_4(xscmpexpqp, void, env, i32, vsr, vsr)
 DEF_HELPER_4(xscmpodp, void, env, i32, vsr, vsr)
@@ -392,11 +436,17 @@ DEF_HELPER_5(xsmaxcdp, void, env, i32, vsr, vsr, vsr)
 DEF_HELPER_5(xsmincdp, void, env, i32, vsr, vsr, vsr)
 DEF_HELPER_5(xsmaxjdp, void, env, i32, vsr, vsr, vsr)
 DEF_HELPER_5(xsminjdp, void, env, i32, vsr, vsr, vsr)
+DEF_HELPER_4(XSMAXCQP, void, env, vsr, vsr, vsr)
+DEF_HELPER_4(XSMINCQP, void, env, vsr, vsr, vsr)
 DEF_HELPER_3(xscvdphp, void, env, vsr, vsr)
 DEF_HELPER_4(xscvdpqp, void, env, i32, vsr, vsr)
 DEF_HELPER_3(xscvdpsp, void, env, vsr, vsr)
 DEF_HELPER_2(xscvdpspn, i64, env, i64)
 DEF_HELPER_4(xscvqpdp, void, env, i32, vsr, vsr)
+DEF_HELPER_3(XSCVQPUQZ, void, env, vsr, vsr)
+DEF_HELPER_3(XSCVQPSQZ, void, env, vsr, vsr)
+DEF_HELPER_3(XSCVUQQP, void, env, vsr, vsr)
+DEF_HELPER_3(XSCVSQQP, void, env, vsr, vsr)
 DEF_HELPER_4(xscvqpsdz, void, env, i32, vsr, vsr)
 DEF_HELPER_4(xscvqpswz, void, env, i32, vsr, vsr)
 DEF_HELPER_4(xscvqpudz, void, env, i32, vsr, vsr)
@@ -496,6 +546,8 @@ DEF_HELPER_FLAGS_4(xvcmpnesp, TCG_CALL_NO_RWG, i32, env, vsr, vsr, vsr)
 DEF_HELPER_3(xvcvspdp, void, env, vsr, vsr)
 DEF_HELPER_3(xvcvsphp, void, env, vsr, vsr)
 DEF_HELPER_3(xvcvhpsp, void, env, vsr, vsr)
+DEF_HELPER_3(XVCVSPBF16, void, env, vsr, vsr)
+DEF_HELPER_FLAGS_2(XVCVBF16SPN, TCG_CALL_NO_RWG, void, vsr, vsr)
 DEF_HELPER_3(xvcvspsxds, void, env, vsr, vsr)
 DEF_HELPER_3(xvcvspsxws, void, env, vsr, vsr)
 DEF_HELPER_3(xvcvspuxds, void, env, vsr, vsr)
@@ -513,9 +565,55 @@ DEF_HELPER_3(xvrspip, void, env, vsr, vsr)
 DEF_HELPER_3(xvrspiz, void, env, vsr, vsr)
 DEF_HELPER_4(xxperm, void, env, vsr, vsr, vsr)
 DEF_HELPER_4(xxpermr, void, env, vsr, vsr, vsr)
+DEF_HELPER_FLAGS_5(XXPERMX, TCG_CALL_NO_RWG, void, vsr, vsr, vsr, vsr, tl)
+DEF_HELPER_FLAGS_2(XXGENPCVBM_be_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVBM_be_comp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVBM_le_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVBM_le_comp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVHM_be_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVHM_be_comp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVHM_le_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVHM_le_comp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVWM_be_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVWM_be_comp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVWM_le_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVWM_le_comp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVDM_be_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVDM_be_comp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVDM_le_exp, TCG_CALL_NO_RWG, void, vsr, avr)
+DEF_HELPER_FLAGS_2(XXGENPCVDM_le_comp, TCG_CALL_NO_RWG, void, vsr, avr)
 DEF_HELPER_4(xxextractuw, void, env, vsr, vsr, i32)
 DEF_HELPER_4(xxinsertw, void, env, vsr, vsr, i32)
 DEF_HELPER_3(xvxsigsp, void, env, vsr, vsr)
+DEF_HELPER_5(XVI4GER8, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI4GER8PP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI8GER4, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI8GER4PP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI8GER4SPP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI16GER2, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI16GER2S, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI16GER2PP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVI16GER2SPP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF16GER2, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF16GER2PP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF16GER2PN, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF16GER2NP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF16GER2NN, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVBF16GER2, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVBF16GER2PP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVBF16GER2PN, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVBF16GER2NP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVBF16GER2NN, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF32GER, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF32GERPP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF32GERPN, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF32GERNP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF32GERNN, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF64GER, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF64GERPP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF64GERPN, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF64GERNP, void, env, vsr, vsr, acc, i32)
+DEF_HELPER_5(XVF64GERNN, void, env, vsr, vsr, acc, i32)
 
 DEF_HELPER_2(efscfsi, i32, env, i32)
 DEF_HELPER_2(efscfui, i32, env, i32)
@@ -616,6 +714,7 @@ DEF_HELPER_2(load_slb_esid, tl, env, tl)
 DEF_HELPER_2(load_slb_vsid, tl, env, tl)
 DEF_HELPER_2(find_slb_vsid, tl, env, tl)
 DEF_HELPER_FLAGS_2(slbia, TCG_CALL_NO_RWG, void, env, i32)
+DEF_HELPER_FLAGS_3(slbiag, TCG_CALL_NO_RWG, void, env, tl, i32)
 DEF_HELPER_FLAGS_2(slbie, TCG_CALL_NO_RWG, void, env, tl)
 DEF_HELPER_FLAGS_2(slbieg, TCG_CALL_NO_RWG, void, env, tl)
 #endif
@@ -732,8 +831,10 @@ DEF_HELPER_3(drsp, void, env, fprp, fprp)
 DEF_HELPER_3(drdpq, void, env, fprp, fprp)
 DEF_HELPER_3(dcffix, void, env, fprp, fprp)
 DEF_HELPER_3(dcffixq, void, env, fprp, fprp)
+DEF_HELPER_3(DCFFIXQQ, void, env, fprp, avr)
 DEF_HELPER_3(dctfix, void, env, fprp, fprp)
 DEF_HELPER_3(dctfixq, void, env, fprp, fprp)
+DEF_HELPER_3(DCTFIXQQ, void, env, avr, fprp)
 DEF_HELPER_4(ddedpd, void, env, fprp, fprp, i32)
 DEF_HELPER_4(ddedpdq, void, env, fprp, fprp, i32)
 DEF_HELPER_4(denbcd, void, env, fprp, fprp, i32)
diff --git a/qemu/target/ppc/int_helper.c b/qemu/target/ppc/int_helper.c
index c6ead3e149..69470a45e1 100644
--- a/qemu/target/ppc/int_helper.c
+++ b/qemu/target/ppc/int_helper.c
@@ -303,6 +303,81 @@ target_ulong helper_popcntw(target_ulong val)
                                            0x0000ffff0000ffffULL);
     return val;
 }
+
+uint64_t helper_CFUGED(uint64_t src, uint64_t mask)
+{
+    target_ulong m, left = 0, right = 0;
+    unsigned int n, i = 64;
+    bool bit = false;
+
+    if (mask == 0 || mask == -1) {
+        return src;
+    }
+
+    while (i) {
+        n = ctz64(mask);
+        if (n > i) {
+            n = i;
+        }
+
+        m = (1ULL << n) - 1;
+        if (bit) {
+            right = ror64(right | (src & m), n);
+        } else {
+            left = ror64(left | (src & m), n);
+        }
+
+        src >>= n;
+        mask >>= n;
+        i -= n;
+        bit = !bit;
+        mask = ~mask;
+    }
+
+    if (bit) {
+        n = ctpop64(mask);
+    } else {
+        n = 64 - ctpop64(mask);
+    }
+
+    return left | (right >> n);
+}
+
+uint64_t helper_PDEPD(uint64_t src, uint64_t mask)
+{
+    int i, o;
+    uint64_t result = 0;
+
+    if (mask == -1) {
+        return src;
+    }
+
+    for (i = 0; mask != 0; i++) {
+        o = ctz64(mask);
+        mask &= mask - 1;
+        result |= ((src >> i) & 1) << o;
+    }
+
+    return result;
+}
+
+uint64_t helper_PEXTD(uint64_t src, uint64_t mask)
+{
+    int i, o;
+    uint64_t result = 0;
+
+    if (mask == -1) {
+        return src;
+    }
+
+    for (o = 0; mask != 0; o++) {
+        i = ctz64(mask);
+        mask &= mask - 1;
+        result |= ((src >> i) & 1) << o;
+    }
+
+    return result;
+}
 #else
 target_ulong helper_popcntb(target_ulong val)
 {
@@ -657,6 +732,149 @@ VCF(ux, uint32_to_float32, u32)
 VCF(sx, int32_to_float32, s32)
 #undef VCF
 
+typedef int64_t do_ger(uint32_t, uint32_t, uint32_t);
+
+static int64_t ger_rank8(uint32_t a, uint32_t b, uint32_t mask)
+{
+    int64_t psum = 0;
+    int i;
+
+    for (i = 0; i < 8; i++, mask >>= 1) {
+        if (mask & 1) {
+            psum += (int64_t)sextract32(a, 4 * i, 4) *
+                    sextract32(b, 4 * i, 4);
+        }
+    }
+    return psum;
+}
+
+static int64_t ger_rank4(uint32_t a, uint32_t b, uint32_t mask)
+{
+    int64_t psum = 0;
+    int i;
+
+    for (i = 0; i < 4; i++, mask >>= 1) {
+        if (mask & 1) {
+            psum += sextract32(a, 8 * i, 8) *
+                    (int64_t)extract32(b, 8 * i, 8);
+        }
+    }
+    return psum;
+}
+
+static int64_t ger_rank2(uint32_t a, uint32_t b, uint32_t mask)
+{
+    int64_t psum = 0;
+    int i;
+
+    for (i = 0; i < 2; i++, mask >>= 1) {
+        if (mask & 1) {
+            psum += (int64_t)sextract32(a, 16 * i, 16) *
+                    sextract32(b, 16 * i, 16);
+        }
+    }
+    return psum;
+}
+
+static void xviger(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                   ppc_acc_t *at, uint32_t mask, bool sat, bool acc,
+                   do_ger ger)
+{
+    uint8_t pmsk = ger_mask_pmsk(mask);
+    uint8_t xmsk = ger_mask_xmsk(mask);
+    uint8_t ymsk = ger_mask_ymsk(mask);
+    uint8_t xmsk_bit;
+    uint8_t ymsk_bit;
+    int64_t psum;
+    int i;
+    int j;
+
+    for (i = 0, xmsk_bit = 1 << 3; i < 4; i++, xmsk_bit >>= 1) {
+        for (j = 0, ymsk_bit = 1 << 3; j < 4; j++, ymsk_bit >>= 1) {
+            if ((xmsk_bit & xmsk) && (ymsk_bit & ymsk)) {
+                psum = ger(a->VsrW(i), b->VsrW(j), pmsk);
+                if (acc) {
+                    psum += at[i].VsrSW(j);
+                }
+                if (sat && psum > INT32_MAX) {
+                    set_vscr_sat(env);
+                    at[i].VsrSW(j) = INT32_MAX;
+                } else if (sat && psum < INT32_MIN) {
+                    set_vscr_sat(env);
+                    at[i].VsrSW(j) = INT32_MIN;
+                } else {
+                    at[i].VsrSW(j) = (int32_t)psum;
+                }
+            } else {
+                at[i].VsrSW(j) = 0;
+            }
+        }
+    }
+}
+
+QEMU_FLATTEN
+void helper_XVI4GER8(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                     ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, false, false, ger_rank8);
+}
+
+QEMU_FLATTEN
+void helper_XVI4GER8PP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, false, true, ger_rank8);
+}
+
+QEMU_FLATTEN
+void helper_XVI8GER4(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                     ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, false, false, ger_rank4);
+}
+
+QEMU_FLATTEN
+void helper_XVI8GER4PP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, false, true, ger_rank4);
+}
+
+QEMU_FLATTEN
+void helper_XVI8GER4SPP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                        ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, true, true, ger_rank4);
+}
+
+QEMU_FLATTEN
+void helper_XVI16GER2(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                      ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, false, false, ger_rank2);
+}
+
+QEMU_FLATTEN
+void helper_XVI16GER2S(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                       ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, true, false, ger_rank2);
+}
+
+QEMU_FLATTEN
+void helper_XVI16GER2PP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                        ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, false, true, ger_rank2);
+}
+
+QEMU_FLATTEN
+void helper_XVI16GER2SPP(CPUPPCState *env, ppc_vsr_t *a, ppc_vsr_t *b,
+                         ppc_acc_t *at, uint32_t mask)
+{
+    xviger(env, a, b, at, mask, true, true, ger_rank2);
+}
+
 #define VCMP_DO(suffix, compare, element, record)                       \
     void helper_vcmp##suffix(CPUPPCState *env, ppc_avr_t *r,            \
                              ppc_avr_t *a, ppc_avr_t *b)                \
@@ -1129,6 +1347,138 @@ void helper_vpermr(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b,
     *r = result;
 }
 
+void helper_XXPERMX(ppc_vsr_t *t, ppc_vsr_t *s0, ppc_vsr_t *s1,
+                    ppc_vsr_t *pcv, target_ulong uim)
+{
+    int i;
+    ppc_vsr_t tmp = { .u64 = { 0, 0 } };
+
+    for (i = 0; i < ARRAY_SIZE(t->u8); i++) {
+        if ((pcv->VsrB(i) >> 5) == uim) {
+            int idx = pcv->VsrB(i) & 0x1f;
+
+            if (idx < ARRAY_SIZE(t->u8)) {
+                tmp.VsrB(i) = s0->VsrB(idx);
+            } else {
+                tmp.VsrB(i) = s1->VsrB(idx - ARRAY_SIZE(t->u8));
+            }
+        }
+    }
+
+    *t = tmp;
+}
+
+void helper_VDIVSQ(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    Int128 neg1 = int128_makes64(-1);
+    Int128 int128_min = int128_make128(0, INT64_MIN);
+
+    if (likely(int128_nz(b->s128) &&
+               (int128_ne(a->s128, int128_min) ||
+                int128_ne(b->s128, neg1)))) {
+        t->s128 = int128_divs(a->s128, b->s128);
+    } else {
+        t->s128 = a->s128;
+    }
+}
+
+void helper_VDIVUQ(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    if (int128_nz(b->s128)) {
+        t->s128 = int128_divu(a->s128, b->s128);
+    } else {
+        t->s128 = a->s128;
+    }
+}
+
+void helper_VDIVESD(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    int i;
+
+    for (i = 0; i < 2; i++) {
+        int64_t high = a->s64[i];
+        uint64_t low = 0;
+
+        if (unlikely((high == INT64_MIN && b->s64[i] == -1) ||
+                     !b->s64[i])) {
+            t->s64[i] = a->s64[i];
+        } else {
+            divs128(&low, &high, b->s64[i]);
+            t->s64[i] = low;
+        }
+    }
+}
+
+void helper_VDIVEUD(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    int i;
+
+    for (i = 0; i < 2; i++) {
+        uint64_t high = a->u64[i];
+        uint64_t low = 0;
+
+        if (unlikely(!b->u64[i])) {
+            t->u64[i] = a->u64[i];
+        } else {
+            divu128(&low, &high, b->u64[i]);
+            t->u64[i] = low;
+        }
+    }
+}
+
+void helper_VDIVESQ(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    Int128 high = a->s128;
+    Int128 low = int128_zero();
+    Int128 int128_min = int128_make128(0, INT64_MIN);
+    Int128 neg1 = int128_makes64(-1);
+
+    if (unlikely(!int128_nz(b->s128) ||
+                 (int128_eq(b->s128, neg1) &&
+                  int128_eq(high, int128_min)))) {
+        t->s128 = a->s128;
+    } else {
+        divs256(&low, &high, b->s128);
+        t->s128 = low;
+    }
+}
+
+void helper_VDIVEUQ(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    Int128 high = a->s128;
+    Int128 low = int128_zero();
+
+    if (unlikely(!int128_nz(b->s128))) {
+        t->s128 = a->s128;
+    } else {
+        divu256(&low, &high, b->s128);
+        t->s128 = low;
+    }
+}
+
+void helper_VMODSQ(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    Int128 neg1 = int128_makes64(-1);
+    Int128 int128_min = int128_make128(0, INT64_MIN);
+
+    if (likely(int128_nz(b->s128) &&
+               (int128_ne(a->s128, int128_min) ||
+                int128_ne(b->s128, neg1)))) {
+        t->s128 = int128_rems(a->s128, b->s128);
+    } else {
+        t->s128 = int128_zero();
+    }
+}
+
+void helper_VMODUQ(ppc_avr_t *t, ppc_avr_t *a, ppc_avr_t *b)
+{
+    if (likely(int128_nz(b->s128))) {
+        t->s128 = int128_remu(a->s128, b->s128);
+    } else {
+        t->s128 = int128_zero();
+    }
+}
+
 #if defined(HOST_WORDS_BIGENDIAN)
 #define VBPERMQ_INDEX(avr, i) ((avr)->u8[(i)])
 #define VBPERMD_INDEX(i) (i)
@@ -1180,6 +1530,99 @@ void helper_vbpermq(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 #undef VBPERMQ_INDEX
 #undef VBPERMQ_DW
 
+#define XXGENPCV_BE_EXP(NAME, SZ) \
+void glue(helper_, glue(NAME, _be_exp))(ppc_vsr_t *t, ppc_vsr_t *b) \
+{                                                                   \
+    ppc_vsr_t tmp;                                                  \
+                                                                    \
+    tmp.VsrD(0) = 0x1011121314151617;                               \
+    tmp.VsrD(1) = 0x18191A1B1C1D1E1F;                               \
+                                                                    \
+    for (int i = 0, j = 0; i < ARRAY_SIZE(b->u8); i += SZ) {        \
+        if (b->VsrB(i) & 0x80) {                                    \
+            for (int k = 0; k < SZ; k++) {                          \
+                tmp.VsrB(i + k) = j + k;                            \
+            }                                                       \
+            j += SZ;                                                \
+        }                                                           \
+    }                                                               \
+                                                                    \
+    *t = tmp;                                                       \
+}
+
+#define XXGENPCV_BE_COMP(NAME, SZ) \
+void glue(helper_, glue(NAME, _be_comp))(ppc_vsr_t *t, ppc_vsr_t *b)\
+{                                                                   \
+    ppc_vsr_t tmp = { .u64 = { 0, 0 } };                            \
+                                                                    \
+    for (int i = 0, j = 0; i < ARRAY_SIZE(b->u8); i += SZ) {        \
+        if (b->VsrB(i) & 0x80) {                                    \
+            for (int k = 0; k < SZ; k++) {                          \
+                tmp.VsrB(j + k) = i + k;                            \
+            }                                                       \
+            j += SZ;                                                \
+        }                                                           \
+    }                                                               \
+                                                                    \
+    *t = tmp;                                                       \
+}
+
+#define XXGENPCV_LE_EXP(NAME, SZ) \
+void glue(helper_, glue(NAME, _le_exp))(ppc_vsr_t *t, ppc_vsr_t *b) \
+{                                                                   \
+    ppc_vsr_t tmp;                                                  \
+                                                                    \
+    tmp.VsrD(0) = 0x1F1E1D1C1B1A1918;                               \
+    tmp.VsrD(1) = 0x1716151413121110;                               \
+                                                                    \
+    for (int i = 0, j = 0; i < ARRAY_SIZE(b->u8); i += SZ) {        \
+        const int idx = ARRAY_SIZE(b->u8) - i - SZ;                 \
+        if (b->VsrB(idx) & 0x80) {                                  \
+            for (int k = 0, rk = SZ - 1; k < SZ; k++, rk--) {       \
+                tmp.VsrB(idx + rk) = j + k;                         \
+            }                                                       \
+            j += SZ;                                                \
+        }                                                           \
+    }                                                               \
+                                                                    \
+    *t = tmp;                                                       \
+}
+
+#define XXGENPCV_LE_COMP(NAME, SZ) \
+void glue(helper_, glue(NAME, _le_comp))(ppc_vsr_t *t, ppc_vsr_t *b)\
+{                                                                   \
+    ppc_vsr_t tmp = { .u64 = { 0, 0 } };                            \
+                                                                    \
+    for (int i = 0, j = 0; i < ARRAY_SIZE(b->u8); i += SZ) {        \
+        if (b->VsrB(ARRAY_SIZE(b->u8) - i - SZ) & 0x80) {           \
+            for (int k = 0, rk = SZ - 1; k < SZ; k++, rk--) {       \
+                const int idx = ARRAY_SIZE(b->u8) - j - SZ;         \
+                tmp.VsrB(idx + rk) = i + k;                         \
+            }                                                       \
+            j += SZ;                                                \
+        }                                                           \
+    }                                                               \
+                                                                    \
+    *t = tmp;                                                       \
+}
+
+#define XXGENPCV(NAME, SZ) \
+    XXGENPCV_BE_EXP(NAME, SZ)  \
+    XXGENPCV_BE_COMP(NAME, SZ) \
+    XXGENPCV_LE_EXP(NAME, SZ)  \
+    XXGENPCV_LE_COMP(NAME, SZ)
+
+XXGENPCV(XXGENPCVBM, 1)
+XXGENPCV(XXGENPCVHM, 2)
+XXGENPCV(XXGENPCVWM, 4)
+XXGENPCV(XXGENPCVDM, 8)
+
+#undef XXGENPCV_BE_EXP
+#undef XXGENPCV_BE_COMP
+#undef XXGENPCV_LE_EXP
+#undef XXGENPCV_LE_COMP
+#undef XXGENPCV
+
 #define PMSUM(name, srcfld, trgfld, trgtyp)                   \
 void helper_##name(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)  \
 {                                                             \
@@ -1421,6 +1864,34 @@ void helper_vlogefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     }
 }
 
+#define VSTRI(NAME, ELEM, NUM_ELEMS, LEFT)                         \
+    uint32_t helper_##NAME(ppc_avr_t *t, ppc_avr_t *b)             \
+    {                                                              \
+        int i, idx, crf = 0;                                       \
+                                                                   \
+        for (i = 0; i < NUM_ELEMS; i++) {                          \
+            idx = LEFT ? i : NUM_ELEMS - i - 1;                    \
+            if (b->Vsr##ELEM(idx)) {                               \
+                t->Vsr##ELEM(idx) = b->Vsr##ELEM(idx);             \
+            } else {                                               \
+                crf = 0b0010;                                      \
+                break;                                             \
+            }                                                      \
+        }                                                          \
+                                                                   \
+        for (; i < NUM_ELEMS; i++) {                               \
+            idx = LEFT ? i : NUM_ELEMS - i - 1;                    \
+            t->Vsr##ELEM(idx) = 0;                                 \
+        }                                                          \
+                                                                   \
+        return crf;                                                \
+    }
+VSTRI(VSTRIBL, B, 16, true)
+VSTRI(VSTRIBR, B, 16, false)
+VSTRI(VSTRIHL, H, 8, true)
+VSTRI(VSTRIHR, H, 8, false)
+#undef VSTRI
+
 #if defined(HOST_WORDS_BIGENDIAN)
 #define VEXTU_X_DO(name, size, left)                                \
     target_ulong glue(helper_, name)(target_ulong a, ppc_avr_t *b)  \
@@ -1519,6 +1990,88 @@ void helper_vslo(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 #endif
 }
 
+#if defined(HOST_WORDS_BIGENDIAN)
+#define ELEM_ADDR(VEC, IDX, SIZE) (&(VEC)->u8[IDX])
+#else
+#define ELEM_ADDR(VEC, IDX, SIZE) (&(VEC)->u8[15 - (IDX)] - (SIZE) + 1)
+#endif
+
+#define VINSX(SUFFIX, TYPE)                                             \
+void glue(glue(helper_VINS, SUFFIX), LX)(CPUPPCState *env,              \
+                                         ppc_avr_t *t, uint64_t val,    \
+                                         target_ulong index)             \
+{                                                                       \
+    const int maxidx = ARRAY_SIZE(t->u8) - sizeof(TYPE);                \
+    target_long idx = index;                                            \
+                                                                        \
+    if (idx < 0 || idx > maxidx) {                                      \
+        idx = idx < 0 ? sizeof(TYPE) - idx : idx;                       \
+        qemu_log_mask(LOG_GUEST_ERROR,                                  \
+                      "Invalid index for Vector Insert Element after "  \
+                      "0x" TARGET_FMT_lx ", RA = " TARGET_FMT_ld       \
+                      " > %d\n", env->nip, idx, maxidx);               \
+    } else {                                                            \
+        TYPE src = val;                                                 \
+        memcpy(ELEM_ADDR(t, idx, sizeof(TYPE)), &src, sizeof(TYPE));    \
+    }                                                                   \
+}
+VINSX(B, uint8_t)
+VINSX(H, uint16_t)
+VINSX(W, uint32_t)
+VINSX(D, uint64_t)
+#undef ELEM_ADDR
+#undef VINSX
+
+#if defined(HOST_WORDS_BIGENDIAN)
+#define VEXTDVLX(NAME, SIZE)                                            \
+void helper_##NAME(CPUPPCState *env, ppc_avr_t *t, ppc_avr_t *a,        \
+                   ppc_avr_t *b, target_ulong index)                    \
+{                                                                       \
+    const target_long idx = index;                                      \
+    ppc_avr_t tmp[2] = { *a, *b };                                      \
+    uint8_t *tmp_bytes = (uint8_t *)tmp;                                \
+                                                                        \
+    memset(t, 0, sizeof(*t));                                           \
+    if (idx >= 0 && idx + SIZE <= sizeof(tmp)) {                        \
+        memcpy(&t->u8[ARRAY_SIZE(t->u8) / 2 - SIZE],                   \
+               tmp_bytes + idx, SIZE);                                 \
+    } else {                                                            \
+        qemu_log_mask(LOG_GUEST_ERROR,                                  \
+                      "Invalid index for " #NAME " after 0x"           \
+                      TARGET_FMT_lx ", RC = " TARGET_FMT_ld            \
+                      " > %d\n", env->nip,                             \
+                      idx < 0 ? SIZE - idx : idx, 32 - SIZE);           \
+    }                                                                   \
+}
+#else
+#define VEXTDVLX(NAME, SIZE)                                            \
+void helper_##NAME(CPUPPCState *env, ppc_avr_t *t, ppc_avr_t *a,        \
+                   ppc_avr_t *b, target_ulong index)                    \
+{                                                                       \
+    const target_long idx = index;                                      \
+    ppc_avr_t tmp[2] = { *b, *a };                                      \
+    uint8_t *tmp_bytes = (uint8_t *)tmp;                                \
+                                                                        \
+    memset(t, 0, sizeof(*t));                                           \
+    if (idx >= 0 && idx + SIZE <= sizeof(tmp)) {                        \
+        memcpy(&t->u8[ARRAY_SIZE(t->u8) / 2],                           \
+               tmp_bytes + sizeof(tmp) - SIZE - idx, SIZE);             \
+    } else {                                                            \
+        qemu_log_mask(LOG_GUEST_ERROR,                                  \
+                      "Invalid index for " #NAME " after 0x"           \
+                      TARGET_FMT_lx ", RC = " TARGET_FMT_ld            \
+                      " > %d\n", env->nip,                             \
+                      idx < 0 ? SIZE - idx : idx, 32 - SIZE);           \
+    }                                                                   \
+}
+#endif
+
+VEXTDVLX(VEXTDUBVLX, 1)
+VEXTDVLX(VEXTDUHVLX, 2)
+VEXTDVLX(VEXTDUWVLX, 4)
+VEXTDVLX(VEXTDDVLX, 8)
+#undef VEXTDVLX
+
 #if defined(HOST_WORDS_BIGENDIAN)
 #define VINSERT(suffix, element)                                            \
     void helper_vinsert##suffix(ppc_avr_t *r, ppc_avr_t *b, uint32_t index) \
diff --git a/qemu/target/ppc/internal.h b/qemu/target/ppc/internal.h
index e4b2d242d1..e28c2988ac 100644
--- a/qemu/target/ppc/internal.h
+++ b/qemu/target/ppc/internal.h
@@ -205,6 +205,32 @@ EXTRACT_HELPER(IMM8, 11, 8);
 EXTRACT_HELPER(DCMX, 16, 7);
 EXTRACT_HELPER_SPLIT_3(DCMX_XV, 5, 16, 0, 1, 2, 5, 1, 6, 6);
 
+#define GER_MSK_XMSK_SHIFT 0
+#define GER_MSK_YMSK_SHIFT 4
+#define GER_MSK_PMSK_SHIFT 8
+
+static inline uint8_t ger_mask_xmsk(uint32_t mask)
+{
+    return extract32(mask, GER_MSK_XMSK_SHIFT, 4);
+}
+
+static inline uint8_t ger_mask_ymsk(uint32_t mask)
+{
+    return extract32(mask, GER_MSK_YMSK_SHIFT, 4);
+}
+
+static inline uint8_t ger_mask_pmsk(uint32_t mask)
+{
+    return extract32(mask, GER_MSK_PMSK_SHIFT, 8);
+}
+
+static inline int ger_pack_masks(int pmsk, int ymsk, int xmsk)
+{
+    return ((xmsk & 0x0f) << GER_MSK_XMSK_SHIFT) |
+           ((ymsk & 0x0f) << GER_MSK_YMSK_SHIFT) |
+           ((pmsk & 0xff) << GER_MSK_PMSK_SHIFT);
+}
+
 void helper_compute_fprf_float16(CPUPPCState *env, float16 arg);
 void helper_compute_fprf_float32(CPUPPCState *env, float32 arg);
 void helper_compute_fprf_float128(CPUPPCState *env, float128 arg);
diff --git a/qemu/target/ppc/mmu-hash64.c b/qemu/target/ppc/mmu-hash64.c
index 8facd2df93..83cdce9e4c 100644
--- a/qemu/target/ppc/mmu-hash64.c
+++ b/qemu/target/ppc/mmu-hash64.c
@@ -165,6 +165,28 @@ void helper_slbia(CPUPPCState *env, uint32_t ih)
     }
 }
 
+void helper_slbiag(CPUPPCState *env, target_ulong rs, uint32_t l)
+{
+    PowerPCCPU *cpu = env_archcpu(env);
+    int n;
+
+    (void)rs;
+    (void)l;
+
+    /*
+     * slbiag must always flush all TLB (which is equivalent to ERAT in ppc
+     * architecture). Matching on SLB_ESID_V is not good enough, because slbmte
+     * can overwrite a valid SLB without flushing its lookaside information.
+     */
+    env->tlb_need_flush |= TLB_NEED_LOCAL_FLUSH;
+
+    for (n = 0; n < cpu->hash64_opts->slb_size; n++) {
+        ppc_slb_t *slb = &env->slb[n];
+
+        slb->esid &= ~SLB_ESID_V;
+    }
+}
+
 static void __helper_slbie(CPUPPCState *env, target_ulong addr,
                            target_ulong global)
 {
diff --git a/qemu/target/ppc/translate.c b/qemu/target/ppc/translate.c
index 0b3e5c8c08..f1d844585d 100644
--- a/qemu/target/ppc/translate.c
+++ b/qemu/target/ppc/translate.c
@@ -151,7 +151,11 @@ void ppc_translate_init(struct uc_struct *uc)
 /* internal defines */
 struct DisasContext {
     DisasContextBase base;
+    target_ulong cia;
     uint32_t opcode;
+#if defined(TARGET_PPC64)
+    uint32_t prefix_opcode;
+#endif
     uint32_t exception;
     /* Routine used to access memory */
     bool pr, hv, dr, le_mode;
@@ -1626,6 +1630,98 @@ static void gen_cnttzw(DisasContext *ctx)
     }
 }
 
+static void gen_set_bool_cond(DisasContext *ctx, bool neg, bool rev)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGCond cond = rev ? TCG_COND_EQ : TCG_COND_NE;
+    uint32_t mask = 0x08 >> (rA(ctx->opcode) & 0x03);
+    TCGv temp = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_extu_i32_tl(tcg_ctx, temp, cpu_crf[rA(ctx->opcode) >> 2]);
+    tcg_gen_andi_tl(tcg_ctx, temp, temp, mask);
+    tcg_gen_setcondi_tl(tcg_ctx, cond, cpu_gpr[rD(ctx->opcode)], temp, 0);
+    if (neg) {
+        tcg_gen_neg_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)],
+                       cpu_gpr[rD(ctx->opcode)]);
+    }
+    tcg_temp_free(tcg_ctx, temp);
+}
+
+static void gen_setbc(DisasContext *ctx)
+{
+    gen_set_bool_cond(ctx, false, false);
+}
+
+static void gen_setbcr(DisasContext *ctx)
+{
+    gen_set_bool_cond(ctx, false, true);
+}
+
+static void gen_setnbc(DisasContext *ctx)
+{
+    gen_set_bool_cond(ctx, true, false);
+}
+
+static void gen_setnbcr(DisasContext *ctx)
+{
+    gen_set_bool_cond(ctx, true, true);
+}
+
+static void gen_addg6s(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    const target_ulong carry_bits = (target_ulong)0x1111111111111111ULL;
+    TCGv t0;
+    TCGv t1;
+    TCGv carry;
+    TCGv zero;
+    int i;
+
+    t0 = tcg_temp_new(tcg_ctx);
+    t1 = tcg_const_tl(tcg_ctx, 0);
+    carry = tcg_const_tl(tcg_ctx, 0);
+    zero = tcg_const_tl(tcg_ctx, 0);
+
+    for (i = 0; i < 16; i++) {
+        tcg_gen_shri_tl(tcg_ctx, t0, cpu_gpr[rA(ctx->opcode)], i * 4);
+        tcg_gen_andi_tl(tcg_ctx, t0, t0, 0xf);
+        tcg_gen_add_tl(tcg_ctx, t1, t1, t0);
+
+        tcg_gen_shri_tl(tcg_ctx, t0, cpu_gpr[rB(ctx->opcode)], i * 4);
+        tcg_gen_andi_tl(tcg_ctx, t0, t0, 0xf);
+        tcg_gen_add_tl(tcg_ctx, t1, t1, t0);
+
+        tcg_gen_andi_tl(tcg_ctx, t1, t1, 0x10);
+        tcg_gen_setcond_tl(tcg_ctx, TCG_COND_NE, t1, t1, zero);
+        tcg_gen_shli_tl(tcg_ctx, t0, t1, i * 4);
+        tcg_gen_or_tl(tcg_ctx, carry, carry, t0);
+    }
+
+    tcg_gen_xori_tl(tcg_ctx, carry, carry, carry_bits);
+    tcg_gen_muli_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)], carry, 6);
+
+    tcg_temp_free(tcg_ctx, t0);
+    tcg_temp_free(tcg_ctx, t1);
+    tcg_temp_free(tcg_ctx, carry);
+    tcg_temp_free(tcg_ctx, zero);
+}
+
+static void gen_cdtbcd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    gen_helper_CDTBCD(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                      cpu_gpr[rS(ctx->opcode)]);
+}
+
+static void gen_cbcdtd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    gen_helper_CBCDTD(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                      cpu_gpr[rS(ctx->opcode)]);
+}
+
 /* eqv & eqv. */
 GEN_LOGICAL2(eqv, tcg_gen_eqv_tl, 0x08, PPC_INTEGER);
 /* extsb & extsb. */
@@ -1905,6 +2001,127 @@ static void gen_cnttzd(DisasContext *ctx)
     }
 }
 
+static void gen_cfuged(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    gen_helper_CFUGED(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                      cpu_gpr[rS(ctx->opcode)], cpu_gpr[rB(ctx->opcode)]);
+}
+
+static void gen_cntzdm_i64(TCGContext *tcg_ctx, TCGv_i64 dst, TCGv_i64 src,
+                           TCGv_i64 mask, int64_t trail)
+{
+    TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 t1 = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_and_i64(tcg_ctx, t0, src, mask);
+    if (trail) {
+        tcg_gen_ctzi_i64(tcg_ctx, t0, t0, -1);
+    } else {
+        tcg_gen_clzi_i64(tcg_ctx, t0, t0, -1);
+    }
+
+    tcg_gen_setcondi_i64(tcg_ctx, TCG_COND_NE, t1, t0, -1);
+    tcg_gen_andi_i64(tcg_ctx, t0, t0, 63);
+    tcg_gen_xori_i64(tcg_ctx, t0, t0, 63);
+    if (trail) {
+        tcg_gen_shl_i64(tcg_ctx, t0, mask, t0);
+        tcg_gen_shl_i64(tcg_ctx, t0, t0, t1);
+    } else {
+        tcg_gen_shr_i64(tcg_ctx, t0, mask, t0);
+        tcg_gen_shr_i64(tcg_ctx, t0, t0, t1);
+    }
+
+    tcg_gen_ctpop_i64(tcg_ctx, dst, t0);
+
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+static void gen_cntzdm(DisasContext *ctx, bool trail)
+{
+    gen_cntzdm_i64(ctx->uc->tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                   cpu_gpr[rS(ctx->opcode)], cpu_gpr[rB(ctx->opcode)],
+                   trail);
+}
+
+static void gen_cntlzdm(DisasContext *ctx)
+{
+    gen_cntzdm(ctx, false);
+}
+
+static void gen_cnttzdm(DisasContext *ctx)
+{
+    gen_cntzdm(ctx, true);
+}
+
+static void gen_pdepd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    gen_helper_PDEPD(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                     cpu_gpr[rS(ctx->opcode)], cpu_gpr[rB(ctx->opcode)]);
+}
+
+static void gen_pextd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    gen_helper_PEXTD(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                     cpu_gpr[rS(ctx->opcode)], cpu_gpr[rB(ctx->opcode)]);
+}
+
+static void gen_hash(DisasContext *ctx, bool priv,
+                     void (*helper)(TCGContext *, TCGv_env, TCGv,
+                                    TCGv, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    uint64_t dw;
+    target_long disp;
+
+    if (!(ctx->insns_flags2 & PPC2_ISA310)) {
+        return;
+    }
+    if (priv && unlikely(ctx->pr)) {
+        gen_priv_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+        return;
+    }
+    if (unlikely(rA(ctx->opcode) == 0)) {
+        gen_invalid(ctx);
+        return;
+    }
+
+    dw = ((uint64_t)Rc(ctx->opcode) << 5) | rD(ctx->opcode);
+    disp = (target_long)(int64_t)(0xfffffffffffffe00ULL | (dw << 3));
+    ea = tcg_temp_new(tcg_ctx);
+    tcg_gen_addi_tl(tcg_ctx, ea, cpu_gpr[rA(ctx->opcode)], disp);
+    helper(tcg_ctx, tcg_ctx->cpu_env, ea, cpu_gpr[rA(ctx->opcode)],
+           cpu_gpr[rB(ctx->opcode)]);
+    tcg_temp_free(tcg_ctx, ea);
+}
+
+static void gen_hashst(DisasContext *ctx)
+{
+    gen_hash(ctx, false, gen_helper_HASHST);
+}
+
+static void gen_hashchk(DisasContext *ctx)
+{
+    gen_hash(ctx, false, gen_helper_HASHCHK);
+}
+
+static void gen_hashstp(DisasContext *ctx)
+{
+    gen_hash(ctx, true, gen_helper_HASHSTP);
+}
+
+static void gen_hashchkp(DisasContext *ctx)
+{
+    gen_hash(ctx, true, gen_helper_HASHCHKP);
+}
+
 /* darn */
 static void gen_darn(DisasContext *ctx)
 {
@@ -2594,6 +2811,546 @@ GEN_QEMU_STORE_64(st64, DEF_MEMOP(MO_Q))
 
 #if defined(TARGET_PPC64)
 GEN_QEMU_STORE_64(st64r, BSWAP_MEMOP(MO_Q))
+
+static bool is_prefix_insn(DisasContext *ctx, uint32_t opcode)
+{
+    return (ctx->insns_flags2 & PPC2_ISA310) && opc1(opcode) == 1;
+}
+
+static target_long prefixed_si(DisasContext *ctx)
+{
+    return ((target_long)sextract32(ctx->prefix_opcode, 0, 18) << 16) |
+           UIMM(ctx->opcode);
+}
+
+static bool prefixed_r(DisasContext *ctx)
+{
+    return extract32(ctx->prefix_opcode, 20, 1);
+}
+
+static bool prefixed_addr(DisasContext *ctx, TCGv ea, int ra, target_long si)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    if (prefixed_r(ctx)) {
+        if (unlikely(ra != 0)) {
+            gen_invalid(ctx);
+            return false;
+        }
+        si += ctx->cia;
+    }
+
+    if (ra == 0) {
+        if (NARROW_MODE(ctx)) {
+            si = (uint32_t)si;
+        }
+        tcg_gen_movi_tl(tcg_ctx, ea, si);
+    } else {
+        tcg_gen_addi_tl(tcg_ctx, ea, cpu_gpr[ra], si);
+        if (NARROW_MODE(ctx)) {
+            tcg_gen_ext32u_tl(tcg_ctx, ea, ea);
+        }
+    }
+    return true;
+}
+
+static void gen_plwz(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld32u(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_plbz(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld8u(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_plhz(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld16u(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_plha(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld16s(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_plwa(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld32s(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_pld(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld64_i64(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_plq(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int ra = rA(ctx->opcode);
+    int rt = rD(ctx->opcode);
+    TCGv ea;
+
+    if (unlikely(ra == rt || rt == 31)) {
+        gen_invalid(ctx);
+        return;
+    }
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, ra, prefixed_si(ctx))) {
+        TCGv first = cpu_gpr[rt];
+        TCGv second = cpu_gpr[rt + 1];
+
+        if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
+#if HAVE_ATOMIC128 == 1
+            TCGv_i32 oi = tcg_temp_new_i32(tcg_ctx);
+            if (ctx->le_mode) {
+                tcg_gen_movi_i32(tcg_ctx, oi,
+                                  make_memop_idx(MO_LEQ, ctx->mem_idx));
+                gen_helper_lq_le_parallel(tcg_ctx, first, tcg_ctx->cpu_env,
+                                           ea, oi);
+                tcg_gen_ld_i64(tcg_ctx, second, tcg_ctx->cpu_env,
+                               offsetof(CPUPPCState, retxh));
+            } else {
+                tcg_gen_movi_i32(tcg_ctx, oi,
+                                  make_memop_idx(MO_BEQ, ctx->mem_idx));
+                gen_helper_lq_be_parallel(tcg_ctx, second, tcg_ctx->cpu_env,
+                                           ea, oi);
+                tcg_gen_ld_i64(tcg_ctx, first, tcg_ctx->cpu_env,
+                               offsetof(CPUPPCState, retxh));
+            }
+            tcg_temp_free_i32(tcg_ctx, oi);
+#else
+            gen_helper_exit_atomic(tcg_ctx, tcg_ctx->cpu_env);
+            ctx->base.is_jmp = DISAS_NORETURN;
+#endif
+        } else {
+            gen_qemu_ld64_i64(ctx, first, ea);
+            gen_addr_add(ctx, ea, ea, 8);
+            gen_qemu_ld64_i64(ctx, second, ea);
+        }
+    }
+    tcg_temp_free(tcg_ctx, ea);
+}
+
+static void gen_pstw(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_st32(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_pstb(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_st8(ctx, cpu_gpr[rS(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_psth(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_st16(ctx, cpu_gpr[rS(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_pstd(DisasContext *ctx)
+{
+    TCGv ea = tcg_temp_new(ctx->uc->tcg_ctx);
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_st64_i64(ctx, cpu_gpr[rD(ctx->opcode)], ea);
+    }
+    tcg_temp_free(ctx->uc->tcg_ctx, ea);
+}
+
+static void gen_pstq(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int rs = rS(ctx->opcode);
+    TCGv ea;
+
+    if (unlikely(rs == 31)) {
+        gen_invalid(ctx);
+        return;
+    }
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        TCGv first = cpu_gpr[rs];
+        TCGv second = cpu_gpr[rs + 1];
+
+        if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
+#if HAVE_ATOMIC128 == 1
+            TCGv_i32 oi = tcg_temp_new_i32(tcg_ctx);
+            if (ctx->le_mode) {
+                tcg_gen_movi_i32(tcg_ctx, oi,
+                                  make_memop_idx(MO_LEQ, ctx->mem_idx));
+                gen_helper_stq_le_parallel(tcg_ctx, tcg_ctx->cpu_env, ea,
+                                            first, second, oi);
+            } else {
+                tcg_gen_movi_i32(tcg_ctx, oi,
+                                  make_memop_idx(MO_BEQ, ctx->mem_idx));
+                gen_helper_stq_be_parallel(tcg_ctx, tcg_ctx->cpu_env, ea,
+                                            second, first, oi);
+            }
+            tcg_temp_free_i32(tcg_ctx, oi);
+#else
+            gen_helper_exit_atomic(tcg_ctx, tcg_ctx->cpu_env);
+            ctx->base.is_jmp = DISAS_NORETURN;
+#endif
+        } else {
+            gen_qemu_st64_i64(ctx, first, ea);
+            gen_addr_add(ctx, ea, ea, 8);
+            gen_qemu_st64_i64(ctx, second, ea);
+        }
+    }
+    tcg_temp_free(tcg_ctx, ea);
+}
+
+static void gen_paddi(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    target_long si = prefixed_si(ctx);
+    int ra = rA(ctx->opcode);
+
+    if (prefixed_r(ctx)) {
+        if (unlikely(ra != 0)) {
+            gen_invalid(ctx);
+            return;
+        }
+        si += ctx->cia;
+    }
+
+    if (ra == 0) {
+        tcg_gen_movi_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)], si);
+    } else {
+        tcg_gen_addi_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)], cpu_gpr[ra], si);
+    }
+}
+
+static void gen_xxeval(DisasContext *ctx);
+static void gen_xxpermx(DisasContext *ctx);
+static void gen_xxblendv(DisasContext *ctx, unsigned vece);
+static void gen_xxsplti32dx(DisasContext *ctx);
+static void gen_xxspltidp(DisasContext *ctx);
+static void gen_xxspltiw(DisasContext *ctx);
+static void gen_plfs(DisasContext *ctx);
+static void gen_plfd(DisasContext *ctx);
+static void gen_pstfs(DisasContext *ctx);
+static void gen_pstfd(DisasContext *ctx);
+static void gen_plxsd(DisasContext *ctx);
+static void gen_plxssp(DisasContext *ctx);
+static void gen_pstxsd(DisasContext *ctx);
+static void gen_pstxssp(DisasContext *ctx);
+static void gen_plxv(DisasContext *ctx, bool paired);
+static void gen_pstxv(DisasContext *ctx, bool paired);
+static bool gen_pmx_ger_insn(DisasContext *ctx);
+
+static bool is_pnop_prefix(DisasContext *ctx)
+{
+    return (ctx->prefix_opcode & 0xfff3ffff) == 0x07000000;
+}
+
+static bool is_pnop_invalid_suffix(DisasContext *ctx)
+{
+    uint32_t xo = extract32(ctx->opcode, 1, 10);
+
+    switch (opc1(ctx->opcode)) {
+    case 0x10:
+    case 0x12:
+        return true;
+    case 0x11:
+        return ctx->opcode & 0x3;
+    case 0x13:
+        switch (xo) {
+        case 0x010: /* bclr */
+        case 0x210: /* bcctr */
+        case 0x230: /* bctar */
+        case 0x092: /* rfebb */
+        case 0x052: /* rfscv */
+        case 0x012: /* rfid */
+        case 0x112: /* hrfid */
+        case 0x132: /* urfid */
+        case 0x172: /* stop */
+            return true;
+        default:
+            return false;
+        }
+    case 0x1f:
+        return (ctx->opcode & 0x00010000) == 0 &&
+               (xo == 0x092 || xo == 0x0b2);
+    case 0x00:
+        return (ctx->opcode & 0x000003fe) == 0x00000200;
+    default:
+        return false;
+    }
+}
+
+static bool gen_prefixed_insn(DisasContext *ctx)
+{
+    uint32_t pfx_class = ctx->prefix_opcode & 0xff800000;
+    uint32_t pfx_8rr = ctx->prefix_opcode & 0xfff00000;
+
+    if (is_pnop_prefix(ctx)) {
+        if (is_pnop_invalid_suffix(ctx)) {
+            gen_invalid(ctx);
+        }
+        return true;
+    }
+
+    if (pfx_8rr == 0x07900000 && opc1(ctx->opcode) == 0x3b) {
+        return gen_pmx_ger_insn(ctx);
+    }
+
+    switch (opc1(ctx->opcode)) {
+    case 0x0e:
+        if (pfx_class == 0x06000000) {
+            gen_paddi(ctx);
+            return true;
+        }
+        break;
+    case 0x20:
+        if (pfx_class == 0x06000000) {
+            gen_plwz(ctx);
+            return true;
+        }
+        if (pfx_8rr == 0x05000000) {
+            if ((ctx->opcode & 0x001c0000) == 0x00000000) {
+                gen_xxsplti32dx(ctx);
+                return true;
+            }
+            switch (ctx->opcode & 0x001e0000) {
+            case 0x00040000:
+                gen_xxspltidp(ctx);
+                return true;
+            case 0x00060000:
+                gen_xxspltiw(ctx);
+                return true;
+            default:
+                break;
+            }
+        }
+        break;
+    case 0x21:
+        if (pfx_8rr == 0x05000000) {
+            gen_xxblendv(ctx, extract32(ctx->opcode, 4, 2));
+            return true;
+        }
+        break;
+    case 0x22:
+        if (pfx_class == 0x06000000) {
+            gen_plbz(ctx);
+            return true;
+        }
+        if (pfx_8rr == 0x05000000) {
+            switch (ctx->opcode & 0x00000030) {
+            case 0x00000000:
+                gen_xxpermx(ctx);
+                return true;
+            case 0x00000010:
+                gen_xxeval(ctx);
+                return true;
+            default:
+                break;
+            }
+        }
+        break;
+    case 0x24:
+        if (pfx_class == 0x06000000) {
+            gen_pstw(ctx);
+            return true;
+        }
+        break;
+    case 0x26:
+        if (pfx_class == 0x06000000) {
+            gen_pstb(ctx);
+            return true;
+        }
+        break;
+    case 0x28:
+        if (pfx_class == 0x06000000) {
+            gen_plhz(ctx);
+            return true;
+        }
+        break;
+    case 0x29:
+        if (pfx_class == 0x04000000) {
+            gen_plwa(ctx);
+            return true;
+        }
+        break;
+    case 0x2a:
+        if (pfx_class == 0x06000000) {
+            gen_plha(ctx);
+            return true;
+        }
+        if (pfx_class == 0x04000000) {
+            gen_plxsd(ctx);
+            return true;
+        }
+        break;
+    case 0x2b:
+        if (pfx_class == 0x04000000) {
+            gen_plxssp(ctx);
+            return true;
+        }
+        break;
+    case 0x2c:
+        if (pfx_class == 0x06000000) {
+            gen_psth(ctx);
+            return true;
+        }
+        break;
+    case 0x2e:
+        if (pfx_class == 0x04000000) {
+            gen_pstxsd(ctx);
+            return true;
+        }
+        break;
+    case 0x2f:
+        if (pfx_class == 0x04000000) {
+            gen_pstxssp(ctx);
+            return true;
+        }
+        break;
+    case 0x30:
+        if (pfx_class == 0x06000000) {
+            gen_plfs(ctx);
+            return true;
+        }
+        break;
+    case 0x32:
+        if (pfx_class == 0x06000000) {
+            gen_plfd(ctx);
+            return true;
+        }
+        if (pfx_class == 0x04000000) {
+            gen_plxv(ctx, false);
+            return true;
+        }
+        break;
+    case 0x33:
+        if (pfx_class == 0x04000000) {
+            gen_plxv(ctx, false);
+            return true;
+        }
+        break;
+    case 0x34:
+        if (pfx_class == 0x06000000) {
+            gen_pstfs(ctx);
+            return true;
+        }
+        break;
+    case 0x36:
+        if (pfx_class == 0x06000000) {
+            gen_pstfd(ctx);
+            return true;
+        }
+        if (pfx_class == 0x04000000) {
+            gen_pstxv(ctx, false);
+            return true;
+        }
+        break;
+    case 0x37:
+        if (pfx_class == 0x04000000) {
+            gen_pstxv(ctx, false);
+            return true;
+        }
+        break;
+    case 0x38:
+        if (pfx_class == 0x04000000) {
+            gen_plq(ctx);
+            return true;
+        }
+        break;
+    case 0x3a:
+        if (pfx_class == 0x04000000) {
+            gen_plxv(ctx, true);
+            return true;
+        }
+        break;
+    case 0x39:
+        if (pfx_class == 0x04000000) {
+            gen_pld(ctx);
+            return true;
+        }
+        break;
+    case 0x3c:
+        if (pfx_class == 0x04000000) {
+            gen_pstq(ctx);
+            return true;
+        }
+        break;
+    case 0x3e:
+        if (pfx_class == 0x04000000) {
+            gen_pstxv(ctx, true);
+            return true;
+        }
+        break;
+    case 0x3d:
+        if (pfx_class == 0x04000000) {
+            gen_pstd(ctx);
+            return true;
+        }
+        break;
+    default:
+        break;
+    }
+
+    return false;
+}
 #endif
 
 #define GEN_LD(name, ldop, opc, type)                                         \
@@ -5060,6 +5817,20 @@ static void gen_slbia(DisasContext *ctx)
     tcg_temp_free_i32(tcg_ctx, t0);
 }
 
+/* slbiag */
+static void gen_slbiag(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    uint32_t l = (ctx->opcode >> 16) & 0x1;
+    TCGv_i32 t0 = tcg_const_i32(tcg_ctx, l);
+
+    CHK_SV;
+
+    gen_helper_slbiag(tcg_ctx, tcg_ctx->cpu_env,
+                      cpu_gpr[rS(ctx->opcode)], t0);
+    tcg_temp_free_i32(tcg_ctx, t0);
+}
+
 /* slbie */
 static void gen_slbie(DisasContext *ctx)
 {
@@ -6654,6 +7425,42 @@ static void gen_msgsync(DisasContext *ctx)
 }
 
 #if defined(TARGET_PPC64)
+static void gen_brd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    tcg_gen_bswap64_i64(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                        cpu_gpr[rS(ctx->opcode)]);
+}
+
+static void gen_brw(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    tcg_gen_bswap64_i64(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                        cpu_gpr[rS(ctx->opcode)]);
+    tcg_gen_rotli_i64(tcg_ctx, cpu_gpr[rA(ctx->opcode)],
+                      cpu_gpr[rA(ctx->opcode)], 32);
+}
+
+static void gen_brh(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 mask = tcg_const_i64(tcg_ctx, 0x00ff00ff00ff00ffull);
+    TCGv_i64 t1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 t2 = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_shri_i64(tcg_ctx, t1, cpu_gpr[rS(ctx->opcode)], 8);
+    tcg_gen_and_i64(tcg_ctx, t2, t1, mask);
+    tcg_gen_and_i64(tcg_ctx, t1, cpu_gpr[rS(ctx->opcode)], mask);
+    tcg_gen_shli_i64(tcg_ctx, t1, t1, 8);
+    tcg_gen_or_i64(tcg_ctx, cpu_gpr[rA(ctx->opcode)], t1, t2);
+
+    tcg_temp_free_i64(tcg_ctx, t1);
+    tcg_temp_free_i64(tcg_ctx, t2);
+    tcg_temp_free_i64(tcg_ctx, mask);
+}
+
 static void gen_maddld(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -6812,6 +7619,19 @@ static inline void set_avr64(TCGContext *tcg_ctx, int regno, TCGv_i64 src, bool
 
 #include "translate/spe-impl.inc.c"
 
+static void gen_lxvp_stxvp(DisasContext *ctx)
+{
+    if (ctx->insns_flags2 & PPC2_ISA310) {
+        switch (ctx->opcode & 0xf) {
+        case 0x0:
+            return gen_lxvp(ctx);
+        case 0x1:
+            return gen_stxvp(ctx);
+        }
+    }
+    return gen_invalid(ctx);
+}
+
 /* Handles lfdp, lxsd, lxssp */
 static void gen_dform39(DisasContext *ctx)
 {
@@ -6880,8 +7700,13 @@ GEN_HANDLER(cmpi, 0x0B, 0xFF, 0xFF, 0x00400000, PPC_INTEGER),
 GEN_HANDLER(cmpl, 0x1F, 0x00, 0x01, 0x00400001, PPC_INTEGER),
 GEN_HANDLER(cmpli, 0x0A, 0xFF, 0xFF, 0x00400000, PPC_INTEGER),
 #if defined(TARGET_PPC64)
+GEN_HANDLER_E(brd, 0x1F, 0x1B, 0x05, 0x0000F801, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(brw, 0x1F, 0x1B, 0x04, 0x0000F801, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(brh, 0x1F, 0x1B, 0x06, 0x0000F801, PPC_NONE, PPC2_ISA310),
 GEN_HANDLER_E(cmpeqb, 0x1F, 0x00, 0x07, 0x00600000, PPC_NONE, PPC2_ISA300),
 #endif
+GEN_HANDLER_E(lxvp_stxvp, 0x06, 0xFF, 0xFF, 0x00000000, PPC_NONE,
+              PPC2_ISA310),
 GEN_HANDLER_E(cmpb, 0x1F, 0x1C, 0x0F, 0x00000001, PPC_NONE, PPC2_ISA205),
 GEN_HANDLER_E(cmprb, 0x1F, 0x00, 0x06, 0x00400001, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER(isel, 0x1F, 0x0F, 0xFF, 0x00000001, PPC_ISEL),
@@ -6908,6 +7733,18 @@ GEN_HANDLER_E(cnttzw, 0x1F, 0x1A, 0x10, 0x00000000, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(copy, 0x1F, 0x06, 0x18, 0x03C00001, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(cp_abort, 0x1F, 0x06, 0x1A, 0x03FFF801, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(paste, 0x1F, 0x06, 0x1C, 0x03C00000, PPC_NONE, PPC2_ISA300),
+GEN_HANDLER_E(setbc, 0x1F, 0x00, 0x0C, 0x0000F801, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(setbcr, 0x1F, 0x00, 0x0D, 0x0000F801, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(setnbc, 0x1F, 0x00, 0x0E, 0x0000F801, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(setnbcr, 0x1F, 0x00, 0x0F, 0x0000F801, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(addg6s, 0x1F, 0x0A, 0x02, 0x00000000, PPC_NONE,
+              PPC2_BCDA_ISA206),
+GEN_HANDLER_E(addg6s, 0x1F, 0x0A, 0x12, 0x00000000, PPC_NONE,
+              PPC2_BCDA_ISA206),
+GEN_HANDLER_E(cdtbcd, 0x1F, 0x1A, 0x08, 0x0000F800, PPC_NONE,
+              PPC2_BCDA_ISA206),
+GEN_HANDLER_E(cbcdtd, 0x1F, 0x1A, 0x09, 0x0000F800, PPC_NONE,
+              PPC2_BCDA_ISA206),
 GEN_HANDLER(or, 0x1F, 0x1C, 0x0D, 0x00000000, PPC_INTEGER),
 GEN_HANDLER(xor, 0x1F, 0x1C, 0x09, 0x00000000, PPC_INTEGER),
 GEN_HANDLER(ori, 0x18, 0xFF, 0xFF, 0x00000000, PPC_INTEGER),
@@ -6921,6 +7758,15 @@ GEN_HANDLER_E(prtyw, 0x1F, 0x1A, 0x04, 0x0000F801, PPC_NONE, PPC2_ISA205),
 GEN_HANDLER(popcntd, 0x1F, 0x1A, 0x0F, 0x0000F801, PPC_POPCNTWD),
 GEN_HANDLER(cntlzd, 0x1F, 0x1A, 0x01, 0x00000000, PPC_64B),
 GEN_HANDLER_E(cnttzd, 0x1F, 0x1A, 0x11, 0x00000000, PPC_NONE, PPC2_ISA300),
+GEN_HANDLER_E(cntlzdm, 0x1F, 0x1B, 0x01, 0x00000001, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(cnttzdm, 0x1F, 0x1B, 0x11, 0x00000001, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(pdepd, 0x1F, 0x1C, 0x04, 0x00000001, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(pextd, 0x1F, 0x1C, 0x05, 0x00000001, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(cfuged, 0x1F, 0x1C, 0x06, 0x00000001, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER(hashst, 0x1F, 0x12, 0x16, 0x00000000, PPC_64B),
+GEN_HANDLER(hashchk, 0x1F, 0x12, 0x17, 0x00000000, PPC_64B),
+GEN_HANDLER(hashstp, 0x1F, 0x12, 0x14, 0x00000000, PPC_64B),
+GEN_HANDLER(hashchkp, 0x1F, 0x12, 0x15, 0x00000000, PPC_64B),
 GEN_HANDLER_E(darn, 0x1F, 0x13, 0x17, 0x001CF801, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(prtyd, 0x1F, 0x1A, 0x05, 0x0000F801, PPC_NONE, PPC2_ISA205),
 GEN_HANDLER_E(bpermd, 0x1F, 0x1C, 0x07, 0x00000001, PPC_NONE, PPC2_PERM_ISA206),
@@ -7061,6 +7907,7 @@ GEN_HANDLER_E(tlbie, 0x1F, 0x12, 0x09, 0x00100001, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER(tlbsync, 0x1F, 0x16, 0x11, 0x03FFF801, PPC_MEM_TLBSYNC),
 #if defined(TARGET_PPC64)
 GEN_HANDLER(slbia, 0x1F, 0x12, 0x0F, 0x031FFC01, PPC_SLBI),
+GEN_HANDLER_E(slbiag, 0x1F, 0x12, 0x1A, 0x001EF801, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER(slbie, 0x1F, 0x12, 0x0D, 0x03FF0001, PPC_SLBI),
 GEN_HANDLER_E(slbieg, 0x1F, 0x12, 0x0E, 0x001F0001, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(slbsync, 0x1F, 0x12, 0x0A, 0x03FFF801, PPC_NONE, PPC2_ISA300),
@@ -7175,7 +8022,8 @@ GEN_HANDLER(lvsl, 0x1f, 0x06, 0x00, 0x00000001, PPC_ALTIVEC),
 GEN_HANDLER(lvsr, 0x1f, 0x06, 0x01, 0x00000001, PPC_ALTIVEC),
 GEN_HANDLER(mfvscr, 0x04, 0x2, 0x18, 0x001ff800, PPC_ALTIVEC),
 GEN_HANDLER(mtvscr, 0x04, 0x2, 0x19, 0x03ff0000, PPC_ALTIVEC),
-GEN_HANDLER(vmladduhm, 0x04, 0x11, 0xFF, 0x00000000, PPC_ALTIVEC),
+GEN_HANDLER_E(vmladduhm_vmsumudm, 0x04, 0x11, 0xFF, 0x00000000,
+              PPC_ALTIVEC, PPC2_ISA300),
 #if defined(TARGET_PPC64)
 GEN_HANDLER_E(maddhd_maddhdu, 0x04, 0x18, 0xFF, 0x00000000, PPC_NONE,
               PPC2_ISA300),
@@ -7620,6 +8468,11 @@ static void ppc_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs)
     CPUPPCState *env = cs->env_ptr;
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     opc_handler_t **table, *handler;
+    target_ulong pc;
+    uint32_t insn_size = 4;
+#if defined(TARGET_PPC64)
+    bool prefixed = false;
+#endif
 
     LOG_DISAS("----------------\n");
     LOG_DISAS("nip=" TARGET_FMT_lx " super=%d ir=%d\n",
@@ -7632,26 +8485,66 @@ static void ppc_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs)
         return;
     }
 
+    pc = ctx->base.pc_next;
+    ctx->cia = pc;
+    ctx->opcode = translator_ldl_swap(tcg_ctx, env, pc, need_byteswap(ctx));
+#if defined(TARGET_PPC64)
+    prefixed = is_prefix_insn(ctx, ctx->opcode);
+    if (prefixed) {
+        insn_size = 8;
+    }
+#endif
     // Unicorn: trace this instruction on request
-    if (HOOK_EXISTS_BOUNDED(uc, UC_HOOK_CODE, ctx->base.pc_next)) {
+    if (HOOK_EXISTS_BOUNDED(uc, UC_HOOK_CODE, pc)) {
 
         // Sypc PC in advance
-        gen_update_nip(ctx, ctx->base.pc_next);
+        gen_update_nip(ctx, pc);
 
-        gen_uc_tracecode(tcg_ctx, 4, UC_HOOK_CODE_IDX, uc, ctx->base.pc_next);
+        gen_uc_tracecode(tcg_ctx, insn_size, UC_HOOK_CODE_IDX, uc, pc);
         // the callback might want to stop emulation immediately
         check_exit_request(tcg_ctx);
     }
 
-    ctx->opcode = translator_ldl_swap(tcg_ctx, env, ctx->base.pc_next,
-                                      need_byteswap(ctx));
+#if defined(TARGET_PPC64)
+    if (prefixed) {
+        ctx->prefix_opcode = ctx->opcode;
+        ctx->base.pc_next = pc + 4;
+        if ((ctx->base.pc_next & 63) == 0) {
+            ctx->base.pc_next = pc + 8;
+            gen_exception_err(ctx, POWERPC_EXCP_ALIGN,
+                              POWERPC_EXCP_ALIGN_INSN);
+            ctx->base.is_jmp = DISAS_NORETURN;
+            return;
+        }
+        ctx->opcode = translator_ldl_swap(tcg_ctx, env, ctx->base.pc_next,
+                                          need_byteswap(ctx));
+        ctx->base.pc_next = pc + 8;
+
+        LOG_DISAS("translate prefixed opcode %08x %08x (%s)\n",
+                  ctx->prefix_opcode, ctx->opcode,
+                  ctx->le_mode ? "little" : "big");
+
+        if (!gen_prefixed_insn(ctx)) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "invalid/unsupported prefixed opcode: "
+                          "%08x %08x " TARGET_FMT_lx " %d\n",
+                          ctx->prefix_opcode, ctx->opcode, pc, (int)msr_ir);
+            gen_invalid(ctx);
+        }
+        goto translated;
+    }
+#endif
 
     LOG_DISAS("translate opcode %08x (%02x %02x %02x %02x) (%s)\n",
               ctx->opcode, opc1(ctx->opcode), opc2(ctx->opcode),
               opc3(ctx->opcode), opc4(ctx->opcode),
               ctx->le_mode ? "little" : "big");
 
-    ctx->base.pc_next += 4;
+    ctx->base.pc_next = pc + 4;
+
+    if (gen_mma_insn(ctx)) {
+        goto translated;
+    }
 
     table = cpu->opcodes;
     handler = table[opc1(ctx->opcode)];
@@ -7701,6 +8594,7 @@ static void ppc_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs)
 #if defined(DO_PPC_STATISTICS)
     handler->count++;
 #endif
+translated:
     /* Check trace mode exceptions */
     if (unlikely(ctx->singlestep_enabled & CPU_SINGLE_STEP &&
                  (ctx->base.pc_next <= 0x100 || ctx->base.pc_next > 0xF00) &&
diff --git a/qemu/target/ppc/translate/dfp-impl.inc.c b/qemu/target/ppc/translate/dfp-impl.inc.c
index 172c1c6ee7..3e64d577fa 100644
--- a/qemu/target/ppc/translate/dfp-impl.inc.c
+++ b/qemu/target/ppc/translate/dfp-impl.inc.c
@@ -218,6 +218,53 @@ GEN_DFP_T_B_Rc(dcffix)
 GEN_DFP_T_B_Rc(dcffixq)
 GEN_DFP_T_B_Rc(dctfix)
 GEN_DFP_T_B_Rc(dctfixq)
+
+static void gen_dfp_fixqq(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr rt;
+    TCGv_ptr rb;
+
+    if (rA(ctx->opcode) == 0) {
+        if (rD(ctx->opcode) & 1) {
+            gen_invalid(ctx);
+            return;
+        }
+    } else if (rA(ctx->opcode) == 1) {
+        if (rB(ctx->opcode) & 1) {
+            gen_invalid(ctx);
+            return;
+        }
+    } else {
+        gen_invalid(ctx);
+        return;
+    }
+
+    if (unlikely(!ctx->fpu_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_FPU);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    gen_update_nip(ctx, ctx->base.pc_next - 4);
+
+    if (rA(ctx->opcode) == 0) {
+        rt = gen_fprp_ptr(tcg_ctx, rD(ctx->opcode));
+        rb = gen_avr_ptr(tcg_ctx, rB(ctx->opcode));
+        gen_helper_DCFFIXQQ(tcg_ctx, tcg_ctx->cpu_env, rt, rb);
+    } else {
+        rt = gen_avr_ptr(tcg_ctx, rD(ctx->opcode));
+        rb = gen_fprp_ptr(tcg_ctx, rB(ctx->opcode));
+        gen_helper_DCTFIXQQ(tcg_ctx, tcg_ctx->cpu_env, rt, rb);
+    }
+
+    tcg_temp_free_ptr(tcg_ctx, rt);
+    tcg_temp_free_ptr(tcg_ctx, rb);
+}
+
 GEN_DFP_T_FPR_I32_Rc(ddedpd, rB, SP)
 GEN_DFP_T_FPR_I32_Rc(ddedpdq, rB, SP)
 GEN_DFP_T_FPR_I32_Rc(denbcd, rB, SP)
diff --git a/qemu/target/ppc/translate/dfp-ops.inc.c b/qemu/target/ppc/translate/dfp-ops.inc.c
index 6ef38e5712..b7b0b78653 100644
--- a/qemu/target/ppc/translate/dfp-ops.inc.c
+++ b/qemu/target/ppc/translate/dfp-ops.inc.c
@@ -151,6 +151,8 @@ GEN_DFP_T_B_Rc(dcffix, 0x02, 0x19),
 GEN_DFP_Tp_B_Rc(dcffixq, 0x02, 0x19),
 GEN_DFP_T_B_Rc(dctfix, 0x02, 0x09),
 GEN_DFP_T_Bp_Rc(dctfixq, 0x02, 0x09),
+GEN_HANDLER_E(dfp_fixqq, 0x3F, 0x02, 0x1F, 0x00000001,
+              PPC_NONE, PPC2_DFP),
 GEN_DFP_SP_T_B_Rc(ddedpd, 0x02, 0x0a),
 GEN_DFP_SP_Tp_Bp_Rc(ddedpdq, 0x02, 0x0a),
 GEN_DFP_S_T_B_Rc(denbcd, 0x02, 0x1a),
diff --git a/qemu/target/ppc/translate/fp-impl.inc.c b/qemu/target/ppc/translate/fp-impl.inc.c
index 58155f21eb..2e247d4dbe 100644
--- a/qemu/target/ppc/translate/fp-impl.inc.c
+++ b/qemu/target/ppc/translate/fp-impl.inc.c
@@ -658,7 +658,8 @@ static void gen_mffsl(DisasContext *ctx)
     gen_reset_fpstatus(tcg_ctx);
     tcg_gen_extu_tl_i64(tcg_ctx, t0, cpu_fpscr);
     /* Mask everything except mode, status, and enables.  */
-    tcg_gen_andi_i64(tcg_ctx, t0, t0, FP_DRN | FP_STATUS | FP_ENABLES | FP_RN);
+    tcg_gen_andi_i64(tcg_ctx, t0, t0,
+                     FP_DRN | FP_STATUS | FP_ENABLES | FP_NI | FP_RN);
     set_fpr(tcg_ctx, rD(ctx->opcode), t0);
     tcg_temp_free_i64(tcg_ctx, t0);
 }
@@ -694,21 +695,20 @@ static void gen_mffsce(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, t0);
 }
 
-static void gen_helper_mffscrn(DisasContext *ctx, TCGv_i64 t1)
+static void gen_helper_mffsc(DisasContext *ctx, TCGv_i64 t1,
+                             uint64_t field, uint32_t store_mask)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);
-    TCGv_i32 mask = tcg_const_i32(tcg_ctx, 0x0001);
+    TCGv_i32 mask = tcg_const_i32(tcg_ctx, store_mask);
 
     gen_reset_fpstatus(tcg_ctx);
     tcg_gen_extu_tl_i64(tcg_ctx, t0, cpu_fpscr);
-    tcg_gen_andi_i64(tcg_ctx, t0, t0, FP_DRN | FP_ENABLES | FP_RN);
+    tcg_gen_andi_i64(tcg_ctx, t0, t0, FP_DRN | FP_ENABLES | FP_NI | FP_RN);
     set_fpr(tcg_ctx, rD(ctx->opcode), t0);
 
-    /* Mask FPSCR value to clear RN.  */
-    tcg_gen_andi_i64(tcg_ctx, t0, t0, ~FP_RN);
+    tcg_gen_andi_i64(tcg_ctx, t0, t0, ~field);
 
-    /* Merge RN into FPSCR value.  */
     tcg_gen_or_i64(tcg_ctx, t0, t0, t1);
 
     gen_helper_store_fpscr(tcg_ctx, tcg_ctx->cpu_env, t0, mask);
@@ -737,7 +737,31 @@ static void gen_mffscrn(DisasContext *ctx)
     /* Mask FRB to get just RN.  */
     tcg_gen_andi_i64(tcg_ctx, t1, t1, FP_RN);
 
-    gen_helper_mffscrn(ctx, t1);
+    gen_helper_mffsc(ctx, t1, FP_RN, 0x0001);
+
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+/* mffscdrn */
+static void gen_mffscdrn(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 t1;
+
+    if (unlikely(!(ctx->insns_flags2 & PPC2_ISA300))) {
+        return;
+    }
+
+    if (unlikely(!ctx->fpu_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_FPU);
+        return;
+    }
+
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    get_fpr(tcg_ctx, t1, rB(ctx->opcode));
+    tcg_gen_andi_i64(tcg_ctx, t1, t1, FP_DRN);
+
+    gen_helper_mffsc(ctx, t1, FP_DRN, 0x0100);
 
     tcg_temp_free_i64(tcg_ctx, t1);
 }
@@ -759,7 +783,30 @@ static void gen_mffscrni(DisasContext *ctx)
 
     t1 = tcg_const_i64(tcg_ctx, (uint64_t)RM(ctx->opcode));
 
-    gen_helper_mffscrn(ctx, t1);
+    gen_helper_mffsc(ctx, t1, FP_RN, 0x0001);
+
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+/* mffscdrni */
+static void gen_mffscdrni(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 t1;
+
+    if (unlikely(!(ctx->insns_flags2 & PPC2_ISA300))) {
+        return;
+    }
+
+    if (unlikely(!ctx->fpu_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_FPU);
+        return;
+    }
+
+    t1 = tcg_const_i64(tcg_ctx,
+                       (uint64_t)(rB(ctx->opcode) & 7) << FPSCR_DRN0);
+
+    gen_helper_mffsc(ctx, t1, FP_DRN, 0x0100);
 
     tcg_temp_free_i64(tcg_ctx, t1);
 }
@@ -1242,6 +1289,94 @@ GEN_STFS(stfd, st64_i64, 0x16, PPC_FLOAT);
 /* stfs stfsu stfsux stfsx */
 GEN_STFS(stfs, st32fs, 0x14, PPC_FLOAT);
 
+static void gen_plfs(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 t0;
+
+    if (unlikely(!ctx->fpu_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_FPU);
+        return;
+    }
+
+    gen_set_access_type(ctx, ACCESS_FLOAT);
+    ea = tcg_temp_new(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld32fs(ctx, t0, ea);
+        set_fpr(tcg_ctx, rD(ctx->opcode), t0);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, t0);
+}
+
+static void gen_plfd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 t0;
+
+    if (unlikely(!ctx->fpu_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_FPU);
+        return;
+    }
+
+    gen_set_access_type(ctx, ACCESS_FLOAT);
+    ea = tcg_temp_new(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld64_i64(ctx, t0, ea);
+        set_fpr(tcg_ctx, rD(ctx->opcode), t0);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, t0);
+}
+
+static void gen_pstfs(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 t0;
+
+    if (unlikely(!ctx->fpu_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_FPU);
+        return;
+    }
+
+    gen_set_access_type(ctx, ACCESS_FLOAT);
+    ea = tcg_temp_new(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        get_fpr(tcg_ctx, t0, rS(ctx->opcode));
+        gen_qemu_st32fs(ctx, t0, ea);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, t0);
+}
+
+static void gen_pstfd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 t0;
+
+    if (unlikely(!ctx->fpu_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_FPU);
+        return;
+    }
+
+    gen_set_access_type(ctx, ACCESS_FLOAT);
+    ea = tcg_temp_new(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        get_fpr(tcg_ctx, t0, rS(ctx->opcode));
+        gen_qemu_st64_i64(ctx, t0, ea);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, t0);
+}
+
 /* stfdepx (external PID lfdx) */
 static void gen_stfdepx(DisasContext *ctx)
 {
diff --git a/qemu/target/ppc/translate/fp-ops.inc.c b/qemu/target/ppc/translate/fp-ops.inc.c
index 88fab65628..f9d0f34808 100644
--- a/qemu/target/ppc/translate/fp-ops.inc.c
+++ b/qemu/target/ppc/translate/fp-ops.inc.c
@@ -105,14 +105,18 @@ GEN_HANDLER_E(fmrgew, 0x3F, 0x06, 0x1E, 0x00000001, PPC_NONE, PPC2_VSX207),
 GEN_HANDLER_E(fmrgow, 0x3F, 0x06, 0x1A, 0x00000001, PPC_NONE, PPC2_VSX207),
 GEN_HANDLER(mcrfs, 0x3F, 0x00, 0x02, 0x0063F801, PPC_FLOAT),
 GEN_HANDLER_E_2(mffs, 0x3F, 0x07, 0x12, 0x00, 0x00000000, PPC_FLOAT, PPC_NONE),
-GEN_HANDLER_E_2(mffsce, 0x3F, 0x07, 0x12, 0x01, 0x00000000, PPC_FLOAT,
+GEN_HANDLER_E_2(mffsce, 0x3F, 0x07, 0x12, 0x01, 0x00000000, PPC_NONE,
     PPC2_ISA300),
-GEN_HANDLER_E_2(mffsl, 0x3F, 0x07, 0x12, 0x18, 0x00000000, PPC_FLOAT,
+GEN_HANDLER_E_2(mffsl, 0x3F, 0x07, 0x12, 0x18, 0x00000000, PPC_NONE,
+    PPC2_ISA300),
+GEN_HANDLER_E_2(mffscdrn, 0x3F, 0x07, 0x12, 0x14, 0x00000000, PPC_NONE,
+    PPC2_ISA300),
+GEN_HANDLER_E_2(mffscdrni, 0x3F, 0x07, 0x12, 0x15, 0x00000000, PPC_NONE,
+    PPC2_ISA300),
+GEN_HANDLER_E_2(mffscrn, 0x3F, 0x07, 0x12, 0x16, 0x00000000, PPC_NONE,
+    PPC2_ISA300),
+GEN_HANDLER_E_2(mffscrni, 0x3F, 0x07, 0x12, 0x17, 0x00000000, PPC_NONE,
     PPC2_ISA300),
-GEN_HANDLER_E_2(mffscrn, 0x3F, 0x07, 0x12, 0x16, 0x00000000, PPC_FLOAT,
-    PPC_NONE),
-GEN_HANDLER_E_2(mffscrni, 0x3F, 0x07, 0x12, 0x17, 0x00000000, PPC_FLOAT,
-    PPC_NONE),
 GEN_HANDLER(mtfsb0, 0x3F, 0x06, 0x02, 0x001FF800, PPC_FLOAT),
 GEN_HANDLER(mtfsb1, 0x3F, 0x06, 0x01, 0x001FF800, PPC_FLOAT),
 GEN_HANDLER(mtfsf, 0x3F, 0x07, 0x16, 0x00000000, PPC_FLOAT),
diff --git a/qemu/target/ppc/translate/vmx-impl.inc.c b/qemu/target/ppc/translate/vmx-impl.inc.c
index 9d4211dd6e..148d80c234 100644
--- a/qemu/target/ppc/translate/vmx-impl.inc.c
+++ b/qemu/target/ppc/translate/vmx-impl.inc.c
@@ -834,18 +834,504 @@ GEN_VXFORM(vmuleuw, 4, 10);
 GEN_VXFORM(vmulesb, 4, 12);
 GEN_VXFORM(vmulesh, 4, 13);
 GEN_VXFORM(vmulesw, 4, 14);
+
+static void gen_vmuleo_dword(DisasContext *ctx, bool even, bool sign)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 vra;
+    TCGv_i64 vrb;
+    TCGv_i64 lo;
+    TCGv_i64 hi;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    vra = tcg_temp_new_i64(tcg_ctx);
+    vrb = tcg_temp_new_i64(tcg_ctx);
+    lo = tcg_temp_new_i64(tcg_ctx);
+    hi = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, vra, rA(ctx->opcode), even);
+    get_avr64(tcg_ctx, vrb, rB(ctx->opcode), even);
+    if (sign) {
+        tcg_gen_muls2_i64(tcg_ctx, lo, hi, vra, vrb);
+    } else {
+        tcg_gen_mulu2_i64(tcg_ctx, lo, hi, vra, vrb);
+    }
+    set_avr64(tcg_ctx, rD(ctx->opcode), lo, false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), hi, true);
+
+    tcg_temp_free_i64(tcg_ctx, vra);
+    tcg_temp_free_i64(tcg_ctx, vrb);
+    tcg_temp_free_i64(tcg_ctx, lo);
+    tcg_temp_free_i64(tcg_ctx, hi);
+}
+
+static void gen_vmulesd(DisasContext *ctx)
+{
+    gen_vmuleo_dword(ctx, true, true);
+}
+
+static void gen_vmulosd(DisasContext *ctx)
+{
+    gen_vmuleo_dword(ctx, false, true);
+}
+
+static void gen_vmuleud(DisasContext *ctx)
+{
+    gen_vmuleo_dword(ctx, true, false);
+}
+
+static void gen_vmuloud(DisasContext *ctx)
+{
+    gen_vmuleo_dword(ctx, false, false);
+}
+
+static void gen_vmulld(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tcg_gen_gvec_mul(tcg_ctx, MO_64, avr_full_offset(rD(ctx->opcode)),
+                     avr_full_offset(rA(ctx->opcode)),
+                     avr_full_offset(rB(ctx->opcode)), 16, 16);
+}
+
+static void gen_vmulhw_i64(TCGContext *tcg_ctx, TCGv_i64 t, TCGv_i64 a,
+                           TCGv_i64 b, bool sign)
+{
+    TCGv_i64 hh = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 lh = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tmp = tcg_temp_new_i64(tcg_ctx);
+
+    if (sign) {
+        tcg_gen_ext32s_i64(tcg_ctx, lh, a);
+        tcg_gen_ext32s_i64(tcg_ctx, tmp, b);
+    } else {
+        tcg_gen_ext32u_i64(tcg_ctx, lh, a);
+        tcg_gen_ext32u_i64(tcg_ctx, tmp, b);
+    }
+    tcg_gen_mul_i64(tcg_ctx, lh, lh, tmp);
+
+    if (sign) {
+        tcg_gen_sari_i64(tcg_ctx, hh, a, 32);
+        tcg_gen_sari_i64(tcg_ctx, tmp, b, 32);
+    } else {
+        tcg_gen_shri_i64(tcg_ctx, hh, a, 32);
+        tcg_gen_shri_i64(tcg_ctx, tmp, b, 32);
+    }
+    tcg_gen_mul_i64(tcg_ctx, hh, hh, tmp);
+
+    tcg_gen_shri_i64(tcg_ctx, lh, lh, 32);
+    tcg_gen_deposit_i64(tcg_ctx, t, hh, lh, 0, 32);
+
+    tcg_temp_free_i64(tcg_ctx, hh);
+    tcg_temp_free_i64(tcg_ctx, lh);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+}
+
+static void gen_vmulhd_i64(TCGContext *tcg_ctx, TCGv_i64 t, TCGv_i64 a,
+                           TCGv_i64 b, bool sign)
+{
+    TCGv_i64 lo = tcg_temp_new_i64(tcg_ctx);
+
+    if (sign) {
+        tcg_gen_muls2_i64(tcg_ctx, lo, t, a, b);
+    } else {
+        tcg_gen_mulu2_i64(tcg_ctx, lo, t, a, b);
+    }
+
+    tcg_temp_free_i64(tcg_ctx, lo);
+}
+
+static void gen_vmulh(DisasContext *ctx, bool sign, bool dword)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 vra;
+    TCGv_i64 vrb;
+    TCGv_i64 vrt;
+    int i;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    vra = tcg_temp_new_i64(tcg_ctx);
+    vrb = tcg_temp_new_i64(tcg_ctx);
+    vrt = tcg_temp_new_i64(tcg_ctx);
+
+    for (i = 0; i < 2; i++) {
+        get_avr64(tcg_ctx, vra, rA(ctx->opcode), i);
+        get_avr64(tcg_ctx, vrb, rB(ctx->opcode), i);
+        if (dword) {
+            gen_vmulhd_i64(tcg_ctx, vrt, vra, vrb, sign);
+        } else {
+            gen_vmulhw_i64(tcg_ctx, vrt, vra, vrb, sign);
+        }
+        set_avr64(tcg_ctx, rD(ctx->opcode), vrt, i);
+    }
+
+    tcg_temp_free_i64(tcg_ctx, vra);
+    tcg_temp_free_i64(tcg_ctx, vrb);
+    tcg_temp_free_i64(tcg_ctx, vrt);
+}
+
+static void gen_vmulhsw(DisasContext *ctx)
+{
+    gen_vmulh(ctx, true, false);
+}
+
+static void gen_vmulhuw(DisasContext *ctx)
+{
+    gen_vmulh(ctx, false, false);
+}
+
+static void gen_vmulhsd(DisasContext *ctx)
+{
+    gen_vmulh(ctx, true, true);
+}
+
+static void gen_vmulhud(DisasContext *ctx)
+{
+    gen_vmulh(ctx, false, true);
+}
+
+GEN_VXFORM_DUAL(vmuleuw, PPC_NONE, PPC2_ALTIVEC_207, \
+                vmulhuw, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vmulesw, PPC_NONE, PPC2_ALTIVEC_207, \
+                vmulhsw, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vmulosd, PPC_NONE, PPC2_ISA310, \
+                vmulld, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vmuleud, PPC_NONE, PPC2_ISA310, \
+                vmulhud, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vmulesd, PPC_NONE, PPC2_ISA310, \
+                vmulhsd, PPC_NONE, PPC2_ISA310)
+
+static void gen_vector_shift_quad(DisasContext *ctx, bool right, bool alg)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 hi;
+    TCGv_i64 lo;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 n;
+    TCGv_i64 zero;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    hi = tcg_temp_new_i64(tcg_ctx);
+    lo = tcg_temp_new_i64(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    n = tcg_temp_new_i64(tcg_ctx);
+    zero = tcg_const_i64(tcg_ctx, 0);
+
+    get_avr64(tcg_ctx, lo, rA(ctx->opcode), false);
+    get_avr64(tcg_ctx, hi, rA(ctx->opcode), true);
+    get_avr64(tcg_ctx, n, rB(ctx->opcode), true);
+
+    tcg_gen_andi_i64(tcg_ctx, t0, n, 64);
+    if (right) {
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, lo, t0, zero, hi, lo);
+        tcg_gen_movi_i64(tcg_ctx, t1, 0);
+        if (alg) {
+            tcg_gen_sari_i64(tcg_ctx, t1, lo, 63);
+        }
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, hi, t0, zero, t1, hi);
+    } else {
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, hi, t0, zero, lo, hi);
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, lo, t0, zero, zero, lo);
+    }
+    tcg_gen_andi_i64(tcg_ctx, n, n, 0x3f);
+
+    if (right) {
+        if (alg) {
+            tcg_gen_sar_i64(tcg_ctx, t0, hi, n);
+        } else {
+            tcg_gen_shr_i64(tcg_ctx, t0, hi, n);
+        }
+    } else {
+        tcg_gen_shl_i64(tcg_ctx, t0, lo, n);
+    }
+    set_avr64(tcg_ctx, rD(ctx->opcode), t0, right);
+
+    if (right) {
+        tcg_gen_shr_i64(tcg_ctx, lo, lo, n);
+    } else {
+        tcg_gen_shl_i64(tcg_ctx, hi, hi, n);
+    }
+    tcg_gen_xori_i64(tcg_ctx, n, n, 63);
+    if (right) {
+        tcg_gen_shl_i64(tcg_ctx, hi, hi, n);
+        tcg_gen_shli_i64(tcg_ctx, hi, hi, 1);
+    } else {
+        tcg_gen_shr_i64(tcg_ctx, lo, lo, n);
+        tcg_gen_shri_i64(tcg_ctx, lo, lo, 1);
+    }
+    tcg_gen_or_i64(tcg_ctx, hi, hi, lo);
+    set_avr64(tcg_ctx, rD(ctx->opcode), hi, !right);
+
+    tcg_temp_free_i64(tcg_ctx, hi);
+    tcg_temp_free_i64(tcg_ctx, lo);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+    tcg_temp_free_i64(tcg_ctx, n);
+    tcg_temp_free_i64(tcg_ctx, zero);
+}
+
+static void gen_vrlq_mask(TCGContext *tcg_ctx, TCGv_i64 mh, TCGv_i64 ml,
+                          TCGv_i64 b, TCGv_i64 e)
+{
+    TCGv_i64 th = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tl = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 t1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 zero = tcg_const_i64(tcg_ctx, 0);
+    TCGv_i64 ones = tcg_const_i64(tcg_ctx, -1);
+
+    tcg_gen_andi_i64(tcg_ctx, t0, b, 64);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, t1, t0, zero, zero, ones);
+    tcg_gen_andi_i64(tcg_ctx, t0, b, 0x3f);
+    tcg_gen_shr_i64(tcg_ctx, mh, t1, t0);
+    tcg_gen_shr_i64(tcg_ctx, ml, ones, t0);
+    tcg_gen_xori_i64(tcg_ctx, t0, t0, 63);
+    tcg_gen_shl_i64(tcg_ctx, t1, t1, t0);
+    tcg_gen_shli_i64(tcg_ctx, t1, t1, 1);
+    tcg_gen_or_i64(tcg_ctx, ml, t1, ml);
+
+    tcg_gen_andi_i64(tcg_ctx, t0, e, 64);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, t1, t0, zero, zero, ones);
+    tcg_gen_andi_i64(tcg_ctx, t0, e, 0x3f);
+    tcg_gen_shr_i64(tcg_ctx, th, t1, t0);
+    tcg_gen_shr_i64(tcg_ctx, tl, ones, t0);
+    tcg_gen_xori_i64(tcg_ctx, t0, t0, 63);
+    tcg_gen_shl_i64(tcg_ctx, t1, t1, t0);
+    tcg_gen_shli_i64(tcg_ctx, t1, t1, 1);
+    tcg_gen_or_i64(tcg_ctx, tl, t1, tl);
+
+    tcg_gen_extract2_i64(tcg_ctx, tl, tl, th, 1);
+    tcg_gen_shri_i64(tcg_ctx, th, th, 1);
+
+    tcg_gen_xor_i64(tcg_ctx, mh, mh, th);
+    tcg_gen_xor_i64(tcg_ctx, ml, ml, tl);
+
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_GT, t0, b, e, ones, zero);
+    tcg_gen_xor_i64(tcg_ctx, mh, mh, t0);
+    tcg_gen_xor_i64(tcg_ctx, ml, ml, t0);
+
+    tcg_temp_free_i64(tcg_ctx, th);
+    tcg_temp_free_i64(tcg_ctx, tl);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+    tcg_temp_free_i64(tcg_ctx, zero);
+    tcg_temp_free_i64(tcg_ctx, ones);
+}
+
+static void gen_vector_rotl_quad(DisasContext *ctx, bool mask, bool insert)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 ah;
+    TCGv_i64 al;
+    TCGv_i64 vrb;
+    TCGv_i64 n;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 zero;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    ah = tcg_temp_new_i64(tcg_ctx);
+    al = tcg_temp_new_i64(tcg_ctx);
+    vrb = tcg_temp_new_i64(tcg_ctx);
+    n = tcg_temp_new_i64(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    zero = tcg_const_i64(tcg_ctx, 0);
+
+    get_avr64(tcg_ctx, ah, rA(ctx->opcode), true);
+    get_avr64(tcg_ctx, al, rA(ctx->opcode), false);
+    get_avr64(tcg_ctx, vrb, rB(ctx->opcode), true);
+
+    tcg_gen_mov_i64(tcg_ctx, t0, ah);
+    tcg_gen_andi_i64(tcg_ctx, t1, vrb, 64);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, ah, t1, zero, al, ah);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, al, t1, zero, t0, al);
+    tcg_gen_andi_i64(tcg_ctx, n, vrb, 0x3f);
+
+    tcg_gen_shl_i64(tcg_ctx, t0, ah, n);
+    tcg_gen_shl_i64(tcg_ctx, t1, al, n);
+
+    tcg_gen_xori_i64(tcg_ctx, n, n, 63);
+
+    tcg_gen_shr_i64(tcg_ctx, al, al, n);
+    tcg_gen_shri_i64(tcg_ctx, al, al, 1);
+    tcg_gen_or_i64(tcg_ctx, t0, al, t0);
+
+    tcg_gen_shr_i64(tcg_ctx, ah, ah, n);
+    tcg_gen_shri_i64(tcg_ctx, ah, ah, 1);
+    tcg_gen_or_i64(tcg_ctx, t1, ah, t1);
+
+    if (mask || insert) {
+        tcg_gen_extract_i64(tcg_ctx, n, vrb, 8, 7);
+        tcg_gen_extract_i64(tcg_ctx, vrb, vrb, 16, 7);
+
+        gen_vrlq_mask(tcg_ctx, ah, al, vrb, n);
+
+        tcg_gen_and_i64(tcg_ctx, t0, t0, ah);
+        tcg_gen_and_i64(tcg_ctx, t1, t1, al);
+
+        if (insert) {
+            get_avr64(tcg_ctx, n, rD(ctx->opcode), true);
+            get_avr64(tcg_ctx, vrb, rD(ctx->opcode), false);
+            tcg_gen_andc_i64(tcg_ctx, n, n, ah);
+            tcg_gen_andc_i64(tcg_ctx, vrb, vrb, al);
+            tcg_gen_or_i64(tcg_ctx, t0, t0, n);
+            tcg_gen_or_i64(tcg_ctx, t1, t1, vrb);
+        }
+    }
+
+    set_avr64(tcg_ctx, rD(ctx->opcode), t0, true);
+    set_avr64(tcg_ctx, rD(ctx->opcode), t1, false);
+
+    tcg_temp_free_i64(tcg_ctx, ah);
+    tcg_temp_free_i64(tcg_ctx, al);
+    tcg_temp_free_i64(tcg_ctx, vrb);
+    tcg_temp_free_i64(tcg_ctx, n);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+    tcg_temp_free_i64(tcg_ctx, zero);
+}
+
+static void gen_vslq(DisasContext *ctx)
+{
+    gen_vector_shift_quad(ctx, false, false);
+}
+
+static void gen_vsrq(DisasContext *ctx)
+{
+    gen_vector_shift_quad(ctx, true, false);
+}
+
+static void gen_vsraq(DisasContext *ctx)
+{
+    gen_vector_shift_quad(ctx, true, true);
+}
+
+static void gen_vrlq(DisasContext *ctx)
+{
+    gen_vector_rotl_quad(ctx, false, false);
+}
+
+static void gen_vrlqnm(DisasContext *ctx)
+{
+    gen_vector_rotl_quad(ctx, true, false);
+}
+
+static void gen_vrlqmi(DisasContext *ctx)
+{
+    gen_vector_rotl_quad(ctx, false, true);
+}
+
+static void gen_vmsumcud(DisasContext *ctx);
+
+static void gen_vsldbi_vsrdbi(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 t2;
+    int opc = opc3(ctx->opcode);
+    int sh = opc & 7;
+
+    if (Rc(ctx->opcode)) {
+        gen_vmsumcud(ctx);
+        return;
+    }
+    if (opc >= 16) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+
+    if (opc & 8) {
+        get_avr64(tcg_ctx, t0, rB(ctx->opcode), false);
+        get_avr64(tcg_ctx, t1, rB(ctx->opcode), true);
+
+        if (sh != 0) {
+            t2 = tcg_temp_new_i64(tcg_ctx);
+
+            get_avr64(tcg_ctx, t2, rA(ctx->opcode), false);
+
+            tcg_gen_extract2_i64(tcg_ctx, t0, t0, t1, sh);
+            tcg_gen_extract2_i64(tcg_ctx, t1, t1, t2, sh);
+
+            tcg_temp_free_i64(tcg_ctx, t2);
+        }
+
+        set_avr64(tcg_ctx, rD(ctx->opcode), t0, false);
+        set_avr64(tcg_ctx, rD(ctx->opcode), t1, true);
+    } else {
+        get_avr64(tcg_ctx, t0, rA(ctx->opcode), true);
+        get_avr64(tcg_ctx, t1, rA(ctx->opcode), false);
+
+        if (sh != 0) {
+            t2 = tcg_temp_new_i64(tcg_ctx);
+
+            get_avr64(tcg_ctx, t2, rB(ctx->opcode), true);
+
+            tcg_gen_extract2_i64(tcg_ctx, t0, t1, t0, 64 - sh);
+            tcg_gen_extract2_i64(tcg_ctx, t1, t2, t1, 64 - sh);
+
+            tcg_temp_free_i64(tcg_ctx, t2);
+        }
+
+        set_avr64(tcg_ctx, rD(ctx->opcode), t0, true);
+        set_avr64(tcg_ctx, rD(ctx->opcode), t1, false);
+    }
+
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
 GEN_VXFORM_V(vslb, MO_8, tcg_gen_gvec_shlv, 2, 4);
 GEN_VXFORM_V(vslh, MO_16, tcg_gen_gvec_shlv, 2, 5);
+GEN_VXFORM_DUAL(vslb, PPC_ALTIVEC, PPC_NONE, \
+                vslq, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vslh, PPC_ALTIVEC, PPC_NONE, \
+                vrlqnm, PPC_NONE, PPC2_ISA310)
 GEN_VXFORM_V(vslw, MO_32, tcg_gen_gvec_shlv, 2, 6);
 GEN_VXFORM(vrlwnm, 2, 6);
 GEN_VXFORM_DUAL(vslw, PPC_ALTIVEC, PPC_NONE, \
                 vrlwnm, PPC_NONE, PPC2_ISA300)
 GEN_VXFORM_V(vsld, MO_64, tcg_gen_gvec_shlv, 2, 23);
 GEN_VXFORM_V(vsrb, MO_8, tcg_gen_gvec_shrv, 2, 8);
+GEN_VXFORM_DUAL(vsrb, PPC_ALTIVEC, PPC_NONE, \
+                vsrq, PPC_NONE, PPC2_ISA310)
 GEN_VXFORM_V(vsrh, MO_16, tcg_gen_gvec_shrv, 2, 9);
 GEN_VXFORM_V(vsrw, MO_32, tcg_gen_gvec_shrv, 2, 10);
 GEN_VXFORM_V(vsrd, MO_64, tcg_gen_gvec_shrv, 2, 27);
 GEN_VXFORM_V(vsrab, MO_8, tcg_gen_gvec_sarv, 2, 12);
+GEN_VXFORM_DUAL(vsrab, PPC_ALTIVEC, PPC_NONE, \
+                vsraq, PPC_NONE, PPC2_ISA310)
 GEN_VXFORM_V(vsrah, MO_16, tcg_gen_gvec_sarv, 2, 13);
 GEN_VXFORM_V(vsraw, MO_32, tcg_gen_gvec_sarv, 2, 14);
 GEN_VXFORM_V(vsrad, MO_64, tcg_gen_gvec_sarv, 2, 15);
@@ -910,8 +1396,98 @@ GEN_VXFORM_SAT(vsubuws, MO_32, sub, ussub, 0, 26);
 GEN_VXFORM_SAT(vsubsbs, MO_8, sub, sssub, 0, 28);
 GEN_VXFORM_SAT(vsubshs, MO_16, sub, sssub, 0, 29);
 GEN_VXFORM_SAT(vsubsws, MO_32, sub, sssub, 0, 30);
+
+static void gen_vcmpq_cr(DisasContext *ctx, bool sign)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 ah;
+    TCGv_i64 al;
+    TCGv_i64 bh;
+    TCGv_i64 bl;
+    TCGv_i64 gt;
+    TCGv_i64 lt;
+    TCGv_i64 eq;
+    TCGv_i64 tmp;
+    TCGv_i64 crf;
+    TCGv_i64 zero;
+    TCGv_i64 cr_gt;
+    TCGv_i64 cr_lt;
+
+    if (unlikely(ctx->opcode & 0x00600000)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    ah = tcg_temp_new_i64(tcg_ctx);
+    al = tcg_temp_new_i64(tcg_ctx);
+    bh = tcg_temp_new_i64(tcg_ctx);
+    bl = tcg_temp_new_i64(tcg_ctx);
+    gt = tcg_temp_new_i64(tcg_ctx);
+    lt = tcg_temp_new_i64(tcg_ctx);
+    eq = tcg_temp_new_i64(tcg_ctx);
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    crf = tcg_const_i64(tcg_ctx, CRF_EQ);
+    zero = tcg_const_i64(tcg_ctx, 0);
+    cr_gt = tcg_const_i64(tcg_ctx, CRF_GT);
+    cr_lt = tcg_const_i64(tcg_ctx, CRF_LT);
+
+    get_avr64(tcg_ctx, ah, rA(ctx->opcode), true);
+    get_avr64(tcg_ctx, al, rA(ctx->opcode), false);
+    get_avr64(tcg_ctx, bh, rB(ctx->opcode), true);
+    get_avr64(tcg_ctx, bl, rB(ctx->opcode), false);
+
+    tcg_gen_setcond_i64(tcg_ctx, sign ? TCG_COND_GT : TCG_COND_GTU,
+                        gt, ah, bh);
+    tcg_gen_setcond_i64(tcg_ctx, sign ? TCG_COND_LT : TCG_COND_LTU,
+                        lt, ah, bh);
+    tcg_gen_setcond_i64(tcg_ctx, TCG_COND_EQ, eq, ah, bh);
+
+    tcg_gen_setcond_i64(tcg_ctx, TCG_COND_GTU, tmp, al, bl);
+    tcg_gen_and_i64(tcg_ctx, tmp, tmp, eq);
+    tcg_gen_or_i64(tcg_ctx, gt, gt, tmp);
+
+    tcg_gen_setcond_i64(tcg_ctx, TCG_COND_LTU, tmp, al, bl);
+    tcg_gen_and_i64(tcg_ctx, tmp, tmp, eq);
+    tcg_gen_or_i64(tcg_ctx, lt, lt, tmp);
+
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, crf, gt, zero, cr_gt, crf);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, crf, lt, zero, cr_lt, crf);
+    tcg_gen_extrl_i64_i32(tcg_ctx, cpu_crf[BF(ctx->opcode)], crf);
+
+    tcg_temp_free_i64(tcg_ctx, ah);
+    tcg_temp_free_i64(tcg_ctx, al);
+    tcg_temp_free_i64(tcg_ctx, bh);
+    tcg_temp_free_i64(tcg_ctx, bl);
+    tcg_temp_free_i64(tcg_ctx, gt);
+    tcg_temp_free_i64(tcg_ctx, lt);
+    tcg_temp_free_i64(tcg_ctx, eq);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+    tcg_temp_free_i64(tcg_ctx, crf);
+    tcg_temp_free_i64(tcg_ctx, zero);
+    tcg_temp_free_i64(tcg_ctx, cr_gt);
+    tcg_temp_free_i64(tcg_ctx, cr_lt);
+}
+
+static void gen_vcmpuq(DisasContext *ctx)
+{
+    gen_vcmpq_cr(ctx, false);
+}
+
+static void gen_vcmpsq(DisasContext *ctx)
+{
+    gen_vcmpq_cr(ctx, true);
+}
+
 GEN_VXFORM(vadduqm, 0, 4);
 GEN_VXFORM(vaddcuq, 0, 5);
+GEN_VXFORM_DUAL(vadduqm, PPC_NONE, PPC2_ALTIVEC_207, \
+                vcmpuq, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vaddcuq, PPC_NONE, PPC2_ALTIVEC_207, \
+                vcmpsq, PPC_NONE, PPC2_ISA310)
 GEN_VXFORM3(vaddeuqm, 30, 0);
 GEN_VXFORM3(vaddecuq, 30, 0);
 GEN_VXFORM_DUAL(vaddeuqm, PPC_NONE, PPC2_ALTIVEC_207, \
@@ -923,7 +1499,11 @@ GEN_VXFORM3(vsubecuq, 31, 0);
 GEN_VXFORM_DUAL(vsubeuqm, PPC_NONE, PPC2_ALTIVEC_207, \
             vsubecuq, PPC_NONE, PPC2_ALTIVEC_207)
 GEN_VXFORM(vrlb, 2, 0);
+GEN_VXFORM_DUAL(vrlb, PPC_ALTIVEC, PPC_NONE, \
+                vrlq, PPC_NONE, PPC2_ISA310)
 GEN_VXFORM(vrlh, 2, 1);
+GEN_VXFORM_DUAL(vrlh, PPC_ALTIVEC, PPC_NONE, \
+                vrlqmi, PPC_NONE, PPC2_ISA310)
 GEN_VXFORM(vrlw, 2, 2);
 GEN_VXFORM(vrlwmi, 2, 2);
 GEN_VXFORM_DUAL(vrlw, PPC_ALTIVEC, PPC_NONE, \
@@ -1045,6 +1625,127 @@ GEN_VXRFORM(vcmpneb, 3, 0)
 GEN_VXRFORM(vcmpneh, 3, 1)
 GEN_VXRFORM(vcmpnew, 3, 2)
 
+static void gen_vcmpq_record(TCGContext *tcg_ctx, TCGv_i64 result)
+{
+    tcg_gen_extrl_i64_i32(tcg_ctx, cpu_crf[6], result);
+    tcg_gen_andi_i32(tcg_ctx, cpu_crf[6], cpu_crf[6], 0xa);
+    tcg_gen_xori_i32(tcg_ctx, cpu_crf[6], cpu_crf[6], 0x2);
+}
+
+static void gen_vcmpequq_common(DisasContext *ctx, bool record)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 t2;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    t2 = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, t0, rA(ctx->opcode), true);
+    get_avr64(tcg_ctx, t1, rB(ctx->opcode), true);
+    tcg_gen_xor_i64(tcg_ctx, t2, t0, t1);
+
+    get_avr64(tcg_ctx, t0, rA(ctx->opcode), false);
+    get_avr64(tcg_ctx, t1, rB(ctx->opcode), false);
+    tcg_gen_xor_i64(tcg_ctx, t1, t0, t1);
+
+    tcg_gen_or_i64(tcg_ctx, t1, t1, t2);
+    tcg_gen_setcondi_i64(tcg_ctx, TCG_COND_EQ, t1, t1, 0);
+    tcg_gen_neg_i64(tcg_ctx, t1, t1);
+
+    set_avr64(tcg_ctx, rD(ctx->opcode), t1, true);
+    set_avr64(tcg_ctx, rD(ctx->opcode), t1, false);
+
+    if (record) {
+        gen_vcmpq_record(tcg_ctx, t1);
+    }
+
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+    tcg_temp_free_i64(tcg_ctx, t2);
+}
+
+static void gen_vcmpgtq_common(DisasContext *ctx, bool sign, bool record)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 t2;
+    TCGv_i64 zero;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    t2 = tcg_temp_new_i64(tcg_ctx);
+    zero = tcg_const_i64(tcg_ctx, 0);
+
+    get_avr64(tcg_ctx, t0, rA(ctx->opcode), false);
+    get_avr64(tcg_ctx, t1, rB(ctx->opcode), false);
+    tcg_gen_setcond_i64(tcg_ctx, TCG_COND_GTU, t2, t0, t1);
+
+    get_avr64(tcg_ctx, t0, rA(ctx->opcode), true);
+    get_avr64(tcg_ctx, t1, rB(ctx->opcode), true);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_EQ, t2, t0, t1, t2, zero);
+    tcg_gen_setcond_i64(tcg_ctx, sign ? TCG_COND_GT : TCG_COND_GTU,
+                        t1, t0, t1);
+
+    tcg_gen_or_i64(tcg_ctx, t1, t1, t2);
+    tcg_gen_neg_i64(tcg_ctx, t1, t1);
+
+    set_avr64(tcg_ctx, rD(ctx->opcode), t1, true);
+    set_avr64(tcg_ctx, rD(ctx->opcode), t1, false);
+
+    if (record) {
+        gen_vcmpq_record(tcg_ctx, t1);
+    }
+
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+    tcg_temp_free_i64(tcg_ctx, t2);
+    tcg_temp_free_i64(tcg_ctx, zero);
+}
+
+static void gen_vcmpequq(DisasContext *ctx)
+{
+    gen_vcmpequq_common(ctx, false);
+}
+
+static void gen_vcmpequq_(DisasContext *ctx)
+{
+    gen_vcmpequq_common(ctx, true);
+}
+
+static void gen_vcmpgtsq(DisasContext *ctx)
+{
+    gen_vcmpgtq_common(ctx, true, false);
+}
+
+static void gen_vcmpgtsq_(DisasContext *ctx)
+{
+    gen_vcmpgtq_common(ctx, true, true);
+}
+
+static void gen_vcmpgtuq(DisasContext *ctx)
+{
+    gen_vcmpgtq_common(ctx, false, false);
+}
+
+static void gen_vcmpgtuq_(DisasContext *ctx)
+{
+    gen_vcmpgtq_common(ctx, false, true);
+}
+
 GEN_VXRFORM_DUAL(vcmpequb, PPC_ALTIVEC, PPC_NONE, \
                  vcmpneb, PPC_NONE, PPC2_ISA300)
 GEN_VXRFORM_DUAL(vcmpequh, PPC_ALTIVEC, PPC_NONE, \
@@ -1053,8 +1754,14 @@ GEN_VXRFORM_DUAL(vcmpequw, PPC_ALTIVEC, PPC_NONE, \
                  vcmpnew, PPC_NONE, PPC2_ISA300)
 GEN_VXRFORM_DUAL(vcmpeqfp, PPC_ALTIVEC, PPC_NONE, \
                  vcmpequd, PPC_NONE, PPC2_ALTIVEC_207)
+GEN_VXRFORM_DUAL(vcmpgefp, PPC_ALTIVEC, PPC_NONE, \
+                 vcmpequq, PPC_NONE, PPC2_ISA310)
 GEN_VXRFORM_DUAL(vcmpbfp, PPC_ALTIVEC, PPC_NONE, \
                  vcmpgtsd, PPC_NONE, PPC2_ALTIVEC_207)
+GEN_VXRFORM_DUAL(vcmpgtsw, PPC_ALTIVEC, PPC_NONE, \
+                 vcmpgtsq, PPC_NONE, PPC2_ISA310)
+GEN_VXRFORM_DUAL(vcmpgtuw, PPC_ALTIVEC, PPC_NONE, \
+                 vcmpgtuq, PPC_NONE, PPC2_ISA310)
 GEN_VXRFORM_DUAL(vcmpgtfp, PPC_ALTIVEC, PPC_NONE, \
                  vcmpgtud, PPC_NONE, PPC2_ALTIVEC_207)
 
@@ -1243,12 +1950,380 @@ GEN_VXFORM_UIMM_ENV(vcfux, 5, 12);
 GEN_VXFORM_UIMM_ENV(vcfsx, 5, 13);
 GEN_VXFORM_UIMM_ENV(vctuxs, 5, 14);
 GEN_VXFORM_UIMM_ENV(vctsxs, 5, 15);
-GEN_VXFORM_DUAL(vspltb, PPC_ALTIVEC, PPC_NONE,
-                vextractub, PPC_NONE, PPC2_ISA300);
-GEN_VXFORM_DUAL(vsplth, PPC_ALTIVEC, PPC_NONE,
-                vextractuh, PPC_NONE, PPC2_ISA300);
-GEN_VXFORM_DUAL(vspltw, PPC_ALTIVEC, PPC_NONE,
-                vextractuw, PPC_NONE, PPC2_ISA300);
+
+static void gen_vgnb(DisasContext *ctx)
+{
+    static const uint64_t mask[6][5] = {
+        {
+            0xAAAAAAAAAAAAAAAAULL, 0xccccccccccccccccULL,
+            0xf0f0f0f0f0f0f0f0ULL, 0xff00ff00ff00ff00ULL,
+            0xffff0000ffff0000ULL
+        },
+        {
+            0x9249249249249249ULL, 0xC30C30C30C30C30CULL,
+            0xF00F00F00F00F00FULL, 0xFF0000FF0000FF00ULL,
+            0xFFFF00000000FFFFULL
+        },
+        {
+            0x8888888888888888ULL, 0,
+            0xf000f000f000f000ULL, 0, 0xFFFF000000000000ULL
+        },
+        {
+            0x8421084210842108ULL, 0, 0xF0000F0000F0000FULL, 0, 0
+        },
+        {
+            0x8208208208208208ULL, 0, 0xF00000F00000F000ULL, 0, 0
+        },
+        {
+            0x8102040810204081ULL, 0, 0xF000000F000000F0ULL, 0, 0
+        }
+    };
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 hi;
+    TCGv_i64 lo;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    uint64_t m;
+    int i;
+    int n = rA(ctx->opcode) & 7;
+    int nbits;
+    int sh;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    if (n < 2) {
+        return;
+    }
+
+    nbits = (64 + n - 1) / n;
+    hi = tcg_temp_new_i64(tcg_ctx);
+    lo = tcg_temp_new_i64(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, hi, rB(ctx->opcode), true);
+    get_avr64(tcg_ctx, lo, rB(ctx->opcode), false);
+
+    tcg_gen_shli_i64(tcg_ctx, lo, lo, n * nbits - 64);
+
+    for (i = 0, sh = n - 1; i < 5; i++, sh <<= 1) {
+        m = mask[n - 2][i];
+        if (m) {
+            tcg_gen_andi_i64(tcg_ctx, hi, hi, m);
+            tcg_gen_andi_i64(tcg_ctx, lo, lo, m);
+        }
+        if (sh < 64) {
+            tcg_gen_shli_i64(tcg_ctx, t0, hi, sh);
+            tcg_gen_shli_i64(tcg_ctx, t1, lo, sh);
+            tcg_gen_or_i64(tcg_ctx, hi, t0, hi);
+            tcg_gen_or_i64(tcg_ctx, lo, t1, lo);
+        }
+    }
+
+    m = ~(~0ULL >> nbits);
+    tcg_gen_andi_i64(tcg_ctx, hi, hi, m);
+    tcg_gen_andi_i64(tcg_ctx, lo, lo, m);
+    tcg_gen_shri_i64(tcg_ctx, lo, lo, nbits);
+    tcg_gen_or_i64(tcg_ctx, hi, hi, lo);
+    tcg_gen_trunc_i64_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)], hi);
+
+    tcg_temp_free_i64(tcg_ctx, hi);
+    tcg_temp_free_i64(tcg_ctx, lo);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+static void gen_vdiv_vmod(DisasContext *ctx, int vece,
+                          void (*func_32)(TCGContext *, TCGv_i32,
+                                          TCGv_i32, TCGv_i32),
+                          void (*func_64)(TCGContext *, TCGv_i64,
+                                          TCGv_i64, TCGv_i64))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    const GVecGen3 op = {
+        .fni4 = func_32,
+        .fni8 = func_64,
+        .vece = vece,
+    };
+
+    if (!Rc(ctx->opcode)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tcg_gen_gvec_3(tcg_ctx, avr_full_offset(rD(ctx->opcode)),
+                   avr_full_offset(rA(ctx->opcode)),
+                   avr_full_offset(rB(ctx->opcode)), 16, 16, &op);
+}
+
+#define DIVU32(NAME, DIV)                                               \
+static void NAME(TCGContext *tcg_ctx, TCGv_i32 t, TCGv_i32 a,           \
+                 TCGv_i32 b)                                            \
+{                                                                       \
+    TCGv_i32 zero = tcg_const_i32(tcg_ctx, 0);                          \
+    TCGv_i32 one = tcg_const_i32(tcg_ctx, 1);                           \
+                                                                        \
+    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_EQ, b, b, zero, one, b);      \
+    DIV(tcg_ctx, t, a, b);                                              \
+                                                                        \
+    tcg_temp_free_i32(tcg_ctx, one);                                    \
+    tcg_temp_free_i32(tcg_ctx, zero);                                   \
+}
+
+#define DIVS32(NAME, DIV)                                               \
+static void NAME(TCGContext *tcg_ctx, TCGv_i32 t, TCGv_i32 a,           \
+                 TCGv_i32 b)                                            \
+{                                                                       \
+    TCGv_i32 t0 = tcg_temp_new_i32(tcg_ctx);                            \
+    TCGv_i32 t1 = tcg_temp_new_i32(tcg_ctx);                            \
+                                                                        \
+    tcg_gen_setcondi_i32(tcg_ctx, TCG_COND_EQ, t0, a, INT32_MIN);       \
+    tcg_gen_setcondi_i32(tcg_ctx, TCG_COND_EQ, t1, b, -1);              \
+    tcg_gen_and_i32(tcg_ctx, t0, t0, t1);                               \
+    tcg_gen_setcondi_i32(tcg_ctx, TCG_COND_EQ, t1, b, 0);               \
+    tcg_gen_or_i32(tcg_ctx, t0, t0, t1);                                \
+    tcg_gen_movi_i32(tcg_ctx, t1, 0);                                   \
+    tcg_gen_movcond_i32(tcg_ctx, TCG_COND_NE, b, t0, t1, t0, b);        \
+    DIV(tcg_ctx, t, a, b);                                              \
+                                                                        \
+    tcg_temp_free_i32(tcg_ctx, t1);                                     \
+    tcg_temp_free_i32(tcg_ctx, t0);                                     \
+}
+
+#define DIVU64(NAME, DIV)                                               \
+static void NAME(TCGContext *tcg_ctx, TCGv_i64 t, TCGv_i64 a,           \
+                 TCGv_i64 b)                                            \
+{                                                                       \
+    TCGv_i64 zero = tcg_const_i64(tcg_ctx, 0);                          \
+    TCGv_i64 one = tcg_const_i64(tcg_ctx, 1);                           \
+                                                                        \
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_EQ, b, b, zero, one, b);      \
+    DIV(tcg_ctx, t, a, b);                                              \
+                                                                        \
+    tcg_temp_free_i64(tcg_ctx, one);                                    \
+    tcg_temp_free_i64(tcg_ctx, zero);                                   \
+}
+
+#define DIVS64(NAME, DIV)                                               \
+static void NAME(TCGContext *tcg_ctx, TCGv_i64 t, TCGv_i64 a,           \
+                 TCGv_i64 b)                                            \
+{                                                                       \
+    TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);                            \
+    TCGv_i64 t1 = tcg_temp_new_i64(tcg_ctx);                            \
+                                                                        \
+    tcg_gen_setcondi_i64(tcg_ctx, TCG_COND_EQ, t0, a, INT64_MIN);       \
+    tcg_gen_setcondi_i64(tcg_ctx, TCG_COND_EQ, t1, b, -1);              \
+    tcg_gen_and_i64(tcg_ctx, t0, t0, t1);                               \
+    tcg_gen_setcondi_i64(tcg_ctx, TCG_COND_EQ, t1, b, 0);               \
+    tcg_gen_or_i64(tcg_ctx, t0, t0, t1);                                \
+    tcg_gen_movi_i64(tcg_ctx, t1, 0);                                   \
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_NE, b, t0, t1, t0, b);        \
+    DIV(tcg_ctx, t, a, b);                                              \
+                                                                        \
+    tcg_temp_free_i64(tcg_ctx, t1);                                     \
+    tcg_temp_free_i64(tcg_ctx, t0);                                     \
+}
+
+DIVS32(do_vdivsw, tcg_gen_div_i32)
+DIVU32(do_vdivuw, tcg_gen_divu_i32)
+DIVS64(do_vdivsd, tcg_gen_div_i64)
+DIVU64(do_vdivud, tcg_gen_divu_i64)
+
+static void gen_vdivsw(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_32, do_vdivsw, NULL);
+}
+
+static void gen_vdivuw(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_32, do_vdivuw, NULL);
+}
+
+static void gen_vdivsd(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_64, NULL, do_vdivsd);
+}
+
+static void gen_vdivud(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_64, NULL, do_vdivud);
+}
+
+static void gen_vx_helper_rc1(DisasContext *ctx,
+                              void (*gen_helper)(TCGContext *, TCGv_ptr,
+                                                 TCGv_ptr, TCGv_ptr))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr ra;
+    TCGv_ptr rb;
+    TCGv_ptr rd;
+
+    if (!Rc(ctx->opcode)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    ra = gen_avr_ptr(tcg_ctx, rA(ctx->opcode));
+    rb = gen_avr_ptr(tcg_ctx, rB(ctx->opcode));
+    rd = gen_avr_ptr(tcg_ctx, rD(ctx->opcode));
+    gen_helper(tcg_ctx, rd, ra, rb);
+    tcg_temp_free_ptr(tcg_ctx, rd);
+    tcg_temp_free_ptr(tcg_ctx, rb);
+    tcg_temp_free_ptr(tcg_ctx, ra);
+}
+
+static void gen_vdivsq(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VDIVSQ);
+}
+
+static void gen_vdivuq(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VDIVUQ);
+}
+
+static void do_vdives_i32(TCGContext *tcg_ctx, TCGv_i32 t, TCGv_i32 a,
+                          TCGv_i32 b)
+{
+    TCGv_i64 val1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 val2 = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_ext_i32_i64(tcg_ctx, val1, a);
+    tcg_gen_ext_i32_i64(tcg_ctx, val2, b);
+    tcg_gen_shli_i64(tcg_ctx, val1, val1, 32);
+    tcg_gen_div_i64(tcg_ctx, val1, val1, val2);
+    tcg_gen_extrl_i64_i32(tcg_ctx, t, val1);
+
+    tcg_temp_free_i64(tcg_ctx, val2);
+    tcg_temp_free_i64(tcg_ctx, val1);
+}
+
+static void do_vdiveu_i32(TCGContext *tcg_ctx, TCGv_i32 t, TCGv_i32 a,
+                          TCGv_i32 b)
+{
+    TCGv_i64 val1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 val2 = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_extu_i32_i64(tcg_ctx, val1, a);
+    tcg_gen_extu_i32_i64(tcg_ctx, val2, b);
+    tcg_gen_shli_i64(tcg_ctx, val1, val1, 32);
+    tcg_gen_divu_i64(tcg_ctx, val1, val1, val2);
+    tcg_gen_extrl_i64_i32(tcg_ctx, t, val1);
+
+    tcg_temp_free_i64(tcg_ctx, val2);
+    tcg_temp_free_i64(tcg_ctx, val1);
+}
+
+DIVS32(do_vdivesw, do_vdives_i32)
+DIVU32(do_vdiveuw, do_vdiveu_i32)
+DIVS32(do_vmodsw, tcg_gen_rem_i32)
+DIVU32(do_vmoduw, tcg_gen_remu_i32)
+DIVS64(do_vmodsd, tcg_gen_rem_i64)
+DIVU64(do_vmodud, tcg_gen_remu_i64)
+
+static void gen_vdivesw(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_32, do_vdivesw, NULL);
+}
+
+static void gen_vdiveuw(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_32, do_vdiveuw, NULL);
+}
+
+static void gen_vdivesd(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VDIVESD);
+}
+
+static void gen_vdiveud(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VDIVEUD);
+}
+
+static void gen_vdivesq(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VDIVESQ);
+}
+
+static void gen_vdiveuq(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VDIVEUQ);
+}
+
+static void gen_vmodsw(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_32, do_vmodsw, NULL);
+}
+
+static void gen_vmoduw(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_32, do_vmoduw, NULL);
+}
+
+static void gen_vmodsd(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_64, NULL, do_vmodsd);
+}
+
+static void gen_vmodud(DisasContext *ctx)
+{
+    gen_vdiv_vmod(ctx, MO_64, NULL, do_vmodud);
+}
+
+static void gen_vmodsq(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VMODSQ);
+}
+
+static void gen_vmoduq(DisasContext *ctx)
+{
+    gen_vx_helper_rc1(ctx, gen_helper_VMODUQ);
+}
+
+GEN_VXFORM_DUAL(vaddfp, PPC_ALTIVEC, PPC_NONE,
+                vdivuq, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL_EXT(vrefp, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vdivsq, PPC_NONE, PPC2_ISA310, 0)
+GEN_VXFORM_DUAL_EXT(vexptefp, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vdivsw, PPC_NONE, PPC2_ISA310, 0)
+GEN_VXFORM_DUAL_EXT(vlogefp, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vdivsd, PPC_NONE, PPC2_ISA310, 0)
+GEN_VXFORM_DUAL_EXT(vrfin, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vdiveuq, PPC_NONE, PPC2_ISA310, 0)
+GEN_VXFORM_DUAL_EXT(vrfip, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vdiveuw, PPC_NONE, PPC2_ISA310, 0)
+GEN_VXFORM_DUAL_EXT(vrfim, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vdiveud, PPC_NONE, PPC2_ISA310, 0)
+GEN_VXFORM_DUAL(vcfux, PPC_ALTIVEC, PPC_NONE,
+                vdivesq, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vctuxs, PPC_ALTIVEC, PPC_NONE,
+                vdivesw, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vctsxs, PPC_ALTIVEC, PPC_NONE,
+                vdivesd, PPC_NONE, PPC2_ISA310)
+
+#undef DIVS32
+#undef DIVU32
+#undef DIVS64
+#undef DIVU64
+
+GEN_VXFORM_DUAL(vspltb, PPC_ALTIVEC, PPC_NONE,
+                vextractub, PPC_NONE, PPC2_ISA300);
+GEN_VXFORM_DUAL(vsplth, PPC_ALTIVEC, PPC_NONE,
+                vextractuh, PPC_NONE, PPC2_ISA300);
+GEN_VXFORM_DUAL(vspltw, PPC_ALTIVEC, PPC_NONE,
+                vextractuw, PPC_NONE, PPC2_ISA300);
 GEN_VXFORM_DUAL(vspltisb, PPC_ALTIVEC, PPC_NONE,
                 vinsertb, PPC_NONE, PPC2_ISA300);
 GEN_VXFORM_DUAL(vspltish, PPC_ALTIVEC, PPC_NONE,
@@ -1256,6 +2331,478 @@ GEN_VXFORM_DUAL(vspltish, PPC_ALTIVEC, PPC_NONE,
 GEN_VXFORM_DUAL(vspltisw, PPC_ALTIVEC, PPC_NONE,
                 vinsertw, PPC_NONE, PPC2_ISA300);
 
+static void gen_vextdx(DisasContext *ctx, int size, bool right,
+                       void (*gen_helper)(TCGContext *, TCGv_ptr,
+                                          TCGv_ptr, TCGv_ptr,
+                                          TCGv_ptr, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr vrt;
+    TCGv_ptr vra;
+    TCGv_ptr vrb;
+    TCGv rc;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    vrt = gen_avr_ptr(tcg_ctx, rD(ctx->opcode));
+    vra = gen_avr_ptr(tcg_ctx, rA(ctx->opcode));
+    vrb = gen_avr_ptr(tcg_ctx, rB(ctx->opcode));
+    rc = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_andi_tl(tcg_ctx, rc, cpu_gpr[rC(ctx->opcode)], 0x1f);
+    if (right) {
+        tcg_gen_subfi_tl(tcg_ctx, rc, 32 - size, rc);
+    }
+    gen_helper(tcg_ctx, tcg_ctx->cpu_env, vrt, vra, vrb, rc);
+
+    tcg_temp_free_ptr(tcg_ctx, vrt);
+    tcg_temp_free_ptr(tcg_ctx, vra);
+    tcg_temp_free_ptr(tcg_ctx, vrb);
+    tcg_temp_free(tcg_ctx, rc);
+}
+
+static void gen_vextdubvlx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 1, false, gen_helper_VEXTDUBVLX);
+}
+
+static void gen_vextduhvlx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 2, false, gen_helper_VEXTDUHVLX);
+}
+
+static void gen_vextduwvlx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 4, false, gen_helper_VEXTDUWVLX);
+}
+
+static void gen_vextddvlx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 8, false, gen_helper_VEXTDDVLX);
+}
+
+static void gen_vextdubvrx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 1, true, gen_helper_VEXTDUBVLX);
+}
+
+static void gen_vextduhvrx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 2, true, gen_helper_VEXTDUHVLX);
+}
+
+static void gen_vextduwvrx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 4, true, gen_helper_VEXTDUWVLX);
+}
+
+static void gen_vextddvrx(DisasContext *ctx)
+{
+    gen_vextdx(ctx, 8, true, gen_helper_VEXTDDVLX);
+}
+
+static void gen_vextdubv(DisasContext *ctx)
+{
+    if (Rc(ctx->opcode)) {
+        gen_vextdubvrx(ctx);
+    } else {
+        gen_vextdubvlx(ctx);
+    }
+}
+
+static void gen_vextduhv(DisasContext *ctx)
+{
+    if (Rc(ctx->opcode)) {
+        gen_vextduhvrx(ctx);
+    } else {
+        gen_vextduhvlx(ctx);
+    }
+}
+
+static void gen_vextduwv(DisasContext *ctx)
+{
+    if (Rc(ctx->opcode)) {
+        gen_vextduwvrx(ctx);
+    } else {
+        gen_vextduwvlx(ctx);
+    }
+}
+
+static void gen_vextddv(DisasContext *ctx)
+{
+    if (Rc(ctx->opcode)) {
+        gen_vextddvrx(ctx);
+    } else {
+        gen_vextddvlx(ctx);
+    }
+}
+
+static void gen_vstri(DisasContext *ctx,
+                      void (*gen_helper)(TCGContext *, TCGv_i32,
+                                         TCGv_ptr, TCGv_ptr))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr vrt;
+    TCGv_ptr vrb;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    vrt = gen_avr_ptr(tcg_ctx, rD(ctx->opcode));
+    vrb = gen_avr_ptr(tcg_ctx, rB(ctx->opcode));
+
+    if (Rc21(ctx->opcode)) {
+        gen_helper(tcg_ctx, cpu_crf[6], vrt, vrb);
+    } else {
+        TCGv_i32 discard = tcg_temp_new_i32(tcg_ctx);
+        gen_helper(tcg_ctx, discard, vrt, vrb);
+        tcg_temp_free_i32(tcg_ctx, discard);
+    }
+
+    tcg_temp_free_ptr(tcg_ctx, vrt);
+    tcg_temp_free_ptr(tcg_ctx, vrb);
+}
+
+static void gen_vstri_isa310(DisasContext *ctx)
+{
+    switch (rA(ctx->opcode)) {
+    case 0:
+        gen_vstri(ctx, gen_helper_VSTRIBL);
+        return;
+    case 1:
+        gen_vstri(ctx, gen_helper_VSTRIBR);
+        return;
+    case 2:
+        gen_vstri(ctx, gen_helper_VSTRIHL);
+        return;
+    case 3:
+        gen_vstri(ctx, gen_helper_VSTRIHR);
+        return;
+    default:
+        gen_invalid(ctx);
+        return;
+    }
+}
+
+static void gen_vmrghb_vstri(DisasContext *ctx)
+{
+    if (!Rc(ctx->opcode) && (ctx->insns_flags & PPC_ALTIVEC)) {
+        gen_vmrghb(ctx);
+    } else if (Rc(ctx->opcode) && (ctx->insns_flags2 & PPC2_ISA310)) {
+        gen_vstri_isa310(ctx);
+    } else {
+        gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
+    }
+}
+
+static void gen_vslo_vstri(DisasContext *ctx)
+{
+    if (!Rc(ctx->opcode) && (ctx->insns_flags & PPC_ALTIVEC)) {
+        gen_vslo(ctx);
+    } else if (Rc(ctx->opcode) && (ctx->insns_flags2 & PPC2_ISA310)) {
+        gen_vstri_isa310(ctx);
+    } else {
+        gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
+    }
+}
+
+static void gen_vclrb(DisasContext *ctx, bool right)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 rb;
+    TCGv_i64 mh;
+    TCGv_i64 ml;
+    TCGv_i64 tmp;
+    TCGv_i64 ones;
+    TCGv_i64 zero;
+    TCGv_i64 eight;
+    TCGv_i64 sixteen;
+
+    if (!Rc(ctx->opcode) || !(ctx->insns_flags2 & PPC2_ISA310)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    rb = tcg_temp_new_i64(tcg_ctx);
+    mh = tcg_temp_new_i64(tcg_ctx);
+    ml = tcg_temp_new_i64(tcg_ctx);
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    ones = tcg_const_i64(tcg_ctx, -1);
+    zero = tcg_const_i64(tcg_ctx, 0);
+    eight = tcg_const_i64(tcg_ctx, 8);
+    sixteen = tcg_const_i64(tcg_ctx, 16);
+
+    tcg_gen_extu_tl_i64(tcg_ctx, rb, cpu_gpr[rB(ctx->opcode)]);
+    tcg_gen_andi_i64(tcg_ctx, tmp, rb, 7);
+    tcg_gen_shli_i64(tcg_ctx, tmp, tmp, 3);
+    if (right) {
+        tcg_gen_shr_i64(tcg_ctx, tmp, ones, tmp);
+    } else {
+        tcg_gen_shl_i64(tcg_ctx, tmp, ones, tmp);
+    }
+    tcg_gen_not_i64(tcg_ctx, tmp, tmp);
+
+    if (right) {
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_LTU, mh, rb, eight, tmp, ones);
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_LTU, ml, rb, eight, zero, tmp);
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_LTU, ml, rb, sixteen, ml, ones);
+    } else {
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_LTU, ml, rb, eight, tmp, ones);
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_LTU, mh, rb, eight, zero, tmp);
+        tcg_gen_movcond_i64(tcg_ctx, TCG_COND_LTU, mh, rb, sixteen, mh, ones);
+    }
+
+    get_avr64(tcg_ctx, tmp, rA(ctx->opcode), true);
+    tcg_gen_and_i64(tcg_ctx, tmp, tmp, mh);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, true);
+
+    get_avr64(tcg_ctx, tmp, rA(ctx->opcode), false);
+    tcg_gen_and_i64(tcg_ctx, tmp, tmp, ml);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, false);
+
+    tcg_temp_free_i64(tcg_ctx, rb);
+    tcg_temp_free_i64(tcg_ctx, mh);
+    tcg_temp_free_i64(tcg_ctx, ml);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+    tcg_temp_free_i64(tcg_ctx, ones);
+    tcg_temp_free_i64(tcg_ctx, zero);
+    tcg_temp_free_i64(tcg_ctx, eight);
+    tcg_temp_free_i64(tcg_ctx, sixteen);
+}
+
+static void gen_vmrglw_vclrlb(DisasContext *ctx)
+{
+    if (!Rc(ctx->opcode) && (ctx->insns_flags & PPC_ALTIVEC)) {
+        gen_vmrglw(ctx);
+    } else if (Rc(ctx->opcode) && (ctx->insns_flags2 & PPC2_ISA310)) {
+        gen_vclrb(ctx, false);
+    } else {
+        gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
+    }
+}
+
+static void gen_vclrrb(DisasContext *ctx)
+{
+    gen_vclrb(ctx, true);
+}
+
+static void gen_vinsx(DisasContext *ctx, int vrt, int size, bool right,
+                      TCGv ra, TCGv_i64 rb,
+                      void (*gen_helper)(TCGContext *, TCGv_ptr,
+                                         TCGv_ptr, TCGv_i64, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr t;
+    TCGv idx;
+
+    t = gen_avr_ptr(tcg_ctx, vrt);
+    idx = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_andi_tl(tcg_ctx, idx, ra, 0xf);
+    if (right) {
+        tcg_gen_subfi_tl(tcg_ctx, idx, 16 - size, idx);
+    }
+    gen_helper(tcg_ctx, tcg_ctx->cpu_env, t, rb, idx);
+
+    tcg_temp_free_ptr(tcg_ctx, t);
+    tcg_temp_free(tcg_ctx, idx);
+}
+
+static void gen_vinsvx(DisasContext *ctx, int vrt, int size, bool right,
+                       TCGv ra, int vrb,
+                       void (*gen_helper)(TCGContext *, TCGv_ptr,
+                                          TCGv_ptr, TCGv_i64, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 val;
+
+    val = tcg_temp_new_i64(tcg_ctx);
+    get_avr64(tcg_ctx, val, vrb, true);
+    gen_vinsx(ctx, vrt, size, right, ra, val, gen_helper);
+    tcg_temp_free_i64(tcg_ctx, val);
+}
+
+static void gen_vinsx_vx(DisasContext *ctx, int size, bool right,
+                         void (*gen_helper)(TCGContext *, TCGv_ptr,
+                                            TCGv_ptr, TCGv_i64, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 val;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    val = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_extu_tl_i64(tcg_ctx, val, cpu_gpr[rB(ctx->opcode)]);
+    gen_vinsx(ctx, rD(ctx->opcode), size, right, cpu_gpr[rA(ctx->opcode)],
+              val, gen_helper);
+    tcg_temp_free_i64(tcg_ctx, val);
+}
+
+static void gen_vinsvx_vx(DisasContext *ctx, int size, bool right,
+                          void (*gen_helper)(TCGContext *, TCGv_ptr,
+                                             TCGv_ptr, TCGv_i64, TCGv))
+{
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    gen_vinsvx(ctx, rD(ctx->opcode), size, right, cpu_gpr[rA(ctx->opcode)],
+               rB(ctx->opcode), gen_helper);
+}
+
+static void gen_vins_uim4(DisasContext *ctx, int size,
+                          void (*gen_helper)(TCGContext *, TCGv_ptr,
+                                             TCGv_ptr, TCGv_i64, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 val;
+    TCGv uim;
+    uint8_t uimm = UIMM4(ctx->opcode);
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    if (uimm > (16 - size)) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "Invalid index for VINS* at 0x" TARGET_FMT_lx
+                      ", UIM = %d > %d\n", ctx->cia, uimm, 16 - size);
+        return;
+    }
+
+    val = tcg_temp_new_i64(tcg_ctx);
+    uim = tcg_const_tl(tcg_ctx, uimm);
+    tcg_gen_extu_tl_i64(tcg_ctx, val, cpu_gpr[rB(ctx->opcode)]);
+    gen_vinsx(ctx, rD(ctx->opcode), size, false, uim, val, gen_helper);
+    tcg_temp_free(tcg_ctx, uim);
+    tcg_temp_free_i64(tcg_ctx, val);
+}
+
+static void gen_vinsblx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 1, false, gen_helper_VINSBLX);
+}
+
+static void gen_vinshlx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 2, false, gen_helper_VINSHLX);
+}
+
+static void gen_vinswlx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 4, false, gen_helper_VINSWLX);
+}
+
+static void gen_vinsdlx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 8, false, gen_helper_VINSDLX);
+}
+
+static void gen_vinsbrx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 1, true, gen_helper_VINSBLX);
+}
+
+static void gen_vinshrx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 2, true, gen_helper_VINSHLX);
+}
+
+static void gen_vinswrx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 4, true, gen_helper_VINSWLX);
+}
+
+static void gen_vinsdrx(DisasContext *ctx)
+{
+    gen_vinsx_vx(ctx, 8, true, gen_helper_VINSDLX);
+}
+
+static void gen_vinsw(DisasContext *ctx)
+{
+    gen_vins_uim4(ctx, 4, gen_helper_VINSWLX);
+}
+
+static void gen_vinsd(DisasContext *ctx)
+{
+    gen_vins_uim4(ctx, 8, gen_helper_VINSDLX);
+}
+
+static void gen_vinsbvlx(DisasContext *ctx)
+{
+    gen_vinsvx_vx(ctx, 1, false, gen_helper_VINSBLX);
+}
+
+static void gen_vinshvlx(DisasContext *ctx)
+{
+    gen_vinsvx_vx(ctx, 2, false, gen_helper_VINSHLX);
+}
+
+static void gen_vinswvlx(DisasContext *ctx)
+{
+    gen_vinsvx_vx(ctx, 4, false, gen_helper_VINSWLX);
+}
+
+static void gen_vinsbvrx(DisasContext *ctx)
+{
+    gen_vinsvx_vx(ctx, 1, true, gen_helper_VINSBLX);
+}
+
+static void gen_vinshvrx(DisasContext *ctx)
+{
+    gen_vinsvx_vx(ctx, 2, true, gen_helper_VINSHLX);
+}
+
+static void gen_vinswvrx(DisasContext *ctx)
+{
+    gen_vinsvx_vx(ctx, 4, true, gen_helper_VINSWLX);
+}
+
+GEN_VXFORM_DUAL(vpkuhum, PPC_ALTIVEC, PPC_NONE,
+                vinsbvlx, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vpkuwum, PPC_ALTIVEC, PPC_NONE,
+                vinshvlx, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vpkuhus, PPC_ALTIVEC, PPC_NONE,
+                vinswvlx, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vpkuwus, PPC_ALTIVEC, PPC_NONE,
+                vinsw, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vpkshus, PPC_ALTIVEC, PPC_NONE,
+                vinsbvrx, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vpkswus, PPC_ALTIVEC, PPC_NONE,
+                vinshvrx, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vpkshss, PPC_ALTIVEC, PPC_NONE,
+                vinswvrx, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vpkswss, PPC_ALTIVEC, PPC_NONE,
+                vinsd, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL_EXT(vupkhsb, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vinsblx, PPC_NONE, PPC2_ISA310, 0x00000000)
+GEN_VXFORM_DUAL_EXT(vupkhsh, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vinshlx, PPC_NONE, PPC2_ISA310, 0x00000000)
+GEN_VXFORM_DUAL_EXT(vupklsb, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vinswlx, PPC_NONE, PPC2_ISA310, 0x00000000)
+GEN_VXFORM_DUAL_EXT(vupklsh, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vinsdlx, PPC_NONE, PPC2_ISA310, 0x00000000)
+GEN_VXFORM_DUAL(vpkpx, PPC_ALTIVEC, PPC_NONE,
+                vinsbrx, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL_EXT(vupkhpx, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vinshrx, PPC_NONE, PPC2_ISA310, 0x00000000)
+GEN_VXFORM_DUAL_EXT(vupklpx, PPC_ALTIVEC, PPC_NONE, 0x001f0000,
+                    vinsdrx, PPC_NONE, PPC2_ISA310, 0x00000000)
+
 static void gen_vsldoi(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -1321,6 +2868,112 @@ static void gen_vmladduhm(DisasContext *ctx)
     tcg_temp_free_ptr(tcg_ctx, rd);
 }
 
+static void gen_vmsumudm(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 rl;
+    TCGv_i64 rh;
+    TCGv_i64 src1;
+    TCGv_i64 src2;
+    int dw;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    rh = tcg_temp_new_i64(tcg_ctx);
+    rl = tcg_temp_new_i64(tcg_ctx);
+    src1 = tcg_temp_new_i64(tcg_ctx);
+    src2 = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, rl, rC(ctx->opcode), false);
+    get_avr64(tcg_ctx, rh, rC(ctx->opcode), true);
+
+    for (dw = 0; dw < 2; dw++) {
+        get_avr64(tcg_ctx, src1, rA(ctx->opcode), dw);
+        get_avr64(tcg_ctx, src2, rB(ctx->opcode), dw);
+        tcg_gen_mulu2_i64(tcg_ctx, src1, src2, src1, src2);
+        tcg_gen_add2_i64(tcg_ctx, rl, rh, rl, rh, src1, src2);
+    }
+
+    set_avr64(tcg_ctx, rD(ctx->opcode), rl, false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), rh, true);
+
+    tcg_temp_free_i64(tcg_ctx, rl);
+    tcg_temp_free_i64(tcg_ctx, rh);
+    tcg_temp_free_i64(tcg_ctx, src1);
+    tcg_temp_free_i64(tcg_ctx, src2);
+}
+
+static void gen_vmsumcud(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 tmp0;
+    TCGv_i64 tmp1;
+    TCGv_i64 prod1h;
+    TCGv_i64 prod1l;
+    TCGv_i64 prod0h;
+    TCGv_i64 prod0l;
+    TCGv_i64 zero;
+
+    if (!Rc(ctx->opcode) || !(ctx->insns_flags2 & PPC2_ISA310)) {
+        gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tmp0 = tcg_temp_new_i64(tcg_ctx);
+    tmp1 = tcg_temp_new_i64(tcg_ctx);
+    prod1h = tcg_temp_new_i64(tcg_ctx);
+    prod1l = tcg_temp_new_i64(tcg_ctx);
+    prod0h = tcg_temp_new_i64(tcg_ctx);
+    prod0l = tcg_temp_new_i64(tcg_ctx);
+    zero = tcg_const_i64(tcg_ctx, 0);
+
+    get_avr64(tcg_ctx, tmp0, rA(ctx->opcode), false);
+    get_avr64(tcg_ctx, tmp1, rB(ctx->opcode), false);
+    tcg_gen_mulu2_i64(tcg_ctx, prod1l, prod1h, tmp0, tmp1);
+
+    get_avr64(tcg_ctx, tmp0, rA(ctx->opcode), true);
+    get_avr64(tcg_ctx, tmp1, rB(ctx->opcode), true);
+    tcg_gen_mulu2_i64(tcg_ctx, prod0l, prod0h, tmp0, tmp1);
+
+    get_avr64(tcg_ctx, tmp1, rC(ctx->opcode), false);
+    tcg_gen_add2_i64(tcg_ctx, tmp1, tmp0, tmp1, zero, prod1l, zero);
+    tcg_gen_add2_i64(tcg_ctx, tmp1, tmp0, tmp1, tmp0, prod0l, zero);
+
+    get_avr64(tcg_ctx, tmp1, rC(ctx->opcode), true);
+    tcg_gen_add2_i64(tcg_ctx, tmp1, tmp0, tmp0, zero, tmp1, zero);
+    tcg_gen_add2_i64(tcg_ctx, tmp1, tmp0, tmp1, tmp0, prod1h, zero);
+    tcg_gen_add2_i64(tcg_ctx, tmp1, tmp0, tmp1, tmp0, prod0h, zero);
+
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp0, false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), zero, true);
+
+    tcg_temp_free_i64(tcg_ctx, tmp0);
+    tcg_temp_free_i64(tcg_ctx, tmp1);
+    tcg_temp_free_i64(tcg_ctx, prod1h);
+    tcg_temp_free_i64(tcg_ctx, prod1l);
+    tcg_temp_free_i64(tcg_ctx, prod0h);
+    tcg_temp_free_i64(tcg_ctx, prod0l);
+    tcg_temp_free_i64(tcg_ctx, zero);
+}
+
+static void gen_vmladduhm_vmsumudm(DisasContext *ctx)
+{
+    if (!Rc(ctx->opcode) && (ctx->insns_flags & PPC_ALTIVEC)) {
+        gen_vmladduhm(ctx);
+    } else if (Rc(ctx->opcode) && (ctx->insns_flags2 & PPC2_ISA300)) {
+        gen_vmsumudm(ctx);
+    } else {
+        gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
+    }
+}
+
 static void gen_vpermr(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -1357,6 +3010,27 @@ GEN_VXFORM_NOA_2(vextsh2w, 1, 24, 17)
 GEN_VXFORM_NOA_2(vextsb2d, 1, 24, 24)
 GEN_VXFORM_NOA_2(vextsh2d, 1, 24, 25)
 GEN_VXFORM_NOA_2(vextsw2d, 1, 24, 26)
+
+static void gen_vextsd2q(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 tmp;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, tmp, rB(ctx->opcode), false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, false);
+    tcg_gen_sari_i64(tcg_ctx, tmp, tmp, 63);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, true);
+
+    tcg_temp_free_i64(tcg_ctx, tmp);
+}
+
 GEN_VXFORM_NOA_2(vctzb, 1, 24, 28)
 GEN_VXFORM_NOA_2(vctzh, 1, 24, 29)
 GEN_VXFORM_NOA_2(vctzw, 1, 24, 30)
@@ -1375,8 +3049,460 @@ GEN_VXFORM_DUAL(vclzw, PPC_NONE, PPC2_ALTIVEC_207, \
                 vpopcntw, PPC_NONE, PPC2_ALTIVEC_207)
 GEN_VXFORM_DUAL(vclzd, PPC_NONE, PPC2_ALTIVEC_207, \
                 vpopcntd, PPC_NONE, PPC2_ALTIVEC_207)
+
+#if defined(TARGET_PPC64)
+static void gen_vcfuged(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    static const GVecGen3 g = {
+        .fni8 = gen_helper_CFUGED,
+        .vece = MO_64,
+    };
+
+    if (!Rc(ctx->opcode)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tcg_gen_gvec_3(tcg_ctx, avr_full_offset(rD(ctx->opcode)),
+                   avr_full_offset(rA(ctx->opcode)),
+                   avr_full_offset(rB(ctx->opcode)), 16, 16, &g);
+}
+
+static void gen_vcntzdm(DisasContext *ctx, bool trail)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    static const GVecGen3i g = {
+        .fni8 = gen_cntzdm_i64,
+        .vece = MO_64,
+    };
+
+    if (Rc(ctx->opcode)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tcg_gen_gvec_3i(tcg_ctx, avr_full_offset(rD(ctx->opcode)),
+                    avr_full_offset(rA(ctx->opcode)),
+                    avr_full_offset(rB(ctx->opcode)), 16, 16, trail, &g);
+}
+
+static void gen_vclzdm(DisasContext *ctx)
+{
+    gen_vcntzdm(ctx, false);
+}
+
+static void gen_vctzdm(DisasContext *ctx)
+{
+    gen_vcntzdm(ctx, true);
+}
+
+static void gen_vpdepd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    static const GVecGen3 g = {
+        .fni8 = gen_helper_PDEPD,
+        .vece = MO_64,
+    };
+
+    if (!Rc(ctx->opcode)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tcg_gen_gvec_3(tcg_ctx, avr_full_offset(rD(ctx->opcode)),
+                   avr_full_offset(rA(ctx->opcode)),
+                   avr_full_offset(rB(ctx->opcode)), 16, 16, &g);
+}
+
+static void gen_vpextd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    static const GVecGen3 g = {
+        .fni8 = gen_helper_PEXTD,
+        .vece = MO_64,
+    };
+
+    if (!Rc(ctx->opcode)) {
+        gen_invalid(ctx);
+        return;
+    }
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tcg_gen_gvec_3(tcg_ctx, avr_full_offset(rD(ctx->opcode)),
+                   avr_full_offset(rA(ctx->opcode)),
+                   avr_full_offset(rB(ctx->opcode)), 16, 16, &g);
+}
+#endif
+
+static void gen_vexpandm(DisasContext *ctx, unsigned vece)
+{
+    const uint64_t elem_width = 8 << vece;
+    const uint64_t elem_count_half = 8 >> vece;
+    const uint64_t mask = dup_const(vece, 1ull << (elem_width - 1));
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 lo;
+    TCGv_i64 hi;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    uint64_t c;
+    uint64_t i;
+    uint64_t j;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    hi = tcg_temp_new_i64(tcg_ctx);
+    lo = tcg_temp_new_i64(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, lo, rB(ctx->opcode), false);
+    get_avr64(tcg_ctx, hi, rB(ctx->opcode), true);
+
+    tcg_gen_andi_i64(tcg_ctx, lo, lo, mask);
+    tcg_gen_andi_i64(tcg_ctx, hi, hi, mask);
+
+    for (i = elem_count_half / 2, j = 32; i > 0; i >>= 1, j >>= 1) {
+        tcg_gen_shli_i64(tcg_ctx, t0, hi, j - i);
+        tcg_gen_shli_i64(tcg_ctx, t1, lo, j - i);
+        tcg_gen_or_i64(tcg_ctx, hi, hi, t0);
+        tcg_gen_or_i64(tcg_ctx, lo, lo, t1);
+    }
+
+    tcg_gen_shri_i64(tcg_ctx, hi, hi, 64 - elem_count_half);
+    tcg_gen_extract2_i64(tcg_ctx, lo, lo, hi, 64 - elem_count_half);
+    tcg_gen_extract_i64(tcg_ctx, hi, lo, elem_count_half, elem_count_half);
+    tcg_gen_extract_i64(tcg_ctx, lo, lo, 0, elem_count_half);
+
+    for (i = elem_count_half / 2, j = 32; i > 0; i >>= 1, j >>= 1) {
+        tcg_gen_shli_i64(tcg_ctx, t0, hi, j - i);
+        tcg_gen_shli_i64(tcg_ctx, t1, lo, j - i);
+        tcg_gen_or_i64(tcg_ctx, hi, hi, t0);
+        tcg_gen_or_i64(tcg_ctx, lo, lo, t1);
+    }
+
+    c = dup_const(vece, 1);
+    tcg_gen_andi_i64(tcg_ctx, hi, hi, c);
+    tcg_gen_andi_i64(tcg_ctx, lo, lo, c);
+
+    c = MAKE_64BIT_MASK(0, elem_width);
+    tcg_gen_muli_i64(tcg_ctx, hi, hi, c);
+    tcg_gen_muli_i64(tcg_ctx, lo, lo, c);
+
+    set_avr64(tcg_ctx, rD(ctx->opcode), lo, false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), hi, true);
+
+    tcg_temp_free_i64(tcg_ctx, hi);
+    tcg_temp_free_i64(tcg_ctx, lo);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+static void gen_vexpandqm(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 tmp;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, tmp, rB(ctx->opcode), true);
+    tcg_gen_sari_i64(tcg_ctx, tmp, tmp, 63);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, true);
+
+    tcg_temp_free_i64(tcg_ctx, tmp);
+}
+
+static void gen_vextractm(DisasContext *ctx, unsigned vece)
+{
+    const uint64_t elem_width = 8 << vece;
+    const uint64_t elem_count_half = 8 >> vece;
+    const uint64_t mask = dup_const(vece, 1ull << (elem_width - 1));
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 lo;
+    TCGv_i64 hi;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    uint64_t i;
+    uint64_t j;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    hi = tcg_temp_new_i64(tcg_ctx);
+    lo = tcg_temp_new_i64(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, lo, rB(ctx->opcode), false);
+    get_avr64(tcg_ctx, hi, rB(ctx->opcode), true);
+
+    tcg_gen_andi_i64(tcg_ctx, lo, lo, mask);
+    tcg_gen_andi_i64(tcg_ctx, hi, hi, mask);
+
+    for (i = elem_count_half / 2, j = 32; i > 0; i >>= 1, j >>= 1) {
+        tcg_gen_shli_i64(tcg_ctx, t0, hi, j - i);
+        tcg_gen_shli_i64(tcg_ctx, t1, lo, j - i);
+        tcg_gen_or_i64(tcg_ctx, hi, hi, t0);
+        tcg_gen_or_i64(tcg_ctx, lo, lo, t1);
+    }
+
+    tcg_gen_shri_i64(tcg_ctx, hi, hi, 64 - elem_count_half);
+    tcg_gen_extract2_i64(tcg_ctx, lo, lo, hi, 64 - elem_count_half);
+    tcg_gen_trunc_i64_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)], lo);
+
+    tcg_temp_free_i64(tcg_ctx, hi);
+    tcg_temp_free_i64(tcg_ctx, lo);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+static void gen_vextractqm(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 tmp;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+
+    get_avr64(tcg_ctx, tmp, rB(ctx->opcode), true);
+    tcg_gen_shri_i64(tcg_ctx, tmp, tmp, 63);
+    tcg_gen_trunc_i64_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)], tmp);
+
+    tcg_temp_free_i64(tcg_ctx, tmp);
+}
+
+static void gen_mtvsrm(DisasContext *ctx, unsigned vece)
+{
+    const uint64_t elem_width = 8 << vece;
+    const uint64_t elem_count_half = 8 >> vece;
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 hi;
+    TCGv_i64 lo;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    uint64_t c;
+    int i;
+    int j;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    hi = tcg_temp_new_i64(tcg_ctx);
+    lo = tcg_temp_new_i64(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_extu_tl_i64(tcg_ctx, t0, cpu_gpr[rB(ctx->opcode)]);
+    tcg_gen_extract_i64(tcg_ctx, hi, t0, elem_count_half, elem_count_half);
+    tcg_gen_extract_i64(tcg_ctx, lo, t0, 0, elem_count_half);
+
+    for (i = elem_count_half / 2, j = 32; i > 0; i >>= 1, j >>= 1) {
+        tcg_gen_shli_i64(tcg_ctx, t0, hi, j - i);
+        tcg_gen_shli_i64(tcg_ctx, t1, lo, j - i);
+        tcg_gen_or_i64(tcg_ctx, hi, hi, t0);
+        tcg_gen_or_i64(tcg_ctx, lo, lo, t1);
+    }
+
+    c = dup_const(vece, 1);
+    tcg_gen_andi_i64(tcg_ctx, hi, hi, c);
+    tcg_gen_andi_i64(tcg_ctx, lo, lo, c);
+
+    c = MAKE_64BIT_MASK(0, elem_width);
+    tcg_gen_muli_i64(tcg_ctx, hi, hi, c);
+    tcg_gen_muli_i64(tcg_ctx, lo, lo, c);
+
+    set_avr64(tcg_ctx, rD(ctx->opcode), lo, false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), hi, true);
+
+    tcg_temp_free_i64(tcg_ctx, hi);
+    tcg_temp_free_i64(tcg_ctx, lo);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+static void gen_mtvsrqm(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 tmp;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_ext_tl_i64(tcg_ctx, tmp, cpu_gpr[rB(ctx->opcode)]);
+    tcg_gen_sextract_i64(tcg_ctx, tmp, tmp, 0, 1);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, false);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, true);
+
+    tcg_temp_free_i64(tcg_ctx, tmp);
+}
+
+static void gen_mtvsrbmi(DisasContext *ctx)
+{
+    const uint64_t mask = dup_const(MO_8, 1);
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 tmp;
+    uint16_t b = DX(ctx->opcode);
+    uint64_t hi;
+    uint64_t lo;
+    int i;
+    int j;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    hi = extract16(b, 8, 8);
+    lo = extract16(b, 0, 8);
+
+    for (i = 4, j = 32; i > 0; i >>= 1, j >>= 1) {
+        hi |= hi << (j - i);
+        lo |= lo << (j - i);
+    }
+
+    hi = (hi & mask) * 0xff;
+    lo = (lo & mask) * 0xff;
+
+    tmp = tcg_const_i64(tcg_ctx, lo);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, false);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+
+    tmp = tcg_const_i64(tcg_ctx, hi);
+    set_avr64(tcg_ctx, rD(ctx->opcode), tmp, true);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+}
+
+static void gen_vcntmb(DisasContext *ctx, unsigned vece, bool match)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 rt;
+    TCGv_i64 vrb;
+    TCGv_i64 mask;
+    int i;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    rt = tcg_const_i64(tcg_ctx, 0);
+    vrb = tcg_temp_new_i64(tcg_ctx);
+    mask = tcg_const_i64(tcg_ctx, dup_const(vece, 1ull << ((8 << vece) - 1)));
+
+    for (i = 0; i < 2; i++) {
+        get_avr64(tcg_ctx, vrb, rB(ctx->opcode), i);
+        if (match) {
+            tcg_gen_and_i64(tcg_ctx, vrb, mask, vrb);
+        } else {
+            tcg_gen_andc_i64(tcg_ctx, vrb, mask, vrb);
+        }
+        tcg_gen_ctpop_i64(tcg_ctx, vrb, vrb);
+        tcg_gen_add_i64(tcg_ctx, rt, rt, vrb);
+    }
+
+    tcg_gen_shli_i64(tcg_ctx, rt, rt, TARGET_LONG_BITS - 8 + vece);
+    tcg_gen_trunc_i64_tl(tcg_ctx, cpu_gpr[rD(ctx->opcode)], rt);
+
+    tcg_temp_free_i64(tcg_ctx, vrb);
+    tcg_temp_free_i64(tcg_ctx, rt);
+    tcg_temp_free_i64(tcg_ctx, mask);
+}
+
+static void gen_vmask_isa310(DisasContext *ctx)
+{
+    switch (rA(ctx->opcode)) {
+    case 0x00:
+        return gen_vexpandm(ctx, MO_8);
+    case 0x01:
+        return gen_vexpandm(ctx, MO_16);
+    case 0x02:
+        return gen_vexpandm(ctx, MO_32);
+    case 0x03:
+        return gen_vexpandm(ctx, MO_64);
+    case 0x04:
+        return gen_vexpandqm(ctx);
+    case 0x08:
+        return gen_vextractm(ctx, MO_8);
+    case 0x09:
+        return gen_vextractm(ctx, MO_16);
+    case 0x0a:
+        return gen_vextractm(ctx, MO_32);
+    case 0x0b:
+        return gen_vextractm(ctx, MO_64);
+    case 0x0c:
+        return gen_vextractqm(ctx);
+    case 0x10:
+        return gen_mtvsrm(ctx, MO_8);
+    case 0x11:
+        return gen_mtvsrm(ctx, MO_16);
+    case 0x12:
+        return gen_mtvsrm(ctx, MO_32);
+    case 0x13:
+        return gen_mtvsrm(ctx, MO_64);
+    case 0x14:
+        return gen_mtvsrqm(ctx);
+    case 0x18:
+    case 0x19:
+        return gen_vcntmb(ctx, MO_8, rA(ctx->opcode) & 1);
+    case 0x1a:
+    case 0x1b:
+        return gen_vcntmb(ctx, MO_16, rA(ctx->opcode) & 1);
+    case 0x1c:
+    case 0x1d:
+        return gen_vcntmb(ctx, MO_32, rA(ctx->opcode) & 1);
+    case 0x1e:
+    case 0x1f:
+        return gen_vcntmb(ctx, MO_64, rA(ctx->opcode) & 1);
+    default:
+        gen_invalid(ctx);
+        return;
+    }
+}
+
 GEN_VXFORM(vbpermd, 6, 23);
 GEN_VXFORM(vbpermq, 6, 21);
+#if defined(TARGET_PPC64)
+GEN_VXFORM_DUAL(vbpermd, PPC_NONE, PPC2_ISA300,
+                vpdepd, PPC_NONE, PPC2_ISA310)
+GEN_VXFORM_DUAL(vbpermq, PPC_NONE, PPC2_ALTIVEC_207,
+                vcfuged, PPC_NONE, PPC2_ISA310)
+#endif
 GEN_VXFORM_TRANS(vgbbd, 6, 20);
 GEN_VXFORM(vpmsumb, 4, 16)
 GEN_VXFORM(vpmsumh, 4, 17)
diff --git a/qemu/target/ppc/translate/vmx-ops.inc.c b/qemu/target/ppc/translate/vmx-ops.inc.c
index 84e05fb827..eccee3f51e 100644
--- a/qemu/target/ppc/translate/vmx-ops.inc.c
+++ b/qemu/target/ppc/translate/vmx-ops.inc.c
@@ -41,6 +41,9 @@ GEN_HANDLER_E(name, 0x04, opc2, opc3, 0x00000000, PPC_NONE, PPC2_ALTIVEC_207)
 #define GEN_VXFORM_300(name, opc2, opc3)                                \
 GEN_HANDLER_E(name, 0x04, opc2, opc3, 0x00000000, PPC_NONE, PPC2_ISA300)
 
+#define GEN_VXFORM_310(name, opc2, opc3)                                \
+GEN_HANDLER_E(name, 0x04, opc2, opc3, 0x00000000, PPC_NONE, PPC2_ISA310)
+
 #define GEN_VXFORM_300_EXT(name, opc2, opc3, inval)                     \
 GEN_HANDLER_E(name, 0x04, opc2, opc3, inval, PPC_NONE, PPC2_ISA300)
 
@@ -86,12 +89,13 @@ GEN_VXFORM_DUAL(vavguw, vabsduw, 1, 18, PPC_ALTIVEC, PPC_NONE),
 GEN_VXFORM(vavgsb, 1, 20),
 GEN_VXFORM(vavgsh, 1, 21),
 GEN_VXFORM(vavgsw, 1, 22),
-GEN_VXFORM(vmrghb, 6, 0),
+GEN_VXFORM_DUAL(vmrghb, vstri, 6, 0, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM(vmrghh, 6, 1),
 GEN_VXFORM(vmrghw, 6, 2),
 GEN_VXFORM(vmrglb, 6, 4),
 GEN_VXFORM(vmrglh, 6, 5),
-GEN_VXFORM(vmrglw, 6, 6),
+GEN_VXFORM_DUAL(vmrglw, vclrlb, 6, 6, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_310(vclrrb, 6, 7),
 GEN_VXFORM_300(vextublx, 6, 24),
 GEN_VXFORM_300(vextuhlx, 6, 25),
 GEN_VXFORM_DUAL(vmrgow, vextuwlx, 6, 26, PPC_NONE, PPC2_ALTIVEC_207),
@@ -101,30 +105,36 @@ GEN_VXFORM_DUAL(vmrgew, vextuwrx, 6, 30, PPC_NONE, PPC2_ALTIVEC_207),
 GEN_VXFORM(vmuloub, 4, 0),
 GEN_VXFORM(vmulouh, 4, 1),
 GEN_VXFORM_DUAL(vmulouw, vmuluwm, 4, 2, PPC_ALTIVEC, PPC_NONE),
+GEN_VXFORM_310(vmuloud, 4, 3),
 GEN_VXFORM(vmulosb, 4, 4),
 GEN_VXFORM(vmulosh, 4, 5),
 GEN_VXFORM_207(vmulosw, 4, 6),
+GEN_VXFORM_DUAL(vmulosd, vmulld, 4, 7, PPC_NONE, PPC2_ISA310),
 GEN_VXFORM(vmuleub, 4, 8),
 GEN_VXFORM(vmuleuh, 4, 9),
-GEN_VXFORM_207(vmuleuw, 4, 10),
+GEN_VXFORM_DUAL(vmuleuw, vmulhuw, 4, 10, PPC_NONE,
+                PPC2_ALTIVEC_207 | PPC2_ISA310),
+GEN_VXFORM_DUAL(vmuleud, vmulhud, 4, 11, PPC_NONE, PPC2_ISA310),
 GEN_VXFORM(vmulesb, 4, 12),
 GEN_VXFORM(vmulesh, 4, 13),
-GEN_VXFORM_207(vmulesw, 4, 14),
-GEN_VXFORM(vslb, 2, 4),
-GEN_VXFORM(vslh, 2, 5),
+GEN_VXFORM_DUAL(vmulesw, vmulhsw, 4, 14, PPC_NONE,
+                PPC2_ALTIVEC_207 | PPC2_ISA310),
+GEN_VXFORM_DUAL(vmulesd, vmulhsd, 4, 15, PPC_NONE, PPC2_ISA310),
+GEN_VXFORM_DUAL(vslb, vslq, 2, 4, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vslh, vrlqnm, 2, 5, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_DUAL(vslw, vrlwnm, 2, 6, PPC_ALTIVEC, PPC_NONE),
 GEN_VXFORM_207(vsld, 2, 23),
-GEN_VXFORM(vsrb, 2, 8),
+GEN_VXFORM_DUAL(vsrb, vsrq, 2, 8, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM(vsrh, 2, 9),
 GEN_VXFORM(vsrw, 2, 10),
 GEN_VXFORM_207(vsrd, 2, 27),
-GEN_VXFORM(vsrab, 2, 12),
+GEN_VXFORM_DUAL(vsrab, vsraq, 2, 12, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM(vsrah, 2, 13),
 GEN_VXFORM(vsraw, 2, 14),
 GEN_VXFORM_207(vsrad, 2, 15),
 GEN_VXFORM_300(vsrv, 2, 28),
 GEN_VXFORM_300(vslv, 2, 29),
-GEN_VXFORM(vslo, 6, 16),
+GEN_VXFORM_DUAL(vslo, vstri, 6, 16, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM(vsro, 6, 17),
 GEN_VXFORM(vaddcuw, 0, 6),
 GEN_HANDLER_E_2(vprtybw, 0x4, 0x1, 0x18, 8, 0, PPC_NONE, PPC2_ISA300),
@@ -146,38 +156,43 @@ GEN_VXFORM(vsubuws, 0, 26),
 GEN_VXFORM_DUAL(vsubsbs, bcdtrunc, 0, 28, PPC_ALTIVEC, PPC2_ISA300),
 GEN_VXFORM(vsubshs, 0, 29),
 GEN_VXFORM_DUAL(vsubsws, xpnd04_2, 0, 30, PPC_ALTIVEC, PPC_NONE),
-GEN_VXFORM_207(vadduqm, 0, 4),
-GEN_VXFORM_207(vaddcuq, 0, 5),
+GEN_VXFORM_DUAL(vadduqm, vcmpuq, 0, 4, PPC_NONE,
+                PPC2_ALTIVEC_207 | PPC2_ISA310),
+GEN_VXFORM_DUAL(vaddcuq, vcmpsq, 0, 5, PPC_NONE,
+                PPC2_ALTIVEC_207 | PPC2_ISA310),
 GEN_VXFORM_DUAL(vaddeuqm, vaddecuq, 30, 0xFF, PPC_NONE, PPC2_ALTIVEC_207),
 GEN_VXFORM_DUAL(vsubuqm, bcdtrunc, 0, 20, PPC2_ALTIVEC_207, PPC2_ISA300),
 GEN_VXFORM_DUAL(vsubcuq, bcdutrunc, 0, 21, PPC2_ALTIVEC_207, PPC2_ISA300),
 GEN_VXFORM_DUAL(vsubeuqm, vsubecuq, 31, 0xFF, PPC_NONE, PPC2_ALTIVEC_207),
-GEN_VXFORM(vrlb, 2, 0),
-GEN_VXFORM(vrlh, 2, 1),
+GEN_VXFORM_DUAL(vrlb, vrlq, 2, 0, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vrlh, vrlqmi, 2, 1, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_DUAL(vrlw, vrlwmi, 2, 2, PPC_ALTIVEC, PPC_NONE),
 GEN_VXFORM_DUAL(vrld, vrldmi, 2, 3, PPC_NONE, PPC2_ALTIVEC_207),
 GEN_VXFORM_DUAL(vsl, vrldnm, 2, 7, PPC_ALTIVEC, PPC_NONE),
 GEN_VXFORM(vsr, 2, 11),
-GEN_VXFORM(vpkuhum, 7, 0),
-GEN_VXFORM(vpkuwum, 7, 1),
+GEN_VXFORM_DUAL(vpkuhum, vinsbvlx, 7, 0, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vpkuwum, vinshvlx, 7, 1, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_207(vpkudum, 7, 17),
-GEN_VXFORM(vpkuhus, 7, 2),
-GEN_VXFORM(vpkuwus, 7, 3),
+GEN_VXFORM_DUAL(vpkuhus, vinswvlx, 7, 2, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vpkuwus, vinsw, 7, 3, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_207(vpkudus, 7, 19),
-GEN_VXFORM(vpkshus, 7, 4),
-GEN_VXFORM(vpkswus, 7, 5),
+GEN_VXFORM_DUAL(vpkshus, vinsbvrx, 7, 4, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vpkswus, vinshvrx, 7, 5, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_207(vpksdus, 7, 21),
-GEN_VXFORM(vpkshss, 7, 6),
-GEN_VXFORM(vpkswss, 7, 7),
+GEN_VXFORM_DUAL(vpkshss, vinswvrx, 7, 6, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vpkswss, vinsd, 7, 7, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_207(vpksdss, 7, 23),
-GEN_VXFORM(vpkpx, 7, 12),
+GEN_VXFORM_DUAL(vpkpx, vinsbrx, 7, 12, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_310(vinswrx, 7, 14),
 GEN_VXFORM(vsum4ubs, 4, 24),
 GEN_VXFORM(vsum4sbs, 4, 28),
 GEN_VXFORM(vsum4shs, 4, 25),
 GEN_VXFORM(vsum2sws, 4, 26),
 GEN_VXFORM(vsumsws, 4, 30),
-GEN_VXFORM(vaddfp, 5, 0),
+GEN_VXFORM_DUAL(vaddfp, vdivuq, 5, 0, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM(vsubfp, 5, 1),
+GEN_VXFORM_310(vdivuw, 5, 2),
+GEN_VXFORM_310(vdivud, 5, 3),
 GEN_VXFORM(vmaxfp, 5, 16),
 GEN_VXFORM(vminfp, 5, 17),
 
@@ -197,14 +212,14 @@ GEN_VXRFORM_300(vcmpnezh, 3, 5)
 GEN_VXRFORM_300(vcmpnezw, 3, 6)
 GEN_VXRFORM(vcmpgtsb, 3, 12)
 GEN_VXRFORM(vcmpgtsh, 3, 13)
-GEN_VXRFORM(vcmpgtsw, 3, 14)
 GEN_VXRFORM(vcmpgtub, 3, 8)
 GEN_VXRFORM(vcmpgtuh, 3, 9)
-GEN_VXRFORM(vcmpgtuw, 3, 10)
+GEN_VXRFORM_DUAL(vcmpgtuw, vcmpgtuq, 3, 10, PPC_ALTIVEC, PPC2_ISA310)
 GEN_VXRFORM_DUAL(vcmpeqfp, vcmpequd, 3, 3, PPC_ALTIVEC, PPC_NONE)
-GEN_VXRFORM(vcmpgefp, 3, 7)
+GEN_VXRFORM_DUAL(vcmpgefp, vcmpequq, 3, 7, PPC_ALTIVEC, PPC2_ISA310)
 GEN_VXRFORM_DUAL(vcmpgtfp, vcmpgtud, 3, 11, PPC_ALTIVEC, PPC_NONE)
 GEN_VXRFORM_DUAL(vcmpbfp, vcmpgtsd, 3, 15, PPC_ALTIVEC, PPC_NONE)
+GEN_VXRFORM_DUAL(vcmpgtsw, vcmpgtsq, 3, 14, PPC_ALTIVEC, PPC2_ISA310)
 GEN_VXRFORM_DUAL(vcmpequb, vcmpneb, 3, 0, PPC_ALTIVEC, PPC_NONE)
 GEN_VXRFORM_DUAL(vcmpequh, vcmpneh, 3, 1, PPC_ALTIVEC, PPC_NONE)
 GEN_VXRFORM_DUAL(vcmpequw, vcmpnew, 3, 2, PPC_ALTIVEC, PPC_NONE)
@@ -233,6 +248,8 @@ GEN_VXFORM_300_EO(vextsh2w, 0x01, 0x18, 0x11),
 GEN_VXFORM_300_EO(vextsb2d, 0x01, 0x18, 0x18),
 GEN_VXFORM_300_EO(vextsh2d, 0x01, 0x18, 0x19),
 GEN_VXFORM_300_EO(vextsw2d, 0x01, 0x18, 0x1A),
+GEN_HANDLER_E_2(vextsd2q, 0x04, 0x01, 0x18, 0x1B, 0, PPC_NONE,
+                PPC2_ISA310),
 GEN_VXFORM_300_EO(vctzb, 0x01, 0x18, 0x1C),
 GEN_VXFORM_300_EO(vctzh, 0x01, 0x18, 0x1D),
 GEN_VXFORM_300_EO(vctzw, 0x01, 0x18, 0x1E),
@@ -243,29 +260,36 @@ GEN_VXFORM_300(vpermr, 0x1D, 0xFF),
 
 #define GEN_VXFORM_NOA(name, opc2, opc3)                                \
     GEN_HANDLER(name, 0x04, opc2, opc3, 0x001f0000, PPC_ALTIVEC)
-GEN_VXFORM_NOA(vupkhsb, 7, 8),
-GEN_VXFORM_NOA(vupkhsh, 7, 9),
+GEN_VXFORM_DUAL(vupkhsb, vinsblx, 7, 8, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vupkhsh, vinshlx, 7, 9, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_207(vupkhsw, 7, 25),
-GEN_VXFORM_NOA(vupklsb, 7, 10),
-GEN_VXFORM_NOA(vupklsh, 7, 11),
+GEN_VXFORM_DUAL(vupklsb, vinswlx, 7, 10, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vupklsh, vinsdlx, 7, 11, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_207(vupklsw, 7, 27),
-GEN_VXFORM_NOA(vupkhpx, 7, 13),
-GEN_VXFORM_NOA(vupklpx, 7, 15),
-GEN_VXFORM_NOA(vrefp, 5, 4),
+GEN_VXFORM_DUAL(vupkhpx, vinshrx, 7, 13, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vupklpx, vinsdrx, 7, 15, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vrefp, vdivsq, 5, 4, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_NOA(vrsqrtefp, 5, 5),
-GEN_VXFORM_NOA(vexptefp, 5, 6),
-GEN_VXFORM_NOA(vlogefp, 5, 7),
-GEN_VXFORM_NOA(vrfim, 5, 11),
-GEN_VXFORM_NOA(vrfin, 5, 8),
-GEN_VXFORM_NOA(vrfip, 5, 10),
+GEN_VXFORM_DUAL(vexptefp, vdivsw, 5, 6, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vlogefp, vdivsd, 5, 7, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vrfim, vdiveud, 5, 11, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vrfin, vdiveuq, 5, 8, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vrfip, vdiveuw, 5, 10, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_NOA(vrfiz, 5, 9),
 
 #define GEN_VXFORM_UIMM(name, opc2, opc3)                               \
     GEN_HANDLER(name, 0x04, opc2, opc3, 0x00000000, PPC_ALTIVEC)
-GEN_VXFORM_UIMM(vcfux, 5, 12),
+GEN_VXFORM_DUAL(vcfux, vdivesq, 5, 12, PPC_ALTIVEC, PPC2_ISA310),
 GEN_VXFORM_UIMM(vcfsx, 5, 13),
-GEN_VXFORM_UIMM(vctuxs, 5, 14),
-GEN_VXFORM_UIMM(vctsxs, 5, 15),
+GEN_VXFORM_DUAL(vctuxs, vdivesw, 5, 14, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_DUAL(vctsxs, vdivesd, 5, 15, PPC_ALTIVEC, PPC2_ISA310),
+GEN_VXFORM_310(vmoduq, 5, 24),
+GEN_VXFORM_310(vmoduw, 5, 26),
+GEN_VXFORM_310(vmodud, 5, 27),
+GEN_VXFORM_310(vmodsq, 5, 28),
+GEN_VXFORM_310(vmodsw, 5, 30),
+GEN_VXFORM_310(vmodsd, 5, 31),
+GEN_HANDLER_E(vgnb, 0x04, 6, 19, 0x00180000, PPC_NONE, PPC2_ISA310),
 
 
 #define GEN_VAFORM_PAIRED(name0, name1, opc2)                           \
@@ -282,8 +306,29 @@ GEN_VXFORM_DUAL(vclzh, vpopcnth, 1, 29, PPC_NONE, PPC2_ALTIVEC_207),
 GEN_VXFORM_DUAL(vclzw, vpopcntw, 1, 30, PPC_NONE, PPC2_ALTIVEC_207),
 GEN_VXFORM_DUAL(vclzd, vpopcntd, 1, 31, PPC_NONE, PPC2_ALTIVEC_207),
 
+#if defined(TARGET_PPC64)
+GEN_VXFORM_310(vclzdm, 0x02, 0x1E),
+GEN_VXFORM_310(vctzdm, 0x02, 0x1F),
+GEN_VXFORM_310(vpextd, 0x06, 0x16),
+GEN_VXFORM_310(vmask_isa310, 0x01, 0x19),
+GEN_HANDLER_E(vextdubv, 0x04, 0x0C, 0xFF, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(vextduhv, 0x04, 0x0D, 0xFF, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(vextduwv, 0x04, 0x0E, 0xFF, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(vextddv, 0x04, 0x0F, 0xFF, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(vsldbi_vsrdbi, 0x04, 0x0B, 0xFF, 0,
+              PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(mtvsrbmi, 0x04, 0x0A, 0xFF, 0, PPC_NONE, PPC2_ISA310),
+#endif
+
+#if defined(TARGET_PPC64)
+GEN_VXFORM_DUAL(vbpermd, vpdepd, 6, 23, PPC_NONE,
+                PPC2_ISA300 | PPC2_ISA310),
+GEN_VXFORM_DUAL(vbpermq, vcfuged, 6, 21, PPC_NONE,
+                PPC2_ALTIVEC_207 | PPC2_ISA310),
+#else
 GEN_VXFORM_300(vbpermd, 6, 23),
 GEN_VXFORM_207(vbpermq, 6, 21),
+#endif
 GEN_VXFORM_207(vgbbd, 6, 20),
 GEN_VXFORM_207(vpmsumb, 4, 16),
 GEN_VXFORM_207(vpmsumh, 4, 17),
diff --git a/qemu/target/ppc/translate/vsx-impl.inc.c b/qemu/target/ppc/translate/vsx-impl.inc.c
index 679da14902..54cf755a71 100644
--- a/qemu/target/ppc/translate/vsx-impl.inc.c
+++ b/qemu/target/ppc/translate/vsx-impl.inc.c
@@ -27,6 +27,13 @@ static inline TCGv_ptr gen_vsr_ptr(TCGContext *tcg_ctx, int reg)
     return r;
 }
 
+static inline TCGv_ptr gen_acc_ptr(TCGContext *tcg_ctx, int reg)
+{
+    TCGv_ptr r = tcg_temp_new_ptr(tcg_ctx);
+    tcg_gen_addi_ptr(tcg_ctx, r, tcg_ctx->cpu_env, acc_full_offset(reg));
+    return r;
+}
+
 #define VSX_LOAD_SCALAR(name, operation)                      \
 static void gen_##name(DisasContext *ctx)                     \
 {                                                             \
@@ -143,6 +150,46 @@ static void gen_lxvw4x(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
 
+static void gen_lxvwsx(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv EA;
+    TCGv_i32 data;
+    TCGv_i64 word;
+    TCGv_i64 dup;
+
+    if (xT(ctx->opcode) < 32) {
+        if (unlikely(!ctx->vsx_enabled)) {
+            gen_exception(ctx, POWERPC_EXCP_VSXU);
+            return;
+        }
+    } else {
+        if (unlikely(!ctx->altivec_enabled)) {
+            gen_exception(ctx, POWERPC_EXCP_VPU);
+            return;
+        }
+    }
+
+    gen_set_access_type(ctx, ACCESS_INT);
+    EA = tcg_temp_new(tcg_ctx);
+    gen_addr_reg_index(ctx, EA);
+
+    data = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_qemu_ld_i32(tcg_ctx, data, EA, ctx->mem_idx, DEF_MEMOP(MO_UL));
+    word = tcg_temp_new_i64(tcg_ctx);
+    dup = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_extu_i32_i64(tcg_ctx, word, data);
+    tcg_gen_shli_i64(tcg_ctx, dup, word, 32);
+    tcg_gen_or_i64(tcg_ctx, dup, dup, word);
+    set_cpu_vsrh(tcg_ctx, xT(ctx->opcode), dup);
+    set_cpu_vsrl(tcg_ctx, xT(ctx->opcode), dup);
+
+    tcg_temp_free(tcg_ctx, EA);
+    tcg_temp_free_i32(tcg_ctx, data);
+    tcg_temp_free_i64(tcg_ctx, word);
+    tcg_temp_free_i64(tcg_ctx, dup);
+}
+
 static void gen_bswap16x8(TCGContext *tcg_ctx, TCGv_i64 outh, TCGv_i64 outl,
                           TCGv_i64 inh, TCGv_i64 inl)
 {
@@ -298,6 +345,168 @@ static void gen_##name(DisasContext *ctx)                   \
 VSX_VECTOR_LOAD(lxv, ld_i64, 0)
 VSX_VECTOR_LOAD(lxvx, ld_i64, 1)
 
+static int prefixed_8ls_xt(DisasContext *ctx)
+{
+    return ((opc1(ctx->opcode) & 1) << 5) | rD(ctx->opcode);
+}
+
+static int vsx_tsxp_rt(DisasContext *ctx)
+{
+    return ((((ctx->opcode >> 21) & 1) << 4) |
+            ((ctx->opcode >> 22) & 0xf)) * 2;
+}
+
+static void gen_vsx_load_vector(DisasContext *ctx, TCGv ea, int xt,
+                                TCGv_i64 xth, TCGv_i64 xtl)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    if (ctx->le_mode) {
+        tcg_gen_qemu_ld_i64(tcg_ctx, xtl, ea, ctx->mem_idx, MO_LEQ);
+        set_cpu_vsrl(tcg_ctx, xt, xtl);
+        tcg_gen_addi_tl(tcg_ctx, ea, ea, 8);
+        tcg_gen_qemu_ld_i64(tcg_ctx, xth, ea, ctx->mem_idx, MO_LEQ);
+        set_cpu_vsrh(tcg_ctx, xt, xth);
+    } else {
+        tcg_gen_qemu_ld_i64(tcg_ctx, xth, ea, ctx->mem_idx, MO_BEQ);
+        set_cpu_vsrh(tcg_ctx, xt, xth);
+        tcg_gen_addi_tl(tcg_ctx, ea, ea, 8);
+        tcg_gen_qemu_ld_i64(tcg_ctx, xtl, ea, ctx->mem_idx, MO_BEQ);
+        set_cpu_vsrl(tcg_ctx, xt, xtl);
+    }
+}
+
+static void gen_vsx_store_vector(DisasContext *ctx, TCGv ea, int xt,
+                                 TCGv_i64 xth, TCGv_i64 xtl)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    get_cpu_vsrh(tcg_ctx, xth, xt);
+    get_cpu_vsrl(tcg_ctx, xtl, xt);
+    if (ctx->le_mode) {
+        tcg_gen_qemu_st_i64(tcg_ctx, xtl, ea, ctx->mem_idx, MO_LEQ);
+        tcg_gen_addi_tl(tcg_ctx, ea, ea, 8);
+        tcg_gen_qemu_st_i64(tcg_ctx, xth, ea, ctx->mem_idx, MO_LEQ);
+    } else {
+        tcg_gen_qemu_st_i64(tcg_ctx, xth, ea, ctx->mem_idx, MO_BEQ);
+        tcg_gen_addi_tl(tcg_ctx, ea, ea, 8);
+        tcg_gen_qemu_st_i64(tcg_ctx, xtl, ea, ctx->mem_idx, MO_BEQ);
+    }
+}
+
+static void gen_vsx_load_vector_pair(DisasContext *ctx, TCGv ea, int xt,
+                                     TCGv_i64 xth, TCGv_i64 xtl)
+{
+    int xt1;
+    int xt2;
+
+    if (ctx->le_mode) {
+        xt1 = xt + 1;
+        xt2 = xt;
+    } else {
+        xt1 = xt;
+        xt2 = xt + 1;
+    }
+    gen_vsx_load_vector(ctx, ea, xt1, xth, xtl);
+    tcg_gen_addi_tl(ctx->uc->tcg_ctx, ea, ea, 8);
+    gen_vsx_load_vector(ctx, ea, xt2, xth, xtl);
+}
+
+static void gen_vsx_store_vector_pair(DisasContext *ctx, TCGv ea, int xt,
+                                      TCGv_i64 xth, TCGv_i64 xtl)
+{
+    int xt1;
+    int xt2;
+
+    if (ctx->le_mode) {
+        xt1 = xt + 1;
+        xt2 = xt;
+    } else {
+        xt1 = xt;
+        xt2 = xt + 1;
+    }
+    gen_vsx_store_vector(ctx, ea, xt1, xth, xtl);
+    tcg_gen_addi_tl(ctx->uc->tcg_ctx, ea, ea, 8);
+    gen_vsx_store_vector(ctx, ea, xt2, xth, xtl);
+}
+
+static void gen_lxvp(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = vsx_tsxp_rt(ctx);
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    gen_addr_imm_index(ctx, ea, 0x0F);
+    gen_vsx_load_vector_pair(ctx, ea, xt, xth, xtl);
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_lxvpx(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = vsx_tsxp_rt(ctx);
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    gen_addr_reg_index(ctx, ea);
+    gen_vsx_load_vector_pair(ctx, ea, xt, xth, xtl);
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_plxv(DisasContext *ctx, bool paired)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = paired ? vsx_tsxp_rt(ctx) : prefixed_8ls_xt(ctx);
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        if (paired) {
+            gen_vsx_load_vector_pair(ctx, ea, xt, xth, xtl);
+        } else {
+            gen_vsx_load_vector(ctx, ea, xt, xth, xtl);
+        }
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
 #define VSX_VECTOR_STORE(name, op, indexed)                 \
 static void gen_##name(DisasContext *ctx)                   \
 {                                                           \
@@ -352,6 +561,168 @@ static void gen_##name(DisasContext *ctx)                   \
 VSX_VECTOR_STORE(stxv, st_i64, 0)
 VSX_VECTOR_STORE(stxvx, st_i64, 1)
 
+static void gen_stxvp(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = vsx_tsxp_rt(ctx);
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    gen_addr_imm_index(ctx, ea, 0x0F);
+    gen_vsx_store_vector_pair(ctx, ea, xt, xth, xtl);
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_stxvpx(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = vsx_tsxp_rt(ctx);
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    gen_addr_reg_index(ctx, ea);
+    gen_vsx_store_vector_pair(ctx, ea, xt, xth, xtl);
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_pstxv(DisasContext *ctx, bool paired)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = paired ? vsx_tsxp_rt(ctx) : prefixed_8ls_xt(ctx);
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        if (paired) {
+            gen_vsx_store_vector_pair(ctx, ea, xt, xth, xtl);
+        } else {
+            gen_vsx_store_vector(ctx, ea, xt, xth, xtl);
+        }
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_lxvrx(DisasContext *ctx, MemOp mop)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = xT(ctx->opcode);
+    TCGv ea;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    gen_addr_reg_index(ctx, ea);
+    tcg_gen_qemu_ld_i64(tcg_ctx, xtl, ea, ctx->mem_idx, mop);
+    set_cpu_vsrl(tcg_ctx, xt, xtl);
+    tcg_gen_movi_i64(tcg_ctx, xtl, 0);
+    set_cpu_vsrh(tcg_ctx, xt, xtl);
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_stxvrx(DisasContext *ctx, MemOp mop)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 xtl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    get_cpu_vsrl(tcg_ctx, xtl, xT(ctx->opcode));
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    gen_addr_reg_index(ctx, ea);
+    tcg_gen_qemu_st_i64(tcg_ctx, xtl, ea, ctx->mem_idx, mop);
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_lxvrbx(DisasContext *ctx)
+{
+    gen_lxvrx(ctx, DEF_MEMOP(MO_UB));
+}
+
+static void gen_lxvrhx(DisasContext *ctx)
+{
+    gen_lxvrx(ctx, DEF_MEMOP(MO_UW));
+}
+
+static void gen_lxvrwx(DisasContext *ctx)
+{
+    gen_lxvrx(ctx, DEF_MEMOP(MO_UL));
+}
+
+static void gen_lxvrdx(DisasContext *ctx)
+{
+    gen_lxvrx(ctx, DEF_MEMOP(MO_UQ));
+}
+
+static void gen_stxvrbx(DisasContext *ctx)
+{
+    gen_stxvrx(ctx, DEF_MEMOP(MO_UB));
+}
+
+static void gen_stxvrhx(DisasContext *ctx)
+{
+    gen_stxvrx(ctx, DEF_MEMOP(MO_UW));
+}
+
+static void gen_stxvrwx(DisasContext *ctx)
+{
+    gen_stxvrx(ctx, DEF_MEMOP(MO_UL));
+}
+
+static void gen_stxvrdx(DisasContext *ctx)
+{
+    gen_stxvrx(ctx, DEF_MEMOP(MO_UQ));
+}
+
 #ifdef TARGET_PPC64
 #define VSX_VECTOR_LOAD_STORE_LENGTH(name)                         \
 static void gen_##name(DisasContext *ctx)                          \
@@ -411,6 +782,62 @@ static void gen_##name(DisasContext *ctx)                         \
 VSX_LOAD_SCALAR_DS(lxsd, ld64_i64)
 VSX_LOAD_SCALAR_DS(lxssp, ld32fs)
 
+static void gen_plxsd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+    int xt = rD(ctx->opcode) + 32;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld64_i64(ctx, xth, ea);
+        set_cpu_vsrh(tcg_ctx, xt, xth);
+        tcg_gen_movi_i64(tcg_ctx, xtl, 0);
+        set_cpu_vsrl(tcg_ctx, xt, xtl);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
+static void gen_plxssp(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 xth;
+    TCGv_i64 xtl;
+    int xt = rD(ctx->opcode) + 32;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    xtl = tcg_temp_new_i64(tcg_ctx);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_ld32fs(ctx, xth, ea);
+        set_cpu_vsrh(tcg_ctx, xt, xth);
+        tcg_gen_movi_i64(tcg_ctx, xtl, 0);
+        set_cpu_vsrl(tcg_ctx, xt, xtl);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+    tcg_temp_free_i64(tcg_ctx, xtl);
+}
+
 #define VSX_STORE_SCALAR(name, operation)                     \
 static void gen_##name(DisasContext *ctx)                     \
 {                                                             \
@@ -590,6 +1017,50 @@ static void gen_##name(DisasContext *ctx)                         \
 VSX_STORE_SCALAR_DS(stxsd, st64_i64)
 VSX_STORE_SCALAR_DS(stxssp, st32fs)
 
+static void gen_pstxsd(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 xth;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    get_cpu_vsrh(tcg_ctx, xth, rD(ctx->opcode) + 32);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_st64_i64(ctx, xth, ea);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+}
+
+static void gen_pstxssp(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv ea;
+    TCGv_i64 xth;
+
+    if (unlikely(!ctx->altivec_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VPU);
+        return;
+    }
+
+    xth = tcg_temp_new_i64(tcg_ctx);
+    get_cpu_vsrh(tcg_ctx, xth, rD(ctx->opcode) + 32);
+    gen_set_access_type(ctx, ACCESS_INT);
+    ea = tcg_temp_new(tcg_ctx);
+    if (prefixed_addr(ctx, ea, rA(ctx->opcode), prefixed_si(ctx))) {
+        gen_qemu_st32fs(ctx, xth, ea);
+    }
+    tcg_temp_free(tcg_ctx, ea);
+    tcg_temp_free_i64(tcg_ctx, xth);
+}
+
 static void gen_mfvsrwz(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -1191,6 +1662,62 @@ static void gen_##name(DisasContext *ctx)                                     \
     tcg_temp_free_ptr(tcg_ctx, xb);                                                    \
 }
 
+#define GEN_VSX_HELPER_R3_NOOPC(name)                                         \
+static void gen_##name(DisasContext *ctx)                                     \
+{                                                                             \
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;                                   \
+    TCGv_ptr xt, xa, xb;                                                      \
+    if (unlikely(!ctx->vsx_enabled)) {                                        \
+        gen_exception(ctx, POWERPC_EXCP_VSXU);                                \
+        return;                                                               \
+    }                                                                         \
+    xt = gen_vsr_ptr(tcg_ctx, rD(ctx->opcode) + 32);                          \
+    xa = gen_vsr_ptr(tcg_ctx, rA(ctx->opcode) + 32);                          \
+    xb = gen_vsr_ptr(tcg_ctx, rB(ctx->opcode) + 32);                          \
+    gen_helper_##name(tcg_ctx, tcg_ctx->cpu_env, xt, xa, xb);                 \
+    tcg_temp_free_ptr(tcg_ctx, xt);                                           \
+    tcg_temp_free_ptr(tcg_ctx, xa);                                           \
+    tcg_temp_free_ptr(tcg_ctx, xb);                                           \
+}
+
+#define GEN_VSX_HELPER_R2_NOOPC(name)                                         \
+static void gen_##name(DisasContext *ctx)                                     \
+{                                                                             \
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;                                   \
+    TCGv_ptr xt, xb;                                                          \
+    if (unlikely(!ctx->vsx_enabled)) {                                        \
+        gen_exception(ctx, POWERPC_EXCP_VSXU);                                \
+        return;                                                               \
+    }                                                                         \
+    xt = gen_vsr_ptr(tcg_ctx, rD(ctx->opcode) + 32);                          \
+    xb = gen_vsr_ptr(tcg_ctx, rB(ctx->opcode) + 32);                          \
+    gen_helper_##name(tcg_ctx, tcg_ctx->cpu_env, xt, xb);                     \
+    tcg_temp_free_ptr(tcg_ctx, xt);                                           \
+    tcg_temp_free_ptr(tcg_ctx, xb);                                           \
+}
+
+#define GEN_VSX_HELPER_XSMADDQP(name)                                         \
+static void gen_##name(DisasContext *ctx)                                     \
+{                                                                             \
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;                                   \
+    TCGv_ptr xt, xa, xb;                                                      \
+    if (unlikely(!ctx->vsx_enabled)) {                                        \
+        gen_exception(ctx, POWERPC_EXCP_VSXU);                                \
+        return;                                                               \
+    }                                                                         \
+    xt = gen_vsr_ptr(tcg_ctx, rD(ctx->opcode) + 32);                          \
+    xa = gen_vsr_ptr(tcg_ctx, rA(ctx->opcode) + 32);                          \
+    xb = gen_vsr_ptr(tcg_ctx, rB(ctx->opcode) + 32);                          \
+    if (Rc(ctx->opcode)) {                                                    \
+        gen_helper_##name##O(tcg_ctx, tcg_ctx->cpu_env, xt, xa, xt, xb);       \
+    } else {                                                                  \
+        gen_helper_##name(tcg_ctx, tcg_ctx->cpu_env, xt, xa, xt, xb);          \
+    }                                                                         \
+    tcg_temp_free_ptr(tcg_ctx, xt);                                           \
+    tcg_temp_free_ptr(tcg_ctx, xa);                                           \
+    tcg_temp_free_ptr(tcg_ctx, xb);                                           \
+}
+
 #define GEN_VSX_HELPER_XT_XB_ENV(name, op1, op2, inval, type) \
 static void gen_##name(DisasContext *ctx)                     \
 {                                                             \
@@ -1212,6 +1739,10 @@ static void gen_##name(DisasContext *ctx)                     \
 
 GEN_VSX_HELPER_X3(xsadddp, 0x00, 0x04, 0, PPC2_VSX)
 GEN_VSX_HELPER_R3(xsaddqp, 0x04, 0x00, 0, PPC2_ISA300)
+GEN_VSX_HELPER_XSMADDQP(XSMADDQP)
+GEN_VSX_HELPER_XSMADDQP(XSMSUBQP)
+GEN_VSX_HELPER_XSMADDQP(XSNMADDQP)
+GEN_VSX_HELPER_XSMADDQP(XSNMSUBQP)
 GEN_VSX_HELPER_X3(xssubdp, 0x00, 0x05, 0, PPC2_VSX)
 GEN_VSX_HELPER_X3(xsmuldp, 0x00, 0x06, 0, PPC2_VSX)
 GEN_VSX_HELPER_R3(xsmulqp, 0x04, 0x01, 0, PPC2_ISA300)
@@ -1226,6 +1757,9 @@ GEN_VSX_HELPER_X3(xscmpeqdp, 0x0C, 0x00, 0, PPC2_ISA300)
 GEN_VSX_HELPER_X3(xscmpgtdp, 0x0C, 0x01, 0, PPC2_ISA300)
 GEN_VSX_HELPER_X3(xscmpgedp, 0x0C, 0x02, 0, PPC2_ISA300)
 GEN_VSX_HELPER_X3(xscmpnedp, 0x0C, 0x03, 0, PPC2_ISA300)
+GEN_VSX_HELPER_R3_NOOPC(XSCMPEQQP)
+GEN_VSX_HELPER_R3_NOOPC(XSCMPGEQP)
+GEN_VSX_HELPER_R3_NOOPC(XSCMPGTQP)
 GEN_VSX_HELPER_X2_AB(xscmpexpdp, 0x0C, 0x07, 0, PPC2_ISA300)
 GEN_VSX_HELPER_R2_AB(xscmpexpqp, 0x04, 0x05, 0, PPC2_ISA300)
 GEN_VSX_HELPER_X2_AB(xscmpodp, 0x0C, 0x05, 0, PPC2_VSX)
@@ -1238,9 +1772,15 @@ GEN_VSX_HELPER_R3(xsmaxcdp, 0x00, 0x10, 0, PPC2_ISA300)
 GEN_VSX_HELPER_R3(xsmincdp, 0x00, 0x11, 0, PPC2_ISA300)
 GEN_VSX_HELPER_R3(xsmaxjdp, 0x00, 0x12, 0, PPC2_ISA300)
 GEN_VSX_HELPER_R3(xsminjdp, 0x00, 0x12, 0, PPC2_ISA300)
+GEN_VSX_HELPER_R3_NOOPC(XSMAXCQP)
+GEN_VSX_HELPER_R3_NOOPC(XSMINCQP)
 GEN_VSX_HELPER_X2(xscvdphp, 0x16, 0x15, 0x11, PPC2_ISA300)
 GEN_VSX_HELPER_X2(xscvdpsp, 0x12, 0x10, 0, PPC2_VSX)
 GEN_VSX_HELPER_R2(xscvdpqp, 0x04, 0x1A, 0x16, PPC2_ISA300)
+GEN_VSX_HELPER_R2_NOOPC(XSCVQPUQZ)
+GEN_VSX_HELPER_R2_NOOPC(XSCVQPSQZ)
+GEN_VSX_HELPER_R2_NOOPC(XSCVUQQP)
+GEN_VSX_HELPER_R2_NOOPC(XSCVSQQP)
 GEN_VSX_HELPER_XT_XB_ENV(xscvdpspn, 0x16, 0x10, 0, PPC2_VSX207)
 GEN_VSX_HELPER_R2(xscvqpsdz, 0x04, 0x1A, 0x19, PPC2_ISA300)
 GEN_VSX_HELPER_R2(xscvqpswz, 0x04, 0x1A, 0x09, PPC2_ISA300)
@@ -1320,6 +1860,439 @@ GEN_VSX_HELPER_X3(xvminsp, 0x00, 0x19, 0, PPC2_VSX)
 GEN_VSX_HELPER_X2(xvcvspdp, 0x12, 0x1C, 0, PPC2_VSX)
 GEN_VSX_HELPER_X2(xvcvhpsp, 0x16, 0x1D, 0x18, PPC2_ISA300)
 GEN_VSX_HELPER_X2(xvcvsphp, 0x16, 0x1D, 0x19, PPC2_ISA300)
+GEN_VSX_HELPER_X2(XVCVSPBF16, 0x16, 0x1D, 0x11, PPC2_ISA310)
+
+static void gen_XVCVBF16SPN(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr xt, xb;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xt = gen_vsr_ptr(tcg_ctx, xT(ctx->opcode));
+    xb = gen_vsr_ptr(tcg_ctx, xB(ctx->opcode));
+    gen_helper_XVCVBF16SPN(tcg_ctx, xt, xb);
+    tcg_temp_free_ptr(tcg_ctx, xt);
+    tcg_temp_free_ptr(tcg_ctx, xb);
+}
+
+typedef void (*mma_ger_genfn)(TCGContext *, TCGv_env, TCGv_ptr, TCGv_ptr,
+                              TCGv_ptr, TCGv_i32);
+
+static int mma_xt(uint32_t opcode)
+{
+    return extract32(opcode, 23, 3);
+}
+
+static int mma_xa_pair(uint32_t opcode)
+{
+    return (((extract32(opcode, 2, 1) << 4) | extract32(opcode, 17, 4)) * 2);
+}
+
+static bool mma_require_isa310_vsx(DisasContext *ctx)
+{
+    if (unlikely(!(ctx->insns_flags2 & PPC2_ISA310))) {
+        gen_invalid(ctx);
+        return false;
+    }
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return false;
+    }
+    return true;
+}
+
+static void gen_mma_ger_helper(DisasContext *ctx, mma_ger_genfn helper,
+                               int xt, int xa, int xb, int pmsk, int xmsk,
+                               int ymsk)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr xtp;
+    TCGv_ptr xap;
+    TCGv_ptr xbp;
+    TCGv_i32 mask;
+
+    if (!mma_require_isa310_vsx(ctx)) {
+        return;
+    }
+    if (unlikely((xa / 4 == xt) || (xb / 4 == xt))) {
+        gen_invalid(ctx);
+        return;
+    }
+
+    xtp = gen_acc_ptr(tcg_ctx, xt);
+    xap = gen_vsr_ptr(tcg_ctx, xa);
+    xbp = gen_vsr_ptr(tcg_ctx, xb);
+    mask = tcg_const_i32(tcg_ctx, ger_pack_masks(pmsk, ymsk, xmsk));
+    helper(tcg_ctx, tcg_ctx->cpu_env, xap, xbp, xtp, mask);
+    tcg_temp_free_i32(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, xtp);
+    tcg_temp_free_ptr(tcg_ctx, xap);
+    tcg_temp_free_ptr(tcg_ctx, xbp);
+}
+
+static bool gen_mma_ger_insn(DisasContext *ctx)
+{
+    uint32_t opcode = ctx->opcode;
+    int fixed = extract32(opcode, 3, 8);
+    int xt = mma_xt(opcode);
+    int xa = xA(opcode);
+    int xb = xB(opcode);
+    int pmsk = 0xff;
+    int xmsk = 0x0f;
+    int ymsk = 0x0f;
+
+    switch (fixed) {
+    case 0x23:
+        gen_mma_ger_helper(ctx, gen_helper_XVI4GER8, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x22:
+        gen_mma_ger_helper(ctx, gen_helper_XVI4GER8PP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x03:
+        gen_mma_ger_helper(ctx, gen_helper_XVI8GER4, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x02:
+        gen_mma_ger_helper(ctx, gen_helper_XVI8GER4PP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x4b:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x6b:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2PP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x63:
+        gen_mma_ger_helper(ctx, gen_helper_XVI8GER4SPP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x2b:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2S, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x2a:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2SPP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x33:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x32:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2PP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0xb2:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2PN, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x72:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2NP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0xf2:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2NN, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x13:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x12:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2PP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x92:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2PN, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x52:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2NP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0xd2:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2NN, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x1b:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GER, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x1a:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERPP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x9a:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERPN, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x5a:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERNP, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0xda:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERNN, xt, xa, xb,
+                           pmsk, xmsk, ymsk);
+        return true;
+    case 0x3b:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GER, xt, mma_xa_pair(opcode),
+                           xb, pmsk, xmsk, ymsk);
+        return true;
+    case 0x3a:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERPP, xt, mma_xa_pair(opcode),
+                           xb, pmsk, xmsk, ymsk);
+        return true;
+    case 0xba:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERPN, xt, mma_xa_pair(opcode),
+                           xb, pmsk, xmsk, ymsk);
+        return true;
+    case 0x7a:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERNP, xt, mma_xa_pair(opcode),
+                           xb, pmsk, xmsk, ymsk);
+        return true;
+    case 0xfa:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERNN, xt, mma_xa_pair(opcode),
+                           xb, pmsk, xmsk, ymsk);
+        return true;
+    default:
+        return false;
+    }
+}
+
+#ifdef TARGET_PPC64
+static bool gen_pmx_ger_insn(DisasContext *ctx)
+{
+    uint32_t prefix = ctx->prefix_opcode;
+    uint32_t opcode = ctx->opcode;
+    int fixed = extract32(opcode, 3, 8);
+    int xt = mma_xt(opcode);
+    int xa = xA(opcode);
+    int xb = xB(opcode);
+    int xmsk = extract32(prefix, 4, 4);
+    int ymsk = extract32(prefix, 0, 4);
+    int pmsk8 = extract32(prefix, 8, 8);
+    int pmsk4 = extract32(prefix, 12, 4);
+    int pmsk2 = extract32(prefix, 14, 2);
+
+    switch (fixed) {
+    case 0x23:
+        gen_mma_ger_helper(ctx, gen_helper_XVI4GER8, xt, xa, xb,
+                           pmsk8, xmsk, ymsk);
+        return true;
+    case 0x22:
+        gen_mma_ger_helper(ctx, gen_helper_XVI4GER8PP, xt, xa, xb,
+                           pmsk8, xmsk, ymsk);
+        return true;
+    case 0x03:
+        gen_mma_ger_helper(ctx, gen_helper_XVI8GER4, xt, xa, xb,
+                           pmsk4, xmsk, ymsk);
+        return true;
+    case 0x02:
+        gen_mma_ger_helper(ctx, gen_helper_XVI8GER4PP, xt, xa, xb,
+                           pmsk4, xmsk, ymsk);
+        return true;
+    case 0x63:
+        gen_mma_ger_helper(ctx, gen_helper_XVI8GER4SPP, xt, xa, xb,
+                           pmsk4, xmsk, ymsk);
+        return true;
+    case 0x4b:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x6b:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2PP, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x2b:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2S, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x2a:
+        gen_mma_ger_helper(ctx, gen_helper_XVI16GER2SPP, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x33:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x32:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2PP, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0xb2:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2PN, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x72:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2NP, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0xf2:
+        gen_mma_ger_helper(ctx, gen_helper_XVBF16GER2NN, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x13:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x12:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2PP, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x92:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2PN, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x52:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2NP, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0xd2:
+        gen_mma_ger_helper(ctx, gen_helper_XVF16GER2NN, xt, xa, xb,
+                           pmsk2, xmsk, ymsk);
+        return true;
+    case 0x1b:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GER, xt, xa, xb,
+                           1, xmsk, ymsk);
+        return true;
+    case 0x1a:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERPP, xt, xa, xb,
+                           1, xmsk, ymsk);
+        return true;
+    case 0x9a:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERPN, xt, xa, xb,
+                           1, xmsk, ymsk);
+        return true;
+    case 0x5a:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERNP, xt, xa, xb,
+                           1, xmsk, ymsk);
+        return true;
+    case 0xda:
+        gen_mma_ger_helper(ctx, gen_helper_XVF32GERNN, xt, xa, xb,
+                           1, xmsk, ymsk);
+        return true;
+    case 0x3b:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GER, xt, mma_xa_pair(opcode),
+                           xb, 1, xmsk, extract32(prefix, 2, 2));
+        return true;
+    case 0x3a:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERPP, xt,
+                           mma_xa_pair(opcode), xb, 1, xmsk,
+                           extract32(prefix, 2, 2));
+        return true;
+    case 0xba:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERPN, xt,
+                           mma_xa_pair(opcode), xb, 1, xmsk,
+                           extract32(prefix, 2, 2));
+        return true;
+    case 0x7a:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERNP, xt,
+                           mma_xa_pair(opcode), xb, 1, xmsk,
+                           extract32(prefix, 2, 2));
+        return true;
+    case 0xfa:
+        if (unlikely(extract32(opcode, 16, 1))) {
+            gen_invalid(ctx);
+            return true;
+        }
+        gen_mma_ger_helper(ctx, gen_helper_XVF64GERNN, xt,
+                           mma_xa_pair(opcode), xb, 1, xmsk,
+                           extract32(prefix, 2, 2));
+        return true;
+    default:
+        return false;
+    }
+}
+#endif
+
+static bool gen_mma_acc_insn(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int op;
+
+    if (extract32(ctx->opcode, 1, 10) != 0x0b1) {
+        return false;
+    }
+
+    op = rA(ctx->opcode);
+    if (op != 0 && op != 1 && op != 3) {
+        return false;
+    }
+
+    if (!mma_require_isa310_vsx(ctx)) {
+        return true;
+    }
+
+    if (op == 3) {
+        int base = mma_xt(ctx->opcode) * 4;
+        int i;
+        TCGv_i64 zero;
+
+        zero = tcg_const_i64(tcg_ctx, 0);
+        for (i = 0; i < 4; i++) {
+            set_cpu_vsrh(tcg_ctx, base + i, zero);
+            set_cpu_vsrl(tcg_ctx, base + i, zero);
+        }
+        tcg_temp_free_i64(tcg_ctx, zero);
+    }
+    return true;
+}
+
+static bool gen_mma_insn(DisasContext *ctx)
+{
+    if (opc1(ctx->opcode) == 0x1f) {
+        return gen_mma_acc_insn(ctx);
+    }
+    if (opc1(ctx->opcode) == 0x3b) {
+        return gen_mma_ger_insn(ctx);
+    }
+    return false;
+}
+
 GEN_VSX_HELPER_X2(xvcvspsxds, 0x10, 0x19, 0, PPC2_VSX)
 GEN_VSX_HELPER_X2(xvcvspsxws, 0x10, 0x09, 0, PPC2_VSX)
 GEN_VSX_HELPER_X2(xvcvspuxds, 0x10, 0x18, 0, PPC2_VSX)
@@ -1530,6 +2503,55 @@ VSX_LOGICAL(xxleqv, MO_64, tcg_gen_gvec_eqv)
 VSX_LOGICAL(xxlnand, MO_64, tcg_gen_gvec_nand)
 VSX_LOGICAL(xxlorc, MO_64, tcg_gen_gvec_orc)
 
+static void gen_xvtlsbb(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 xb;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 all_true;
+    TCGv_i64 all_false;
+    TCGv_i64 mask;
+    TCGv_i64 zero;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xb = tcg_temp_new_i64(tcg_ctx);
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    all_true = tcg_temp_new_i64(tcg_ctx);
+    all_false = tcg_temp_new_i64(tcg_ctx);
+    mask = tcg_const_i64(tcg_ctx, dup_const(MO_8, 1));
+    zero = tcg_const_i64(tcg_ctx, 0);
+
+    get_cpu_vsrh(tcg_ctx, xb, xB(ctx->opcode));
+    tcg_gen_and_i64(tcg_ctx, t0, mask, xb);
+    get_cpu_vsrl(tcg_ctx, xb, xB(ctx->opcode));
+    tcg_gen_and_i64(tcg_ctx, t1, mask, xb);
+
+    tcg_gen_or_i64(tcg_ctx, all_false, t0, t1);
+    tcg_gen_and_i64(tcg_ctx, all_true, t0, t1);
+
+    tcg_gen_setcond_i64(tcg_ctx, TCG_COND_EQ, all_false, all_false, zero);
+    tcg_gen_shli_i64(tcg_ctx, all_false, all_false, 1);
+    tcg_gen_setcond_i64(tcg_ctx, TCG_COND_EQ, all_true, all_true, mask);
+    tcg_gen_shli_i64(tcg_ctx, all_true, all_true, 3);
+
+    tcg_gen_or_i64(tcg_ctx, t0, all_false, all_true);
+    tcg_gen_extrl_i64_i32(tcg_ctx, cpu_crf[BF(ctx->opcode)], t0);
+
+    tcg_temp_free_i64(tcg_ctx, xb);
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+    tcg_temp_free_i64(tcg_ctx, all_true);
+    tcg_temp_free_i64(tcg_ctx, all_false);
+    tcg_temp_free_i64(tcg_ctx, mask);
+    tcg_temp_free_i64(tcg_ctx, zero);
+}
+
 #define VSX_XXMRG(name, high)                               \
 static void glue(gen_, name)(DisasContext *ctx)             \
     {                                                       \
@@ -1612,12 +2634,27 @@ static void gen_xxspltw(DisasContext *ctx)
 
 #define pattern(x) (((x) & 0xff) * (~(uint64_t)0 / 0xff))
 
+#ifdef TARGET_PPC64
+static void gen_lxvkq(DisasContext *ctx);
+#endif
+
 static void gen_xxspltib(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     uint8_t uim8 = IMM8(ctx->opcode);
     int rt = xT(ctx->opcode);
 
+    if (unlikely(rA(ctx->opcode) > 7)) {
+#ifdef TARGET_PPC64
+        if ((ctx->insns_flags2 & PPC2_ISA310) && rA(ctx->opcode) == 31) {
+            gen_lxvkq(ctx);
+            return;
+        }
+#endif
+        gen_invalid(ctx);
+        return;
+    }
+
     if (rt < 32) {
         if (unlikely(!ctx->vsx_enabled)) {
             gen_exception(ctx, POWERPC_EXCP_VSXU);
@@ -1632,6 +2669,403 @@ static void gen_xxspltib(DisasContext *ctx)
     tcg_gen_gvec_dup8i(tcg_ctx, vsr_full_offset(rt), 16, 16, uim8);
 }
 
+#ifdef TARGET_PPC64
+static int prefixed_8rr_xt(DisasContext *ctx)
+{
+    return (extract32(ctx->opcode, 16, 1) << 5) | rD(ctx->opcode);
+}
+
+static int32_t prefixed_8rr_si(DisasContext *ctx)
+{
+    return (int32_t)(((ctx->prefix_opcode & 0xffff) << 16) |
+                     UIMM(ctx->opcode));
+}
+
+static void gen_xxspltiw(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = prefixed_8rr_xt(ctx);
+    int32_t si = prefixed_8rr_si(ctx);
+    uint64_t val = deposit64((uint32_t)si, 32, 32, (uint32_t)si);
+    TCGv_i64 t;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    t = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_movi_i64(tcg_ctx, t, val);
+    set_cpu_vsrh(tcg_ctx, xt, t);
+    set_cpu_vsrl(tcg_ctx, xt, t);
+    tcg_temp_free_i64(tcg_ctx, t);
+}
+
+static void gen_xxspltidp(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = prefixed_8rr_xt(ctx);
+    int32_t si = prefixed_8rr_si(ctx);
+    uint64_t val;
+    TCGv_i64 t;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    val = helper_todouble((uint32_t)si);
+    t = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_movi_i64(tcg_ctx, t, val);
+    set_cpu_vsrh(tcg_ctx, xt, t);
+    set_cpu_vsrl(tcg_ctx, xt, t);
+    tcg_temp_free_i64(tcg_ctx, t);
+}
+
+static void gen_xxsplti32dx(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = prefixed_8rr_xt(ctx);
+    int ix = extract32(ctx->opcode, 17, 1);
+    int32_t si = prefixed_8rr_si(ctx);
+    TCGv_i32 imm;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    imm = tcg_temp_new_i32(tcg_ctx);
+    tcg_gen_movi_i32(tcg_ctx, imm, si);
+    tcg_gen_st_i32(tcg_ctx, imm, tcg_ctx->cpu_env,
+                   offsetof(CPUPPCState, vsr[xt].VsrW(0 + ix)));
+    tcg_gen_st_i32(tcg_ctx, imm, tcg_ctx->cpu_env,
+                   offsetof(CPUPPCState, vsr[xt].VsrW(2 + ix)));
+    tcg_temp_free_i32(tcg_ctx, imm);
+}
+
+static int prefixed_8rr_xx_xt(DisasContext *ctx)
+{
+    return (extract32(ctx->opcode, 0, 1) << 5) | rD(ctx->opcode);
+}
+
+static int prefixed_8rr_xx_xa(DisasContext *ctx)
+{
+    return (extract32(ctx->opcode, 2, 1) << 5) | rA(ctx->opcode);
+}
+
+static int prefixed_8rr_xx_xb(DisasContext *ctx)
+{
+    return (extract32(ctx->opcode, 1, 1) << 5) | rB(ctx->opcode);
+}
+
+static int prefixed_8rr_xx_xc(DisasContext *ctx)
+{
+    return (extract32(ctx->opcode, 3, 1) << 5) | rC(ctx->opcode);
+}
+
+static void gen_xxeval_i64(TCGContext *tcg_ctx, TCGv_i64 t, TCGv_i64 a,
+                           TCGv_i64 b, TCGv_i64 c, uint32_t imm)
+{
+    TCGv_i64 conj;
+    TCGv_i64 disj;
+
+    conj = tcg_temp_new_i64(tcg_ctx);
+    disj = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_movi_i64(tcg_ctx, disj, 0);
+
+    while (imm != 0) {
+        int bit = 7 - ctz32(imm);
+
+        if (bit & 0x4) {
+            tcg_gen_mov_i64(tcg_ctx, conj, a);
+        } else {
+            tcg_gen_not_i64(tcg_ctx, conj, a);
+        }
+        if (bit & 0x2) {
+            tcg_gen_and_i64(tcg_ctx, conj, conj, b);
+        } else {
+            tcg_gen_andc_i64(tcg_ctx, conj, conj, b);
+        }
+        if (bit & 0x1) {
+            tcg_gen_and_i64(tcg_ctx, conj, conj, c);
+        } else {
+            tcg_gen_andc_i64(tcg_ctx, conj, conj, c);
+        }
+        tcg_gen_or_i64(tcg_ctx, disj, disj, conj);
+        imm &= imm - 1;
+    }
+
+    tcg_gen_mov_i64(tcg_ctx, t, disj);
+    tcg_temp_free_i64(tcg_ctx, conj);
+    tcg_temp_free_i64(tcg_ctx, disj);
+}
+
+static void gen_xxeval(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = prefixed_8rr_xx_xt(ctx);
+    int xa = prefixed_8rr_xx_xa(ctx);
+    int xb = prefixed_8rr_xx_xb(ctx);
+    int xc = prefixed_8rr_xx_xc(ctx);
+    uint32_t imm = ctx->prefix_opcode & 0xff;
+    TCGv_i64 ah, al, bh, bl, ch, cl, rh, rl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    ah = tcg_temp_new_i64(tcg_ctx);
+    al = tcg_temp_new_i64(tcg_ctx);
+    bh = tcg_temp_new_i64(tcg_ctx);
+    bl = tcg_temp_new_i64(tcg_ctx);
+    ch = tcg_temp_new_i64(tcg_ctx);
+    cl = tcg_temp_new_i64(tcg_ctx);
+    rh = tcg_temp_new_i64(tcg_ctx);
+    rl = tcg_temp_new_i64(tcg_ctx);
+
+    get_cpu_vsrh(tcg_ctx, ah, xa);
+    get_cpu_vsrl(tcg_ctx, al, xa);
+    get_cpu_vsrh(tcg_ctx, bh, xb);
+    get_cpu_vsrl(tcg_ctx, bl, xb);
+    get_cpu_vsrh(tcg_ctx, ch, xc);
+    get_cpu_vsrl(tcg_ctx, cl, xc);
+
+    gen_xxeval_i64(tcg_ctx, rh, ah, bh, ch, imm);
+    gen_xxeval_i64(tcg_ctx, rl, al, bl, cl, imm);
+    set_cpu_vsrh(tcg_ctx, xt, rh);
+    set_cpu_vsrl(tcg_ctx, xt, rl);
+
+    tcg_temp_free_i64(tcg_ctx, ah);
+    tcg_temp_free_i64(tcg_ctx, al);
+    tcg_temp_free_i64(tcg_ctx, bh);
+    tcg_temp_free_i64(tcg_ctx, bl);
+    tcg_temp_free_i64(tcg_ctx, ch);
+    tcg_temp_free_i64(tcg_ctx, cl);
+    tcg_temp_free_i64(tcg_ctx, rh);
+    tcg_temp_free_i64(tcg_ctx, rl);
+}
+
+static void gen_xxpermx(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = prefixed_8rr_xx_xt(ctx);
+    int xa = prefixed_8rr_xx_xa(ctx);
+    int xb = prefixed_8rr_xx_xb(ctx);
+    int xc = prefixed_8rr_xx_xc(ctx);
+    TCGv_ptr xtp;
+    TCGv_ptr xap;
+    TCGv_ptr xbp;
+    TCGv_ptr xcp;
+    TCGv uim;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    xtp = gen_vsr_ptr(tcg_ctx, xt);
+    xap = gen_vsr_ptr(tcg_ctx, xa);
+    xbp = gen_vsr_ptr(tcg_ctx, xb);
+    xcp = gen_vsr_ptr(tcg_ctx, xc);
+    uim = tcg_const_tl(tcg_ctx, extract32(ctx->prefix_opcode, 0, 3));
+
+    gen_helper_XXPERMX(tcg_ctx, xtp, xap, xbp, xcp, uim);
+
+    tcg_temp_free_ptr(tcg_ctx, xtp);
+    tcg_temp_free_ptr(tcg_ctx, xap);
+    tcg_temp_free_ptr(tcg_ctx, xbp);
+    tcg_temp_free_ptr(tcg_ctx, xcp);
+    tcg_temp_free(tcg_ctx, uim);
+}
+
+typedef void (*xxgenpcv_genfn)(TCGContext *tcg_ctx, TCGv_ptr xt,
+                               TCGv_ptr vrb);
+
+static void do_xxgenpcv(DisasContext *ctx, const xxgenpcv_genfn fn[4])
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr xt;
+    TCGv_ptr vrb;
+    int imm = rA(ctx->opcode);
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    if (unlikely(imm & ~0x3)) {
+        gen_invalid(ctx);
+        return;
+    }
+
+    xt = gen_vsr_ptr(tcg_ctx, xT(ctx->opcode));
+    vrb = gen_avr_ptr(tcg_ctx, rB(ctx->opcode));
+
+    fn[imm](tcg_ctx, xt, vrb);
+
+    tcg_temp_free_ptr(tcg_ctx, xt);
+    tcg_temp_free_ptr(tcg_ctx, vrb);
+}
+
+#define XXGENPCV(NAME, HELPER)                                          \
+static void gen_##NAME(DisasContext *ctx)                               \
+{                                                                       \
+    static const xxgenpcv_genfn fn[4] = {                                \
+        gen_helper_##HELPER##_be_exp,                                    \
+        gen_helper_##HELPER##_be_comp,                                   \
+        gen_helper_##HELPER##_le_exp,                                    \
+        gen_helper_##HELPER##_le_comp,                                   \
+    };                                                                  \
+    do_xxgenpcv(ctx, fn);                                               \
+}
+
+XXGENPCV(xxgenpcvbm, XXGENPCVBM)
+XXGENPCV(xxgenpcvhm, XXGENPCVHM)
+XXGENPCV(xxgenpcvwm, XXGENPCVWM)
+XXGENPCV(xxgenpcvdm, XXGENPCVDM)
+#undef XXGENPCV
+
+static void gen_lxvkq(DisasContext *ctx)
+{
+    static const uint64_t values[32] = {
+        0,
+        0x3FFF000000000000ull,
+        0x4000000000000000ull,
+        0x4000800000000000ull,
+        0x4001000000000000ull,
+        0x4001400000000000ull,
+        0x4001800000000000ull,
+        0x4001C00000000000ull,
+        0x7FFF000000000000ull,
+        0x7FFF800000000000ull,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0x8000000000000000ull,
+        0xBFFF000000000000ull,
+        0xC000000000000000ull,
+        0xC000800000000000ull,
+        0xC001000000000000ull,
+        0xC001400000000000ull,
+        0xC001800000000000ull,
+        0xC001C00000000000ull,
+        0xFFFF000000000000ull,
+    };
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 high;
+    TCGv_i64 zero;
+    int uim = rB(ctx->opcode);
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    if (unlikely(rA(ctx->opcode) != 31 || values[uim] == 0)) {
+        gen_invalid(ctx);
+        return;
+    }
+
+    high = tcg_const_i64(tcg_ctx, values[uim]);
+    zero = tcg_const_i64(tcg_ctx, 0);
+
+    set_cpu_vsrh(tcg_ctx, xT(ctx->opcode), high);
+    set_cpu_vsrl(tcg_ctx, xT(ctx->opcode), zero);
+
+    tcg_temp_free_i64(tcg_ctx, high);
+    tcg_temp_free_i64(tcg_ctx, zero);
+}
+
+static void gen_xxblendv_i64(TCGContext *tcg_ctx, TCGv_i64 t, TCGv_i64 a,
+                             TCGv_i64 b, TCGv_i64 c, unsigned vece)
+{
+    static const uint64_t sign_masks[] = {
+        0x8080808080808080ull,
+        0x8000800080008000ull,
+        0x8000000080000000ull,
+        0x8000000000000000ull,
+    };
+    static const uint64_t lane_masks[] = {
+        0xffull,
+        0xffffull,
+        0xffffffffull,
+        0xffffffffffffffffull,
+    };
+    TCGv_i64 mask;
+    TCGv_i64 tmp;
+
+    mask = tcg_temp_new_i64(tcg_ctx);
+    tmp = tcg_temp_new_i64(tcg_ctx);
+
+    if (vece == MO_64) {
+        tcg_gen_sari_i64(tcg_ctx, mask, c, 63);
+    } else {
+        unsigned shift = (8u << vece) - 1;
+
+        tcg_gen_andi_i64(tcg_ctx, mask, c, sign_masks[vece]);
+        tcg_gen_shri_i64(tcg_ctx, mask, mask, shift);
+        tcg_gen_muli_i64(tcg_ctx, mask, mask, lane_masks[vece]);
+    }
+
+    tcg_gen_and_i64(tcg_ctx, tmp, b, mask);
+    tcg_gen_andc_i64(tcg_ctx, t, a, mask);
+    tcg_gen_or_i64(tcg_ctx, t, t, tmp);
+
+    tcg_temp_free_i64(tcg_ctx, mask);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+}
+
+static void gen_xxblendv(DisasContext *ctx, unsigned vece)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    int xt = prefixed_8rr_xx_xt(ctx);
+    int xa = prefixed_8rr_xx_xa(ctx);
+    int xb = prefixed_8rr_xx_xb(ctx);
+    int xc = prefixed_8rr_xx_xc(ctx);
+    TCGv_i64 ah, al, bh, bl, ch, cl, rh, rl;
+
+    if (unlikely(!ctx->vsx_enabled)) {
+        gen_exception(ctx, POWERPC_EXCP_VSXU);
+        return;
+    }
+
+    ah = tcg_temp_new_i64(tcg_ctx);
+    al = tcg_temp_new_i64(tcg_ctx);
+    bh = tcg_temp_new_i64(tcg_ctx);
+    bl = tcg_temp_new_i64(tcg_ctx);
+    ch = tcg_temp_new_i64(tcg_ctx);
+    cl = tcg_temp_new_i64(tcg_ctx);
+    rh = tcg_temp_new_i64(tcg_ctx);
+    rl = tcg_temp_new_i64(tcg_ctx);
+
+    get_cpu_vsrh(tcg_ctx, ah, xa);
+    get_cpu_vsrl(tcg_ctx, al, xa);
+    get_cpu_vsrh(tcg_ctx, bh, xb);
+    get_cpu_vsrl(tcg_ctx, bl, xb);
+    get_cpu_vsrh(tcg_ctx, ch, xc);
+    get_cpu_vsrl(tcg_ctx, cl, xc);
+
+    gen_xxblendv_i64(tcg_ctx, rh, ah, bh, ch, vece);
+    gen_xxblendv_i64(tcg_ctx, rl, al, bl, cl, vece);
+    set_cpu_vsrh(tcg_ctx, xt, rh);
+    set_cpu_vsrl(tcg_ctx, xt, rl);
+
+    tcg_temp_free_i64(tcg_ctx, ah);
+    tcg_temp_free_i64(tcg_ctx, al);
+    tcg_temp_free_i64(tcg_ctx, bh);
+    tcg_temp_free_i64(tcg_ctx, bl);
+    tcg_temp_free_i64(tcg_ctx, ch);
+    tcg_temp_free_i64(tcg_ctx, cl);
+    tcg_temp_free_i64(tcg_ctx, rh);
+    tcg_temp_free_i64(tcg_ctx, rl);
+}
+#endif
+
 static void gen_xxsldwi(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
diff --git a/qemu/target/ppc/translate/vsx-ops.inc.c b/qemu/target/ppc/translate/vsx-ops.inc.c
index 7fd3942b84..23063f7fb9 100644
--- a/qemu/target/ppc/translate/vsx-ops.inc.c
+++ b/qemu/target/ppc/translate/vsx-ops.inc.c
@@ -5,11 +5,17 @@ GEN_HANDLER_E(lxsibzx, 0x1F, 0x0D, 0x18, 0, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(lxsihzx, 0x1F, 0x0D, 0x19, 0, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(lxsspx, 0x1F, 0x0C, 0x10, 0, PPC_NONE, PPC2_VSX207),
 GEN_HANDLER_E(lxvd2x, 0x1F, 0x0C, 0x1A, 0, PPC_NONE, PPC2_VSX),
+GEN_HANDLER_E(lxvwsx, 0x1F, 0x0C, 0x0B, 0, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(lxvdsx, 0x1F, 0x0C, 0x0A, 0, PPC_NONE, PPC2_VSX),
 GEN_HANDLER_E(lxvw4x, 0x1F, 0x0C, 0x18, 0, PPC_NONE, PPC2_VSX),
 GEN_HANDLER_E(lxvh8x, 0x1F, 0x0C, 0x19, 0, PPC_NONE,  PPC2_ISA300),
 GEN_HANDLER_E(lxvb16x, 0x1F, 0x0C, 0x1B, 0, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(lxvx, 0x1F, 0x0C, 0x08, 0x00000040, PPC_NONE, PPC2_ISA300),
+GEN_HANDLER_E(lxvpx, 0x1F, 0x0D, 0x0A, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(lxvrbx, 0x1F, 0x0D, 0x00, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(lxvrhx, 0x1F, 0x0D, 0x01, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(lxvrwx, 0x1F, 0x0D, 0x02, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(lxvrdx, 0x1F, 0x0D, 0x03, 0, PPC_NONE, PPC2_ISA310),
 #if defined(TARGET_PPC64)
 GEN_HANDLER_E(lxvl, 0x1F, 0x0D, 0x08, 0, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(lxvll, 0x1F, 0x0D, 0x09, 0, PPC_NONE, PPC2_ISA300),
@@ -25,6 +31,11 @@ GEN_HANDLER_E(stxvw4x, 0x1F, 0xC, 0x1C, 0, PPC_NONE, PPC2_VSX),
 GEN_HANDLER_E(stxvh8x, 0x1F, 0x0C, 0x1D, 0, PPC_NONE,  PPC2_ISA300),
 GEN_HANDLER_E(stxvb16x, 0x1F, 0x0C, 0x1F, 0, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(stxvx, 0x1F, 0x0C, 0x0C, 0, PPC_NONE, PPC2_ISA300),
+GEN_HANDLER_E(stxvpx, 0x1F, 0x0D, 0x0E, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(stxvrbx, 0x1F, 0x0D, 0x04, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(stxvrhx, 0x1F, 0x0D, 0x05, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(stxvrwx, 0x1F, 0x0D, 0x06, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(stxvrdx, 0x1F, 0x0D, 0x07, 0, PPC_NONE, PPC2_ISA310),
 #if defined(TARGET_PPC64)
 GEN_HANDLER_E(stxvl, 0x1F, 0x0D, 0x0C, 0, PPC_NONE, PPC2_ISA300),
 GEN_HANDLER_E(stxvll, 0x1F, 0x0D, 0x0D, 0, PPC_NONE, PPC2_ISA300),
@@ -109,6 +120,12 @@ GEN_HANDLER_E(name, 0x3F, opc2, opc3, inval, PPC_NONE, PPC2_ISA300)
 #define GEN_VSX_XFORM_300_EO(name, opc2, opc3, opc4, inval)             \
 GEN_HANDLER_E_2(name, 0x3F, opc2, opc3, opc4, inval, PPC_NONE, PPC2_ISA300)
 
+#define GEN_VSX_XFORM_310(name, opc2, opc3, inval) \
+GEN_HANDLER_E(name, 0x3F, opc2, opc3, inval, PPC_NONE, PPC2_ISA310)
+
+#define GEN_VSX_XFORM_310_EO(name, opc2, opc3, opc4, inval)             \
+GEN_HANDLER_E_2(name, 0x3F, opc2, opc3, opc4, inval, PPC_NONE, PPC2_ISA310)
+
 #define GEN_VSX_Z23FORM_300(name, opc2, opc3, opc4, inval) \
 GEN_VSX_XFORM_300_EO(name, opc2, opc3 | 0x00, opc4 | 0x0, inval), \
 GEN_VSX_XFORM_300_EO(name, opc2, opc3 | 0x08, opc4 | 0x0, inval), \
@@ -123,6 +140,10 @@ GEN_VSX_Z23FORM_300(xsrqpi, 0x05, 0x0, 0x0, 0x0),
 GEN_VSX_Z23FORM_300(xsrqpxp, 0x05, 0x1, 0x0, 0x0),
 GEN_VSX_XFORM_300_EO(xssqrtqp, 0x04, 0x19, 0x1B, 0x0),
 GEN_VSX_XFORM_300(xssubqp, 0x04, 0x10, 0x0),
+GEN_VSX_XFORM_300(XSMADDQP, 0x04, 0x0C, 0x0),
+GEN_VSX_XFORM_300(XSMSUBQP, 0x04, 0x0D, 0x0),
+GEN_VSX_XFORM_300(XSNMADDQP, 0x04, 0x0E, 0x0),
+GEN_VSX_XFORM_300(XSNMSUBQP, 0x04, 0x0F, 0x0),
 
 GEN_XX2FORM(xsabsdp, 0x12, 0x15, PPC2_VSX),
 GEN_XX2FORM(xsnabsdp, 0x12, 0x16, PPC2_VSX),
@@ -135,6 +156,10 @@ GEN_VSX_XFORM_300_EO(xsnegqp, 0x04, 0x19, 0x10, 0x00000001),
 GEN_VSX_XFORM_300(xscpsgnqp, 0x04, 0x03, 0x00000001),
 GEN_VSX_XFORM_300_EO(xscvdpqp, 0x04, 0x1A, 0x16, 0x00000001),
 GEN_VSX_XFORM_300_EO(xscvqpdp, 0x04, 0x1A, 0x14, 0x0),
+GEN_VSX_XFORM_310_EO(XSCVQPUQZ, 0x04, 0x1A, 0x00, 0x00000001),
+GEN_VSX_XFORM_310_EO(XSCVQPSQZ, 0x04, 0x1A, 0x08, 0x00000001),
+GEN_VSX_XFORM_310_EO(XSCVUQQP, 0x04, 0x1A, 0x03, 0x00000001),
+GEN_VSX_XFORM_310_EO(XSCVSQQP, 0x04, 0x1A, 0x0B, 0x00000001),
 GEN_VSX_XFORM_300_EO(xscvqpsdz, 0x04, 0x1A, 0x19, 0x00000001),
 GEN_VSX_XFORM_300_EO(xscvqpswz, 0x04, 0x1A, 0x09, 0x00000001),
 GEN_VSX_XFORM_300_EO(xscvqpudz, 0x04, 0x1A, 0x11, 0x00000001),
@@ -147,6 +172,10 @@ GEN_XX2FORM_EO(xsxsigdp, 0x16, 0x15, 0x01, PPC2_ISA300),
 GEN_VSX_XFORM_300_EO(xsxsigqp, 0x04, 0x19, 0x12, 0x00000001),
 GEN_HANDLER_E(xsiexpdp, 0x3C, 0x16, 0x1C, 0, PPC_NONE, PPC2_ISA300),
 GEN_VSX_XFORM_300(xsiexpqp, 0x4, 0x1B, 0x00000001),
+GEN_HANDLER_E(xxgenpcvbm, 0x3C, 0x14, 0x1C, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(xxgenpcvhm, 0x3C, 0x15, 0x1C, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(xxgenpcvwm, 0x3C, 0x14, 0x1D, 0, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER_E(xxgenpcvdm, 0x3C, 0x15, 0x1D, 0, PPC_NONE, PPC2_ISA310),
 #endif
 
 GEN_XX2FORM(xststdcdp, 0x14, 0x16, PPC2_ISA300),
@@ -159,6 +188,10 @@ GEN_XX2FORM_EO(xvxexpdp, 0x16, 0x1D, 0x00, PPC2_ISA300),
 GEN_XX2FORM_EO(xvxsigdp, 0x16, 0x1D, 0x01, PPC2_ISA300),
 GEN_XX2FORM_EO(xvxexpsp, 0x16, 0x1D, 0x08, PPC2_ISA300),
 GEN_XX2FORM_EO(xvxsigsp, 0x16, 0x1D, 0x09, PPC2_ISA300),
+GEN_HANDLER2_E_2(xvtlsbb, "xvtlsbb", 0x3C, 0x16, 0x1D, 0x02,
+                 0x00600001, PPC_NONE, PPC2_ISA310),
+GEN_HANDLER2_E_2(xvtlsbb, "xvtlsbb", 0x3C, 0x17, 0x1D, 0x02,
+                 0x00600001, PPC_NONE, PPC2_ISA310),
 
 /* DCMX  =  bit[25] << 6 | bit[29] << 5 | bit[11:15] */
 #define GEN_XX2FORM_DCMX(name, opc2, opc3, fl2) \
@@ -202,6 +235,9 @@ GEN_XX3FORM(xscmpgedp, 0x0C, 0x02, PPC2_ISA300),
 GEN_XX3FORM(xscmpnedp, 0x0C, 0x03, PPC2_ISA300),
 GEN_XX3FORM(xscmpexpdp, 0x0C, 0x07, PPC2_ISA300),
 GEN_VSX_XFORM_300(xscmpexpqp, 0x04, 0x05, 0x00600001),
+GEN_VSX_XFORM_310(XSCMPEQQP, 0x04, 0x02, 0x00000001),
+GEN_VSX_XFORM_310(XSCMPGEQP, 0x04, 0x06, 0x00000001),
+GEN_VSX_XFORM_310(XSCMPGTQP, 0x04, 0x07, 0x00000001),
 GEN_XX2IFORM(xscmpodp,  0x0C, 0x05, PPC2_VSX),
 GEN_XX2IFORM(xscmpudp,  0x0C, 0x04, PPC2_VSX),
 GEN_VSX_XFORM_300(xscmpoqp, 0x04, 0x04, 0x00600001),
@@ -212,6 +248,8 @@ GEN_XX3FORM(xsmaxcdp, 0x00, 0x10, PPC2_ISA300),
 GEN_XX3FORM(xsmincdp, 0x00, 0x11, PPC2_ISA300),
 GEN_XX3FORM(xsmaxjdp, 0x00, 0x12, PPC2_ISA300),
 GEN_XX3FORM(xsminjdp, 0x00, 0x13, PPC2_ISA300),
+GEN_VSX_XFORM_310(XSMAXCQP, 0x04, 0x15, 0x00000001),
+GEN_VSX_XFORM_310(XSMINCQP, 0x04, 0x17, 0x00000001),
 GEN_XX2FORM_EO(xscvdphp, 0x16, 0x15, 0x11, PPC2_ISA300),
 GEN_XX2FORM(xscvdpsp, 0x12, 0x10, PPC2_VSX),
 GEN_XX2FORM(xscvdpspn, 0x16, 0x10, PPC2_VSX207),
@@ -332,6 +370,8 @@ GEN_XX2FORM_EO(xxbrw, 0x16, 0x1D, 0x0F, PPC2_ISA300),
 GEN_XX2FORM_EO(xxbrd, 0x16, 0x1D, 0x17, PPC2_ISA300),
 GEN_XX2FORM_EO(xvcvhpsp, 0x16, 0x1D, 0x18, PPC2_ISA300),
 GEN_XX2FORM_EO(xvcvsphp, 0x16, 0x1D, 0x19, PPC2_ISA300),
+GEN_XX2FORM_EO(XVCVBF16SPN, 0x16, 0x1D, 0x10, PPC2_ISA310),
+GEN_XX2FORM_EO(XVCVSPBF16, 0x16, 0x1D, 0x11, PPC2_ISA310),
 GEN_XX2FORM_EO(xxbrq, 0x16, 0x1D, 0x1F, PPC2_ISA300),
 
 #define VSX_LOGICAL(name, opc2, opc3, fl2) \
diff --git a/qemu/target/ppc/translate_init.inc.c b/qemu/target/ppc/translate_init.inc.c
index 15ec4bce04..16d1026625 100644
--- a/qemu/target/ppc/translate_init.inc.c
+++ b/qemu/target/ppc/translate_init.inc.c
@@ -8410,7 +8410,7 @@ POWERPC_FAMILY(POWER7)(CPUClass *oc, void *data)
                         PPC2_PERM_ISA206 | PPC2_DIVE_ISA206 |
                         PPC2_ATOMIC_ISA206 | PPC2_FP_CVT_ISA206 |
                         PPC2_FP_TST_ISA206 | PPC2_FP_CVT_S64 |
-                        PPC2_PM_ISA206;
+                        PPC2_PM_ISA206 | PPC2_BCDA_ISA206;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_VR) |
                     (1ull << MSR_VSX) |
@@ -8578,7 +8578,7 @@ POWERPC_FAMILY(POWER8)(CPUClass *oc, void *data)
                         PPC2_FP_TST_ISA206 | PPC2_BCTAR_ISA207 |
                         PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 |
                         PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64 |
-                        PPC2_TM | PPC2_PM_ISA206;
+                        PPC2_TM | PPC2_PM_ISA206 | PPC2_BCDA_ISA206;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_HV) |
                     (1ull << MSR_TM) |
@@ -8789,7 +8789,8 @@ POWERPC_FAMILY(POWER9)(CPUClass *oc, void *data)
                         PPC2_FP_TST_ISA206 | PPC2_BCTAR_ISA207 |
                         PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 |
                         PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64 |
-                        PPC2_TM | PPC2_ISA300 | PPC2_PRCNTL;
+                        PPC2_TM | PPC2_ISA300 | PPC2_PRCNTL |
+                        PPC2_BCDA_ISA206;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_HV) |
                     (1ull << MSR_TM) |
@@ -8851,6 +8852,17 @@ static struct ppc_radix_page_info POWER10_radix_page_info = {
     }
 };
 
+static void register_power10_hash_sprs(CPUPPCState *env)
+{
+    spr_register(env, SPR_HASHKEYR, "HASHKEYR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 spr_read_generic, spr_write_generic, 0);
+    spr_register_hv(env, SPR_HASHPKEYR, "HASHPKEYR",
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    spr_read_generic, spr_write_generic, 0);
+}
+
 static void init_proc_POWER10(CPUPPCState *env)
 {
     /* Common Registers */
@@ -8890,6 +8902,7 @@ static void init_proc_POWER10(CPUPPCState *env)
     spr_register_kvm_hv(env, SPR_PSSCR, "PSSCR", NULL, NULL, NULL, NULL,
                         spr_read_generic, spr_write_generic,
                         KVM_REG_PPC_PSSCR, 0);
+    register_power10_hash_sprs(env);
 
     /* env variables */
     env->dcache_line_size = 128;
@@ -9000,7 +9013,8 @@ POWERPC_FAMILY(POWER10)(CPUClass *oc, void *data)
                         PPC2_FP_TST_ISA206 | PPC2_BCTAR_ISA207 |
                         PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 |
                         PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64 |
-                        PPC2_TM | PPC2_ISA300 | PPC2_PRCNTL;
+                        PPC2_TM | PPC2_ISA300 | PPC2_PRCNTL |
+                        PPC2_ISA310 | PPC2_BCDA_ISA206;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_HV) |
                     (1ull << MSR_TM) |
@@ -10121,6 +10135,14 @@ static void ppc_cpu_reset(CPUState *dev)
     if (env->mmu_model & POWERPC_MMU_64) {
         msr |= (1ULL << MSR_SF);
     }
+    /*
+     * Power-on reset enters HV mode on server CPUs. The compact init path
+     * reaches here with env->msr cleared, while hreg_store_msr() only changes
+     * HV from an existing HV state.
+     */
+    if (msr & MSR_HVB) {
+        env->msr |= MSR_HVB;
+    }
 #endif
 
     hreg_store_msr(env, msr, 1);
@@ -11015,6 +11037,8 @@ PowerPCCPU *cpu_ppc_init(struct uc_struct *uc)
     } else if (uc->cpu_model + UC_CPU_PPC32_7457A_V1_2 + 1 >= ARRAY_SIZE(ppc_cpus)) {
         free(cpu);
         return NULL;
+    } else {
+        uc->cpu_model += UC_CPU_PPC32_7457A_V1_2 + 1;
     }
 #else
     if (uc->cpu_model == INT_MAX) {
diff --git a/qemu/target/riscv/bitmanip_helper.c b/qemu/target/riscv/bitmanip_helper.c
new file mode 100644
index 0000000000..c91233eafc
--- /dev/null
+++ b/qemu/target/riscv/bitmanip_helper.c
@@ -0,0 +1,133 @@
+/*
+ * RISC-V Bitmanip Extension Helpers for QEMU.
+ *
+ * Copyright (c) 2020 Kito Cheng, kito.cheng@sifive.com
+ * Copyright (c) 2020 Frank Chang, frank.chang@sifive.com
+ * Copyright (c) 2021 Philipp Tomsich, philipp.tomsich@vrull.eu
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/helper-proto.h"
+
+target_ulong HELPER(clmul)(target_ulong rs1, target_ulong rs2)
+{
+    target_ulong result = 0;
+    int i;
+
+    for (i = 0; i < TARGET_LONG_BITS; i++) {
+        if ((rs2 >> i) & 1) {
+            result ^= (rs1 << i);
+        }
+    }
+
+    return result;
+}
+
+target_ulong HELPER(clmulr)(target_ulong rs1, target_ulong rs2)
+{
+    target_ulong result = 0;
+    int i;
+
+    for (i = 0; i < TARGET_LONG_BITS; i++) {
+        if ((rs2 >> i) & 1) {
+            result ^= (rs1 >> (TARGET_LONG_BITS - i - 1));
+        }
+    }
+
+    return result;
+}
+
+static inline target_ulong do_swap(target_ulong x, uint64_t mask, int shift)
+{
+    return ((x & mask) << shift) | ((x & ~mask) >> shift);
+}
+
+target_ulong HELPER(brev8)(target_ulong rs1)
+{
+    target_ulong x = rs1;
+
+    x = do_swap(x, 0x5555555555555555ull, 1);
+    x = do_swap(x, 0x3333333333333333ull, 2);
+    x = do_swap(x, 0x0f0f0f0f0f0f0f0full, 4);
+    return x;
+}
+
+static const uint64_t shuf_masks[] = {
+    0x4444444444444444ull,
+    0x3030303030303030ull,
+    0x0f000f000f000f00ull,
+    0x00ff000000ff0000ull
+};
+
+static inline target_ulong do_shuf_stage(target_ulong src, uint64_t mask_l,
+                                         uint64_t mask_r, int shift)
+{
+    target_ulong x = src & ~(mask_l | mask_r);
+
+    x |= ((src << shift) & mask_l) | ((src >> shift) & mask_r);
+    return x;
+}
+
+target_ulong HELPER(unzip)(target_ulong rs1)
+{
+    target_ulong x = rs1;
+
+    x = do_shuf_stage(x, shuf_masks[0], shuf_masks[0] >> 1, 1);
+    x = do_shuf_stage(x, shuf_masks[1], shuf_masks[1] >> 2, 2);
+    x = do_shuf_stage(x, shuf_masks[2], shuf_masks[2] >> 4, 4);
+    x = do_shuf_stage(x, shuf_masks[3], shuf_masks[3] >> 8, 8);
+    return x;
+}
+
+target_ulong HELPER(zip)(target_ulong rs1)
+{
+    target_ulong x = rs1;
+
+    x = do_shuf_stage(x, shuf_masks[3], shuf_masks[3] >> 8, 8);
+    x = do_shuf_stage(x, shuf_masks[2], shuf_masks[2] >> 4, 4);
+    x = do_shuf_stage(x, shuf_masks[1], shuf_masks[1] >> 2, 2);
+    x = do_shuf_stage(x, shuf_masks[0], shuf_masks[0] >> 1, 1);
+    return x;
+}
+
+static inline target_ulong do_xperm(target_ulong rs1, target_ulong rs2,
+                                    uint32_t sz_log2)
+{
+    target_ulong r = 0;
+    target_ulong sz = 1ull << sz_log2;
+    target_ulong mask = (1ull << sz) - 1;
+    target_ulong pos;
+    int i;
+
+    for (i = 0; i < TARGET_LONG_BITS; i += sz) {
+        pos = ((rs2 >> i) & mask) << sz_log2;
+        if (pos < sizeof(target_ulong) * 8) {
+            r |= ((rs1 >> pos) & mask) << i;
+        }
+    }
+
+    return r;
+}
+
+target_ulong HELPER(xperm4)(target_ulong rs1, target_ulong rs2)
+{
+    return do_xperm(rs1, rs2, 2);
+}
+
+target_ulong HELPER(xperm8)(target_ulong rs1, target_ulong rs2)
+{
+    return do_xperm(rs1, rs2, 3);
+}
diff --git a/qemu/target/riscv/cpu-param.h b/qemu/target/riscv/cpu-param.h
index 664fc1d371..80eb615f93 100644
--- a/qemu/target/riscv/cpu-param.h
+++ b/qemu/target/riscv/cpu-param.h
@@ -18,6 +18,15 @@
 # define TARGET_VIRT_ADDR_SPACE_BITS 32 /* sv32 */
 #endif
 #define TARGET_PAGE_BITS 12 /* 4 KiB Pages */
-#define NB_MMU_MODES 4
+/*
+ * The current MMU Modes are:
+ *  - U mode 0b000
+ *  - S mode 0b001
+ *  - M mode 0b011
+ *  - U mode HLV/HLVX/HSV 0b100
+ *  - S mode HLV/HLVX/HSV 0b101
+ *  - M mode HLV/HLVX/HSV 0b111
+ */
+#define NB_MMU_MODES 8
 
 #endif
diff --git a/qemu/target/riscv/cpu.c b/qemu/target/riscv/cpu.c
index 2313cfc6cc..c2cc03acb4 100644
--- a/qemu/target/riscv/cpu.c
+++ b/qemu/target/riscv/cpu.c
@@ -20,6 +20,7 @@
 #include "qemu/osdep.h"
 #include "qemu/ctype.h"
 #include "qemu/log.h"
+#include "qemu/timer.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "fpu/softfloat-helpers.h"
@@ -67,10 +68,15 @@ static void set_resetvec(CPURISCVState *env, int resetvec)
     env->resetvec = resetvec;
 }
 
+static uint64_t riscv_default_rdtime(void)
+{
+    return cpu_get_host_ticks();
+}
+
 static void riscv_any_cpu_init(CPUState *obj)
 {
     CPURISCVState *env = &RISCV_CPU(obj)->env;
-    set_misa(env, RVXLEN | RVI | RVM | RVA | RVF | RVD | RVC | RVU);
+    set_misa(env, RVXLEN | RVI | RVM | RVA | RVF | RVD | RVV | RVC | RVU);
     set_priv_version(env, PRIV_VERSION_1_11_0);
     set_resetvec(env, DEFAULT_RSTVEC);
 }
@@ -88,7 +94,8 @@ static void riscv_base32_cpu_init(CPUState *obj)
 static void rv32gcsu_priv1_10_0_cpu_init(CPUState *obj)
 {
     CPURISCVState *env = &RISCV_CPU(obj)->env;
-    set_misa(env, RV32 | RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU);
+    set_misa(env, RV32 | RVI | RVM | RVA | RVF | RVD | RVV | RVC | RVS |
+             RVU);
     set_priv_version(env, PRIV_VERSION_1_10_0);
     set_resetvec(env, DEFAULT_RSTVEC);
     set_feature(env, RISCV_FEATURE_MMU);
@@ -118,7 +125,8 @@ static void riscv_base64_cpu_init(CPUState *obj)
 static void rv64gcsu_priv1_10_0_cpu_init(CPUState *obj)
 {
     CPURISCVState *env = &RISCV_CPU(obj)->env;
-    set_misa(env, RV64 | RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU);
+    set_misa(env, RV64 | RVI | RVM | RVA | RVF | RVD | RVV | RVC | RVS |
+             RVU);
     set_priv_version(env, PRIV_VERSION_1_10_0);
     set_resetvec(env, DEFAULT_RSTVEC);
     set_feature(env, RISCV_FEATURE_MMU);
@@ -165,6 +173,7 @@ void restore_state_to_opc(CPURISCVState *env, TranslationBlock *tb,
                           target_ulong *data)
 {
     env->pc = data[0];
+    env->bins = data[1];
 }
 
 static void riscv_cpu_reset(CPUState *dev)
@@ -180,6 +189,15 @@ static void riscv_cpu_reset(CPUState *dev)
     env->mstatus &= ~(MSTATUS_MIE | MSTATUS_MPRV);
     env->mcause = 0;
     env->pc = env->resetvec;
+    env->bins = 0;
+    env->two_stage_lookup = false;
+    env->two_stage_indirect_lookup = false;
+    env->vxrm = 0;
+    env->vxsat = 0;
+    env->vl = 0;
+    env->vstart = 0;
+    env->vtype = 0;
+    env->vill = false;
 
     cs->exception_index = EXCP_NONE;
     env->load_res = -1;
@@ -274,6 +292,9 @@ static void riscv_cpu_realize(struct uc_struct *uc, CPUState *dev)
         if (cpu->cfg.ext_h) {
             target_misa |= RVH;
         }
+        if (cpu->cfg.ext_v) {
+            target_misa |= RVV;
+        }
 
         set_misa(env, RVXLEN | target_misa);
     }
@@ -334,8 +355,9 @@ RISCVCPU *cpu_riscv_init(struct uc_struct *uc)
     RISCVCPU *cpu;
     CPUState *cs;
     CPUClass *cc;
+    CPURISCVState *env;
 
-    cpu = qemu_memalign(8, sizeof(*cpu));
+    cpu = qemu_memalign(16, sizeof(*cpu));
     if (cpu == NULL) {
         return NULL;
     }
@@ -380,10 +402,37 @@ RISCVCPU *cpu_riscv_init(struct uc_struct *uc)
     cpu->cfg.ext_c = true;
     cpu->cfg.ext_s = true;
     cpu->cfg.ext_u = true;
-    cpu->cfg.ext_h = false;
+    cpu->cfg.ext_h = true;
+    cpu->cfg.ext_v = true;
     cpu->cfg.ext_counters = true;
     cpu->cfg.ext_ifencei = true;
     cpu->cfg.ext_icsr = true;
+    cpu->cfg.ext_zihintpause = true;
+    cpu->cfg.ext_zba = true;
+    cpu->cfg.ext_zbb = true;
+    cpu->cfg.ext_zbc = true;
+    cpu->cfg.ext_zbkb = true;
+    cpu->cfg.ext_zbkc = false;
+    cpu->cfg.ext_zbkx = true;
+    cpu->cfg.ext_zbs = true;
+    cpu->cfg.ext_zfh = true;
+    cpu->cfg.ext_zfhmin = true;
+    cpu->cfg.ext_zknd = true;
+    cpu->cfg.ext_zkne = true;
+    cpu->cfg.ext_zknh = true;
+    cpu->cfg.ext_zkr = true;
+    cpu->cfg.ext_zksed = true;
+    cpu->cfg.ext_zksh = true;
+    cpu->cfg.ext_svinval = true;
+    cpu->cfg.ext_xventanacondops = true;
+    cpu->cfg.ext_sstc = true;
+    cpu->cfg.ext_zmmul = true;
+    cpu->cfg.ext_zve32f = true;
+    cpu->cfg.ext_zve64f = true;
+    cpu->cfg.rvv_ta_all_1s = true;
+    cpu->cfg.rvv_ma_all_1s = true;
+    cpu->cfg.vlen = 128;
+    cpu->cfg.elen = 64;
     cpu->cfg.priv_spec = "v1.11.0";
     cpu->cfg.mmu = true;
     cpu->cfg.pmp = true;
@@ -396,6 +445,27 @@ RISCVCPU *cpu_riscv_init(struct uc_struct *uc)
 
     /* init specific CPU model */
     cpu_models[uc->cpu_model].initfn(cs);
+    env = &cpu->env;
+    if (env->misa != 0) {
+        cpu->cfg.ext_s = riscv_has_ext(env, RVS);
+        cpu->cfg.ext_h = riscv_has_ext(env, RVH);
+        if (!cpu->cfg.ext_s) {
+            cpu->cfg.ext_svinval = false;
+            cpu->cfg.ext_sstc = false;
+        }
+        cpu->cfg.ext_f = riscv_has_ext(env, RVF);
+        cpu->cfg.ext_d = riscv_has_ext(env, RVD);
+        if (!cpu->cfg.ext_f) {
+            cpu->cfg.ext_zfh = false;
+            cpu->cfg.ext_zfhmin = false;
+        }
+        if (!riscv_has_ext(env, RVV)) {
+            cpu->cfg.ext_v = false;
+            cpu->cfg.ext_zve32f = false;
+            cpu->cfg.ext_zve64f = false;
+        }
+    }
+    riscv_cpu_set_rdtime_fn(env, riscv_default_rdtime);
 
     /* realize CPU */
     riscv_cpu_realize(uc, cs);
diff --git a/qemu/target/riscv/cpu.h b/qemu/target/riscv/cpu.h
index b94516eb7c..3e59e1ea84 100644
--- a/qemu/target/riscv/cpu.h
+++ b/qemu/target/riscv/cpu.h
@@ -20,7 +20,10 @@
 #ifndef RISCV_CPU_H
 #define RISCV_CPU_H
 
+#define TARGET_INSN_START_EXTRA_WORDS 1
+
 #include "hw/core/cpu.h"
+#include "hw/registerfields.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat-types.h"
 
@@ -57,6 +60,7 @@ typedef struct TCGContext TCGContext;
 #define RVA RV('A')
 #define RVF RV('F')
 #define RVD RV('D')
+#define RVV RV('V')
 #define RVC RV('C')
 #define RVS RV('S')
 #define RVU RV('U')
@@ -79,9 +83,25 @@ enum {
 #define TRANSLATE_PMP_FAIL 2
 #define TRANSLATE_FAIL 1
 #define TRANSLATE_SUCCESS 0
+#define TRANSLATE_G_STAGE_FAIL 3
 #define MMU_USER_IDX 3
 
 #define MAX_RISCV_PMPS (16)
+#define RV_VLEN_MAX 1024
+
+FIELD(VTYPE, VLMUL, 0, 3)
+FIELD(VTYPE, VSEW, 3, 3)
+FIELD(VTYPE, VTA, 6, 1)
+FIELD(VTYPE, VMA, 7, 1)
+FIELD(VTYPE, VEDIV, 8, 2)
+#define R_VTYPE_RESERVED_SHIFT 10
+
+FIELD(VDATA, VM, 0, 1)
+FIELD(VDATA, LMUL, 1, 3)
+FIELD(VDATA, VTA, 4, 1)
+FIELD(VDATA, VTA_ALL_1S, 5, 1)
+FIELD(VDATA, VMA, 6, 1)
+FIELD(VDATA, NF, 7, 4)
 
 typedef struct CPURISCVState CPURISCVState;
 
@@ -90,6 +110,13 @@ typedef struct CPURISCVState CPURISCVState;
 struct CPURISCVState {
     target_ulong gpr[32];
     uint64_t fpr[32]; /* assume both F and D extensions */
+    QEMU_ALIGN(16, uint64_t vreg[32 * RV_VLEN_MAX / 64]);
+    target_ulong vxrm;
+    target_ulong vxsat;
+    target_ulong vl;
+    target_ulong vstart;
+    target_ulong vtype;
+    bool vill;
     target_ulong pc;
     target_ulong load_res;
     target_ulong load_val;
@@ -97,7 +124,10 @@ struct CPURISCVState {
     target_ulong frm;
 
     target_ulong badaddr;
+    target_ulong bins;
     target_ulong guest_phys_fault_addr;
+    bool two_stage_lookup;
+    bool two_stage_indirect_lookup;
 
     target_ulong priv_ver;
     target_ulong misa;
@@ -186,6 +216,8 @@ struct CPURISCVState {
     uint64_t mfromhost;
     uint64_t mtohost;
     uint64_t timecmp;
+    uint64_t stimecmp;
+    uint64_t vstimecmp;
 
     /* physical memory protection */
     pmp_table_t pmp_state;
@@ -230,7 +262,7 @@ typedef struct RISCVCPU {
     CPUState parent_obj;
     /*< public >*/
     CPUNegativeOffsetState neg;
-    CPURISCVState env;
+    QEMU_ALIGN(16, CPURISCVState env);
 
     /* Configuration Settings */
     struct {
@@ -245,9 +277,36 @@ typedef struct RISCVCPU {
         bool ext_s;
         bool ext_u;
         bool ext_h;
+        bool ext_v;
         bool ext_counters;
         bool ext_ifencei;
         bool ext_icsr;
+        bool ext_zihintpause;
+        bool ext_zba;
+        bool ext_zbb;
+        bool ext_zbc;
+        bool ext_zbkb;
+        bool ext_zbkc;
+        bool ext_zbkx;
+        bool ext_zbs;
+        bool ext_zfh;
+        bool ext_zfhmin;
+        bool ext_zknd;
+        bool ext_zkne;
+        bool ext_zknh;
+        bool ext_zkr;
+        bool ext_zksed;
+        bool ext_zksh;
+        bool ext_svinval;
+        bool ext_xventanacondops;
+        bool ext_sstc;
+        bool ext_zmmul;
+        bool ext_zve32f;
+        bool ext_zve64f;
+        bool rvv_ta_all_1s;
+        bool rvv_ma_all_1s;
+        uint16_t vlen;
+        uint16_t elen;
 
         char *priv_spec;
         char *user_spec;
@@ -272,6 +331,14 @@ static inline bool riscv_feature(CPURISCVState *env, int feature)
     return env->features & (1ULL << feature);
 }
 
+static inline uint32_t vext_get_vlmax(RISCVCPU *cpu, target_ulong vtype)
+{
+    uint8_t sew = FIELD_EX64(vtype, VTYPE, VSEW);
+    int8_t lmul = sextract32(FIELD_EX64(vtype, VTYPE, VLMUL), 0, 3);
+
+    return cpu->cfg.vlen >> (sew + 3 - lmul);
+}
+
 #include "cpu_user.h"
 #include "cpu_bits.h"
 
@@ -285,6 +352,7 @@ int riscv_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg);
 int riscv_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request);
 bool riscv_cpu_fp_enabled(CPURISCVState *env);
+bool riscv_cpu_vector_enabled(CPURISCVState *env);
 bool riscv_cpu_virt_enabled(CPURISCVState *env);
 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable);
 bool riscv_cpu_force_hs_excep_enabled(CPURISCVState *env);
@@ -320,18 +388,53 @@ void QEMU_NORETURN riscv_raise_exception(CPURISCVState *env,
 target_ulong riscv_cpu_get_fflags(CPURISCVState *env);
 void riscv_cpu_set_fflags(CPURISCVState *env, target_ulong);
 
-#define TB_FLAGS_MMU_MASK   3
+#define TB_FLAGS_PRIV_MMU_MASK          3
+#define TB_FLAGS_PRIV_HYP_ACCESS_MASK   (1 << 2)
+#define TB_FLAGS_MMU_MASK               7
 #define TB_FLAGS_MSTATUS_FS MSTATUS_FS
+#define TB_FLAGS_MSTATUS_VS MSTATUS_VS
+
+FIELD(TB_FLAGS, LMUL, 3, 3)
+FIELD(TB_FLAGS, SEW, 6, 3)
+FIELD(TB_FLAGS, VILL, 12, 1)
+FIELD(TB_FLAGS, HLSX, 15, 1)
+FIELD(TB_FLAGS, VTA, 24, 1)
+FIELD(TB_FLAGS, VMA, 25, 1)
 
 static inline void cpu_get_tb_cpu_state(CPURISCVState *env, target_ulong *pc,
                                         target_ulong *cs_base, uint32_t *flags)
 {
+    uint32_t tb_flags;
+    uint32_t vtype;
+
     *pc = env->pc;
     *cs_base = 0;
-    *flags = cpu_mmu_index(env, 0);
+    tb_flags = cpu_mmu_index(env, 0);
     if (riscv_cpu_fp_enabled(env)) {
-        *flags |= env->mstatus & MSTATUS_FS;
+        tb_flags |= env->mstatus & MSTATUS_FS;
+    }
+    if (riscv_cpu_vector_enabled(env)) {
+        tb_flags |= env->mstatus & MSTATUS_VS;
+        vtype = env->vtype;
+        FIELD_DP32(tb_flags, TB_FLAGS, LMUL,
+                   FIELD_EX64(vtype, VTYPE, VLMUL), tb_flags);
+        FIELD_DP32(tb_flags, TB_FLAGS, SEW,
+                   FIELD_EX64(vtype, VTYPE, VSEW), tb_flags);
+        FIELD_DP32(tb_flags, TB_FLAGS, VILL, env->vill, tb_flags);
+        FIELD_DP32(tb_flags, TB_FLAGS, VTA,
+                   FIELD_EX64(vtype, VTYPE, VTA), tb_flags);
+        FIELD_DP32(tb_flags, TB_FLAGS, VMA,
+                   FIELD_EX64(vtype, VTYPE, VMA), tb_flags);
+    }
+    if (riscv_has_ext(env, RVH)) {
+        bool hlsx = env->priv == PRV_M ||
+                    (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) ||
+                    (env->priv == PRV_U && !riscv_cpu_virt_enabled(env) &&
+                     get_field(env->hstatus, HSTATUS_HU));
+
+        FIELD_DP32(tb_flags, TB_FLAGS, HLSX, hlsx, tb_flags);
     }
+    *flags = tb_flags;
 }
 
 int riscv_csrrw(CPURISCVState *env, int csrno, target_ulong *ret_value,
diff --git a/qemu/target/riscv/cpu_bits.h b/qemu/target/riscv/cpu_bits.h
index ffa73864a9..873388e8c2 100644
--- a/qemu/target/riscv/cpu_bits.h
+++ b/qemu/target/riscv/cpu_bits.h
@@ -12,6 +12,10 @@
 /* Floating point round mode */
 #define FSR_RD_SHIFT        5
 #define FSR_RD              (0x7 << FSR_RD_SHIFT)
+#define FSR_VXRM_SHIFT      9
+#define FSR_VXRM            (0x3 << FSR_VXRM_SHIFT)
+#define FSR_VXSAT_SHIFT     8
+#define FSR_VXSAT           (0x1 << FSR_VXSAT_SHIFT)
 
 /* Floating point accrued exception flags */
 #define FPEXC_NX            0x01
@@ -47,6 +51,22 @@
 #define CSR_FFLAGS          0x001
 #define CSR_FRM             0x002
 #define CSR_FCSR            0x003
+#define CSR_SEED            0x015
+
+/* User Vector CSRs */
+#define CSR_VSTART          0x008
+#define CSR_VXSAT           0x009
+#define CSR_VXRM            0x00a
+#define CSR_VCSR            0x00f
+#define CSR_VL              0xc20
+#define CSR_VTYPE           0xc21
+#define CSR_VLENB           0xc22
+
+/* VCSR fields */
+#define VCSR_VXSAT_SHIFT    0
+#define VCSR_VXSAT          (0x1 << VCSR_VXSAT_SHIFT)
+#define VCSR_VXRM_SHIFT     1
+#define VCSR_VXRM           (0x3 << VCSR_VXRM_SHIFT)
 
 /* User Timers and Counters */
 #define CSR_CYCLE           0xc00
@@ -171,6 +191,8 @@
 
 /* Legacy Supervisor Trap Handling (priv v1.9.1) */
 #define CSR_SBADADDR        0x143
+#define CSR_STIMECMP        0x14D
+#define CSR_STIMECMPH       0x15D
 
 /* Supervisor Protection and Translation */
 #define CSR_SPTBR           0x180
@@ -209,6 +231,8 @@
 #define CSR_VSCAUSE         0x242
 #define CSR_VSTVAL          0x243
 #define CSR_VSIP            0x244
+#define CSR_VSTIMECMP       0x24D
+#define CSR_VSTIMECMPH      0x25D
 #define CSR_VSATP           0x280
 
 #define CSR_MTINST          0x34a
@@ -306,6 +330,14 @@
 #define CSR_MHPMEVENT29     0x33d
 #define CSR_MHPMEVENT30     0x33e
 #define CSR_MHPMEVENT31     0x33f
+
+/* seed CSR bits */
+#define SEED_OPST           (0x3u << 30)
+#define SEED_OPST_BIST      (0x0u << 30)
+#define SEED_OPST_WAIT      (0x1u << 30)
+#define SEED_OPST_ES16      (0x2u << 30)
+#define SEED_OPST_DEAD      (0x3u << 30)
+
 #define CSR_MHPMCOUNTER3H   0xb83
 #define CSR_MHPMCOUNTER4H   0xb84
 #define CSR_MHPMCOUNTER5H   0xb85
@@ -352,6 +384,7 @@
 #define MSTATUS_SPIE        0x00000020
 #define MSTATUS_MPIE        0x00000080
 #define MSTATUS_SPP         0x00000100
+#define MSTATUS_VS          0x00000600
 #define MSTATUS_MPP         0x00001800
 #define MSTATUS_FS          0x00006000
 #define MSTATUS_XS          0x00018000
@@ -364,10 +397,10 @@
 #define MSTATUS_TW          0x20000000 /* since: priv-1.10 */
 #define MSTATUS_TSR         0x40000000 /* since: priv-1.10 */
 #if defined(TARGET_RISCV64)
-#define MSTATUS_MTL         0x4000000000ULL
+#define MSTATUS_GVA         0x4000000000ULL
 #define MSTATUS_MPV         0x8000000000ULL
 #elif defined(TARGET_RISCV32)
-#define MSTATUS_MTL         0x00000040
+#define MSTATUS_GVA         0x00000040
 #define MSTATUS_MPV         0x00000080
 #endif
 
@@ -406,6 +439,7 @@
 #define SSTATUS_UPIE        0x00000010
 #define SSTATUS_SPIE        0x00000020
 #define SSTATUS_SPP         0x00000100
+#define SSTATUS_VS          0x00000600
 #define SSTATUS_FS          0x00006000
 #define SSTATUS_XS          0x00018000
 #define SSTATUS_PUM         0x00040000 /* until: priv-1.9.1 */
@@ -422,13 +456,16 @@
 #endif
 
 /* hstatus CSR bits */
-#define HSTATUS_SPRV         0x00000001
+#define HSTATUS_VSBE         0x00000020
+#define HSTATUS_GVA          0x00000040
 #define HSTATUS_SPV          0x00000080
-#define HSTATUS_SP2P         0x00000100
-#define HSTATUS_SP2V         0x00000200
+#define HSTATUS_SPVP         0x00000100
+#define HSTATUS_HU           0x00000200
+#define HSTATUS_VGEIN        0x0003F000
 #define HSTATUS_VTVM         0x00100000
 #define HSTATUS_VTW          0x00200000
 #define HSTATUS_VTSR         0x00400000
+#define HSTATUS_VSXL         0x300000000ULL
 
 #define HSTATUS32_WPRI       0xFF8FF87E
 #define HSTATUS64_WPRI       0xFFFFFFFFFF8FF87EULL
@@ -530,6 +567,7 @@
 #define RISCV_EXCP_STORE_PAGE_FAULT              0xf /* since: priv-1.10.0 */
 #define RISCV_EXCP_INST_GUEST_PAGE_FAULT         0x14
 #define RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT       0x15
+#define RISCV_EXCP_VIRT_INSTRUCTION_FAULT        0x16
 #define RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT  0x17
 #define RISCV_EXCP_UNICORN_END                   0x8888
 
diff --git a/qemu/target/riscv/cpu_helper.c b/qemu/target/riscv/cpu_helper.c
index bdc3926d8a..2dd53ad074 100644
--- a/qemu/target/riscv/cpu_helper.c
+++ b/qemu/target/riscv/cpu_helper.c
@@ -22,6 +22,7 @@
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "tcg/tcg-op.h"
+#include "instmap.h"
 
 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
 {
@@ -104,10 +105,23 @@ bool riscv_cpu_fp_enabled(CPURISCVState *env)
     return false;
 }
 
+bool riscv_cpu_vector_enabled(CPURISCVState *env)
+{
+    if (env->mstatus & MSTATUS_VS) {
+        if (riscv_cpu_virt_enabled(env) && !(env->mstatus_hs & MSTATUS_VS)) {
+            return false;
+        }
+        return true;
+    }
+
+    return false;
+}
+
 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
 {
     target_ulong mstatus_mask = MSTATUS_MXR | MSTATUS_SUM | MSTATUS_FS |
-                                MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE;
+                                MSTATUS_VS | MSTATUS_SPP | MSTATUS_SPIE |
+                                MSTATUS_SIE;
     bool current_virt = riscv_cpu_virt_enabled(env);
 
     g_assert(riscv_has_ext(env, RVH));
@@ -271,6 +285,24 @@ void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
     env->load_res = -1;
 }
 
+static bool riscv_cpu_mmu_two_stage_lookup(CPURISCVState *env, int mmu_idx,
+                                           MMUAccessType access_type)
+{
+    int mode = mmu_idx & TB_FLAGS_PRIV_MMU_MASK;
+
+    if (!riscv_has_ext(env, RVH)) {
+        return false;
+    }
+
+    if (riscv_cpu_virt_enabled(env) ||
+        (mmu_idx & TB_FLAGS_PRIV_HYP_ACCESS_MASK)) {
+        return true;
+    }
+
+    return mode == PRV_M && access_type != MMU_INST_FETCH &&
+           get_field(env->mstatus, MSTATUS_MPRV) && MSTATUS_MPV_ISSET(env);
+}
+
 /* get_physical_address - get the physical address for this virtual address
  *
  * Do a page table walk to obtain the physical address corresponding to a
@@ -282,6 +314,7 @@ void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
  * @physical: This will be set to the calculated physical address
  * @prot: The returned protection attributes
  * @addr: The virtual address to be translated
+ * @fault_pte_addr: Optional shifted PTE address for indirect G-stage faults
  * @access_type: The type of MMU access
  * @mmu_idx: Indicates current privilege level
  * @first_stage: Are we in first stage translation?
@@ -290,6 +323,7 @@ void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
  */
 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
                                 int *prot, target_ulong addr,
+                                target_ulong *fault_pte_addr,
                                 int access_type, int mmu_idx,
                                 bool first_stage, bool two_stage)
 {
@@ -298,7 +332,7 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
      * (riscv_cpu_do_interrupt) is correct */
     MemTxResult res;
     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
-    int mode = mmu_idx;
+    int mode = mmu_idx & TB_FLAGS_PRIV_MMU_MASK;
     bool use_background = false;
     hwaddr base;
     int levels = 0, ptidxbits = 0, ptesize = 0, vm, sum, mxr, widened;
@@ -311,7 +345,13 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
      * was called. Background registers will be used if the guest has
      * forced a two stage translation to be on (in HS or M mode).
      */
-    if (mode == PRV_M && access_type != MMU_INST_FETCH) {
+    if (!riscv_cpu_virt_enabled(env) && two_stage) {
+        use_background = true;
+    }
+
+    if (mmu_idx & TB_FLAGS_PRIV_HYP_ACCESS_MASK) {
+        mode = get_field(env->hstatus, HSTATUS_SPVP);
+    } else if (mode == PRV_M && access_type != MMU_INST_FETCH) {
         if (get_field(env->mstatus, MSTATUS_MPRV)) {
             mode = get_field(env->mstatus, MSTATUS_MPP);
 
@@ -322,14 +362,6 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
         }
     }
 
-    if (mode == PRV_S && access_type != MMU_INST_FETCH &&
-        riscv_has_ext(env, RVH) && !riscv_cpu_virt_enabled(env)) {
-        if (get_field(env->hstatus, HSTATUS_SPRV)) {
-            mode = get_field(env->mstatus, SSTATUS_SPP);
-            use_background = true;
-        }
-    }
-
     if (first_stage == false) {
         /* We are in stage 2 translation, this is similar to stage 1. */
         /* Stage 2 is always taken as U-mode */
@@ -365,7 +397,7 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
             vm = get_field(env->hgatp, HGATP_MODE);
             widened = 2;
         }
-        sum = get_field(env->mstatus, MSTATUS_SUM);
+        sum = get_field(env->mstatus, MSTATUS_SUM) || use_background;
         switch (vm) {
         case VM_1_10_SV32:
           levels = 2; ptidxbits = 10; ptesize = 4; break;
@@ -438,11 +470,21 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
         hwaddr pte_addr;
 
         if (two_stage && first_stage) {
+            int vbase_prot;
+            int vbase_ret;
             hwaddr vbase;
 
             /* Do the second stage translation on the base PTE address. */
-            get_physical_address(env, &vbase, prot, base, access_type,
-                                 mmu_idx, false, true);
+            vbase_ret = get_physical_address(env, &vbase, &vbase_prot, base,
+                                             NULL, MMU_DATA_LOAD, mmu_idx,
+                                             false, true);
+
+            if (vbase_ret != TRANSLATE_SUCCESS) {
+                if (fault_pte_addr) {
+                    *fault_pte_addr = (base + idx * ptesize) >> 2;
+                }
+                return TRANSLATE_G_STAGE_FAIL;
+            }
 
             pte_addr = vbase + idx * ptesize;
         } else {
@@ -592,7 +634,8 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
 
 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
                                 MMUAccessType access_type, bool pmp_violation,
-                                bool first_stage)
+                                bool first_stage, bool two_stage,
+                                bool two_stage_indirect)
 {
     CPUState *cs = env_cpu(env);
     int page_fault_exceptions;
@@ -608,7 +651,7 @@ static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
     }
     switch (access_type) {
     case MMU_INST_FETCH:
-        if (riscv_cpu_virt_enabled(env) && !first_stage) {
+        if (two_stage && !first_stage) {
             cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
         } else {
             cs->exception_index = page_fault_exceptions ?
@@ -616,7 +659,7 @@ static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
         }
         break;
     case MMU_DATA_LOAD:
-        if (riscv_cpu_virt_enabled(env) && !first_stage) {
+        if (two_stage && !first_stage) {
             cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
         } else {
             cs->exception_index = page_fault_exceptions ?
@@ -624,7 +667,7 @@ static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
         }
         break;
     case MMU_DATA_STORE:
-        if (riscv_cpu_virt_enabled(env) && !first_stage) {
+        if (two_stage && !first_stage) {
             cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
         } else {
             cs->exception_index = page_fault_exceptions ?
@@ -635,6 +678,8 @@ static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
         g_assert_not_reached();
     }
     env->badaddr = address;
+    env->two_stage_lookup = two_stage;
+    env->two_stage_indirect_lookup = two_stage_indirect;
 }
 
 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
@@ -645,13 +690,13 @@ hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     int prot;
     int mmu_idx = cpu_mmu_index(&cpu->env, false);
 
-    if (get_physical_address(env, &phys_addr, &prot, addr, 0, mmu_idx,
+    if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
                              true, riscv_cpu_virt_enabled(env))) {
         return -1;
     }
 
     if (riscv_cpu_virt_enabled(env)) {
-        if (get_physical_address(env, &phys_addr, &prot, phys_addr,
+        if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
                                  0, mmu_idx, false, true)) {
             return -1;
         }
@@ -671,11 +716,16 @@ void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
 
     if (access_type == MMU_DATA_STORE) {
         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
-    } else {
+    } else if (access_type == MMU_DATA_LOAD) {
         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
+    } else {
+        cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
     }
 
     env->badaddr = addr;
+    env->two_stage_lookup =
+        riscv_cpu_mmu_two_stage_lookup(env, mmu_idx, access_type);
+    env->two_stage_indirect_lookup = false;
     riscv_raise_exception(&cpu->env, cs->exception_index, retaddr);
 }
 
@@ -699,6 +749,9 @@ void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
         g_assert_not_reached();
     }
     env->badaddr = addr;
+    env->two_stage_lookup =
+        riscv_cpu_mmu_two_stage_lookup(env, mmu_idx, access_type);
+    env->two_stage_indirect_lookup = false;
     riscv_raise_exception(env, cs->exception_index, retaddr);
 }
 
@@ -710,13 +763,15 @@ bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
     CPURISCVState *env = &cpu->env;
     vaddr im_address;
     hwaddr pa = 0;
-    int prot;
+    int prot = 0;
+    int prot2 = 0;
     bool pmp_violation = false;
-    bool m_mode_two_stage = false;
-    bool hs_mode_two_stage = false;
     bool first_stage_error = true;
+    bool two_stage_lookup = false;
+    bool two_stage_indirect_error = false;
     int ret = TRANSLATE_FAIL;
-    int mode = mmu_idx;
+    int mode = mmu_idx & TB_FLAGS_PRIV_MMU_MASK;
+    bool hyp_access = (mmu_idx & TB_FLAGS_PRIV_HYP_ACCESS_MASK) != 0;
 
     env->guest_phys_fault_addr = 0;
 
@@ -728,45 +783,48 @@ bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
      * set and we want to access a virtulisation address.
      */
     if (riscv_has_ext(env, RVH)) {
-        m_mode_two_stage = env->priv == PRV_M &&
-                           access_type != MMU_INST_FETCH &&
-                           get_field(env->mstatus, MSTATUS_MPRV) &&
-                           MSTATUS_MPV_ISSET(env);
-
-        hs_mode_two_stage = env->priv == PRV_S &&
-                            !riscv_cpu_virt_enabled(env) &&
-                            access_type != MMU_INST_FETCH &&
-                            get_field(env->hstatus, HSTATUS_SPRV) &&
-                            get_field(env->hstatus, HSTATUS_SPV);
+        two_stage_lookup = riscv_cpu_mmu_two_stage_lookup(env, mmu_idx,
+                                                          access_type);
     }
 
-    if (mode == PRV_M && access_type != MMU_INST_FETCH) {
+    if (hyp_access) {
+        mode = get_field(env->hstatus, HSTATUS_SPVP);
+    } else if (mode == PRV_M && access_type != MMU_INST_FETCH) {
         if (get_field(env->mstatus, MSTATUS_MPRV)) {
             mode = get_field(env->mstatus, MSTATUS_MPP);
         }
     }
 
-    if (riscv_cpu_virt_enabled(env) || m_mode_two_stage || hs_mode_two_stage) {
+    if (two_stage_lookup) {
         /* Two stage lookup */
-        ret = get_physical_address(env, &pa, &prot, address, access_type,
+        ret = get_physical_address(env, &pa, &prot, address,
+                                   &env->guest_phys_fault_addr, access_type,
                                    mmu_idx, true, true);
 
+        if (ret == TRANSLATE_G_STAGE_FAIL) {
+            first_stage_error = false;
+            two_stage_indirect_error = true;
+            access_type = MMU_DATA_LOAD;
+        }
+
         qemu_log_mask(CPU_LOG_MMU,
                       "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
                       TARGET_FMT_plx " prot %d\n",
                       __func__, address, ret, pa, prot);
 
-        if (ret != TRANSLATE_FAIL) {
+        if (ret == TRANSLATE_SUCCESS) {
             /* Second stage lookup */
             im_address = pa;
 
-            ret = get_physical_address(env, &pa, &prot, im_address,
+            ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
                                        access_type, mmu_idx, false, true);
 
             qemu_log_mask(CPU_LOG_MMU,
                     "%s 2nd-stage address=%" VADDR_PRIx " ret %d physical "
                     TARGET_FMT_plx " prot %d\n",
-                    __func__, im_address, ret, pa, prot);
+                    __func__, im_address, ret, pa, prot2);
+
+            prot &= prot2;
 
             if (riscv_feature(env, RISCV_FEATURE_PMP) &&
                 (ret == TRANSLATE_SUCCESS) &&
@@ -787,8 +845,8 @@ bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
         }
     } else {
         /* Single stage lookup */
-        ret = get_physical_address(env, &pa, &prot, address, access_type,
-                                   mmu_idx, true, false);
+        ret = get_physical_address(env, &pa, &prot, address, NULL,
+                                   access_type, mmu_idx, true, false);
 
         qemu_log_mask(CPU_LOG_MMU,
                       "%s address=%" VADDR_PRIx " ret %d physical "
@@ -812,13 +870,208 @@ bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
     } else if (probe) {
         return false;
     } else {
-        raise_mmu_exception(env, address, access_type, pmp_violation, first_stage_error);
+        raise_mmu_exception(env, address, access_type, pmp_violation,
+                            first_stage_error, two_stage_lookup,
+                            two_stage_indirect_error);
         riscv_raise_exception(env, cs->exception_index, retaddr);
     }
 
     return true;
 }
 
+static target_ulong riscv_transformed_insn(CPURISCVState *env,
+                                           target_ulong insn,
+                                           target_ulong taddr)
+{
+    target_ulong xinsn = 0;
+    target_ulong access_rs1 = 0;
+    target_ulong access_imm = 0;
+    target_ulong access_size = 0;
+
+    if ((insn & 0x3) != 0x3) {
+        switch (GET_C_OP(insn)) {
+        case OPC_RISC_C_OP_QUAD0:
+            switch (GET_C_FUNC(insn)) {
+            case OPC_RISC_C_FUNC_FLD_LQ:
+                if (TARGET_LONG_BITS != 128) {
+                    xinsn = OPC_RISC_FLD;
+                    xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
+                    access_rs1 = GET_C_RS1S(insn);
+                    access_imm = GET_C_LD_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            case OPC_RISC_C_FUNC_LW:
+                xinsn = OPC_RISC_LW;
+                xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
+                access_rs1 = GET_C_RS1S(insn);
+                access_imm = GET_C_LW_IMM(insn);
+                access_size = 4;
+                break;
+            case OPC_RISC_C_FUNC_FLW_LD:
+                if (TARGET_LONG_BITS == 32) {
+                    xinsn = OPC_RISC_FLW;
+                    xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
+                    access_rs1 = GET_C_RS1S(insn);
+                    access_imm = GET_C_LW_IMM(insn);
+                    access_size = 4;
+                } else {
+                    xinsn = OPC_RISC_LD;
+                    xinsn = SET_RD(xinsn, GET_C_RS2S(insn));
+                    access_rs1 = GET_C_RS1S(insn);
+                    access_imm = GET_C_LD_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            case OPC_RISC_C_FUNC_FSD_SQ:
+                if (TARGET_LONG_BITS != 128) {
+                    xinsn = OPC_RISC_FSD;
+                    xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
+                    access_rs1 = GET_C_RS1S(insn);
+                    access_imm = GET_C_SD_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            case OPC_RISC_C_FUNC_SW:
+                xinsn = OPC_RISC_SW;
+                xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
+                access_rs1 = GET_C_RS1S(insn);
+                access_imm = GET_C_SW_IMM(insn);
+                access_size = 4;
+                break;
+            case OPC_RISC_C_FUNC_FSW_SD:
+                if (TARGET_LONG_BITS == 32) {
+                    xinsn = OPC_RISC_FSW;
+                    xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
+                    access_rs1 = GET_C_RS1S(insn);
+                    access_imm = GET_C_SW_IMM(insn);
+                    access_size = 4;
+                } else {
+                    xinsn = OPC_RISC_SD;
+                    xinsn = SET_RS2(xinsn, GET_C_RS2S(insn));
+                    access_rs1 = GET_C_RS1S(insn);
+                    access_imm = GET_C_SD_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            default:
+                break;
+            }
+            break;
+        case OPC_RISC_C_OP_QUAD2:
+            switch (GET_C_FUNC(insn)) {
+            case OPC_RISC_C_FUNC_FLDSP_LQSP:
+                if (TARGET_LONG_BITS != 128) {
+                    xinsn = OPC_RISC_FLD;
+                    xinsn = SET_RD(xinsn, GET_C_RD(insn));
+                    access_rs1 = 2;
+                    access_imm = GET_C_LDSP_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            case OPC_RISC_C_FUNC_LWSP:
+                xinsn = OPC_RISC_LW;
+                xinsn = SET_RD(xinsn, GET_C_RD(insn));
+                access_rs1 = 2;
+                access_imm = GET_C_LWSP_IMM(insn);
+                access_size = 4;
+                break;
+            case OPC_RISC_C_FUNC_FLWSP_LDSP:
+                if (TARGET_LONG_BITS == 32) {
+                    xinsn = OPC_RISC_FLW;
+                    xinsn = SET_RD(xinsn, GET_C_RD(insn));
+                    access_rs1 = 2;
+                    access_imm = GET_C_LWSP_IMM(insn);
+                    access_size = 4;
+                } else {
+                    xinsn = OPC_RISC_LD;
+                    xinsn = SET_RD(xinsn, GET_C_RD(insn));
+                    access_rs1 = 2;
+                    access_imm = GET_C_LDSP_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            case OPC_RISC_C_FUNC_FSDSP_SQSP:
+                if (TARGET_LONG_BITS != 128) {
+                    xinsn = OPC_RISC_FSD;
+                    xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
+                    access_rs1 = 2;
+                    access_imm = GET_C_SDSP_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            case OPC_RISC_C_FUNC_SWSP:
+                xinsn = OPC_RISC_SW;
+                xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
+                access_rs1 = 2;
+                access_imm = GET_C_SWSP_IMM(insn);
+                access_size = 4;
+                break;
+            case OPC_RISC_C_FUNC_FSWSP_SDSP:
+                if (TARGET_LONG_BITS == 32) {
+                    xinsn = OPC_RISC_FSW;
+                    xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
+                    access_rs1 = 2;
+                    access_imm = GET_C_SWSP_IMM(insn);
+                    access_size = 4;
+                } else {
+                    xinsn = OPC_RISC_SD;
+                    xinsn = SET_RS2(xinsn, GET_C_RS2(insn));
+                    access_rs1 = 2;
+                    access_imm = GET_C_SDSP_IMM(insn);
+                    access_size = 8;
+                }
+                break;
+            default:
+                break;
+            }
+            break;
+        default:
+            break;
+        }
+        xinsn &= ~((target_ulong)0x2);
+    } else {
+        switch (MASK_OP_MAJOR(insn)) {
+        case OPC_RISC_ATOMIC:
+            xinsn = insn;
+            access_rs1 = GET_RS1(insn);
+            access_size = 1 << GET_FUNCT3(insn);
+            break;
+        case OPC_RISC_LOAD:
+        case OPC_RISC_FP_LOAD:
+            xinsn = SET_I_IMM(insn, 0);
+            access_rs1 = GET_RS1(insn);
+            access_imm = GET_IMM(insn);
+            access_size = 1 << GET_FUNCT3(insn);
+            break;
+        case OPC_RISC_STORE:
+        case OPC_RISC_FP_STORE:
+            xinsn = SET_S_IMM(insn, 0);
+            access_rs1 = GET_RS1(insn);
+            access_imm = GET_STORE_IMM(insn);
+            access_size = 1 << GET_FUNCT3(insn);
+            break;
+        case OPC_RISC_SYSTEM:
+            if (MASK_OP_SYSTEM(insn) == OPC_RISC_HLVHSV) {
+                xinsn = insn;
+                access_rs1 = GET_RS1(insn);
+                access_size = 1 << ((GET_FUNCT7(insn) >> 1) & 0x3);
+                access_size = 1 << access_size;
+            }
+            break;
+        default:
+            break;
+        }
+    }
+
+    if (access_size) {
+        xinsn = SET_RS1(xinsn, (taddr - (env->gpr[access_rs1] +
+                                         access_imm)) & (access_size - 1));
+    }
+
+    return xinsn;
+}
+
 /*
  * Handle Traps
  *
@@ -830,6 +1083,7 @@ void riscv_cpu_do_interrupt(CPUState *cs)
     RISCVCPU *cpu = RISCV_CPU(cs);
     CPURISCVState *env = &cpu->env;
     bool force_hs_execp = riscv_cpu_force_hs_excep_enabled(env);
+    bool write_gva = false;
     target_ulong s;
 
     /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
@@ -839,27 +1093,46 @@ void riscv_cpu_do_interrupt(CPUState *cs)
     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
     target_ulong deleg = async ? env->mideleg : env->medeleg;
     target_ulong tval = 0;
+    target_ulong tinst = 0;
     target_ulong htval = 0;
     target_ulong mtval2 = 0;
 
     if (!async) {
         /* set tval to badaddr for traps with address information */
         switch (cause) {
-        case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
             force_hs_execp = true;
             /* fallthrough */
-        case RISCV_EXCP_INST_ADDR_MIS:
-        case RISCV_EXCP_INST_ACCESS_FAULT:
         case RISCV_EXCP_LOAD_ADDR_MIS:
         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
         case RISCV_EXCP_LOAD_ACCESS_FAULT:
         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
-        case RISCV_EXCP_INST_PAGE_FAULT:
         case RISCV_EXCP_LOAD_PAGE_FAULT:
         case RISCV_EXCP_STORE_PAGE_FAULT:
+            write_gva = env->two_stage_lookup;
+            tval = env->badaddr;
+            if (env->two_stage_indirect_lookup) {
+                tinst = TARGET_LONG_BITS == 32 ? 0x00002000 : 0x00003000;
+            } else {
+                tinst = riscv_transformed_insn(env, env->bins, tval);
+            }
+            break;
+        case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
+            force_hs_execp = true;
+            /* fallthrough */
+        case RISCV_EXCP_INST_ADDR_MIS:
+        case RISCV_EXCP_INST_ACCESS_FAULT:
+        case RISCV_EXCP_INST_PAGE_FAULT:
+            write_gva = env->two_stage_lookup;
             tval = env->badaddr;
+            if (env->two_stage_indirect_lookup) {
+                tinst = TARGET_LONG_BITS == 32 ? 0x00002000 : 0x00003000;
+            }
+            break;
+        case RISCV_EXCP_ILLEGAL_INST:
+        case RISCV_EXCP_VIRT_INSTRUCTION_FAULT:
+            tval = env->bins;
             break;
         default:
             break;
@@ -893,16 +1166,16 @@ void riscv_cpu_do_interrupt(CPUState *cs)
                  * no if hypervisor has delegated one of hs mode's interrupt
                  */
                 if (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
-                    cause == IRQ_VS_EXT)
+                    cause == IRQ_VS_EXT) {
                     cause = cause - 1;
+                }
+                write_gva = false;
                 /* Trap to VS mode */
             } else if (riscv_cpu_virt_enabled(env)) {
                 /* Trap into HS mode, from virt */
                 riscv_cpu_swap_hypervisor_regs(env);
-                env->hstatus = set_field(env->hstatus, HSTATUS_SP2V,
-                                         get_field(env->hstatus, HSTATUS_SPV));
-                env->hstatus = set_field(env->hstatus, HSTATUS_SP2P,
-                                         get_field(env->mstatus, SSTATUS_SPP));
+                env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
+                                         env->priv);
                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
                                          riscv_cpu_virt_enabled(env));
 
@@ -912,15 +1185,11 @@ void riscv_cpu_do_interrupt(CPUState *cs)
                 riscv_cpu_set_force_hs_excep(env, 0);
             } else {
                 /* Trap into HS mode */
-                env->hstatus = set_field(env->hstatus, HSTATUS_SP2V,
-                                         get_field(env->hstatus, HSTATUS_SPV));
-                env->hstatus = set_field(env->hstatus, HSTATUS_SP2P,
-                                         get_field(env->mstatus, SSTATUS_SPP));
-                env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
-                                         riscv_cpu_virt_enabled(env));
+                env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false);
 
                 htval = env->guest_phys_fault_addr;
             }
+            env->hstatus = set_field(env->hstatus, HSTATUS_GVA, write_gva);
         }
 
         s = env->mstatus;
@@ -933,6 +1202,7 @@ void riscv_cpu_do_interrupt(CPUState *cs)
         env->sepc = env->pc;
         env->sbadaddr = tval;
         env->htval = htval;
+        env->htinst = tinst;
         env->pc = (env->stvec >> 2 << 2) +
             ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
         riscv_cpu_set_mode(env, PRV_S);
@@ -945,13 +1215,11 @@ void riscv_cpu_do_interrupt(CPUState *cs)
 #ifdef TARGET_RISCV32
             env->mstatush = set_field(env->mstatush, MSTATUS_MPV,
                                        riscv_cpu_virt_enabled(env));
-            env->mstatush = set_field(env->mstatush, MSTATUS_MTL,
-                                       riscv_cpu_force_hs_excep_enabled(env));
+            env->mstatush = set_field(env->mstatush, MSTATUS_GVA, write_gva);
 #else
             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
                                       riscv_cpu_virt_enabled(env));
-            env->mstatus = set_field(env->mstatus, MSTATUS_MTL,
-                                      riscv_cpu_force_hs_excep_enabled(env));
+            env->mstatus = set_field(env->mstatus, MSTATUS_GVA, write_gva);
 #endif
 
             mtval2 = env->guest_phys_fault_addr;
@@ -971,6 +1239,7 @@ void riscv_cpu_do_interrupt(CPUState *cs)
         env->mepc = env->pc;
         env->mbadaddr = tval;
         env->mtval2 = mtval2;
+        env->mtinst = tinst;
         env->pc = (env->mtvec >> 2 << 2) +
             ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
         riscv_cpu_set_mode(env, PRV_M);
@@ -982,5 +1251,7 @@ void riscv_cpu_do_interrupt(CPUState *cs)
      * RISC-V ISA Specification.
      */
 
+    env->two_stage_lookup = false;
+    env->two_stage_indirect_lookup = false;
     cs->exception_index = EXCP_NONE; /* mark handled to qemu */
 }
diff --git a/qemu/target/riscv/crypto_helper.c b/qemu/target/riscv/crypto_helper.c
new file mode 100644
index 0000000000..2ef30281b1
--- /dev/null
+++ b/qemu/target/riscv/crypto_helper.c
@@ -0,0 +1,302 @@
+/*
+ * RISC-V Crypto Emulation Helpers for QEMU.
+ *
+ * Copyright (c) 2021 Ruibo Lu, luruibo2000@163.com
+ * Copyright (c) 2021 Zewen Ye, lustrew@foxmail.com
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
+#include "exec/helper-proto.h"
+#include "crypto/aes.h"
+#include "crypto/sm4.h"
+
+#define AES_XTIME(a) \
+    ((a << 1) ^ ((a & 0x80) ? 0x1b : 0))
+
+#define AES_GFMUL(a, b) (( \
+    (((b) & 0x1) ? (a) : 0) ^ \
+    (((b) & 0x2) ? AES_XTIME(a) : 0) ^ \
+    (((b) & 0x4) ? AES_XTIME(AES_XTIME(a)) : 0) ^ \
+    (((b) & 0x8) ? AES_XTIME(AES_XTIME(AES_XTIME(a))) : 0)) & 0xFF)
+
+static inline uint32_t aes_mixcolumn_byte(uint8_t x, bool fwd)
+{
+    uint32_t u;
+
+    if (fwd) {
+        u = (AES_GFMUL(x, 3) << 24) | (x << 16) | (x << 8) |
+            (AES_GFMUL(x, 2) << 0);
+    } else {
+        u = (AES_GFMUL(x, 0xb) << 24) | (AES_GFMUL(x, 0xd) << 16) |
+            (AES_GFMUL(x, 0x9) << 8) | (AES_GFMUL(x, 0xe) << 0);
+    }
+    return u;
+}
+
+#define sext32_xlen(x) (target_ulong)(int32_t)(x)
+
+static inline target_ulong aes32_operation(target_ulong shamt,
+                                           target_ulong rs1, target_ulong rs2,
+                                           bool enc, bool mix)
+{
+    uint8_t si = rs2 >> shamt;
+    uint8_t so;
+    uint32_t mixed;
+    target_ulong res;
+
+    if (enc) {
+        so = AES_sbox[si];
+        if (mix) {
+            mixed = aes_mixcolumn_byte(so, true);
+        } else {
+            mixed = so;
+        }
+    } else {
+        so = AES_isbox[si];
+        if (mix) {
+            mixed = aes_mixcolumn_byte(so, false);
+        } else {
+            mixed = so;
+        }
+    }
+    mixed = rol32(mixed, shamt);
+    res = rs1 ^ mixed;
+
+    return sext32_xlen(res);
+}
+
+target_ulong HELPER(aes32esmi)(target_ulong rs1, target_ulong rs2,
+                               target_ulong shamt)
+{
+    return aes32_operation(shamt, rs1, rs2, true, true);
+}
+
+target_ulong HELPER(aes32esi)(target_ulong rs1, target_ulong rs2,
+                              target_ulong shamt)
+{
+    return aes32_operation(shamt, rs1, rs2, true, false);
+}
+
+target_ulong HELPER(aes32dsmi)(target_ulong rs1, target_ulong rs2,
+                               target_ulong shamt)
+{
+    return aes32_operation(shamt, rs1, rs2, false, true);
+}
+
+target_ulong HELPER(aes32dsi)(target_ulong rs1, target_ulong rs2,
+                              target_ulong shamt)
+{
+    return aes32_operation(shamt, rs1, rs2, false, false);
+}
+
+#define BY(X, I) ((X >> (8 * I)) & 0xFF)
+
+#define AES_SHIFROWS_LO(RS1, RS2) ( \
+    (((RS1 >> 24) & 0xFF) << 56) | (((RS2 >> 48) & 0xFF) << 48) | \
+    (((RS2 >> 8) & 0xFF) << 40) | (((RS1 >> 32) & 0xFF) << 32) | \
+    (((RS2 >> 56) & 0xFF) << 24) | (((RS2 >> 16) & 0xFF) << 16) | \
+    (((RS1 >> 40) & 0xFF) << 8) | (((RS1 >> 0) & 0xFF) << 0))
+
+#define AES_INVSHIFROWS_LO(RS1, RS2) ( \
+    (((RS2 >> 24) & 0xFF) << 56) | (((RS2 >> 48) & 0xFF) << 48) | \
+    (((RS1 >> 8) & 0xFF) << 40) | (((RS1 >> 32) & 0xFF) << 32) | \
+    (((RS1 >> 56) & 0xFF) << 24) | (((RS2 >> 16) & 0xFF) << 16) | \
+    (((RS2 >> 40) & 0xFF) << 8) | (((RS1 >> 0) & 0xFF) << 0))
+
+#define AES_MIXBYTE(COL, B0, B1, B2, B3) ( \
+    BY(COL, B3) ^ BY(COL, B2) ^ AES_GFMUL(BY(COL, B1), 3) ^ \
+    AES_GFMUL(BY(COL, B0), 2))
+
+#define AES_MIXCOLUMN(COL) ( \
+    AES_MIXBYTE(COL, 3, 0, 1, 2) << 24 | \
+    AES_MIXBYTE(COL, 2, 3, 0, 1) << 16 | \
+    AES_MIXBYTE(COL, 1, 2, 3, 0) << 8 | AES_MIXBYTE(COL, 0, 1, 2, 3) << 0)
+
+#define AES_INVMIXBYTE(COL, B0, B1, B2, B3) ( \
+    AES_GFMUL(BY(COL, B3), 0x9) ^ AES_GFMUL(BY(COL, B2), 0xd) ^ \
+    AES_GFMUL(BY(COL, B1), 0xb) ^ AES_GFMUL(BY(COL, B0), 0xe))
+
+#define AES_INVMIXCOLUMN(COL) ( \
+    AES_INVMIXBYTE(COL, 3, 0, 1, 2) << 24 | \
+    AES_INVMIXBYTE(COL, 2, 3, 0, 1) << 16 | \
+    AES_INVMIXBYTE(COL, 1, 2, 3, 0) << 8 | \
+    AES_INVMIXBYTE(COL, 0, 1, 2, 3) << 0)
+
+static inline target_ulong aes64_operation(target_ulong rs1, target_ulong rs2,
+                                           bool enc, bool mix)
+{
+    uint64_t RS1 = rs1;
+    uint64_t RS2 = rs2;
+    uint64_t result;
+    uint64_t temp;
+    uint32_t col_0;
+    uint32_t col_1;
+
+    if (enc) {
+        temp = AES_SHIFROWS_LO(RS1, RS2);
+        temp = (((uint64_t)AES_sbox[(temp >> 0) & 0xFF] << 0) |
+                ((uint64_t)AES_sbox[(temp >> 8) & 0xFF] << 8) |
+                ((uint64_t)AES_sbox[(temp >> 16) & 0xFF] << 16) |
+                ((uint64_t)AES_sbox[(temp >> 24) & 0xFF] << 24) |
+                ((uint64_t)AES_sbox[(temp >> 32) & 0xFF] << 32) |
+                ((uint64_t)AES_sbox[(temp >> 40) & 0xFF] << 40) |
+                ((uint64_t)AES_sbox[(temp >> 48) & 0xFF] << 48) |
+                ((uint64_t)AES_sbox[(temp >> 56) & 0xFF] << 56));
+        if (mix) {
+            col_0 = temp & 0xFFFFFFFF;
+            col_1 = temp >> 32;
+
+            col_0 = AES_MIXCOLUMN(col_0);
+            col_1 = AES_MIXCOLUMN(col_1);
+
+            result = ((uint64_t)col_1 << 32) | col_0;
+        } else {
+            result = temp;
+        }
+    } else {
+        temp = AES_INVSHIFROWS_LO(RS1, RS2);
+        temp = (((uint64_t)AES_isbox[(temp >> 0) & 0xFF] << 0) |
+                ((uint64_t)AES_isbox[(temp >> 8) & 0xFF] << 8) |
+                ((uint64_t)AES_isbox[(temp >> 16) & 0xFF] << 16) |
+                ((uint64_t)AES_isbox[(temp >> 24) & 0xFF] << 24) |
+                ((uint64_t)AES_isbox[(temp >> 32) & 0xFF] << 32) |
+                ((uint64_t)AES_isbox[(temp >> 40) & 0xFF] << 40) |
+                ((uint64_t)AES_isbox[(temp >> 48) & 0xFF] << 48) |
+                ((uint64_t)AES_isbox[(temp >> 56) & 0xFF] << 56));
+        if (mix) {
+            col_0 = temp & 0xFFFFFFFF;
+            col_1 = temp >> 32;
+
+            col_0 = AES_INVMIXCOLUMN(col_0);
+            col_1 = AES_INVMIXCOLUMN(col_1);
+
+            result = ((uint64_t)col_1 << 32) | col_0;
+        } else {
+            result = temp;
+        }
+    }
+
+    return result;
+}
+
+target_ulong HELPER(aes64esm)(target_ulong rs1, target_ulong rs2)
+{
+    return aes64_operation(rs1, rs2, true, true);
+}
+
+target_ulong HELPER(aes64es)(target_ulong rs1, target_ulong rs2)
+{
+    return aes64_operation(rs1, rs2, true, false);
+}
+
+target_ulong HELPER(aes64ds)(target_ulong rs1, target_ulong rs2)
+{
+    return aes64_operation(rs1, rs2, false, false);
+}
+
+target_ulong HELPER(aes64dsm)(target_ulong rs1, target_ulong rs2)
+{
+    return aes64_operation(rs1, rs2, false, true);
+}
+
+target_ulong HELPER(aes64ks2)(target_ulong rs1, target_ulong rs2)
+{
+    uint64_t RS1 = rs1;
+    uint64_t RS2 = rs2;
+    uint32_t rs1_hi = RS1 >> 32;
+    uint32_t rs2_lo = RS2;
+    uint32_t rs2_hi = RS2 >> 32;
+
+    uint32_t r_lo = (rs1_hi ^ rs2_lo);
+    uint32_t r_hi = (rs1_hi ^ rs2_lo ^ rs2_hi);
+    target_ulong result = ((uint64_t)r_hi << 32) | r_lo;
+
+    return result;
+}
+
+target_ulong HELPER(aes64ks1i)(target_ulong rs1, target_ulong rnum)
+{
+    uint64_t RS1 = rs1;
+    static const uint8_t round_consts[10] = {
+        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36
+    };
+
+    uint8_t enc_rnum = rnum;
+    uint32_t temp = (RS1 >> 32) & 0xFFFFFFFF;
+    uint8_t rcon_ = 0;
+    target_ulong result;
+
+    if (enc_rnum != 0xA) {
+        temp = ror32(temp, 8); /* Rotate right by 8 */
+        rcon_ = round_consts[enc_rnum];
+    }
+
+    temp = ((uint32_t)AES_sbox[(temp >> 24) & 0xFF] << 24) |
+           ((uint32_t)AES_sbox[(temp >> 16) & 0xFF] << 16) |
+           ((uint32_t)AES_sbox[(temp >> 8) & 0xFF] << 8) |
+           ((uint32_t)AES_sbox[(temp >> 0) & 0xFF] << 0);
+
+    temp ^= rcon_;
+
+    result = ((uint64_t)temp << 32) | temp;
+
+    return result;
+}
+
+target_ulong HELPER(aes64im)(target_ulong rs1)
+{
+    uint64_t RS1 = rs1;
+    uint32_t col_0 = RS1 & 0xFFFFFFFF;
+    uint32_t col_1 = RS1 >> 32;
+    target_ulong result;
+
+    col_0 = AES_INVMIXCOLUMN(col_0);
+    col_1 = AES_INVMIXCOLUMN(col_1);
+
+    result = ((uint64_t)col_1 << 32) | col_0;
+
+    return result;
+}
+
+target_ulong HELPER(sm4ed)(target_ulong rs1, target_ulong rs2,
+                           target_ulong shamt)
+{
+    uint32_t sb_in = (uint8_t)(rs2 >> shamt);
+    uint32_t sb_out = (uint32_t)sm4_sbox[sb_in];
+
+    uint32_t x = sb_out ^ (sb_out << 8) ^ (sb_out << 2) ^ (sb_out << 18) ^
+                 ((sb_out & 0x3f) << 26) ^ ((sb_out & 0xC0) << 10);
+
+    uint32_t rotl = rol32(x, shamt);
+
+    return sext32_xlen(rotl ^ (uint32_t)rs1);
+}
+
+target_ulong HELPER(sm4ks)(target_ulong rs1, target_ulong rs2,
+                           target_ulong shamt)
+{
+    uint32_t sb_in = (uint8_t)(rs2 >> shamt);
+    uint32_t sb_out = sm4_sbox[sb_in];
+
+    uint32_t x = sb_out ^ ((sb_out & 0x07) << 29) ^ ((sb_out & 0xFE) << 7) ^
+                 ((sb_out & 0x01) << 23) ^ ((sb_out & 0xF8) << 13);
+
+    uint32_t rotl = rol32(x, shamt);
+
+    return sext32_xlen(rotl ^ (uint32_t)rs1);
+}
+#undef sext32_xlen
diff --git a/qemu/target/riscv/csr.c b/qemu/target/riscv/csr.c
index 785ef26dc4..997a682ca4 100644
--- a/qemu/target/riscv/csr.c
+++ b/qemu/target/riscv/csr.c
@@ -19,6 +19,7 @@
 
 #include "qemu/osdep.h"
 #include "qemu/log.h"
+#include "qemu/guest-random.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 
@@ -29,10 +30,23 @@ static int read_frm(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_frm(CPURISCVState *env, int csrno, target_ulong val);
 static int read_fcsr(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_fcsr(CPURISCVState *env, int csrno, target_ulong val);
+static int read_vtype(CPURISCVState *env, int csrno, target_ulong *val);
+static int read_vl(CPURISCVState *env, int csrno, target_ulong *val);
+static int read_vlenb(CPURISCVState *env, int csrno, target_ulong *val);
+static int read_vxrm(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_vxrm(CPURISCVState *env, int csrno, target_ulong val);
+static int read_vxsat(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_vxsat(CPURISCVState *env, int csrno, target_ulong val);
+static int read_vstart(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_vstart(CPURISCVState *env, int csrno, target_ulong val);
+static int read_vcsr(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_vcsr(CPURISCVState *env, int csrno, target_ulong val);
 static int ctr(CPURISCVState *env, int csrno);
 static int read_instret(CPURISCVState *env, int csrno, target_ulong *val);
 static int read_time(CPURISCVState *env, int csrno, target_ulong *val);
 static int any(CPURISCVState *env, int csrno);
+static int vs(CPURISCVState *env, int csrno);
+static int seed(CPURISCVState *env, int csrno);
 static int read_zero(CPURISCVState *env, int csrno, target_ulong *val);
 static int read_mhartid(CPURISCVState *env, int csrno, target_ulong *val);
 static int read_mstatus(CPURISCVState *env, int csrno, target_ulong *val);
@@ -103,6 +117,11 @@ static int read_htinst(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_htinst(CPURISCVState *env, int csrno, target_ulong val);
 static int read_hgatp(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_hgatp(CPURISCVState *env, int csrno, target_ulong val);
+static int sstc(CPURISCVState *env, int csrno);
+static int read_stimecmp(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_stimecmp(CPURISCVState *env, int csrno, target_ulong val);
+static int read_vstimecmp(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_vstimecmp(CPURISCVState *env, int csrno, target_ulong val);
 static int read_htimedelta(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_htimedelta(CPURISCVState *env, int csrno, target_ulong val);
 static int read_vsstatus(CPURISCVState *env, int csrno, target_ulong *val);
@@ -132,6 +151,8 @@ static int write_pmpcfg(CPURISCVState *env, int csrno, target_ulong val);
 static int read_pmpaddr(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_pmpaddr(CPURISCVState *env, int csrno, target_ulong val);
 static int pmp(CPURISCVState *env, int csrno);
+static int rmw_seed(CPURISCVState *env, int csrno, target_ulong *ret_value,
+                    target_ulong new_value, target_ulong write_mask);
 
 #if defined(TARGET_RISCV32)
 static int read_instreth(CPURISCVState *env, int csrno, target_ulong *val);
@@ -140,6 +161,11 @@ static int read_mstatush(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_mstatush(CPURISCVState *env, int csrno, target_ulong val);
 static int read_htimedeltah(CPURISCVState *env, int csrno, target_ulong *val);
 static int write_htimedeltah(CPURISCVState *env, int csrno, target_ulong val);
+static int sstc_32(CPURISCVState *env, int csrno);
+static int read_stimecmph(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_stimecmph(CPURISCVState *env, int csrno, target_ulong val);
+static int read_vstimecmph(CPURISCVState *env, int csrno, target_ulong *val);
+static int write_vstimecmph(CPURISCVState *env, int csrno, target_ulong val);
 #endif
 
 /* CSR function table constants */
@@ -154,6 +180,18 @@ static riscv_csr_operations csr_ops[CSR_TABLE_SIZE] = {
     [CSR_FRM] =                 { fs,   read_frm,         write_frm         },
     [CSR_FCSR] =                { fs,   read_fcsr,        write_fcsr        },
 
+    /* User Vector CSRs */
+    [CSR_VSTART] =              { vs,   read_vstart,      write_vstart      },
+    [CSR_VXSAT] =               { vs,   read_vxsat,       write_vxsat       },
+    [CSR_VXRM] =                { vs,   read_vxrm,        write_vxrm        },
+    [CSR_VCSR] =                { vs,   read_vcsr,        write_vcsr        },
+    [CSR_VL] =                  { vs,   read_vl                             },
+    [CSR_VTYPE] =               { vs,   read_vtype                          },
+    [CSR_VLENB] =               { vs,   read_vlenb                          },
+
+    /* Crypto Extension */
+    [CSR_SEED] =                { seed, NULL,             NULL, rmw_seed    },
+
     /* User Timers and Counters */
     [CSR_CYCLE] =               { ctr,  read_instret                        },
     [CSR_INSTRET] =             { ctr,  read_instret                        },
@@ -219,6 +257,10 @@ static riscv_csr_operations csr_ops[CSR_TABLE_SIZE] = {
     [CSR_SCAUSE] =              { smode, read_scause,      write_scause      },
     [CSR_SBADADDR] =            { smode, read_sbadaddr,    write_sbadaddr    },
     [CSR_SIP] =                 { smode, NULL,     NULL,     rmw_sip         },
+    [CSR_STIMECMP] =            { sstc,  read_stimecmp,    write_stimecmp    },
+#if defined(TARGET_RISCV32)
+    [CSR_STIMECMPH] =           { sstc_32, read_stimecmph, write_stimecmph   },
+#endif
 
     /* Supervisor Protection and Translation */
     [CSR_SATP] =                { smode, read_satp,        write_satp        },
@@ -246,6 +288,10 @@ static riscv_csr_operations csr_ops[CSR_TABLE_SIZE] = {
     [CSR_VSCAUSE] =             { hmode,   read_vscause,     write_vscause    },
     [CSR_VSTVAL] =              { hmode,   read_vstval,      write_vstval     },
     [CSR_VSATP] =               { hmode,   read_vsatp,       write_vsatp      },
+    [CSR_VSTIMECMP] =           { sstc,    read_vstimecmp,   write_vstimecmp  },
+#if defined(TARGET_RISCV32)
+    [CSR_VSTIMECMPH] =          { sstc_32, read_vstimecmph,  write_vstimecmph },
+#endif
 
     [CSR_MTVAL2] =              { hmode,   read_mtval2,      write_mtval2     },
     [CSR_MTINST] =              { hmode,   read_mtinst,      write_mtinst     },
@@ -484,6 +530,31 @@ static int any(CPURISCVState *env, int csrno)
     return 0;
 }
 
+static int vs(CPURISCVState *env, int csrno)
+{
+    RISCVCPU *cpu = env_archcpu(env);
+
+    if (!(riscv_has_ext(env, RVV) || cpu->cfg.ext_zve32f ||
+          cpu->cfg.ext_zve64f)) {
+        return -1;
+    }
+    if (!env->debugger && !riscv_cpu_vector_enabled(env)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int seed(CPURISCVState *env, int csrno)
+{
+    RISCVCPU *cpu = env_archcpu(env);
+
+    if (!cpu->cfg.ext_zkr || env->priv != PRV_M) {
+        return -1;
+    }
+    return 0;
+}
+
 static int smode(CPURISCVState *env, int csrno)
 {
     return -!riscv_has_ext(env, RVS);
@@ -568,6 +639,84 @@ static int write_fcsr(CPURISCVState *env, int csrno, target_ulong val)
     return 0;
 }
 
+/* User Vector CSRs */
+static int read_vtype(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    target_ulong vill = env->vill ?
+        ((target_ulong)1 << (TARGET_LONG_BITS - 1)) : 0;
+
+    *val = vill | env->vtype;
+    return 0;
+}
+
+static int read_vl(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    *val = env->vl;
+    return 0;
+}
+
+static int read_vlenb(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    *val = env_archcpu(env)->cfg.vlen >> 3;
+    return 0;
+}
+
+static int read_vxrm(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    *val = env->vxrm;
+    return 0;
+}
+
+static int write_vxrm(CPURISCVState *env, int csrno, target_ulong val)
+{
+    env->mstatus |= MSTATUS_VS | MSTATUS_SD;
+    env->vxrm = val & (VCSR_VXRM >> VCSR_VXRM_SHIFT);
+    return 0;
+}
+
+static int read_vxsat(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    *val = env->vxsat & (VCSR_VXSAT >> VCSR_VXSAT_SHIFT);
+    return 0;
+}
+
+static int write_vxsat(CPURISCVState *env, int csrno, target_ulong val)
+{
+    env->mstatus |= MSTATUS_VS | MSTATUS_SD;
+    env->vxsat = val & (VCSR_VXSAT >> VCSR_VXSAT_SHIFT);
+    return 0;
+}
+
+static int read_vstart(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    *val = env->vstart;
+    return 0;
+}
+
+static int write_vstart(CPURISCVState *env, int csrno, target_ulong val)
+{
+    RISCVCPU *cpu = env_archcpu(env);
+
+    env->mstatus |= MSTATUS_VS | MSTATUS_SD;
+    env->vstart = val & (cpu->cfg.vlen - 1);
+    return 0;
+}
+
+static int read_vcsr(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    *val = (env->vxrm << VCSR_VXRM_SHIFT) |
+           (env->vxsat << VCSR_VXSAT_SHIFT);
+    return 0;
+}
+
+static int write_vcsr(CPURISCVState *env, int csrno, target_ulong val)
+{
+    env->mstatus |= MSTATUS_VS | MSTATUS_SD;
+    env->vxrm = (val & VCSR_VXRM) >> VCSR_VXRM_SHIFT;
+    env->vxsat = (val & VCSR_VXSAT) >> VCSR_VXSAT_SHIFT;
+    return 0;
+}
+
 /* User Timers and Counters */
 static int read_instret(CPURISCVState *env, int csrno, target_ulong *val)
 {
@@ -611,6 +760,117 @@ static int read_timeh(CPURISCVState *env, int csrno, target_ulong *val)
 }
 #endif
 
+static int sstc(CPURISCVState *env, int csrno)
+{
+    CPUState *cs = env_cpu(env);
+    RISCVCPU *cpu = RISCV_CPU(cs);
+
+    if (!cpu->cfg.ext_sstc || !env->rdtime_fn) {
+        return -1;
+    }
+
+    if (csrno == CSR_VSTIMECMP || csrno == CSR_VSTIMECMPH) {
+        return hmode(env, csrno);
+    }
+
+    return (env->priv == PRV_M) ? 0 : -1;
+}
+
+#if defined(TARGET_RISCV32)
+static int sstc_32(CPURISCVState *env, int csrno)
+{
+    return sstc(env, csrno);
+}
+#endif
+
+static int read_vstimecmp(CPURISCVState *env, int csrno, target_ulong *val)
+{
+#if defined(TARGET_RISCV32)
+    *val = env->vstimecmp & 0xffffffff;
+#else
+    *val = env->vstimecmp;
+#endif
+    return 0;
+}
+
+#if defined(TARGET_RISCV32)
+static int read_vstimecmph(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    *val = env->vstimecmp >> 32;
+    return 0;
+}
+#endif
+
+static int write_vstimecmp(CPURISCVState *env, int csrno, target_ulong val)
+{
+#if defined(TARGET_RISCV32)
+    env->vstimecmp = deposit64(env->vstimecmp, 0, 32, (uint64_t)val);
+#else
+    env->vstimecmp = val;
+#endif
+    return 0;
+}
+
+#if defined(TARGET_RISCV32)
+static int write_vstimecmph(CPURISCVState *env, int csrno, target_ulong val)
+{
+    env->vstimecmp = deposit64(env->vstimecmp, 32, 32, (uint64_t)val);
+    return 0;
+}
+#endif
+
+static int read_stimecmp(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    if (riscv_cpu_virt_enabled(env)) {
+        return read_vstimecmp(env, csrno, val);
+    }
+
+#if defined(TARGET_RISCV32)
+    *val = env->stimecmp & 0xffffffff;
+#else
+    *val = env->stimecmp;
+#endif
+    return 0;
+}
+
+#if defined(TARGET_RISCV32)
+static int read_stimecmph(CPURISCVState *env, int csrno, target_ulong *val)
+{
+    if (riscv_cpu_virt_enabled(env)) {
+        return read_vstimecmph(env, csrno, val);
+    }
+
+    *val = env->stimecmp >> 32;
+    return 0;
+}
+#endif
+
+static int write_stimecmp(CPURISCVState *env, int csrno, target_ulong val)
+{
+    if (riscv_cpu_virt_enabled(env)) {
+        return write_vstimecmp(env, csrno, val);
+    }
+
+#if defined(TARGET_RISCV32)
+    env->stimecmp = deposit64(env->stimecmp, 0, 32, (uint64_t)val);
+#else
+    env->stimecmp = val;
+#endif
+    return 0;
+}
+
+#if defined(TARGET_RISCV32)
+static int write_stimecmph(CPURISCVState *env, int csrno, target_ulong val)
+{
+    if (riscv_cpu_virt_enabled(env)) {
+        return write_vstimecmph(env, csrno, val);
+    }
+
+    env->stimecmp = deposit64(env->stimecmp, 32, 32, (uint64_t)val);
+    return 0;
+}
+#endif
+
 /* Machine constants */
 
 #define M_MODE_INTERRUPTS  (MIP_MSIP | MIP_MTIP | MIP_MEIP)
@@ -621,31 +881,41 @@ static const target_ulong delegable_ints = S_MODE_INTERRUPTS |
                                            VS_MODE_INTERRUPTS;
 static const target_ulong all_ints = M_MODE_INTERRUPTS | S_MODE_INTERRUPTS |
                                      VS_MODE_INTERRUPTS;
-static const target_ulong delegable_excps =
-    (1ULL << (RISCV_EXCP_INST_ADDR_MIS)) |
-    (1ULL << (RISCV_EXCP_INST_ACCESS_FAULT)) |
-    (1ULL << (RISCV_EXCP_ILLEGAL_INST)) |
-    (1ULL << (RISCV_EXCP_BREAKPOINT)) |
-    (1ULL << (RISCV_EXCP_LOAD_ADDR_MIS)) |
-    (1ULL << (RISCV_EXCP_LOAD_ACCESS_FAULT)) |
-    (1ULL << (RISCV_EXCP_STORE_AMO_ADDR_MIS)) |
-    (1ULL << (RISCV_EXCP_STORE_AMO_ACCESS_FAULT)) |
-    (1ULL << (RISCV_EXCP_U_ECALL)) |
-    (1ULL << (RISCV_EXCP_S_ECALL)) |
-    (1ULL << (RISCV_EXCP_VS_ECALL)) |
-    (1ULL << (RISCV_EXCP_M_ECALL)) |
-    (1ULL << (RISCV_EXCP_INST_PAGE_FAULT)) |
-    (1ULL << (RISCV_EXCP_LOAD_PAGE_FAULT)) |
-    (1ULL << (RISCV_EXCP_STORE_PAGE_FAULT)) |
-    (1ULL << (RISCV_EXCP_INST_GUEST_PAGE_FAULT)) |
-    (1ULL << (RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT)) |
-    (1ULL << (RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT));
+#define DELEGABLE_EXCPS \
+    ((1ULL << (RISCV_EXCP_INST_ADDR_MIS)) | \
+     (1ULL << (RISCV_EXCP_INST_ACCESS_FAULT)) | \
+     (1ULL << (RISCV_EXCP_ILLEGAL_INST)) | \
+     (1ULL << (RISCV_EXCP_BREAKPOINT)) | \
+     (1ULL << (RISCV_EXCP_LOAD_ADDR_MIS)) | \
+     (1ULL << (RISCV_EXCP_LOAD_ACCESS_FAULT)) | \
+     (1ULL << (RISCV_EXCP_STORE_AMO_ADDR_MIS)) | \
+     (1ULL << (RISCV_EXCP_STORE_AMO_ACCESS_FAULT)) | \
+     (1ULL << (RISCV_EXCP_U_ECALL)) | \
+     (1ULL << (RISCV_EXCP_S_ECALL)) | \
+     (1ULL << (RISCV_EXCP_VS_ECALL)) | \
+     (1ULL << (RISCV_EXCP_M_ECALL)) | \
+     (1ULL << (RISCV_EXCP_INST_PAGE_FAULT)) | \
+     (1ULL << (RISCV_EXCP_LOAD_PAGE_FAULT)) | \
+     (1ULL << (RISCV_EXCP_STORE_PAGE_FAULT)) | \
+     (1ULL << (RISCV_EXCP_INST_GUEST_PAGE_FAULT)) | \
+     (1ULL << (RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT)) | \
+     (1ULL << (RISCV_EXCP_VIRT_INSTRUCTION_FAULT)) | \
+     (1ULL << (RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT)))
+static const target_ulong delegable_excps = DELEGABLE_EXCPS;
+static const target_ulong vs_delegable_excps = DELEGABLE_EXCPS &
+    ~((1ULL << (RISCV_EXCP_S_ECALL)) |
+      (1ULL << (RISCV_EXCP_VS_ECALL)) |
+      (1ULL << (RISCV_EXCP_M_ECALL)) |
+      (1ULL << (RISCV_EXCP_INST_GUEST_PAGE_FAULT)) |
+      (1ULL << (RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT)) |
+      (1ULL << (RISCV_EXCP_VIRT_INSTRUCTION_FAULT)) |
+      (1ULL << (RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT)));
 static const target_ulong sstatus_v1_9_mask = SSTATUS_SIE | SSTATUS_SPIE |
-    SSTATUS_UIE | SSTATUS_UPIE | SSTATUS_SPP | SSTATUS_FS | SSTATUS_XS |
-    SSTATUS_SUM | SSTATUS_SD;
+    SSTATUS_UIE | SSTATUS_UPIE | SSTATUS_SPP | SSTATUS_VS | SSTATUS_FS |
+    SSTATUS_XS | SSTATUS_SUM | SSTATUS_SD;
 static const target_ulong sstatus_v1_10_mask = SSTATUS_SIE | SSTATUS_SPIE |
-    SSTATUS_UIE | SSTATUS_UPIE | SSTATUS_SPP | SSTATUS_FS | SSTATUS_XS |
-    SSTATUS_SUM | SSTATUS_MXR | SSTATUS_SD;
+    SSTATUS_UIE | SSTATUS_UPIE | SSTATUS_SPP | SSTATUS_VS | SSTATUS_FS |
+    SSTATUS_XS | SSTATUS_SUM | SSTATUS_MXR | SSTATUS_SD;
 static const target_ulong sip_writable_mask = SIP_SSIP | MIP_USIP | MIP_UEIP;
 static const target_ulong hip_writable_mask = MIP_VSSIP | MIP_VSTIP | MIP_VSEIP;
 static const target_ulong vsip_writable_mask = MIP_VSSIP;
@@ -711,7 +981,8 @@ static int write_mstatus(CPURISCVState *env, int csrno, target_ulong val)
             tlb_flush(env_cpu(env));
         }
         mask = MSTATUS_SIE | MSTATUS_SPIE | MSTATUS_MIE | MSTATUS_MPIE |
-            MSTATUS_SPP | MSTATUS_FS | MSTATUS_MPRV | MSTATUS_SUM |
+            MSTATUS_SPP | MSTATUS_VS | MSTATUS_FS | MSTATUS_MPRV |
+            MSTATUS_SUM |
             MSTATUS_MPP | MSTATUS_MXR |
             (validate_vm(env, get_field(val, MSTATUS_VM)) ?
                 MSTATUS_VM : 0);
@@ -722,21 +993,23 @@ static int write_mstatus(CPURISCVState *env, int csrno, target_ulong val)
             tlb_flush(env_cpu(env));
         }
         mask = MSTATUS_SIE | MSTATUS_SPIE | MSTATUS_MIE | MSTATUS_MPIE |
-            MSTATUS_SPP | MSTATUS_FS | MSTATUS_MPRV | MSTATUS_SUM |
-            MSTATUS_MPP | MSTATUS_MXR | MSTATUS_TVM | MSTATUS_TSR |
+            MSTATUS_SPP | MSTATUS_VS | MSTATUS_FS | MSTATUS_MPRV |
+            MSTATUS_SUM | MSTATUS_MPP | MSTATUS_MXR | MSTATUS_TVM |
+            MSTATUS_TSR |
             MSTATUS_TW;
 #if defined(TARGET_RISCV64)
             /*
-             * RV32: MPV and MTL are not in mstatus. The current plan is to
+             * RV32: MPV and GVA are not in mstatus. The current plan is to
              * add them to mstatush. For now, we just don't support it.
              */
-            mask |= MSTATUS_MTL | MSTATUS_MPV;
+            mask |= MSTATUS_GVA | MSTATUS_MPV;
 #endif
     }
 
     mstatus = (mstatus & ~mask) | (val & mask);
 
-    dirty = ((mstatus & MSTATUS_FS) == MSTATUS_FS) |
+    dirty = ((mstatus & MSTATUS_VS) == MSTATUS_VS) |
+            ((mstatus & MSTATUS_FS) == MSTATUS_FS) |
             ((mstatus & MSTATUS_XS) == MSTATUS_XS);
     mstatus = set_field(mstatus, MSTATUS_SD, dirty);
     env->mstatus = mstatus;
@@ -757,7 +1030,7 @@ static int write_mstatush(CPURISCVState *env, int csrno, target_ulong val)
         tlb_flush(env_cpu(env));
     }
 
-    val &= MSTATUS_MPV | MSTATUS_MTL;
+    val &= MSTATUS_MPV | MSTATUS_GVA;
 
     env->mstatush = val;
 
@@ -1201,6 +1474,10 @@ static int write_satp(CPURISCVState *env, int csrno, target_ulong val)
 static int read_hstatus(CPURISCVState *env, int csrno, target_ulong *val)
 {
     *val = env->hstatus;
+#ifdef TARGET_RISCV64
+    *val = set_field(*val, HSTATUS_VSXL, MXL_RV64);
+#endif
+    *val = set_field(*val, HSTATUS_VSBE, 0);
     return 0;
 }
 
@@ -1218,7 +1495,7 @@ static int read_hedeleg(CPURISCVState *env, int csrno, target_ulong *val)
 
 static int write_hedeleg(CPURISCVState *env, int csrno, target_ulong val)
 {
-    env->hedeleg = val;
+    env->hedeleg = val & vs_delegable_excps;
     return 0;
 }
 
@@ -1507,6 +1784,24 @@ static int write_pmpaddr(CPURISCVState *env, int csrno, target_ulong val)
     return 0;
 }
 
+static int rmw_seed(CPURISCVState *env, int csrno, target_ulong *ret_value,
+                    target_ulong new_value, target_ulong write_mask)
+{
+    uint16_t random_v;
+    target_ulong rval;
+
+    if (qemu_guest_getrandom(&random_v, sizeof(random_v)) < 0) {
+        rval = SEED_OPST_DEAD;
+    } else {
+        rval = random_v | SEED_OPST_ES16;
+    }
+
+    if (ret_value) {
+        *ret_value = rval;
+    }
+    return 0;
+}
+
 /*
  * riscv_csrrw - read and/or update control and status register
  *
@@ -1601,4 +1896,3 @@ int riscv_csrrw_debug(CPURISCVState *env, int csrno, target_ulong *ret_value,
     env->debugger = false;
     return ret;
 }
-
diff --git a/qemu/target/riscv/fpu_helper.c b/qemu/target/riscv/fpu_helper.c
index 9efdb5d4da..b81ef50b14 100644
--- a/qemu/target/riscv/fpu_helper.c
+++ b/qemu/target/riscv/fpu_helper.c
@@ -24,12 +24,18 @@
 #include "fpu/softfloat.h"
 
 #define RISCV_NANBOX32_MASK UINT64_C(0xffffffff00000000)
+#define RISCV_NANBOX16_MASK UINT64_C(0xffffffffffff0000)
 
 static uint64_t nanbox_s(float32 f)
 {
     return (uint64_t)f | RISCV_NANBOX32_MASK;
 }
 
+static uint64_t nanbox_h(float16 f)
+{
+    return (uint64_t)f | RISCV_NANBOX16_MASK;
+}
+
 static float32 check_nanbox_s(uint64_t f)
 {
     if ((f & RISCV_NANBOX32_MASK) == RISCV_NANBOX32_MASK) {
@@ -39,6 +45,15 @@ static float32 check_nanbox_s(uint64_t f)
     return 0x7fc00000u;
 }
 
+static float16 check_nanbox_h(uint64_t f)
+{
+    if ((f & RISCV_NANBOX16_MASK) == RISCV_NANBOX16_MASK) {
+        return (uint16_t)f;
+    }
+
+    return 0x7e00u;
+}
+
 target_ulong riscv_cpu_get_fflags(CPURISCVState *env)
 {
     int soft = get_float_exception_flags(&env->fp_status);
@@ -96,6 +111,11 @@ void helper_set_rounding_mode(CPURISCVState *env, uint32_t rm)
     set_float_rounding_mode(softrm, &env->fp_status);
 }
 
+void helper_set_rod_rounding_mode(CPURISCVState *env)
+{
+    set_float_rounding_mode(float_round_to_odd, &env->fp_status);
+}
+
 uint64_t helper_fmadd_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                         uint64_t frs3)
 {
@@ -112,6 +132,16 @@ uint64_t helper_fmadd_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
     return float64_muladd(frs1, frs2, frs3, 0, &env->fp_status);
 }
 
+uint64_t helper_fmadd_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
+                        uint64_t frs3)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+    frs3 = check_nanbox_h(frs3);
+
+    return nanbox_h(float16_muladd(frs1, frs2, frs3, 0, &env->fp_status));
+}
+
 uint64_t helper_fmsub_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                         uint64_t frs3)
 {
@@ -130,6 +160,17 @@ uint64_t helper_fmsub_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                           &env->fp_status);
 }
 
+uint64_t helper_fmsub_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
+                        uint64_t frs3)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+    frs3 = check_nanbox_h(frs3);
+
+    return nanbox_h(float16_muladd(frs1, frs2, frs3,
+                                   float_muladd_negate_c, &env->fp_status));
+}
+
 uint64_t helper_fnmsub_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                          uint64_t frs3)
 {
@@ -149,6 +190,18 @@ uint64_t helper_fnmsub_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                           &env->fp_status);
 }
 
+uint64_t helper_fnmsub_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
+                         uint64_t frs3)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+    frs3 = check_nanbox_h(frs3);
+
+    return nanbox_h(float16_muladd(frs1, frs2, frs3,
+                                   float_muladd_negate_product,
+                                   &env->fp_status));
+}
+
 uint64_t helper_fnmadd_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                          uint64_t frs3)
 {
@@ -169,6 +222,19 @@ uint64_t helper_fnmadd_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
                           float_muladd_negate_product, &env->fp_status);
 }
 
+uint64_t helper_fnmadd_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2,
+                         uint64_t frs3)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+    frs3 = check_nanbox_h(frs3);
+
+    return nanbox_h(float16_muladd(frs1, frs2, frs3,
+                                   float_muladd_negate_c |
+                                   float_muladd_negate_product,
+                                   &env->fp_status));
+}
+
 uint64_t helper_fadd_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
     frs1 = check_nanbox_s(frs1);
@@ -206,7 +272,9 @@ uint64_t helper_fmin_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
     frs1 = check_nanbox_s(frs1);
     frs2 = check_nanbox_s(frs2);
 
-    return nanbox_s(float32_minnum(frs1, frs2, &env->fp_status));
+    return nanbox_s(env->priv_ver < PRIV_VERSION_1_11_0 ?
+                    float32_minnum(frs1, frs2, &env->fp_status) :
+                    float32_minimum_number(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fmax_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
@@ -214,7 +282,9 @@ uint64_t helper_fmax_s(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
     frs1 = check_nanbox_s(frs1);
     frs2 = check_nanbox_s(frs2);
 
-    return nanbox_s(float32_maxnum(frs1, frs2, &env->fp_status));
+    return nanbox_s(env->priv_ver < PRIV_VERSION_1_11_0 ?
+                    float32_maxnum(frs1, frs2, &env->fp_status) :
+                    float32_maximum_number(frs1, frs2, &env->fp_status));
 }
 
 uint64_t helper_fsqrt_s(CPURISCVState *env, uint64_t frs1)
@@ -319,6 +389,186 @@ target_ulong helper_fclass_s(uint64_t frs1)
     }
 }
 
+uint64_t helper_fadd_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return nanbox_h(float16_add(frs1, frs2, &env->fp_status));
+}
+
+uint64_t helper_fsub_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return nanbox_h(float16_sub(frs1, frs2, &env->fp_status));
+}
+
+uint64_t helper_fmul_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return nanbox_h(float16_mul(frs1, frs2, &env->fp_status));
+}
+
+uint64_t helper_fdiv_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return nanbox_h(float16_div(frs1, frs2, &env->fp_status));
+}
+
+uint64_t helper_fmin_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return nanbox_h(env->priv_ver < PRIV_VERSION_1_11_0 ?
+                    float16_minnum(frs1, frs2, &env->fp_status) :
+                    float16_minimum_number(frs1, frs2, &env->fp_status));
+}
+
+uint64_t helper_fmax_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return nanbox_h(env->priv_ver < PRIV_VERSION_1_11_0 ?
+                    float16_maxnum(frs1, frs2, &env->fp_status) :
+                    float16_maximum_number(frs1, frs2, &env->fp_status));
+}
+
+uint64_t helper_fsqrt_h(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_h(frs1);
+
+    return nanbox_h(float16_sqrt(frs1, &env->fp_status));
+}
+
+target_ulong helper_fle_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return float16_le(frs1, frs2, &env->fp_status);
+}
+
+target_ulong helper_flt_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return float16_lt(frs1, frs2, &env->fp_status);
+}
+
+target_ulong helper_feq_h(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
+{
+    frs1 = check_nanbox_h(frs1);
+    frs2 = check_nanbox_h(frs2);
+
+    return float16_eq_quiet(frs1, frs2, &env->fp_status);
+}
+
+target_ulong helper_fcvt_w_h(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_h(frs1);
+
+    return float16_to_int32(frs1, &env->fp_status);
+}
+
+target_ulong helper_fcvt_wu_h(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_h(frs1);
+
+    return (int32_t)float16_to_uint32(frs1, &env->fp_status);
+}
+
+#if defined(TARGET_RISCV64)
+target_ulong helper_fcvt_l_h(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_h(frs1);
+
+    return float16_to_int64(frs1, &env->fp_status);
+}
+
+target_ulong helper_fcvt_lu_h(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_h(frs1);
+
+    return float16_to_uint64(frs1, &env->fp_status);
+}
+#endif
+
+uint64_t helper_fcvt_h_w(CPURISCVState *env, target_ulong rs1)
+{
+    return nanbox_h(int32_to_float16((int32_t)rs1, &env->fp_status));
+}
+
+uint64_t helper_fcvt_h_wu(CPURISCVState *env, target_ulong rs1)
+{
+    return nanbox_h(uint32_to_float16((uint32_t)rs1, &env->fp_status));
+}
+
+#if defined(TARGET_RISCV64)
+uint64_t helper_fcvt_h_l(CPURISCVState *env, target_ulong rs1)
+{
+    return nanbox_h(int64_to_float16(rs1, &env->fp_status));
+}
+
+uint64_t helper_fcvt_h_lu(CPURISCVState *env, target_ulong rs1)
+{
+    return nanbox_h(uint64_to_float16(rs1, &env->fp_status));
+}
+#endif
+
+uint64_t helper_fcvt_h_s(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_s(frs1);
+
+    return nanbox_h(float32_to_float16(frs1, true, &env->fp_status));
+}
+
+uint64_t helper_fcvt_s_h(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_h(frs1);
+
+    return nanbox_s(float16_to_float32(frs1, true, &env->fp_status));
+}
+
+uint64_t helper_fcvt_h_d(CPURISCVState *env, uint64_t frs1)
+{
+    return nanbox_h(float64_to_float16(frs1, true, &env->fp_status));
+}
+
+uint64_t helper_fcvt_d_h(CPURISCVState *env, uint64_t frs1)
+{
+    frs1 = check_nanbox_h(frs1);
+
+    return float16_to_float64(frs1, true, &env->fp_status);
+}
+
+target_ulong helper_fclass_h(CPURISCVState *env, uint64_t frs1)
+{
+    float16 f = check_nanbox_h(frs1);
+    bool sign = float16_is_neg(f);
+
+    if (float16_is_infinity(f)) {
+        return sign ? 1 << 0 : 1 << 7;
+    } else if (float16_is_zero(f)) {
+        return sign ? 1 << 3 : 1 << 4;
+    } else if (float16_is_zero_or_denormal(f)) {
+        return sign ? 1 << 2 : 1 << 5;
+    } else if (float16_is_any_nan(f)) {
+        float_status s = { 0 }; /* for snan_bit_is_one */
+        return float16_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
+    } else {
+        return sign ? 1 << 1 : 1 << 6;
+    }
+}
+
 uint64_t helper_fadd_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
     return float64_add(frs1, frs2, &env->fp_status);
@@ -341,12 +591,16 @@ uint64_t helper_fdiv_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 
 uint64_t helper_fmin_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float64_minnum(frs1, frs2, &env->fp_status);
+    return env->priv_ver < PRIV_VERSION_1_11_0 ?
+            float64_minnum(frs1, frs2, &env->fp_status) :
+            float64_minimum_number(frs1, frs2, &env->fp_status);
 }
 
 uint64_t helper_fmax_d(CPURISCVState *env, uint64_t frs1, uint64_t frs2)
 {
-    return float64_maxnum(frs1, frs2, &env->fp_status);
+    return env->priv_ver < PRIV_VERSION_1_11_0 ?
+            float64_maxnum(frs1, frs2, &env->fp_status) :
+            float64_maximum_number(frs1, frs2, &env->fp_status);
 }
 
 uint64_t helper_fcvt_s_d(CPURISCVState *env, uint64_t rs1)
diff --git a/qemu/target/riscv/helper.h b/qemu/target/riscv/helper.h
index 32e483860f..137a971025 100644
--- a/qemu/target/riscv/helper.h
+++ b/qemu/target/riscv/helper.h
@@ -6,17 +6,22 @@ DEF_HELPER_1(uc_riscv_exit, void, env)
 DEF_HELPER_2(raise_exception, noreturn, env, i32)
 
 /* Floating Point - rounding mode */
+DEF_HELPER_FLAGS_1(set_rod_rounding_mode, TCG_CALL_NO_WG, void, env)
 DEF_HELPER_FLAGS_2(set_rounding_mode, TCG_CALL_NO_WG, void, env, i32)
 
 /* Floating Point - fused */
 DEF_HELPER_FLAGS_4(fmadd_s, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(fmadd_d, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(fmadd_h, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(fmsub_s, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(fmsub_d, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(fmsub_h, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(fnmsub_s, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(fnmsub_d, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(fnmsub_h, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(fnmadd_s, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(fnmadd_d, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
+DEF_HELPER_FLAGS_4(fnmadd_h, TCG_CALL_NO_RWG, i64, env, i64, i64, i64)
 
 /* Floating Point - Single Precision */
 DEF_HELPER_FLAGS_3(fadd_s, TCG_CALL_NO_RWG, i64, env, i64, i64)
@@ -70,11 +75,989 @@ DEF_HELPER_FLAGS_2(fcvt_d_lu, TCG_CALL_NO_RWG, i64, env, tl)
 #endif
 DEF_HELPER_FLAGS_1(fclass_d, TCG_CALL_NO_RWG_SE, tl, i64)
 
+/* Floating Point - Half Precision */
+DEF_HELPER_FLAGS_3(fadd_h, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(fsub_h, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(fmul_h, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(fdiv_h, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(fmin_h, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(fmax_h, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(fsqrt_h, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_3(fle_h, TCG_CALL_NO_RWG, tl, env, i64, i64)
+DEF_HELPER_FLAGS_3(flt_h, TCG_CALL_NO_RWG, tl, env, i64, i64)
+DEF_HELPER_FLAGS_3(feq_h, TCG_CALL_NO_RWG, tl, env, i64, i64)
+DEF_HELPER_FLAGS_2(fcvt_s_h, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(fcvt_h_s, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(fcvt_d_h, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(fcvt_h_d, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(fcvt_w_h, TCG_CALL_NO_RWG, tl, env, i64)
+DEF_HELPER_FLAGS_2(fcvt_wu_h, TCG_CALL_NO_RWG, tl, env, i64)
+#if defined(TARGET_RISCV64)
+DEF_HELPER_FLAGS_2(fcvt_l_h, TCG_CALL_NO_RWG, tl, env, i64)
+DEF_HELPER_FLAGS_2(fcvt_lu_h, TCG_CALL_NO_RWG, tl, env, i64)
+#endif
+DEF_HELPER_FLAGS_2(fcvt_h_w, TCG_CALL_NO_RWG, i64, env, tl)
+DEF_HELPER_FLAGS_2(fcvt_h_wu, TCG_CALL_NO_RWG, i64, env, tl)
+#if defined(TARGET_RISCV64)
+DEF_HELPER_FLAGS_2(fcvt_h_l, TCG_CALL_NO_RWG, i64, env, tl)
+DEF_HELPER_FLAGS_2(fcvt_h_lu, TCG_CALL_NO_RWG, i64, env, tl)
+#endif
+DEF_HELPER_FLAGS_2(fclass_h, TCG_CALL_NO_RWG_SE, tl, env, i64)
+
 /* Special functions */
 DEF_HELPER_3(csrrw, tl, env, tl, tl)
 DEF_HELPER_4(csrrs, tl, env, tl, tl, tl)
 DEF_HELPER_4(csrrc, tl, env, tl, tl, tl)
+DEF_HELPER_3(vsetvl, tl, env, tl, tl)
+DEF_HELPER_5(vle8_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vle16_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vle32_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vle64_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vse8_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vse16_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vse32_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vse64_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_6(vlse8_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vlse16_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vlse32_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vlse64_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vsse8_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vsse16_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vsse32_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vsse64_v, void, ptr, ptr, tl, tl, env, i32)
+DEF_HELPER_6(vlxei8_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei8_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei8_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei8_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei16_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei16_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei16_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei16_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei32_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei32_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei32_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei32_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei64_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei64_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei64_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vlxei64_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei8_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei8_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei8_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei8_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei16_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei16_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei16_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei16_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei32_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei32_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei32_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei32_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei64_8_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei64_16_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei64_32_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsxei64_64_v, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vle8ff_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vle16ff_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vle32ff_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vle64ff_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_4(vl1re8_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl1re16_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl1re32_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl1re64_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl2re8_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl2re16_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl2re32_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl2re64_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl4re8_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl4re16_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl4re32_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl4re64_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl8re8_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl8re16_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl8re32_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vl8re64_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vs1r_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vs2r_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vs4r_v, void, ptr, tl, env, i32)
+DEF_HELPER_4(vs8r_v, void, ptr, tl, env, i32)
+DEF_HELPER_5(vlm_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_5(vsm_v, void, ptr, ptr, tl, env, i32)
+DEF_HELPER_6(vadd_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadd_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsub_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsub_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfadd_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsub_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmul_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmul_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmul_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfdiv_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfdiv_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfdiv_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwsub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwsub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwadd_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwadd_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwsub_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwsub_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwmul_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwmul_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmacc_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmacc_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmacc_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmacc_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmacc_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmacc_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmsac_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmsac_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmsac_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmsac_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmsac_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmsac_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmadd_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmadd_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmsub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmsub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmsub_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmsub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmsub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfnmsub_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwmacc_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwmacc_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwnmacc_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwnmacc_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwmsac_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwmsac_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwnmsac_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwnmsac_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmin_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmin_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmin_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmax_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmax_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfmax_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnj_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnj_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnj_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnjn_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnjn_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnjn_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnjx_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnjx_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfsgnjx_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadc_vvm_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadc_vvm_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadc_vvm_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadc_vvm_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsbc_vvm_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsbc_vvm_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsbc_vvm_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsbc_vvm_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadc_vvm_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadc_vvm_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadc_vvm_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadc_vvm_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vvm_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vvm_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vvm_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vvm_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vand_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vand_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vand_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vand_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vor_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vor_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vor_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vor_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vxor_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vxor_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vxor_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vxor_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmseq_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmseq_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmseq_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmseq_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsne_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsne_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsne_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsne_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmslt_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmslt_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmslt_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmslt_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsle_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsle_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsle_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmsle_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfeq_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfeq_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfeq_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfne_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfne_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfne_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmflt_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmflt_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmflt_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfle_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfle_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmfle_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfsqrt_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfsqrt_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfsqrt_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfrsqrt7_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfrsqrt7_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfrsqrt7_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfrec7_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfrec7_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfrec7_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_xu_f_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_xu_f_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_xu_f_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_x_f_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_x_f_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_x_f_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_f_xu_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_f_xu_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_f_xu_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_f_x_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_f_x_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfcvt_f_x_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_xu_f_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_xu_f_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_x_f_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_x_f_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_xu_v_b, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_xu_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_xu_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_x_v_b, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_x_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_x_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_f_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfwcvt_f_f_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_xu_f_w_b, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_xu_f_w_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_xu_f_w_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_x_f_w_b, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_x_f_w_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_x_f_w_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_f_xu_w_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_f_xu_w_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_f_x_w_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_f_x_w_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_f_f_w_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfncvt_f_f_w_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfclass_v_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfclass_v_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vfclass_v_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vadd_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vadd_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vadd_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vadd_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsub_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsub_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsub_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsub_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrsub_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrsub_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrsub_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrsub_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vfadd_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfadd_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfadd_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsub_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsub_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsub_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfrsub_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfrsub_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfrsub_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmul_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmul_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmul_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfdiv_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfdiv_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfdiv_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfrdiv_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfrdiv_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfrdiv_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwadd_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwadd_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwsub_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwsub_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwadd_wf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwadd_wf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwsub_wf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwsub_wf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwmul_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwmul_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmacc_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmacc_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmacc_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmacc_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmacc_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmacc_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmsac_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmsac_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmsac_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmsac_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmsac_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmsac_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmadd_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmadd_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmadd_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmadd_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmadd_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmadd_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmsub_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmsub_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmsub_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmsub_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmsub_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfnmsub_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwmacc_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwmacc_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwnmacc_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwnmacc_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwmsac_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwmsac_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwnmsac_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfwnmsac_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmin_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmin_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmin_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmax_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmax_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmax_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnj_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnj_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnj_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnjn_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnjn_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnjn_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnjx_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnjx_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfsgnjx_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfeq_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfeq_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfeq_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfne_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfne_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfne_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmflt_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmflt_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmflt_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfle_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfle_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfle_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfgt_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfgt_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfgt_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfge_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfge_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vmfge_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmerge_vfm_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmerge_vfm_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfmerge_vfm_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vslideup_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslideup_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslideup_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslideup_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslidedown_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslidedown_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslidedown_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslidedown_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1up_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1up_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1up_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1up_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1down_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1down_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1down_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vslide1down_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vfslide1up_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfslide1up_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfslide1up_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfslide1down_vf_h, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfslide1down_vf_w, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vfslide1down_vf_d, void, ptr, ptr, i64, ptr, env, i32)
+DEF_HELPER_6(vrgather_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgather_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgather_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgather_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgatherei16_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgatherei16_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgatherei16_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgatherei16_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrgather_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrgather_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrgather_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrgather_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vcompress_vm_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vcompress_vm_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vcompress_vm_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vcompress_vm_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_4(vmvr_v, void, ptr, ptr, env, i32)
+DEF_HELPER_6(vadc_vxm_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vadc_vxm_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vadc_vxm_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vadc_vxm_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsbc_vxm_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsbc_vxm_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsbc_vxm_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsbc_vxm_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadc_vxm_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadc_vxm_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadc_vxm_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadc_vxm_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vxm_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vxm_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vxm_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsbc_vxm_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vand_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vand_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vand_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vand_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vor_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vor_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vor_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vor_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vxor_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vxor_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vxor_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vxor_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsadd_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsadd_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssubu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssubu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssubu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssubu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssub_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssub_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsaddu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsadd_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsadd_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsadd_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsadd_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssubu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssubu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssubu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssubu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssub_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssub_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssub_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssub_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaadd_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaadd_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasub_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasub_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasubu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasubu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasubu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vasubu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vaadd_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaadd_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaadd_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaadd_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vaaddu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasub_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasub_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasub_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasub_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasubu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasubu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasubu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vasubu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsmul_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsmul_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsmul_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsmul_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsmul_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsmul_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsmul_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsmul_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssrl_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssrl_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssrl_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssrl_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssra_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssra_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssra_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssra_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vssrl_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssrl_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssrl_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssrl_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssra_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssra_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssra_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vssra_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnclip_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnclip_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnclip_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnclipu_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnclipu_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnclipu_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnclipu_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnclipu_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnclipu_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnclip_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnclip_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnclip_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwaddu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwaddu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwaddu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsubu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsubu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsubu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwadd_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsub_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwaddu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwaddu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwaddu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsubu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsubu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsubu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwadd_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwadd_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwadd_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsub_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsub_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsub_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwaddu_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwaddu_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwaddu_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsubu_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsubu_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsubu_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwadd_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwadd_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwadd_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsub_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsub_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwsub_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwaddu_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwaddu_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwaddu_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsubu_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsubu_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsubu_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwadd_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwadd_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwadd_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsub_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsub_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwsub_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmseq_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmseq_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmseq_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmseq_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsne_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsne_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsne_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsne_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsltu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmslt_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmslt_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmslt_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmslt_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsleu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsle_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsle_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsle_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsle_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgtu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgtu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgtu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgtu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgt_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgt_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgt_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmsgt_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmand_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmnand_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmandn_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmxor_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmor_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmnor_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmorn_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmxnor_mm, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_4(vcpop_m, tl, ptr, ptr, env, i32)
+DEF_HELPER_4(vfirst_m, tl, ptr, ptr, env, i32)
+DEF_HELPER_5(vmsbf_m, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vmsif_m, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vmsof_m, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(viota_m_b, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(viota_m_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(viota_m_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(viota_m_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_4(vid_v_b, void, ptr, ptr, env, i32)
+DEF_HELPER_4(vid_v_h, void, ptr, ptr, env, i32)
+DEF_HELPER_4(vid_v_w, void, ptr, ptr, env, i32)
+DEF_HELPER_4(vid_v_d, void, ptr, ptr, env, i32)
+DEF_HELPER_6(vredsum_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredsum_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredsum_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredsum_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredand_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredand_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredand_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredand_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredor_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredor_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredor_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredor_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredxor_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredxor_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredxor_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredxor_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredminu_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredminu_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredminu_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredminu_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmin_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmin_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmin_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmin_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmaxu_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmaxu_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmaxu_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmaxu_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmax_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmax_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmax_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vredmax_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwredsumu_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwredsumu_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwredsumu_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwredsum_vs_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwredsum_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwredsum_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredusum_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredusum_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredusum_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredosum_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredosum_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredosum_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredmin_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredmin_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredmin_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredmax_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredmax_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfredmax_vs_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwredusum_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwredusum_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwredosum_vs_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vfwredosum_vs_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vminu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vminu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vminu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vminu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmin_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmin_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmin_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmin_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmax_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmax_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmax_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmax_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vminu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vminu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vminu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vminu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmin_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmin_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmin_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmin_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmaxu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmax_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmax_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmax_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmax_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmul_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmul_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmul_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmul_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulh_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulh_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulh_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulh_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmul_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmul_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmul_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmul_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulh_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulh_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulh_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulh_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmulhsu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdivu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdivu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdivu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdivu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdiv_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdiv_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdiv_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdiv_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vremu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vremu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vremu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vremu_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrem_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrem_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrem_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vrem_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vdivu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdivu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdivu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdivu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdiv_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdiv_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdiv_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vdiv_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vremu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vremu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vremu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vremu_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrem_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrem_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrem_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vrem_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmul_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmul_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmul_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmulu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmulu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmulu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmulsu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmulsu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmulsu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmul_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmul_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmul_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmulu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmulu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmulu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmulsu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmulsu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmulsu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmacc_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmacc_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmacc_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmacc_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadd_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadd_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadd_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmadd_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmacc_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmacc_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmacc_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmacc_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsac_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadd_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadd_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadd_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmadd_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnmsub_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmaccu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmaccu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmacc_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmacc_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmacc_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmaccsu_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmaccsu_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmaccsu_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vwmaccu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmacc_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmacc_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmacc_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccsu_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccsu_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccsu_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccus_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccus_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vwmaccus_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsll_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsll_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsll_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsll_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsrl_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsrl_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsrl_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsrl_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsra_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsra_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsra_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsra_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vsll_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsll_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsll_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsll_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsrl_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsrl_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsrl_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsrl_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsra_vx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsra_vx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsra_vx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vsra_vx_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnsrl_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnsrl_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnsrl_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnsra_wv_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnsra_wv_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnsra_wv_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vnsrl_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnsrl_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnsrl_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnsra_wx_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnsra_wx_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vnsra_wx_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_5(vzext_vf2_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vzext_vf2_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vzext_vf2_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vzext_vf4_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vzext_vf4_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vzext_vf8_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vsext_vf2_h, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vsext_vf2_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vsext_vf2_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vsext_vf4_w, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vsext_vf4_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_5(vsext_vf8_d, void, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmerge_vvm_b, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmerge_vvm_h, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmerge_vvm_w, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmerge_vvm_d, void, ptr, ptr, ptr, ptr, env, i32)
+DEF_HELPER_6(vmerge_vxm_b, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmerge_vxm_h, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmerge_vxm_w, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_6(vmerge_vxm_d, void, ptr, ptr, tl, ptr, env, i32)
+DEF_HELPER_4(vmv_v_v_b, void, ptr, ptr, env, i32)
+DEF_HELPER_4(vmv_v_v_h, void, ptr, ptr, env, i32)
+DEF_HELPER_4(vmv_v_v_w, void, ptr, ptr, env, i32)
+DEF_HELPER_4(vmv_v_v_d, void, ptr, ptr, env, i32)
+DEF_HELPER_4(vmv_v_x_b, void, ptr, i64, env, i32)
+DEF_HELPER_4(vmv_v_x_h, void, ptr, i64, env, i32)
+DEF_HELPER_4(vmv_v_x_w, void, ptr, i64, env, i32)
+DEF_HELPER_4(vmv_v_x_d, void, ptr, i64, env, i32)
+DEF_HELPER_FLAGS_2(clmul, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(clmulr, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_1(brev8, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(unzip, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(zip, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_2(xperm4, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(xperm8, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(aes32esmi, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl)
+DEF_HELPER_FLAGS_3(aes32esi, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl)
+DEF_HELPER_FLAGS_3(aes32dsmi, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl)
+DEF_HELPER_FLAGS_3(aes32dsi, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl)
+DEF_HELPER_FLAGS_2(aes64esm, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(aes64es, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(aes64ds, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(aes64dsm, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(aes64ks2, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(aes64ks1i, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_1(aes64im, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_3(sm4ed, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl)
+DEF_HELPER_FLAGS_3(sm4ks, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl)
 DEF_HELPER_2(sret, tl, env, tl)
 DEF_HELPER_2(mret, tl, env, tl)
 DEF_HELPER_1(wfi, void, env)
 DEF_HELPER_1(tlb_flush, void, env)
+DEF_HELPER_1(hyp_tlb_flush, void, env)
+DEF_HELPER_1(hyp_gvma_tlb_flush, void, env)
+DEF_HELPER_2(hyp_hlvx_hu, tl, env, tl)
+DEF_HELPER_2(hyp_hlvx_wu, tl, env, tl)
diff --git a/qemu/target/riscv/insn_trans/trans_privileged.inc.c b/qemu/target/riscv/insn_trans/trans_privileged.inc.c
index 05662b21e6..e59a19acd8 100644
--- a/qemu/target/riscv/insn_trans/trans_privileged.inc.c
+++ b/qemu/target/riscv/insn_trans/trans_privileged.inc.c
@@ -99,16 +99,9 @@ static bool trans_hfence_gvma(DisasContext *ctx, arg_sfence_vma *a)
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     if (ctx->priv_ver >= PRIV_VERSION_1_10_0 &&
         has_ext(ctx, RVH)) {
-        /* Hpervisor extensions exist */
-        /*
-         * if (env->priv == PRV_M ||
-         *   (env->priv == PRV_S &&
-         *    !riscv_cpu_virt_enabled(env) &&
-         *    get_field(ctx->mstatus_fs, MSTATUS_TVM))) {
-         */
-            gen_helper_tlb_flush(tcg_ctx, tcg_ctx->cpu_env);
-            return true;
-        /* } */
+        tcg_gen_movi_tl(tcg_ctx, tcg_ctx->cpu_pc, ctx->base.pc_next);
+        gen_helper_hyp_gvma_tlb_flush(tcg_ctx, tcg_ctx->cpu_env);
+        return true;
     }
     return false;
 }
@@ -118,16 +111,9 @@ static bool trans_hfence_bvma(DisasContext *ctx, arg_sfence_vma *a)
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     if (ctx->priv_ver >= PRIV_VERSION_1_10_0 &&
         has_ext(ctx, RVH)) {
-        /* Hpervisor extensions exist */
-        /*
-         * if (env->priv == PRV_M ||
-         *   (env->priv == PRV_S &&
-         *    !riscv_cpu_virt_enabled(env) &&
-         *    get_field(ctx->mstatus_fs, MSTATUS_TVM))) {
-         */
-            gen_helper_tlb_flush(tcg_ctx, tcg_ctx->cpu_env);
-            return true;
-        /* } */
+        tcg_gen_movi_tl(tcg_ctx, tcg_ctx->cpu_pc, ctx->base.pc_next);
+        gen_helper_hyp_tlb_flush(tcg_ctx, tcg_ctx->cpu_env);
+        return true;
     }
     return false;
 }
diff --git a/qemu/target/riscv/insn_trans/trans_rvb.inc.c b/qemu/target/riscv/insn_trans/trans_rvb.inc.c
new file mode 100644
index 0000000000..0e09c70034
--- /dev/null
+++ b/qemu/target/riscv/insn_trans/trans_rvb.inc.c
@@ -0,0 +1,680 @@
+/*
+ * RISC-V translation routines for bit manipulation extensions.
+ *
+ * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
+ * Copyright (c) 2018 Peer Adelt, peer.adelt@hni.uni-paderborn.de
+ *                    Bastian Koppelmann, kbastian@mail.uni-paderborn.de
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define REQUIRE_ZBA(ctx) do { \
+    if (!(ctx)->ext_zba) {    \
+        return false;         \
+    }                         \
+} while (0)
+
+#define REQUIRE_ZBB(ctx) do { \
+    if (!(ctx)->ext_zbb) {    \
+        return false;         \
+    }                         \
+} while (0)
+
+#define REQUIRE_ZBC(ctx) do { \
+    if (!(ctx)->ext_zbc) {    \
+        return false;         \
+    }                         \
+} while (0)
+
+#define REQUIRE_ZBC_OR_ZBKC(ctx) do {             \
+    if (!(ctx)->ext_zbc && !(ctx)->ext_zbkc) {    \
+        return false;                             \
+    }                                             \
+} while (0)
+
+#define REQUIRE_ZBKB(ctx) do { \
+    if (!(ctx)->ext_zbkb) {    \
+        return false;          \
+    }                          \
+} while (0)
+
+#define REQUIRE_ZBKX(ctx) do { \
+    if (!(ctx)->ext_zbkx) {    \
+        return false;          \
+    }                          \
+} while (0)
+
+#define REQUIRE_ZBS(ctx) do { \
+    if (!(ctx)->ext_zbs) {    \
+        return false;         \
+    }                         \
+} while (0)
+
+static bool gen_zbb_unary(DisasContext *ctx, arg_decode_insn3213 *a,
+                          void (*func)(TCGContext *, TCGv, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source;
+
+    if (a->rd == 0) {
+        return true;
+    }
+
+    source = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    func(tcg_ctx, source, source);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+    tcg_temp_free(tcg_ctx, source);
+    return true;
+}
+
+static void gen_clz(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    tcg_gen_clzi_tl(tcg_ctx, ret, arg1, TARGET_LONG_BITS);
+}
+
+static bool trans_clz(DisasContext *ctx, arg_clz *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_clz);
+}
+
+static void gen_ctz(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    tcg_gen_ctzi_tl(tcg_ctx, ret, arg1, TARGET_LONG_BITS);
+}
+
+static bool trans_ctz(DisasContext *ctx, arg_ctz *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_ctz);
+}
+
+static bool trans_cpop(DisasContext *ctx, arg_cpop *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, tcg_gen_ctpop_tl);
+}
+
+static bool trans_andn(DisasContext *ctx, arg_andn *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, tcg_gen_andc_tl);
+}
+
+static bool trans_orn(DisasContext *ctx, arg_orn *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, tcg_gen_orc_tl);
+}
+
+static bool trans_xnor(DisasContext *ctx, arg_xnor *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, tcg_gen_eqv_tl);
+}
+
+static bool trans_min(DisasContext *ctx, arg_min *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, tcg_gen_smin_tl);
+}
+
+static bool trans_max(DisasContext *ctx, arg_max *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, tcg_gen_smax_tl);
+}
+
+static bool trans_minu(DisasContext *ctx, arg_minu *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, tcg_gen_umin_tl);
+}
+
+static bool trans_maxu(DisasContext *ctx, arg_maxu *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, tcg_gen_umax_tl);
+}
+
+static bool trans_sext_b(DisasContext *ctx, arg_sext_b *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, tcg_gen_ext8s_tl);
+}
+
+static bool trans_sext_h(DisasContext *ctx, arg_sext_h *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, tcg_gen_ext16s_tl);
+}
+
+static bool trans_rol(DisasContext *ctx, arg_rol *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_shift(ctx, a, tcg_gen_rotl_tl);
+}
+
+static bool trans_ror(DisasContext *ctx, arg_ror *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_shift(ctx, a, tcg_gen_rotr_tl);
+}
+
+static bool trans_rori(DisasContext *ctx, arg_rori *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source;
+
+    REQUIRE_ZBB(ctx);
+
+    if (a->shamt >= TARGET_LONG_BITS) {
+        return false;
+    }
+
+    if (a->rd == 0) {
+        return true;
+    }
+
+    source = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    tcg_gen_rotri_tl(tcg_ctx, source, source, a->shamt);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+    tcg_temp_free(tcg_ctx, source);
+    return true;
+}
+
+static void gen_rev8_32(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    tcg_gen_bswap32_tl(tcg_ctx, ret, arg1);
+}
+
+#ifndef TARGET_RISCV64
+static bool trans_rev8_32(DisasContext *ctx, arg_rev8_32 *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_rev8_32);
+}
+
+static bool trans_zext_h_32(DisasContext *ctx, arg_zext_h_32 *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, tcg_gen_ext16u_tl);
+}
+#endif
+
+static void gen_orc_b(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    TCGv tmp = tcg_temp_new(tcg_ctx);
+#ifdef TARGET_RISCV64
+    TCGv low7 = tcg_const_tl(tcg_ctx, 0x7f7f7f7f7f7f7f7full);
+#else
+    TCGv low7 = tcg_const_tl(tcg_ctx, 0x7f7f7f7f);
+#endif
+
+    tcg_gen_and_tl(tcg_ctx, tmp, arg1, low7);
+    tcg_gen_add_tl(tcg_ctx, tmp, tmp, low7);
+    tcg_gen_or_tl(tcg_ctx, tmp, tmp, arg1);
+    tcg_gen_andc_tl(tcg_ctx, tmp, tmp, low7);
+    tcg_gen_shri_tl(tcg_ctx, tmp, tmp, 7);
+    tcg_gen_muli_tl(tcg_ctx, ret, tmp, 0xff);
+
+    tcg_temp_free(tcg_ctx, tmp);
+    tcg_temp_free(tcg_ctx, low7);
+}
+
+static bool trans_orc_b(DisasContext *ctx, arg_orc_b *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_orc_b);
+}
+
+#ifdef TARGET_RISCV64
+static void gen_rev8_64(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    tcg_gen_bswap64_tl(tcg_ctx, ret, arg1);
+}
+
+static bool trans_rev8_64(DisasContext *ctx, arg_rev8_64 *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_rev8_64);
+}
+
+static bool trans_zext_h_64(DisasContext *ctx, arg_zext_h_64 *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, tcg_gen_ext16u_tl);
+}
+
+static void gen_clzw(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    TCGv tmp = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_shli_tl(tcg_ctx, tmp, arg1, 32);
+    tcg_gen_clzi_tl(tcg_ctx, ret, tmp, 32);
+    tcg_temp_free(tcg_ctx, tmp);
+}
+
+static bool trans_clzw(DisasContext *ctx, arg_clzw *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_clzw);
+}
+
+static void gen_ctzw(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    TCGv tmp = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_ext32u_tl(tcg_ctx, tmp, arg1);
+    tcg_gen_ctzi_tl(tcg_ctx, ret, tmp, 32);
+    tcg_temp_free(tcg_ctx, tmp);
+}
+
+static bool trans_ctzw(DisasContext *ctx, arg_ctzw *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_ctzw);
+}
+
+static void gen_cpopw(TCGContext *tcg_ctx, TCGv ret, TCGv arg1)
+{
+    tcg_gen_ext32u_tl(tcg_ctx, ret, arg1);
+    tcg_gen_ctpop_tl(tcg_ctx, ret, ret);
+}
+
+static bool trans_cpopw(DisasContext *ctx, arg_cpopw *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_unary(ctx, a, gen_cpopw);
+}
+
+static bool gen_zbb_word_shift(DisasContext *ctx, arg_r *a,
+                               void (*func)(TCGContext *, TCGv_i32,
+                                            TCGv_i32, TCGv_i32))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source1 = tcg_temp_new(tcg_ctx);
+    TCGv source2 = tcg_temp_new(tcg_ctx);
+    TCGv_i32 word1 = tcg_temp_new_i32(tcg_ctx);
+    TCGv_i32 word2 = tcg_temp_new_i32(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source1, a->rs1);
+    gen_get_gpr(tcg_ctx, source2, a->rs2);
+    tcg_gen_trunc_tl_i32(tcg_ctx, word1, source1);
+    tcg_gen_trunc_tl_i32(tcg_ctx, word2, source2);
+    tcg_gen_andi_i32(tcg_ctx, word2, word2, 31);
+    func(tcg_ctx, word1, word1, word2);
+    tcg_gen_ext_i32_tl(tcg_ctx, source1, word1);
+    gen_set_gpr(tcg_ctx, a->rd, source1);
+
+    tcg_temp_free(tcg_ctx, source1);
+    tcg_temp_free(tcg_ctx, source2);
+    tcg_temp_free_i32(tcg_ctx, word1);
+    tcg_temp_free_i32(tcg_ctx, word2);
+    return true;
+}
+
+static bool trans_rolw(DisasContext *ctx, arg_rolw *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_word_shift(ctx, a, tcg_gen_rotl_i32);
+}
+
+static bool trans_rorw(DisasContext *ctx, arg_rorw *a)
+{
+    REQUIRE_ZBB(ctx);
+    return gen_zbb_word_shift(ctx, a, tcg_gen_rotr_i32);
+}
+
+static bool trans_roriw(DisasContext *ctx, arg_roriw *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source;
+    TCGv_i32 word;
+
+    REQUIRE_ZBB(ctx);
+
+    if (a->rd == 0) {
+        return true;
+    }
+
+    source = tcg_temp_new(tcg_ctx);
+    word = tcg_temp_new_i32(tcg_ctx);
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    tcg_gen_trunc_tl_i32(tcg_ctx, word, source);
+    tcg_gen_rotri_i32(tcg_ctx, word, word, a->shamt);
+    tcg_gen_ext_i32_tl(tcg_ctx, source, word);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+
+    tcg_temp_free(tcg_ctx, source);
+    tcg_temp_free_i32(tcg_ctx, word);
+    return true;
+}
+#endif
+
+#define GEN_SHADD(SHAMT)                                             \
+static void gen_sh##SHAMT##add(TCGContext *tcg_ctx, TCGv ret,        \
+                               TCGv arg1, TCGv arg2)                 \
+{                                                                    \
+    TCGv tmp = tcg_temp_new(tcg_ctx);                                \
+                                                                     \
+    tcg_gen_shli_tl(tcg_ctx, tmp, arg1, SHAMT);                      \
+    tcg_gen_add_tl(tcg_ctx, ret, tmp, arg2);                         \
+    tcg_temp_free(tcg_ctx, tmp);                                     \
+}
+
+GEN_SHADD(1)
+GEN_SHADD(2)
+GEN_SHADD(3)
+
+#define GEN_TRANS_SHADD(SHAMT)                                      \
+static bool trans_sh##SHAMT##add(DisasContext *ctx,                 \
+                                 arg_sh##SHAMT##add *a)             \
+{                                                                   \
+    REQUIRE_ZBA(ctx);                                               \
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_sh##SHAMT##add);      \
+}
+
+GEN_TRANS_SHADD(1)
+GEN_TRANS_SHADD(2)
+GEN_TRANS_SHADD(3)
+
+#ifdef TARGET_RISCV64
+#define GEN_SHADD_UW(SHAMT)                                          \
+static void gen_sh##SHAMT##add_uw(TCGContext *tcg_ctx, TCGv ret,     \
+                                  TCGv arg1, TCGv arg2)              \
+{                                                                    \
+    TCGv tmp = tcg_temp_new(tcg_ctx);                                \
+                                                                     \
+    tcg_gen_ext32u_tl(tcg_ctx, tmp, arg1);                           \
+    tcg_gen_shli_tl(tcg_ctx, tmp, tmp, SHAMT);                       \
+    tcg_gen_add_tl(tcg_ctx, ret, tmp, arg2);                         \
+    tcg_temp_free(tcg_ctx, tmp);                                     \
+}
+
+GEN_SHADD_UW(1)
+GEN_SHADD_UW(2)
+GEN_SHADD_UW(3)
+
+#define GEN_TRANS_SHADD_UW(SHAMT)                                   \
+static bool trans_sh##SHAMT##add_uw(DisasContext *ctx,              \
+                                    arg_sh##SHAMT##add_uw *a)       \
+{                                                                   \
+    REQUIRE_ZBA(ctx);                                               \
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_sh##SHAMT##add_uw);   \
+}
+
+GEN_TRANS_SHADD_UW(1)
+GEN_TRANS_SHADD_UW(2)
+GEN_TRANS_SHADD_UW(3)
+
+static void gen_add_uw(TCGContext *tcg_ctx, TCGv ret, TCGv arg1,
+                       TCGv arg2)
+{
+    TCGv tmp = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_ext32u_tl(tcg_ctx, tmp, arg1);
+    tcg_gen_add_tl(tcg_ctx, ret, tmp, arg2);
+    tcg_temp_free(tcg_ctx, tmp);
+}
+
+static bool trans_add_uw(DisasContext *ctx, arg_add_uw *a)
+{
+    REQUIRE_ZBA(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_add_uw);
+}
+
+static bool trans_slli_uw(DisasContext *ctx, arg_slli_uw *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source;
+
+    REQUIRE_ZBA(ctx);
+
+    if (a->shamt >= TARGET_LONG_BITS) {
+        return false;
+    }
+
+    if (a->rd == 0) {
+        return true;
+    }
+
+    source = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    tcg_gen_ext32u_tl(tcg_ctx, source, source);
+    tcg_gen_shli_tl(tcg_ctx, source, source, a->shamt);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+    tcg_temp_free(tcg_ctx, source);
+    return true;
+}
+#endif
+
+static bool trans_clmul(DisasContext *ctx, arg_clmul *a)
+{
+    REQUIRE_ZBC_OR_ZBKC(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_clmul);
+}
+
+static void gen_clmulh(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv arg2)
+{
+    gen_helper_clmulr(tcg_ctx, ret, arg1, arg2);
+    tcg_gen_shri_tl(tcg_ctx, ret, ret, 1);
+}
+
+static bool trans_clmulh(DisasContext *ctx, arg_clmulh *a)
+{
+    REQUIRE_ZBC_OR_ZBKC(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_clmulh);
+}
+
+static bool trans_clmulr(DisasContext *ctx, arg_clmulr *a)
+{
+    REQUIRE_ZBC(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_clmulr);
+}
+
+static bool trans_brev8(DisasContext *ctx, arg_brev8 *a)
+{
+    REQUIRE_ZBKB(ctx);
+    return gen_zbb_unary(ctx, a, gen_helper_brev8);
+}
+
+static void gen_pack(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv arg2)
+{
+    tcg_gen_deposit_tl(tcg_ctx, ret, arg1, arg2,
+                       TARGET_LONG_BITS / 2, TARGET_LONG_BITS / 2);
+}
+
+static bool trans_pack(DisasContext *ctx, arg_pack *a)
+{
+    REQUIRE_ZBKB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_pack);
+}
+
+static void gen_packh(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv arg2)
+{
+    TCGv tmp = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_ext8u_tl(tcg_ctx, tmp, arg2);
+    tcg_gen_deposit_tl(tcg_ctx, ret, arg1, tmp, 8, TARGET_LONG_BITS - 8);
+    tcg_temp_free(tcg_ctx, tmp);
+}
+
+static bool trans_packh(DisasContext *ctx, arg_packh *a)
+{
+    REQUIRE_ZBKB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_packh);
+}
+
+#ifdef TARGET_RISCV64
+static void gen_packw(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv arg2)
+{
+    TCGv tmp = tcg_temp_new(tcg_ctx);
+
+    tcg_gen_ext16s_tl(tcg_ctx, tmp, arg2);
+    tcg_gen_deposit_tl(tcg_ctx, ret, arg1, tmp, 16, TARGET_LONG_BITS - 16);
+    tcg_temp_free(tcg_ctx, tmp);
+}
+
+static bool trans_packw(DisasContext *ctx, arg_packw *a)
+{
+    REQUIRE_ZBKB(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_packw);
+}
+#endif
+
+#ifndef TARGET_RISCV64
+static bool trans_unzip(DisasContext *ctx, arg_unzip *a)
+{
+    REQUIRE_ZBKB(ctx);
+    return gen_zbb_unary(ctx, a, gen_helper_unzip);
+}
+
+static bool trans_zip(DisasContext *ctx, arg_zip *a)
+{
+    REQUIRE_ZBKB(ctx);
+    return gen_zbb_unary(ctx, a, gen_helper_zip);
+}
+#endif
+
+static bool trans_xperm4(DisasContext *ctx, arg_xperm4 *a)
+{
+    REQUIRE_ZBKX(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_xperm4);
+}
+
+static bool trans_xperm8(DisasContext *ctx, arg_xperm8 *a)
+{
+    REQUIRE_ZBKX(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_xperm8);
+}
+
+static void gen_sbop_mask(TCGContext *tcg_ctx, TCGv ret, TCGv shamt)
+{
+    tcg_gen_movi_tl(tcg_ctx, ret, 1);
+    tcg_gen_shl_tl(tcg_ctx, ret, ret, shamt);
+}
+
+static void gen_bset(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv shamt)
+{
+    TCGv mask = tcg_temp_new(tcg_ctx);
+
+    gen_sbop_mask(tcg_ctx, mask, shamt);
+    tcg_gen_or_tl(tcg_ctx, ret, arg1, mask);
+    tcg_temp_free(tcg_ctx, mask);
+}
+
+static void gen_bclr(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv shamt)
+{
+    TCGv mask = tcg_temp_new(tcg_ctx);
+
+    gen_sbop_mask(tcg_ctx, mask, shamt);
+    tcg_gen_andc_tl(tcg_ctx, ret, arg1, mask);
+    tcg_temp_free(tcg_ctx, mask);
+}
+
+static void gen_binv(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv shamt)
+{
+    TCGv mask = tcg_temp_new(tcg_ctx);
+
+    gen_sbop_mask(tcg_ctx, mask, shamt);
+    tcg_gen_xor_tl(tcg_ctx, ret, arg1, mask);
+    tcg_temp_free(tcg_ctx, mask);
+}
+
+static void gen_bext(TCGContext *tcg_ctx, TCGv ret, TCGv arg1, TCGv shamt)
+{
+    tcg_gen_shr_tl(tcg_ctx, ret, arg1, shamt);
+    tcg_gen_andi_tl(tcg_ctx, ret, ret, 1);
+}
+
+static bool gen_zbs_imm(DisasContext *ctx, arg_shift *a,
+                        void (*func)(TCGContext *, TCGv, TCGv, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source;
+    TCGv shamt;
+
+    REQUIRE_ZBS(ctx);
+
+    if (a->shamt >= TARGET_LONG_BITS) {
+        return false;
+    }
+
+    if (a->rd == 0) {
+        return true;
+    }
+
+    source = tcg_temp_new(tcg_ctx);
+    shamt = tcg_const_tl(tcg_ctx, a->shamt);
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    func(tcg_ctx, source, source, shamt);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+    tcg_temp_free(tcg_ctx, source);
+    tcg_temp_free(tcg_ctx, shamt);
+    return true;
+}
+
+static bool trans_bset(DisasContext *ctx, arg_bset *a)
+{
+    REQUIRE_ZBS(ctx);
+    return gen_shift(ctx, a, gen_bset);
+}
+
+static bool trans_bseti(DisasContext *ctx, arg_bseti *a)
+{
+    return gen_zbs_imm(ctx, a, gen_bset);
+}
+
+static bool trans_bclr(DisasContext *ctx, arg_bclr *a)
+{
+    REQUIRE_ZBS(ctx);
+    return gen_shift(ctx, a, gen_bclr);
+}
+
+static bool trans_bclri(DisasContext *ctx, arg_bclri *a)
+{
+    return gen_zbs_imm(ctx, a, gen_bclr);
+}
+
+static bool trans_binv(DisasContext *ctx, arg_binv *a)
+{
+    REQUIRE_ZBS(ctx);
+    return gen_shift(ctx, a, gen_binv);
+}
+
+static bool trans_binvi(DisasContext *ctx, arg_binvi *a)
+{
+    return gen_zbs_imm(ctx, a, gen_binv);
+}
+
+static bool trans_bext(DisasContext *ctx, arg_bext *a)
+{
+    REQUIRE_ZBS(ctx);
+    return gen_shift(ctx, a, gen_bext);
+}
+
+static bool trans_bexti(DisasContext *ctx, arg_bexti *a)
+{
+    return gen_zbs_imm(ctx, a, gen_bext);
+}
+
+#undef REQUIRE_ZBA
+#undef REQUIRE_ZBB
+#undef REQUIRE_ZBC
+#undef REQUIRE_ZBC_OR_ZBKC
+#undef REQUIRE_ZBKB
+#undef REQUIRE_ZBKX
+#undef REQUIRE_ZBS
diff --git a/qemu/target/riscv/insn_trans/trans_rvh.inc.c b/qemu/target/riscv/insn_trans/trans_rvh.inc.c
new file mode 100644
index 0000000000..0f948ef5e2
--- /dev/null
+++ b/qemu/target/riscv/insn_trans/trans_rvh.inc.c
@@ -0,0 +1,275 @@
+/*
+ * RISC-V translation routines for the Hypervisor Extension.
+ *
+ * Copyright (c) 2020 Western Digital
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+typedef struct {
+    int rd;
+    int rs1;
+} arg_rvh_load;
+
+typedef struct {
+    int rs1;
+    int rs2;
+} arg_rvh_store;
+
+static bool rvh_check_access(DisasContext *ctx)
+{
+    if (ctx->hlsx) {
+        return true;
+    }
+
+    if (ctx->virt_enabled) {
+        gen_exception_virtual_instruction(ctx);
+    } else {
+        gen_exception_illegal(ctx);
+    }
+    return false;
+}
+
+static bool do_hlv(DisasContext *ctx, arg_rvh_load *a, MemOp mop)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv dest;
+    TCGv addr;
+    int mem_idx;
+
+    if (!rvh_check_access(ctx)) {
+        return true;
+    }
+
+    dest = tcg_temp_new(tcg_ctx);
+    addr = tcg_temp_new(tcg_ctx);
+    mem_idx = ctx->mem_idx | TB_FLAGS_PRIV_HYP_ACCESS_MASK;
+
+    gen_get_gpr(tcg_ctx, addr, a->rs1);
+    tcg_gen_qemu_ld_tl(tcg_ctx, dest, addr, mem_idx, mop);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+
+    tcg_temp_free(tcg_ctx, addr);
+    tcg_temp_free(tcg_ctx, dest);
+    return true;
+}
+
+static bool trans_hlv_b(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlv(ctx, a, MO_SB);
+}
+
+static bool trans_hlv_bu(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlv(ctx, a, MO_UB);
+}
+
+static bool trans_hlv_h(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlv(ctx, a, MO_TESW);
+}
+
+static bool trans_hlv_hu(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlv(ctx, a, MO_TEUW);
+}
+
+static bool trans_hlv_w(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlv(ctx, a, MO_TESL);
+}
+
+static bool do_hsv(DisasContext *ctx, arg_rvh_store *a, MemOp mop)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv data;
+    TCGv addr;
+    int mem_idx;
+
+    if (!rvh_check_access(ctx)) {
+        return true;
+    }
+
+    addr = tcg_temp_new(tcg_ctx);
+    data = tcg_temp_new(tcg_ctx);
+    mem_idx = ctx->mem_idx | TB_FLAGS_PRIV_HYP_ACCESS_MASK;
+
+    gen_get_gpr(tcg_ctx, addr, a->rs1);
+    gen_get_gpr(tcg_ctx, data, a->rs2);
+    tcg_gen_qemu_st_tl(tcg_ctx, data, addr, mem_idx, mop);
+
+    tcg_temp_free(tcg_ctx, data);
+    tcg_temp_free(tcg_ctx, addr);
+    return true;
+}
+
+static bool trans_hsv_b(DisasContext *ctx, arg_rvh_store *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hsv(ctx, a, MO_SB);
+}
+
+static bool trans_hsv_h(DisasContext *ctx, arg_rvh_store *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hsv(ctx, a, MO_TESW);
+}
+
+static bool trans_hsv_w(DisasContext *ctx, arg_rvh_store *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hsv(ctx, a, MO_TESL);
+}
+
+#ifdef TARGET_RISCV64
+static bool trans_hlv_wu(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlv(ctx, a, MO_TEUL);
+}
+
+static bool trans_hlv_d(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlv(ctx, a, MO_TEUQ);
+}
+
+static bool trans_hsv_d(DisasContext *ctx, arg_rvh_store *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hsv(ctx, a, MO_TEUQ);
+}
+#endif
+
+static bool do_hlvx(DisasContext *ctx, arg_rvh_load *a,
+                    void (*func)(TCGContext *, TCGv, TCGv_env, TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv dest;
+    TCGv addr;
+
+    if (!rvh_check_access(ctx)) {
+        return true;
+    }
+
+    dest = tcg_temp_new(tcg_ctx);
+    addr = tcg_temp_new(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, addr, a->rs1);
+    func(tcg_ctx, dest, tcg_ctx->cpu_env, addr);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+
+    tcg_temp_free(tcg_ctx, addr);
+    tcg_temp_free(tcg_ctx, dest);
+    return true;
+}
+
+static bool trans_hlvx_hu(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlvx(ctx, a, gen_helper_hyp_hlvx_hu);
+}
+
+static bool trans_hlvx_wu(DisasContext *ctx, arg_rvh_load *a)
+{
+    REQUIRE_EXT(ctx, RVH);
+    return do_hlvx(ctx, a, gen_helper_hyp_hlvx_wu);
+}
+
+static bool decode_rvh(DisasContext *ctx, uint32_t insn)
+{
+    uint32_t funct7;
+    uint32_t rs2;
+    uint32_t rs1;
+    uint32_t rd;
+    arg_rvh_load load;
+    arg_rvh_store store;
+
+    if ((insn & 0x707f) != 0x4073) {
+        return false;
+    }
+
+    funct7 = extract32(insn, 25, 7);
+    rs2 = extract32(insn, 20, 5);
+    rs1 = extract32(insn, 15, 5);
+    rd = extract32(insn, 7, 5);
+    load.rd = rd;
+    load.rs1 = rs1;
+    store.rs1 = rs1;
+    store.rs2 = rs2;
+
+    switch (funct7) {
+    case 0x30:
+        if (rs2 == 0) {
+            return trans_hlv_b(ctx, &load);
+        } else if (rs2 == 1) {
+            return trans_hlv_bu(ctx, &load);
+        }
+        break;
+    case 0x31:
+        if (rd == 0) {
+            return trans_hsv_b(ctx, &store);
+        }
+        break;
+    case 0x32:
+        if (rs2 == 0) {
+            return trans_hlv_h(ctx, &load);
+        } else if (rs2 == 1) {
+            return trans_hlv_hu(ctx, &load);
+        } else if (rs2 == 3) {
+            return trans_hlvx_hu(ctx, &load);
+        }
+        break;
+    case 0x33:
+        if (rd == 0) {
+            return trans_hsv_h(ctx, &store);
+        }
+        break;
+    case 0x34:
+        if (rs2 == 0) {
+            return trans_hlv_w(ctx, &load);
+        } else if (rs2 == 3) {
+            return trans_hlvx_wu(ctx, &load);
+        }
+#ifdef TARGET_RISCV64
+        else if (rs2 == 1) {
+            return trans_hlv_wu(ctx, &load);
+        }
+#endif
+        break;
+    case 0x35:
+        if (rd == 0) {
+            return trans_hsv_w(ctx, &store);
+        }
+        break;
+#ifdef TARGET_RISCV64
+    case 0x36:
+        if (rs2 == 0) {
+            return trans_hlv_d(ctx, &load);
+        }
+        break;
+    case 0x37:
+        if (rd == 0) {
+            return trans_hsv_d(ctx, &store);
+        }
+        break;
+#endif
+    }
+
+    return false;
+}
diff --git a/qemu/target/riscv/insn_trans/trans_rvi.inc.c b/qemu/target/riscv/insn_trans/trans_rvi.inc.c
index 186ed98e97..e1c13991c0 100644
--- a/qemu/target/riscv/insn_trans/trans_rvi.inc.c
+++ b/qemu/target/riscv/insn_trans/trans_rvi.inc.c
@@ -501,6 +501,20 @@ static bool trans_sraw(DisasContext *ctx, arg_sraw *a)
 }
 #endif
 
+static bool trans_pause(DisasContext *ctx, arg_pause *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    if (!ctx->ext_zihintpause) {
+        return false;
+    }
+
+    tcg_gen_movi_tl(tcg_ctx, tcg_ctx->cpu_pc, ctx->pc_succ_insn);
+    exit_tb(ctx);
+    ctx->base.is_jmp = DISAS_NORETURN;
+    return true;
+}
+
 static bool trans_fence(DisasContext *ctx, arg_fence *a)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
diff --git a/qemu/target/riscv/insn_trans/trans_rvk.inc.c b/qemu/target/riscv/insn_trans/trans_rvk.inc.c
new file mode 100644
index 0000000000..bf0d85548d
--- /dev/null
+++ b/qemu/target/riscv/insn_trans/trans_rvk.inc.c
@@ -0,0 +1,453 @@
+/*
+ * RISC-V translation routines for scalar cryptography extensions.
+ *
+ * Copyright (c) 2021 Ruibo Lu, luruibo2000@163.com
+ * Copyright (c) 2021 Zewen Ye, lustrew@foxmail.com
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define REQUIRE_ZKNH(ctx) do { \
+    if (!(ctx)->ext_zknh) {    \
+        return false;          \
+    }                          \
+} while (0)
+
+#define REQUIRE_ZKND(ctx) do { \
+    if (!(ctx)->ext_zknd) {    \
+        return false;          \
+    }                          \
+} while (0)
+
+#define REQUIRE_ZKNE(ctx) do { \
+    if (!(ctx)->ext_zkne) {    \
+        return false;          \
+    }                          \
+} while (0)
+
+#define REQUIRE_ZKND_OR_ZKNE(ctx) do {                 \
+    if (!(ctx)->ext_zknd && !(ctx)->ext_zkne) {         \
+        return false;                                  \
+    }                                                  \
+} while (0)
+
+#define REQUIRE_ZKSED(ctx) do { \
+    if (!(ctx)->ext_zksed) {    \
+        return false;           \
+    }                           \
+} while (0)
+
+#define REQUIRE_ZKSH(ctx) do { \
+    if (!(ctx)->ext_zksh) {    \
+        return false;          \
+    }                          \
+} while (0)
+
+static void gen_shri_i32(TCGContext *tcg_ctx, TCGv_i32 ret, TCGv_i32 arg,
+                         unsigned shift)
+{
+    tcg_gen_shri_i32(tcg_ctx, ret, arg, shift);
+}
+
+static void gen_shri_i64(TCGContext *tcg_ctx, TCGv_i64 ret, TCGv_i64 arg,
+                         unsigned shift)
+{
+    tcg_gen_shri_i64(tcg_ctx, ret, arg, shift);
+}
+
+static bool gen_aes32_sm4(DisasContext *ctx, arg_k_aes *a,
+                          void (*func)(TCGContext *, TCGv, TCGv, TCGv,
+                                       TCGv))
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv dest = tcg_temp_new(tcg_ctx);
+    TCGv source1 = tcg_temp_new(tcg_ctx);
+    TCGv source2 = tcg_temp_new(tcg_ctx);
+    TCGv shamt = tcg_temp_new(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source1, a->rs1);
+    gen_get_gpr(tcg_ctx, source2, a->rs2);
+    tcg_gen_movi_tl(tcg_ctx, shamt, a->shamt);
+    func(tcg_ctx, dest, source1, source2, shamt);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+
+    tcg_temp_free(tcg_ctx, dest);
+    tcg_temp_free(tcg_ctx, source1);
+    tcg_temp_free(tcg_ctx, source2);
+    tcg_temp_free(tcg_ctx, shamt);
+    return true;
+}
+
+#ifndef TARGET_RISCV64
+static bool trans_aes32esmi(DisasContext *ctx, arg_aes32esmi *a)
+{
+    REQUIRE_ZKNE(ctx);
+    return gen_aes32_sm4(ctx, a, gen_helper_aes32esmi);
+}
+
+static bool trans_aes32esi(DisasContext *ctx, arg_aes32esi *a)
+{
+    REQUIRE_ZKNE(ctx);
+    return gen_aes32_sm4(ctx, a, gen_helper_aes32esi);
+}
+
+static bool trans_aes32dsmi(DisasContext *ctx, arg_aes32dsmi *a)
+{
+    REQUIRE_ZKND(ctx);
+    return gen_aes32_sm4(ctx, a, gen_helper_aes32dsmi);
+}
+
+static bool trans_aes32dsi(DisasContext *ctx, arg_aes32dsi *a)
+{
+    REQUIRE_ZKND(ctx);
+    return gen_aes32_sm4(ctx, a, gen_helper_aes32dsi);
+}
+#else
+static bool trans_aes64es(DisasContext *ctx, arg_aes64es *a)
+{
+    REQUIRE_ZKNE(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_aes64es);
+}
+
+static bool trans_aes64esm(DisasContext *ctx, arg_aes64esm *a)
+{
+    REQUIRE_ZKNE(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_aes64esm);
+}
+
+static bool trans_aes64ds(DisasContext *ctx, arg_aes64ds *a)
+{
+    REQUIRE_ZKND(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_aes64ds);
+}
+
+static bool trans_aes64dsm(DisasContext *ctx, arg_aes64dsm *a)
+{
+    REQUIRE_ZKND(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_aes64dsm);
+}
+
+static bool trans_aes64ks2(DisasContext *ctx, arg_aes64ks2 *a)
+{
+    REQUIRE_ZKND_OR_ZKNE(ctx);
+    return gen_arith(ctx->uc->tcg_ctx, a, gen_helper_aes64ks2);
+}
+
+static bool trans_aes64ks1i(DisasContext *ctx, arg_aes64ks1i *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv dest;
+    TCGv source;
+    TCGv round;
+
+    REQUIRE_ZKND_OR_ZKNE(ctx);
+    if (a->imm > 0xa) {
+        return false;
+    }
+
+    dest = tcg_temp_new(tcg_ctx);
+    source = tcg_temp_new(tcg_ctx);
+    round = tcg_temp_new(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    tcg_gen_movi_tl(tcg_ctx, round, a->imm);
+    gen_helper_aes64ks1i(tcg_ctx, dest, source, round);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+
+    tcg_temp_free(tcg_ctx, dest);
+    tcg_temp_free(tcg_ctx, source);
+    tcg_temp_free(tcg_ctx, round);
+    return true;
+}
+
+static bool trans_aes64im(DisasContext *ctx, arg_aes64im *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv dest = tcg_temp_new(tcg_ctx);
+    TCGv source = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_ZKND(ctx);
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    gen_helper_aes64im(tcg_ctx, dest, source);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+
+    tcg_temp_free(tcg_ctx, dest);
+    tcg_temp_free(tcg_ctx, source);
+    return true;
+}
+#endif
+
+static bool gen_sha256(DisasContext *ctx, arg_decode_insn3213 *a,
+                       void (*func)(TCGContext *, TCGv_i32, TCGv_i32,
+                                    unsigned),
+                       unsigned num1, unsigned num2, unsigned num3)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source = tcg_temp_new(tcg_ctx);
+    TCGv_i32 word = tcg_temp_new_i32(tcg_ctx);
+    TCGv_i32 tmp1 = tcg_temp_new_i32(tcg_ctx);
+    TCGv_i32 tmp2 = tcg_temp_new_i32(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    tcg_gen_trunc_tl_i32(tcg_ctx, word, source);
+    tcg_gen_rotri_i32(tcg_ctx, tmp1, word, num1);
+    tcg_gen_rotri_i32(tcg_ctx, tmp2, word, num2);
+    tcg_gen_xor_i32(tcg_ctx, tmp1, tmp1, tmp2);
+    func(tcg_ctx, tmp2, word, num3);
+    tcg_gen_xor_i32(tcg_ctx, tmp1, tmp1, tmp2);
+    tcg_gen_ext_i32_tl(tcg_ctx, source, tmp1);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+
+    tcg_temp_free(tcg_ctx, source);
+    tcg_temp_free_i32(tcg_ctx, word);
+    tcg_temp_free_i32(tcg_ctx, tmp1);
+    tcg_temp_free_i32(tcg_ctx, tmp2);
+    return true;
+}
+
+static bool trans_sha256sig0(DisasContext *ctx, arg_sha256sig0 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha256(ctx, a, gen_shri_i32, 7, 18, 3);
+}
+
+static bool trans_sha256sig1(DisasContext *ctx, arg_sha256sig1 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha256(ctx, a, gen_shri_i32, 17, 19, 10);
+}
+
+static bool trans_sha256sum0(DisasContext *ctx, arg_sha256sum0 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha256(ctx, a, tcg_gen_rotri_i32, 2, 13, 22);
+}
+
+static bool trans_sha256sum1(DisasContext *ctx, arg_sha256sum1 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha256(ctx, a, tcg_gen_rotri_i32, 6, 11, 25);
+}
+
+#ifndef TARGET_RISCV64
+static bool gen_sha512_rv32(DisasContext *ctx, arg_r *a,
+                            void (*func1)(TCGContext *, TCGv_i64,
+                                          TCGv_i64, unsigned),
+                            void (*func2)(TCGContext *, TCGv_i64,
+                                          TCGv_i64, unsigned),
+                            unsigned num1, unsigned num2, unsigned num3)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source1 = tcg_temp_new(tcg_ctx);
+    TCGv source2 = tcg_temp_new(tcg_ctx);
+    TCGv_i64 dword = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tmp1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tmp2 = tcg_temp_new_i64(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source1, a->rs1);
+    gen_get_gpr(tcg_ctx, source2, a->rs2);
+    tcg_gen_concat_tl_i64(tcg_ctx, dword, source1, source2);
+    func1(tcg_ctx, tmp1, dword, num1);
+    func2(tcg_ctx, tmp2, dword, num2);
+    tcg_gen_xor_i64(tcg_ctx, tmp1, tmp1, tmp2);
+    tcg_gen_rotri_i64(tcg_ctx, tmp2, dword, num3);
+    tcg_gen_xor_i64(tcg_ctx, tmp1, tmp1, tmp2);
+    tcg_gen_trunc_i64_tl(tcg_ctx, source1, tmp1);
+    gen_set_gpr(tcg_ctx, a->rd, source1);
+
+    tcg_temp_free(tcg_ctx, source1);
+    tcg_temp_free(tcg_ctx, source2);
+    tcg_temp_free_i64(tcg_ctx, dword);
+    tcg_temp_free_i64(tcg_ctx, tmp1);
+    tcg_temp_free_i64(tcg_ctx, tmp2);
+    return true;
+}
+
+static bool trans_sha512sum0r(DisasContext *ctx, arg_sha512sum0r *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv32(ctx, a, tcg_gen_rotli_i64,
+                           tcg_gen_rotli_i64, 25, 30, 28);
+}
+
+static bool trans_sha512sum1r(DisasContext *ctx, arg_sha512sum1r *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv32(ctx, a, tcg_gen_rotli_i64,
+                           tcg_gen_rotri_i64, 23, 14, 18);
+}
+
+static bool trans_sha512sig0l(DisasContext *ctx, arg_sha512sig0l *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv32(ctx, a, tcg_gen_rotri_i64,
+                           tcg_gen_rotri_i64, 1, 7, 8);
+}
+
+static bool trans_sha512sig1l(DisasContext *ctx, arg_sha512sig1l *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv32(ctx, a, tcg_gen_rotli_i64,
+                           tcg_gen_rotri_i64, 3, 6, 19);
+}
+
+static bool gen_sha512h_rv32(DisasContext *ctx, arg_r *a,
+                             void (*func)(TCGContext *, TCGv_i64,
+                                          TCGv_i64, unsigned),
+                             unsigned num1, unsigned num2, unsigned num3)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source1 = tcg_temp_new(tcg_ctx);
+    TCGv source2 = tcg_temp_new(tcg_ctx);
+    TCGv_i64 dword = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tmp1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tmp2 = tcg_temp_new_i64(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source1, a->rs1);
+    gen_get_gpr(tcg_ctx, source2, a->rs2);
+    tcg_gen_concat_tl_i64(tcg_ctx, dword, source1, source2);
+    func(tcg_ctx, tmp1, dword, num1);
+    tcg_gen_ext32u_i64(tcg_ctx, tmp2, dword);
+    tcg_gen_shri_i64(tcg_ctx, tmp2, tmp2, num2);
+    tcg_gen_xor_i64(tcg_ctx, tmp1, tmp1, tmp2);
+    tcg_gen_rotri_i64(tcg_ctx, tmp2, dword, num3);
+    tcg_gen_xor_i64(tcg_ctx, tmp1, tmp1, tmp2);
+    tcg_gen_trunc_i64_tl(tcg_ctx, source1, tmp1);
+    gen_set_gpr(tcg_ctx, a->rd, source1);
+
+    tcg_temp_free(tcg_ctx, source1);
+    tcg_temp_free(tcg_ctx, source2);
+    tcg_temp_free_i64(tcg_ctx, dword);
+    tcg_temp_free_i64(tcg_ctx, tmp1);
+    tcg_temp_free_i64(tcg_ctx, tmp2);
+    return true;
+}
+
+static bool trans_sha512sig0h(DisasContext *ctx, arg_sha512sig0h *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512h_rv32(ctx, a, tcg_gen_rotri_i64, 1, 7, 8);
+}
+
+static bool trans_sha512sig1h(DisasContext *ctx, arg_sha512sig1h *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512h_rv32(ctx, a, tcg_gen_rotli_i64, 3, 6, 19);
+}
+#else
+static bool gen_sha512_rv64(DisasContext *ctx, arg_decode_insn3213 *a,
+                            void (*func)(TCGContext *, TCGv_i64,
+                                         TCGv_i64, unsigned),
+                            unsigned num1, unsigned num2, unsigned num3)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source = tcg_temp_new(tcg_ctx);
+    TCGv_i64 dword = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tmp1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 tmp2 = tcg_temp_new_i64(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    tcg_gen_extu_tl_i64(tcg_ctx, dword, source);
+    tcg_gen_rotri_i64(tcg_ctx, tmp1, dword, num1);
+    tcg_gen_rotri_i64(tcg_ctx, tmp2, dword, num2);
+    tcg_gen_xor_i64(tcg_ctx, tmp1, tmp1, tmp2);
+    func(tcg_ctx, tmp2, dword, num3);
+    tcg_gen_xor_i64(tcg_ctx, tmp1, tmp1, tmp2);
+    tcg_gen_trunc_i64_tl(tcg_ctx, source, tmp1);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+
+    tcg_temp_free(tcg_ctx, source);
+    tcg_temp_free_i64(tcg_ctx, dword);
+    tcg_temp_free_i64(tcg_ctx, tmp1);
+    tcg_temp_free_i64(tcg_ctx, tmp2);
+    return true;
+}
+
+static bool trans_sha512sig0(DisasContext *ctx, arg_sha512sig0 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv64(ctx, a, gen_shri_i64, 1, 8, 7);
+}
+
+static bool trans_sha512sig1(DisasContext *ctx, arg_sha512sig1 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv64(ctx, a, gen_shri_i64, 19, 61, 6);
+}
+
+static bool trans_sha512sum0(DisasContext *ctx, arg_sha512sum0 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv64(ctx, a, tcg_gen_rotri_i64, 28, 34, 39);
+}
+
+static bool trans_sha512sum1(DisasContext *ctx, arg_sha512sum1 *a)
+{
+    REQUIRE_ZKNH(ctx);
+    return gen_sha512_rv64(ctx, a, tcg_gen_rotri_i64, 14, 18, 41);
+}
+#endif
+
+static bool gen_sm3(DisasContext *ctx, arg_decode_insn3213 *a,
+                    unsigned num1, unsigned num2)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv source = tcg_temp_new(tcg_ctx);
+    TCGv_i32 word = tcg_temp_new_i32(tcg_ctx);
+    TCGv_i32 tmp = tcg_temp_new_i32(tcg_ctx);
+
+    gen_get_gpr(tcg_ctx, source, a->rs1);
+    tcg_gen_trunc_tl_i32(tcg_ctx, word, source);
+    tcg_gen_rotli_i32(tcg_ctx, tmp, word, num1);
+    tcg_gen_xor_i32(tcg_ctx, tmp, word, tmp);
+    tcg_gen_rotli_i32(tcg_ctx, word, word, num2);
+    tcg_gen_xor_i32(tcg_ctx, tmp, tmp, word);
+    tcg_gen_ext_i32_tl(tcg_ctx, source, tmp);
+    gen_set_gpr(tcg_ctx, a->rd, source);
+
+    tcg_temp_free(tcg_ctx, source);
+    tcg_temp_free_i32(tcg_ctx, word);
+    tcg_temp_free_i32(tcg_ctx, tmp);
+    return true;
+}
+
+static bool trans_sm3p0(DisasContext *ctx, arg_sm3p0 *a)
+{
+    REQUIRE_ZKSH(ctx);
+    return gen_sm3(ctx, a, 9, 17);
+}
+
+static bool trans_sm3p1(DisasContext *ctx, arg_sm3p1 *a)
+{
+    REQUIRE_ZKSH(ctx);
+    return gen_sm3(ctx, a, 15, 23);
+}
+
+static bool trans_sm4ed(DisasContext *ctx, arg_sm4ed *a)
+{
+    REQUIRE_ZKSED(ctx);
+    return gen_aes32_sm4(ctx, a, gen_helper_sm4ed);
+}
+
+static bool trans_sm4ks(DisasContext *ctx, arg_sm4ks *a)
+{
+    REQUIRE_ZKSED(ctx);
+    return gen_aes32_sm4(ctx, a, gen_helper_sm4ks);
+}
+
+#undef REQUIRE_ZKND
+#undef REQUIRE_ZKNE
+#undef REQUIRE_ZKNH
+#undef REQUIRE_ZKND_OR_ZKNE
+#undef REQUIRE_ZKSED
+#undef REQUIRE_ZKSH
diff --git a/qemu/target/riscv/insn_trans/trans_rvm.inc.c b/qemu/target/riscv/insn_trans/trans_rvm.inc.c
index 1346010aff..a31324b276 100644
--- a/qemu/target/riscv/insn_trans/trans_rvm.inc.c
+++ b/qemu/target/riscv/insn_trans/trans_rvm.inc.c
@@ -18,10 +18,16 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define REQUIRE_M_OR_ZMMUL(ctx) do {       \
+    if (!(ctx)->ext_zmmul &&               \
+        !has_ext(ctx, RVM)) {              \
+        return false;                      \
+    }                                      \
+} while (0)
 
 static bool trans_mul(DisasContext *ctx, arg_mul *a)
 {
-    REQUIRE_EXT(ctx, RVM);
+    REQUIRE_M_OR_ZMMUL(ctx);
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     return gen_arith(tcg_ctx, a, &tcg_gen_mul_tl);
 }
@@ -29,7 +35,7 @@ static bool trans_mul(DisasContext *ctx, arg_mul *a)
 static bool trans_mulh(DisasContext *ctx, arg_mulh *a)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
-    REQUIRE_EXT(ctx, RVM);
+    REQUIRE_M_OR_ZMMUL(ctx);
     TCGv source1 = tcg_temp_new(tcg_ctx);
     TCGv source2 = tcg_temp_new(tcg_ctx);
     gen_get_gpr(tcg_ctx, source1, a->rs1);
@@ -45,7 +51,7 @@ static bool trans_mulh(DisasContext *ctx, arg_mulh *a)
 
 static bool trans_mulhsu(DisasContext *ctx, arg_mulhsu *a)
 {
-    REQUIRE_EXT(ctx, RVM);
+    REQUIRE_M_OR_ZMMUL(ctx);
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     return gen_arith(tcg_ctx, a, &gen_mulhsu);
 }
@@ -53,7 +59,7 @@ static bool trans_mulhsu(DisasContext *ctx, arg_mulhsu *a)
 static bool trans_mulhu(DisasContext *ctx, arg_mulhu *a)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
-    REQUIRE_EXT(ctx, RVM);
+    REQUIRE_M_OR_ZMMUL(ctx);
     TCGv source1 = tcg_temp_new(tcg_ctx);
     TCGv source2 = tcg_temp_new(tcg_ctx);
     gen_get_gpr(tcg_ctx, source1, a->rs1);
@@ -98,7 +104,7 @@ static bool trans_remu(DisasContext *ctx, arg_remu *a)
 #ifdef TARGET_RISCV64
 static bool trans_mulw(DisasContext *ctx, arg_mulw *a)
 {
-    REQUIRE_EXT(ctx, RVM);
+    REQUIRE_M_OR_ZMMUL(ctx);
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
     return gen_arith(tcg_ctx, a, &gen_mulw);
 }
diff --git a/qemu/target/riscv/insn_trans/trans_rvv.inc.c b/qemu/target/riscv/insn_trans/trans_rvv.inc.c
new file mode 100644
index 0000000000..cc363f736b
--- /dev/null
+++ b/qemu/target/riscv/insn_trans/trans_rvv.inc.c
@@ -0,0 +1,7895 @@
+/*
+ * RISC-V translation routines for vector configuration instructions.
+ *
+ * Copyright (c) 2020 T-Head Semiconductor Co., Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+typedef struct {
+    int rd;
+    int rs1;
+    int zimm;
+} arg_vsetvli;
+
+typedef arg_vsetvli arg_vsetivli;
+typedef arg_r arg_vsetvl;
+
+typedef struct {
+    int rd;
+    int rs1;
+    int rs2;
+    int nf;
+    int vm;
+} arg_rvv_ldst;
+
+typedef struct {
+    int rd;
+    int rs1;
+    int rs2;
+    int vm;
+} arg_rvv_arith;
+
+static bool require_rvv(DisasContext *ctx)
+{
+    return ctx->mstatus_vs != 0 &&
+           (has_ext(ctx, RVV) || ctx->ext_zve32f || ctx->ext_zve64f);
+}
+
+static bool require_rvv_data(DisasContext *ctx)
+{
+    return require_rvv(ctx) && !ctx->vill && ctx->sew <= MO_64;
+}
+
+static bool require_rvf(DisasContext *ctx)
+{
+    if (ctx->mstatus_fs == 0) {
+        return false;
+    }
+
+    switch (ctx->sew) {
+    case MO_16:
+    case MO_32:
+        return has_ext(ctx, RVF);
+    case MO_64:
+        return has_ext(ctx, RVD);
+    default:
+        return false;
+    }
+}
+
+static bool require_scale_rvf(DisasContext *ctx)
+{
+    if (ctx->mstatus_fs == 0) {
+        return false;
+    }
+
+    switch (ctx->sew) {
+    case MO_8:
+    case MO_16:
+        return has_ext(ctx, RVF);
+    case MO_32:
+        return has_ext(ctx, RVD);
+    default:
+        return false;
+    }
+}
+
+static bool require_zve32f(DisasContext *ctx)
+{
+    if (has_ext(ctx, RVV)) {
+        return true;
+    }
+
+    return ctx->ext_zve32f ? ctx->sew <= MO_32 : true;
+}
+
+static bool require_zve64f(DisasContext *ctx)
+{
+    if (has_ext(ctx, RVV)) {
+        return true;
+    }
+
+    return ctx->ext_zve64f ? ctx->sew <= MO_32 : true;
+}
+
+static bool require_scale_zve32f(DisasContext *ctx)
+{
+    if (has_ext(ctx, RVV)) {
+        return true;
+    }
+
+    return ctx->ext_zve32f ? ctx->sew <= MO_16 : true;
+}
+
+static bool require_scale_zve64f(DisasContext *ctx)
+{
+    if (has_ext(ctx, RVV)) {
+        return true;
+    }
+
+    return ctx->ext_zve64f ? ctx->sew <= MO_16 : true;
+}
+
+static bool require_vm(int vm, int vd)
+{
+    return vm != 0 || vd != 0;
+}
+
+static bool require_nf(int vd, int nf, int lmul)
+{
+    int size = nf << MAX(lmul, 0);
+
+    return size <= 8 && vd + size <= 32;
+}
+
+static bool require_align(int val, int8_t lmul)
+{
+    return lmul <= 0 || extract32(val, 0, lmul) == 0;
+}
+
+static inline bool is_overlapped(int8_t astart, int8_t asize,
+                                 int8_t bstart, int8_t bsize)
+{
+    int8_t aend = astart + asize;
+    int8_t bend = bstart + bsize;
+
+    return MAX(aend, bend) - MIN(astart, bstart) < asize + bsize;
+}
+
+static bool require_noover(int8_t dst, int8_t dst_lmul,
+                           int8_t src, int8_t src_lmul)
+{
+    int8_t dst_size = dst_lmul <= 0 ? 1 : 1 << dst_lmul;
+    int8_t src_size = src_lmul <= 0 ? 1 : 1 << src_lmul;
+
+    if (dst_size > src_size &&
+        dst < src &&
+        src_lmul >= 0 &&
+        is_overlapped(dst, dst_size, src, src_size) &&
+        !is_overlapped(dst, dst_size, src + src_size, src_size)) {
+        return true;
+    }
+
+    return !is_overlapped(dst, dst_size, src, src_size);
+}
+
+static bool vext_check_store(DisasContext *ctx, int vd, int nf, uint8_t eew)
+{
+    int8_t emul = eew - ctx->sew + ctx->lmul;
+
+    return emul >= -3 && emul <= 3 &&
+           require_align(vd, emul) &&
+           require_nf(vd, nf, emul);
+}
+
+static bool vext_check_load(DisasContext *ctx, int vd, int nf, int vm,
+                            uint8_t eew)
+{
+    return vext_check_store(ctx, vd, nf, eew) && require_vm(vm, vd);
+}
+
+static bool vext_check_st_index(DisasContext *ctx, int vd, int vs2,
+                                int nf, uint8_t eew)
+{
+    int8_t emul = eew - ctx->sew + ctx->lmul;
+    bool ret = emul >= -3 && emul <= 3 &&
+               require_align(vs2, emul) &&
+               require_align(vd, ctx->lmul) &&
+               require_nf(vd, nf, ctx->lmul);
+
+#if !defined(TARGET_RISCV64)
+    if (!has_ext(ctx, RVV) && ctx->ext_zve64f && eew == MO_64) {
+        ret = false;
+    }
+#endif
+
+    return ret;
+}
+
+static bool vext_check_ld_index(DisasContext *ctx, int vd, int vs2,
+                                int nf, int vm, uint8_t eew)
+{
+    int8_t emul = eew - ctx->sew + ctx->lmul;
+    int8_t seg_vd;
+    int i;
+    bool ret = vext_check_st_index(ctx, vd, vs2, nf, eew) &&
+               require_vm(vm, vd);
+
+    for (i = 0; i < nf; i++) {
+        seg_vd = vd + (1 << MAX(ctx->lmul, 0)) * i;
+
+        if (eew > ctx->sew) {
+            if (seg_vd != vs2) {
+                ret &= require_noover(seg_vd, ctx->lmul, vs2, emul);
+            }
+        } else if (eew < ctx->sew) {
+            ret &= require_noover(seg_vd, ctx->lmul, vs2, emul);
+        }
+
+        if (nf > 1) {
+            ret &= !is_overlapped(seg_vd, 1 << MAX(ctx->lmul, 0),
+                                  vs2, 1 << MAX(emul, 0));
+        }
+    }
+    return ret;
+}
+
+static bool vext_check_ss(DisasContext *ctx, int vd, int vs, int vm)
+{
+    return require_vm(vm, vd) &&
+           require_align(vd, ctx->lmul) &&
+           require_align(vs, ctx->lmul);
+}
+
+static bool vext_check_sss(DisasContext *ctx, int vd, int vs1, int vs2,
+                           int vm)
+{
+    return vext_check_ss(ctx, vd, vs2, vm) &&
+           require_align(vs1, ctx->lmul);
+}
+
+static bool vext_check_ms(DisasContext *ctx, int vd, int vs)
+{
+    bool ret = require_align(vs, ctx->lmul);
+
+    if (vd != vs) {
+        ret &= require_noover(vd, 0, vs, ctx->lmul);
+    }
+    return ret;
+}
+
+static bool vext_check_mss(DisasContext *ctx, int vd, int vs1, int vs2)
+{
+    bool ret = vext_check_ms(ctx, vd, vs2) &&
+               require_align(vs1, ctx->lmul);
+
+    if (vd != vs1) {
+        ret &= require_noover(vd, 0, vs1, ctx->lmul);
+    }
+    return ret;
+}
+
+static bool vext_wide_check_common(DisasContext *ctx, int vd, int vm)
+{
+    return ctx->lmul <= 2 &&
+           ctx->sew < MO_64 &&
+           require_align(vd, ctx->lmul + 1) &&
+           require_vm(vm, vd);
+}
+
+static bool vext_check_ds(DisasContext *ctx, int vd, int vs, int vm)
+{
+    return vext_wide_check_common(ctx, vd, vm) &&
+           require_align(vs, ctx->lmul) &&
+           require_noover(vd, ctx->lmul + 1, vs, ctx->lmul);
+}
+
+static bool vext_check_dd(DisasContext *ctx, int vd, int vs, int vm)
+{
+    return vext_wide_check_common(ctx, vd, vm) &&
+           require_align(vs, ctx->lmul + 1);
+}
+
+static bool vext_check_dss(DisasContext *ctx, int vd, int vs1, int vs2,
+                           int vm)
+{
+    return vext_check_ds(ctx, vd, vs2, vm) &&
+           require_align(vs1, ctx->lmul) &&
+           require_noover(vd, ctx->lmul + 1, vs1, ctx->lmul);
+}
+
+static bool vext_check_dds(DisasContext *ctx, int vd, int vs1, int vs2,
+                           int vm)
+{
+    return vext_check_ds(ctx, vd, vs1, vm) &&
+           require_align(vs2, ctx->lmul + 1);
+}
+
+static bool vext_narrow_check_common(DisasContext *ctx, int vd, int vs2,
+                                     int vm)
+{
+    return ctx->lmul <= 2 &&
+           ctx->sew < MO_64 &&
+           require_align(vs2, ctx->lmul + 1) &&
+           require_align(vd, ctx->lmul) &&
+           require_vm(vm, vd);
+}
+
+static bool vext_check_sd(DisasContext *ctx, int vd, int vs, int vm)
+{
+    bool ret = vext_narrow_check_common(ctx, vd, vs, vm);
+
+    if (vd != vs) {
+        ret &= require_noover(vd, ctx->lmul, vs, ctx->lmul + 1);
+    }
+    return ret;
+}
+
+static bool vext_check_sds(DisasContext *ctx, int vd, int vs1, int vs2,
+                           int vm)
+{
+    return vext_check_sd(ctx, vd, vs2, vm) &&
+           require_align(vs1, ctx->lmul);
+}
+
+static bool vext_check_reduction(DisasContext *ctx, int vs2)
+{
+    return require_align(vs2, ctx->lmul);
+}
+
+static bool vext_check_slide(DisasContext *ctx, int vd, int vs2, int vm,
+                             bool slide_up)
+{
+    bool ret = require_vm(vm, vd) &&
+               require_align(vd, ctx->lmul) &&
+               require_align(vs2, ctx->lmul);
+
+    if (slide_up) {
+        ret &= vd != vs2;
+    }
+    return ret;
+}
+
+static uint8_t vext_get_emul(DisasContext *ctx, uint8_t eew)
+{
+    int8_t emul = eew - ctx->sew + ctx->lmul;
+
+    return emul < 0 ? 0 : emul;
+}
+
+static uint32_t vreg_ofs(DisasContext *ctx, int reg)
+{
+    return offsetof(CPURISCVState, vreg) + reg * ctx->vlen / 8;
+}
+
+static TCGv_ptr gen_vreg_ptr(DisasContext *ctx, int reg)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr ptr = tcg_temp_new_ptr(tcg_ctx);
+
+    tcg_gen_addi_ptr(tcg_ctx, ptr, tcg_ctx->cpu_env, vreg_ofs(ctx, reg));
+    return ptr;
+}
+
+static TCGv_i32 gen_rvv_desc(DisasContext *ctx, uint32_t data)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    uint32_t vlenb = ctx->vlen >> 3;
+
+    return tcg_const_i32(tcg_ctx, simd_desc(vlenb, vlenb, data));
+}
+
+typedef void gen_helper_ldst_us(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv,
+                                TCGv_env, TCGv_i32);
+typedef void gen_helper_ldst_stride(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv,
+                                    TCGv, TCGv_env, TCGv_i32);
+typedef void gen_helper_ldst_index(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv,
+                                   TCGv_ptr, TCGv_env, TCGv_i32);
+typedef void gen_helper_ldst_whole(TCGContext *, TCGv_ptr, TCGv, TCGv_env,
+                                   TCGv_i32);
+typedef void gen_helper_opivv(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv_ptr,
+                              TCGv_ptr, TCGv_env, TCGv_i32);
+typedef void gen_helper_opivx(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv,
+                              TCGv_ptr, TCGv_env, TCGv_i32);
+typedef void gen_helper_opfvf(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv_i64,
+                              TCGv_ptr, TCGv_env, TCGv_i32);
+typedef void gen_helper_opivm(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv_ptr,
+                              TCGv_env, TCGv_i32);
+typedef void gen_helper_vmv_vv(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv_env,
+                               TCGv_i32);
+typedef void gen_helper_vmv_vx(TCGContext *, TCGv_ptr, TCGv_i64, TCGv_env,
+                               TCGv_i32);
+typedef void gen_helper_vid_v(TCGContext *, TCGv_ptr, TCGv_ptr, TCGv_env,
+                              TCGv_i32);
+typedef void gen_helper_mscalar(TCGContext *, TCGv, TCGv_ptr, TCGv_ptr,
+                                TCGv_env, TCGv_i32);
+
+static bool do_vsetvl(DisasContext *ctx, int rd, int rs1, TCGv s2)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv s1;
+    TCGv dst;
+
+    if (!require_rvv(ctx)) {
+        return false;
+    }
+
+    dst = tcg_temp_new(tcg_ctx);
+    s1 = tcg_temp_new(tcg_ctx);
+    if (rd == 0 && rs1 == 0) {
+        tcg_gen_mov_tl(tcg_ctx, s1, cpu_vl);
+    } else if (rs1 == 0) {
+        tcg_gen_movi_tl(tcg_ctx, s1, RV_VLEN_MAX);
+    } else {
+        gen_get_gpr(tcg_ctx, s1, rs1);
+    }
+
+    gen_helper_vsetvl(tcg_ctx, dst, tcg_ctx->cpu_env, s1, s2);
+    gen_set_gpr(tcg_ctx, rd, dst);
+    mark_vs_dirty(ctx);
+    tcg_gen_movi_tl(tcg_ctx, tcg_ctx->cpu_pc, ctx->pc_succ_insn);
+    exit_tb(ctx);
+    ctx->base.is_jmp = DISAS_NORETURN;
+
+    tcg_temp_free(tcg_ctx, dst);
+    tcg_temp_free(tcg_ctx, s1);
+    return true;
+}
+
+static bool trans_vsetvl(DisasContext *ctx, arg_vsetvl *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv s2;
+    bool ret;
+
+    s2 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, s2, a->rs2);
+    ret = do_vsetvl(ctx, a->rd, a->rs1, s2);
+    tcg_temp_free(tcg_ctx, s2);
+    return ret;
+}
+
+static bool trans_vsetvli(DisasContext *ctx, arg_vsetvli *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv s2;
+    bool ret;
+
+    s2 = tcg_const_tl(tcg_ctx, a->zimm);
+    ret = do_vsetvl(ctx, a->rd, a->rs1, s2);
+    tcg_temp_free(tcg_ctx, s2);
+    return ret;
+}
+
+static bool trans_vsetivli(DisasContext *ctx, arg_vsetivli *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv s1;
+    TCGv s2;
+    TCGv dst;
+
+    if (!require_rvv(ctx)) {
+        return false;
+    }
+
+    dst = tcg_temp_new(tcg_ctx);
+    s1 = tcg_const_tl(tcg_ctx, a->rs1);
+    s2 = tcg_const_tl(tcg_ctx, a->zimm);
+
+    gen_helper_vsetvl(tcg_ctx, dst, tcg_ctx->cpu_env, s1, s2);
+    gen_set_gpr(tcg_ctx, a->rd, dst);
+    mark_vs_dirty(ctx);
+    tcg_gen_movi_tl(tcg_ctx, tcg_ctx->cpu_pc, ctx->pc_succ_insn);
+    exit_tb(ctx);
+    ctx->base.is_jmp = DISAS_NORETURN;
+
+    tcg_temp_free(tcg_ctx, dst);
+    tcg_temp_free(tcg_ctx, s1);
+    tcg_temp_free(tcg_ctx, s2);
+    return true;
+}
+
+static bool ldst_us_trans(DisasContext *ctx, int vd, int rs1, uint32_t data,
+                          gen_helper_ldst_us *fn, bool is_store)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv base;
+    TCGv_i32 desc;
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    dest = gen_vreg_ptr(ctx, vd);
+    mask = gen_vreg_ptr(ctx, 0);
+    base = tcg_temp_new(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    gen_get_gpr(tcg_ctx, base, rs1);
+    fn(tcg_ctx, dest, mask, base, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free(tcg_ctx, base);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    if (!is_store) {
+        mark_vs_dirty(ctx);
+    }
+
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool ldst_stride_trans(DisasContext *ctx, int vd, int rs1, int rs2,
+                              uint32_t data, gen_helper_ldst_stride *fn,
+                              bool is_store)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv base;
+    TCGv stride;
+    TCGv_i32 desc;
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    dest = gen_vreg_ptr(ctx, vd);
+    mask = gen_vreg_ptr(ctx, 0);
+    base = tcg_temp_new(tcg_ctx);
+    stride = tcg_temp_new(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    gen_get_gpr(tcg_ctx, base, rs1);
+    gen_get_gpr(tcg_ctx, stride, rs2);
+    fn(tcg_ctx, dest, mask, base, stride, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free(tcg_ctx, base);
+    tcg_temp_free(tcg_ctx, stride);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    if (!is_store) {
+        mark_vs_dirty(ctx);
+    }
+
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool ldst_index_trans(DisasContext *ctx, int vd, int rs1, int vs2,
+                             uint32_t data, gen_helper_ldst_index *fn,
+                             bool is_store)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr index;
+    TCGv base;
+    TCGv_i32 desc;
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    dest = gen_vreg_ptr(ctx, vd);
+    mask = gen_vreg_ptr(ctx, 0);
+    index = gen_vreg_ptr(ctx, vs2);
+    base = tcg_temp_new(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    gen_get_gpr(tcg_ctx, base, rs1);
+    fn(tcg_ctx, dest, mask, base, index, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, index);
+    tcg_temp_free(tcg_ctx, base);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    if (!is_store) {
+        mark_vs_dirty(ctx);
+    }
+
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool ldst_whole_trans(DisasContext *ctx, int vd, int rs1,
+                             uint32_t nf, uint32_t width,
+                             gen_helper_ldst_whole *fn, bool is_store)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv base;
+    TCGv_i32 desc;
+    uint32_t evl = (ctx->vlen >> 3) * nf / width;
+    uint32_t data = 0;
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, evl, over);
+
+    dest = gen_vreg_ptr(ctx, vd);
+    base = tcg_temp_new(tcg_ctx);
+    FIELD_DP32(data, VDATA, NF, nf, data);
+    desc = gen_rvv_desc(ctx, data);
+
+    gen_get_gpr(tcg_ctx, base, rs1);
+    fn(tcg_ctx, dest, base, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free(tcg_ctx, base);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    if (!is_store) {
+        mark_vs_dirty(ctx);
+    }
+
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool vle_trans(DisasContext *ctx, arg_rvv_ldst *a, uint8_t eew,
+                      gen_helper_ldst_us *fn)
+{
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_load(ctx, a->rd, a->nf, a->vm, eew)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, vext_get_emul(ctx, eew), data);
+    FIELD_DP32(data, VDATA, NF, a->nf, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+    return ldst_us_trans(ctx, a->rd, a->rs1, data, fn, false);
+}
+
+static bool vleff_trans(DisasContext *ctx, arg_rvv_ldst *a, uint8_t eew,
+                        gen_helper_ldst_us *fn)
+{
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_load(ctx, a->rd, a->nf, a->vm, eew)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, vext_get_emul(ctx, eew), data);
+    FIELD_DP32(data, VDATA, NF, a->nf, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+    return ldst_us_trans(ctx, a->rd, a->rs1, data, fn, false);
+}
+
+static bool vse_trans(DisasContext *ctx, arg_rvv_ldst *a, uint8_t eew,
+                      gen_helper_ldst_us *fn)
+{
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_store(ctx, a->rd, a->nf, eew)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, vext_get_emul(ctx, eew), data);
+    FIELD_DP32(data, VDATA, NF, a->nf, data);
+    return ldst_us_trans(ctx, a->rd, a->rs1, data, fn, true);
+}
+
+static bool vlse_trans(DisasContext *ctx, arg_rvv_ldst *a, uint8_t eew,
+                       gen_helper_ldst_stride *fn)
+{
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_load(ctx, a->rd, a->nf, a->vm, eew)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, vext_get_emul(ctx, eew), data);
+    FIELD_DP32(data, VDATA, NF, a->nf, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+    return ldst_stride_trans(ctx, a->rd, a->rs1, a->rs2, data, fn, false);
+}
+
+static bool vsse_trans(DisasContext *ctx, arg_rvv_ldst *a, uint8_t eew,
+                       gen_helper_ldst_stride *fn)
+{
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_store(ctx, a->rd, a->nf, eew)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, vext_get_emul(ctx, eew), data);
+    FIELD_DP32(data, VDATA, NF, a->nf, data);
+    return ldst_stride_trans(ctx, a->rd, a->rs1, a->rs2, data, fn, true);
+}
+
+static bool vlxei_trans(DisasContext *ctx, arg_rvv_ldst *a, uint8_t eew)
+{
+    uint32_t data = 0;
+    gen_helper_ldst_index *fn;
+    static gen_helper_ldst_index * const fns[4][4] = {
+        { gen_helper_vlxei8_8_v, gen_helper_vlxei8_16_v,
+          gen_helper_vlxei8_32_v, gen_helper_vlxei8_64_v },
+        { gen_helper_vlxei16_8_v, gen_helper_vlxei16_16_v,
+          gen_helper_vlxei16_32_v, gen_helper_vlxei16_64_v },
+        { gen_helper_vlxei32_8_v, gen_helper_vlxei32_16_v,
+          gen_helper_vlxei32_32_v, gen_helper_vlxei32_64_v },
+        { gen_helper_vlxei64_8_v, gen_helper_vlxei64_16_v,
+          gen_helper_vlxei64_32_v, gen_helper_vlxei64_64_v },
+    };
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_ld_index(ctx, a->rd, a->rs2, a->nf, a->vm, eew)) {
+        return false;
+    }
+
+    fn = fns[eew][ctx->sew];
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, vext_get_emul(ctx, ctx->sew), data);
+    FIELD_DP32(data, VDATA, NF, a->nf, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+    return ldst_index_trans(ctx, a->rd, a->rs1, a->rs2, data, fn, false);
+}
+
+static bool vsxei_trans(DisasContext *ctx, arg_rvv_ldst *a, uint8_t eew)
+{
+    uint32_t data = 0;
+    gen_helper_ldst_index *fn;
+    static gen_helper_ldst_index * const fns[4][4] = {
+        { gen_helper_vsxei8_8_v, gen_helper_vsxei8_16_v,
+          gen_helper_vsxei8_32_v, gen_helper_vsxei8_64_v },
+        { gen_helper_vsxei16_8_v, gen_helper_vsxei16_16_v,
+          gen_helper_vsxei16_32_v, gen_helper_vsxei16_64_v },
+        { gen_helper_vsxei32_8_v, gen_helper_vsxei32_16_v,
+          gen_helper_vsxei32_32_v, gen_helper_vsxei32_64_v },
+        { gen_helper_vsxei64_8_v, gen_helper_vsxei64_16_v,
+          gen_helper_vsxei64_32_v, gen_helper_vsxei64_64_v },
+    };
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_st_index(ctx, a->rd, a->rs2, a->nf, eew)) {
+        return false;
+    }
+
+    fn = fns[eew][ctx->sew];
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, vext_get_emul(ctx, ctx->sew), data);
+    FIELD_DP32(data, VDATA, NF, a->nf, data);
+    return ldst_index_trans(ctx, a->rd, a->rs1, a->rs2, data, fn, true);
+}
+
+static bool trans_vle8_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vle_trans(ctx, a, MO_8, gen_helper_vle8_v);
+}
+
+static bool trans_vle16_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vle_trans(ctx, a, MO_16, gen_helper_vle16_v);
+}
+
+static bool trans_vle32_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vle_trans(ctx, a, MO_32, gen_helper_vle32_v);
+}
+
+static bool trans_vle64_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vle_trans(ctx, a, MO_64, gen_helper_vle64_v);
+}
+
+static bool trans_vle8ff_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vleff_trans(ctx, a, MO_8, gen_helper_vle8ff_v);
+}
+
+static bool trans_vle16ff_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vleff_trans(ctx, a, MO_16, gen_helper_vle16ff_v);
+}
+
+static bool trans_vle32ff_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vleff_trans(ctx, a, MO_32, gen_helper_vle32ff_v);
+}
+
+static bool trans_vle64ff_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vleff_trans(ctx, a, MO_64, gen_helper_vle64ff_v);
+}
+
+static bool trans_vse8_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vse_trans(ctx, a, MO_8, gen_helper_vse8_v);
+}
+
+static bool trans_vse16_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vse_trans(ctx, a, MO_16, gen_helper_vse16_v);
+}
+
+static bool trans_vse32_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vse_trans(ctx, a, MO_32, gen_helper_vse32_v);
+}
+
+static bool trans_vse64_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vse_trans(ctx, a, MO_64, gen_helper_vse64_v);
+}
+
+static bool trans_vlse8_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlse_trans(ctx, a, MO_8, gen_helper_vlse8_v);
+}
+
+static bool trans_vlse16_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlse_trans(ctx, a, MO_16, gen_helper_vlse16_v);
+}
+
+static bool trans_vlse32_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlse_trans(ctx, a, MO_32, gen_helper_vlse32_v);
+}
+
+static bool trans_vlse64_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlse_trans(ctx, a, MO_64, gen_helper_vlse64_v);
+}
+
+static bool trans_vsse8_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsse_trans(ctx, a, MO_8, gen_helper_vsse8_v);
+}
+
+static bool trans_vsse16_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsse_trans(ctx, a, MO_16, gen_helper_vsse16_v);
+}
+
+static bool trans_vsse32_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsse_trans(ctx, a, MO_32, gen_helper_vsse32_v);
+}
+
+static bool trans_vsse64_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsse_trans(ctx, a, MO_64, gen_helper_vsse64_v);
+}
+
+static bool trans_vlxei8_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlxei_trans(ctx, a, MO_8);
+}
+
+static bool trans_vlxei16_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlxei_trans(ctx, a, MO_16);
+}
+
+static bool trans_vlxei32_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlxei_trans(ctx, a, MO_32);
+}
+
+static bool trans_vlxei64_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vlxei_trans(ctx, a, MO_64);
+}
+
+static bool trans_vsxei8_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsxei_trans(ctx, a, MO_8);
+}
+
+static bool trans_vsxei16_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsxei_trans(ctx, a, MO_16);
+}
+
+static bool trans_vsxei32_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsxei_trans(ctx, a, MO_32);
+}
+
+static bool trans_vsxei64_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    return vsxei_trans(ctx, a, MO_64);
+}
+
+static bool trans_vlm_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, LMUL, 0, data);
+    FIELD_DP32(data, VDATA, NF, 1, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->rvv_ta_all_1s, data);
+    return ldst_us_trans(ctx, a->rd, a->rs1, data, gen_helper_vlm_v, false);
+}
+
+static bool trans_vsm_v(DisasContext *ctx, arg_rvv_ldst *a)
+{
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, LMUL, 0, data);
+    FIELD_DP32(data, VDATA, NF, 1, data);
+    return ldst_us_trans(ctx, a->rd, a->rs1, data, gen_helper_vsm_v, true);
+}
+
+#define GEN_LDST_WHOLE_TRANS(NAME, NF, WIDTH, IS_STORE)                  \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_ldst *a)             \
+{                                                                        \
+    if (require_rvv(ctx) && QEMU_IS_ALIGNED(a->rd, NF)) {                \
+        return ldst_whole_trans(ctx, a->rd, a->rs1, NF, WIDTH,           \
+                                gen_helper_##NAME, IS_STORE);            \
+    }                                                                    \
+    return false;                                                        \
+}
+
+GEN_LDST_WHOLE_TRANS(vl1re8_v, 1, 1, false)
+GEN_LDST_WHOLE_TRANS(vl1re16_v, 1, 2, false)
+GEN_LDST_WHOLE_TRANS(vl1re32_v, 1, 4, false)
+GEN_LDST_WHOLE_TRANS(vl1re64_v, 1, 8, false)
+GEN_LDST_WHOLE_TRANS(vl2re8_v, 2, 1, false)
+GEN_LDST_WHOLE_TRANS(vl2re16_v, 2, 2, false)
+GEN_LDST_WHOLE_TRANS(vl2re32_v, 2, 4, false)
+GEN_LDST_WHOLE_TRANS(vl2re64_v, 2, 8, false)
+GEN_LDST_WHOLE_TRANS(vl4re8_v, 4, 1, false)
+GEN_LDST_WHOLE_TRANS(vl4re16_v, 4, 2, false)
+GEN_LDST_WHOLE_TRANS(vl4re32_v, 4, 4, false)
+GEN_LDST_WHOLE_TRANS(vl4re64_v, 4, 8, false)
+GEN_LDST_WHOLE_TRANS(vl8re8_v, 8, 1, false)
+GEN_LDST_WHOLE_TRANS(vl8re16_v, 8, 2, false)
+GEN_LDST_WHOLE_TRANS(vl8re32_v, 8, 4, false)
+GEN_LDST_WHOLE_TRANS(vl8re64_v, 8, 8, false)
+GEN_LDST_WHOLE_TRANS(vs1r_v, 1, 1, true)
+GEN_LDST_WHOLE_TRANS(vs2r_v, 2, 1, true)
+GEN_LDST_WHOLE_TRANS(vs4r_v, 4, 1, true)
+GEN_LDST_WHOLE_TRANS(vs8r_v, 8, 1, true)
+
+static bool opivv_trans(DisasContext *ctx, arg_rvv_arith *a,
+                        gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_sss(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_trans_common(DisasContext *ctx, arg_rvv_arith *a,
+                               bool is_imm, target_long imm,
+                               gen_helper_opivx *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_ss(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_trans(DisasContext *ctx, arg_rvv_arith *a,
+                        gen_helper_opivx *fn)
+{
+    return opivx_trans_common(ctx, a, false, 0, fn);
+}
+
+static bool opivi_trans(DisasContext *ctx, arg_rvv_arith *a,
+                        target_long scalar_value, gen_helper_opivx *fn)
+{
+    return opivx_trans_common(ctx, a, true, scalar_value, fn);
+}
+
+static bool opivx_slide_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              bool is_imm, target_long imm,
+                              gen_helper_opivx *fn, bool slide_up)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_slide(ctx, a->rd, a->rs2, a->vm, slide_up)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivv_widen_trans_common(DisasContext *ctx, arg_rvv_arith *a,
+                                     gen_helper_opivv *fn, bool wide_vs2)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx)) {
+        return false;
+    }
+    if (wide_vs2) {
+        if (!vext_check_dds(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+            return false;
+        }
+    } else if (!vext_check_dss(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_widen_trans_common(DisasContext *ctx, arg_rvv_arith *a,
+                                     gen_helper_opivx *fn, bool wide_vs2)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx)) {
+        return false;
+    }
+    if (wide_vs2) {
+        if (!vext_check_dd(ctx, a->rd, a->rs2, a->vm)) {
+            return false;
+        }
+    } else if (!vext_check_ds(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    scalar = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, scalar, a->rs1);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivv_reduction_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                  gen_helper_opivv *fn, bool widening)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *call;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) ||
+        !vext_check_reduction(ctx, a->rs2)) {
+        return false;
+    }
+    if (widening &&
+        (ctx->sew >= MO_64 || (ctx->sew + 1) > (ctx->elen >> 4))) {
+        return false;
+    }
+
+    call = gen_new_label(tcg_ctx);
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_NE, cpu_vstart, 0, call);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    gen_set_label(tcg_ctx, call);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvv_reduction_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                  gen_helper_opivv *fn, bool widening)
+{
+    if (fn == NULL || !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        !vext_check_reduction(ctx, a->rs2)) {
+        return false;
+    }
+    if (widening &&
+        (ctx->sew >= MO_64 || (ctx->sew + 1) > (ctx->elen >> 4))) {
+        return false;
+    }
+
+    gen_set_rm(ctx, 7);
+    return opivv_reduction_trans(ctx, a, fn, widening);
+}
+
+#define GEN_WIDEN_VV_TRANS(NAME)                                      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)         \
+{                                                                     \
+    static gen_helper_opivv * const fns[4] = {                        \
+        gen_helper_##NAME##_b,                                        \
+        gen_helper_##NAME##_h,                                        \
+        gen_helper_##NAME##_w,                                        \
+        NULL,                                                         \
+    };                                                                \
+                                                                      \
+    return opivv_widen_trans_common(ctx, a, fns[ctx->sew], false);    \
+}
+
+#define GEN_WIDEN_VX_TRANS(NAME)                                      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)         \
+{                                                                     \
+    static gen_helper_opivx * const fns[4] = {                        \
+        gen_helper_##NAME##_b,                                        \
+        gen_helper_##NAME##_h,                                        \
+        gen_helper_##NAME##_w,                                        \
+        NULL,                                                         \
+    };                                                                \
+                                                                      \
+    return opivx_widen_trans_common(ctx, a, fns[ctx->sew], false);    \
+}
+
+#define GEN_WIDEN_WV_TRANS(NAME)                                      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)         \
+{                                                                     \
+    static gen_helper_opivv * const fns[4] = {                        \
+        gen_helper_##NAME##_b,                                        \
+        gen_helper_##NAME##_h,                                        \
+        gen_helper_##NAME##_w,                                        \
+        NULL,                                                         \
+    };                                                                \
+                                                                      \
+    return opivv_widen_trans_common(ctx, a, fns[ctx->sew], true);     \
+}
+
+#define GEN_WIDEN_WX_TRANS(NAME)                                      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)         \
+{                                                                     \
+    static gen_helper_opivx * const fns[4] = {                        \
+        gen_helper_##NAME##_b,                                        \
+        gen_helper_##NAME##_h,                                        \
+        gen_helper_##NAME##_w,                                        \
+        NULL,                                                         \
+    };                                                                \
+                                                                      \
+    return opivx_widen_trans_common(ctx, a, fns[ctx->sew], true);     \
+}
+
+GEN_WIDEN_VV_TRANS(vwaddu_vv)
+GEN_WIDEN_VV_TRANS(vwadd_vv)
+GEN_WIDEN_VV_TRANS(vwsubu_vv)
+GEN_WIDEN_VV_TRANS(vwsub_vv)
+GEN_WIDEN_VX_TRANS(vwaddu_vx)
+GEN_WIDEN_VX_TRANS(vwadd_vx)
+GEN_WIDEN_VX_TRANS(vwsubu_vx)
+GEN_WIDEN_VX_TRANS(vwsub_vx)
+GEN_WIDEN_WV_TRANS(vwaddu_wv)
+GEN_WIDEN_WV_TRANS(vwadd_wv)
+GEN_WIDEN_WV_TRANS(vwsubu_wv)
+GEN_WIDEN_WV_TRANS(vwsub_wv)
+GEN_WIDEN_WX_TRANS(vwaddu_wx)
+GEN_WIDEN_WX_TRANS(vwadd_wx)
+GEN_WIDEN_WX_TRANS(vwsubu_wx)
+GEN_WIDEN_WX_TRANS(vwsub_wx)
+GEN_WIDEN_VV_TRANS(vwmulu_vv)
+GEN_WIDEN_VV_TRANS(vwmulsu_vv)
+GEN_WIDEN_VV_TRANS(vwmul_vv)
+GEN_WIDEN_VX_TRANS(vwmulu_vx)
+GEN_WIDEN_VX_TRANS(vwmulsu_vx)
+GEN_WIDEN_VX_TRANS(vwmul_vx)
+GEN_WIDEN_VV_TRANS(vwmaccu_vv)
+GEN_WIDEN_VV_TRANS(vwmacc_vv)
+GEN_WIDEN_VV_TRANS(vwmaccsu_vv)
+GEN_WIDEN_VX_TRANS(vwmaccu_vx)
+GEN_WIDEN_VX_TRANS(vwmacc_vx)
+GEN_WIDEN_VX_TRANS(vwmaccsu_vx)
+GEN_WIDEN_VX_TRANS(vwmaccus_vx)
+
+static bool opivv_vadc_trans(DisasContext *ctx, arg_rvv_arith *a,
+                             gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) || a->rd == 0 ||
+        !vext_check_sss(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_vadc_trans_common(DisasContext *ctx, arg_rvv_arith *a,
+                                    bool is_imm, target_long imm,
+                                    gen_helper_opivx *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) || a->rd == 0 ||
+        !vext_check_ss(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_vadc_trans(DisasContext *ctx, arg_rvv_arith *a,
+                             gen_helper_opivx *fn)
+{
+    return opivx_vadc_trans_common(ctx, a, false, 0, fn);
+}
+
+static bool opivi_vadc_trans(DisasContext *ctx, arg_rvv_arith *a,
+                             target_long scalar_value, gen_helper_opivx *fn)
+{
+    return opivx_vadc_trans_common(ctx, a, true, scalar_value, fn);
+}
+
+static bool opivv_vmadc_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_mss(ctx, a->rd, a->rs1, a->rs2)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_vmadc_trans_common(DisasContext *ctx, arg_rvv_arith *a,
+                                     bool is_imm, target_long imm,
+                                     gen_helper_opivx *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) || !vext_check_ms(ctx, a->rd, a->rs2)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_vmadc_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              gen_helper_opivx *fn)
+{
+    return opivx_vmadc_trans_common(ctx, a, false, 0, fn);
+}
+
+static bool opivi_vmadc_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              target_long scalar_value,
+                              gen_helper_opivx *fn)
+{
+    return opivx_vmadc_trans_common(ctx, a, true, scalar_value, fn);
+}
+
+static bool trans_vadd_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vadd_vv_b,
+        gen_helper_vadd_vv_h,
+        gen_helper_vadd_vv_w,
+        gen_helper_vadd_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsub_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vsub_vv_b,
+        gen_helper_vsub_vv_h,
+        gen_helper_vsub_vv_w,
+        gen_helper_vsub_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vand_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vand_vv_b,
+        gen_helper_vand_vv_h,
+        gen_helper_vand_vv_w,
+        gen_helper_vand_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vor_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vor_vv_b,
+        gen_helper_vor_vv_h,
+        gen_helper_vor_vv_w,
+        gen_helper_vor_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vxor_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vxor_vv_b,
+        gen_helper_vxor_vv_h,
+        gen_helper_vxor_vv_w,
+        gen_helper_vxor_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vminu_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vminu_vv_b,
+        gen_helper_vminu_vv_h,
+        gen_helper_vminu_vv_w,
+        gen_helper_vminu_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmin_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmin_vv_b,
+        gen_helper_vmin_vv_h,
+        gen_helper_vmin_vv_w,
+        gen_helper_vmin_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmaxu_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmaxu_vv_b,
+        gen_helper_vmaxu_vv_h,
+        gen_helper_vmaxu_vv_w,
+        gen_helper_vmaxu_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmax_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmax_vv_b,
+        gen_helper_vmax_vv_h,
+        gen_helper_vmax_vv_w,
+        gen_helper_vmax_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmul_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmul_vv_b,
+        gen_helper_vmul_vv_h,
+        gen_helper_vmul_vv_w,
+        gen_helper_vmul_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmulh_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmulh_vv_b,
+        gen_helper_vmulh_vv_h,
+        gen_helper_vmulh_vv_w,
+        gen_helper_vmulh_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmulhu_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmulhu_vv_b,
+        gen_helper_vmulhu_vv_h,
+        gen_helper_vmulhu_vv_w,
+        gen_helper_vmulhu_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmulhsu_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmulhsu_vv_b,
+        gen_helper_vmulhsu_vv_h,
+        gen_helper_vmulhsu_vv_w,
+        gen_helper_vmulhsu_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vdivu_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vdivu_vv_b,
+        gen_helper_vdivu_vv_h,
+        gen_helper_vdivu_vv_w,
+        gen_helper_vdivu_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vdiv_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vdiv_vv_b,
+        gen_helper_vdiv_vv_h,
+        gen_helper_vdiv_vv_w,
+        gen_helper_vdiv_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vremu_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vremu_vv_b,
+        gen_helper_vremu_vv_h,
+        gen_helper_vremu_vv_w,
+        gen_helper_vremu_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vrem_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vrem_vv_b,
+        gen_helper_vrem_vv_h,
+        gen_helper_vrem_vv_w,
+        gen_helper_vrem_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+#define GEN_FIXED_VV_TRANS(NAME)                                     \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivv * const fns[4] = {                       \
+        gen_helper_##NAME##_b,                                       \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+        gen_helper_##NAME##_d,                                       \
+    };                                                               \
+                                                                     \
+    return opivv_trans(ctx, a, fns[ctx->sew]);                       \
+}
+
+#define GEN_FIXED_VX_TRANS(NAME)                                     \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivx * const fns[4] = {                       \
+        gen_helper_##NAME##_b,                                       \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+        gen_helper_##NAME##_d,                                       \
+    };                                                               \
+                                                                     \
+    return opivx_trans(ctx, a, fns[ctx->sew]);                       \
+}
+
+#define GEN_FIXED_VI_TRANS(NAME, HELPER, IMM)                        \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivx * const fns[4] = {                       \
+        gen_helper_##HELPER##_b,                                     \
+        gen_helper_##HELPER##_h,                                     \
+        gen_helper_##HELPER##_w,                                     \
+        gen_helper_##HELPER##_d,                                     \
+    };                                                               \
+                                                                     \
+    return opivi_trans(ctx, a, IMM, fns[ctx->sew]);                  \
+}
+
+GEN_FIXED_VV_TRANS(vsaddu_vv)
+GEN_FIXED_VV_TRANS(vsadd_vv)
+GEN_FIXED_VV_TRANS(vssubu_vv)
+GEN_FIXED_VV_TRANS(vssub_vv)
+GEN_FIXED_VX_TRANS(vsaddu_vx)
+GEN_FIXED_VX_TRANS(vsadd_vx)
+GEN_FIXED_VX_TRANS(vssubu_vx)
+GEN_FIXED_VX_TRANS(vssub_vx)
+GEN_FIXED_VI_TRANS(vsaddu_vi, vsaddu_vx, sextract32(a->rs1, 0, 5))
+GEN_FIXED_VI_TRANS(vsadd_vi, vsadd_vx, sextract32(a->rs1, 0, 5))
+
+GEN_FIXED_VV_TRANS(vaadd_vv)
+GEN_FIXED_VV_TRANS(vaaddu_vv)
+GEN_FIXED_VV_TRANS(vasub_vv)
+GEN_FIXED_VV_TRANS(vasubu_vv)
+GEN_FIXED_VX_TRANS(vaadd_vx)
+GEN_FIXED_VX_TRANS(vaaddu_vx)
+GEN_FIXED_VX_TRANS(vasub_vx)
+GEN_FIXED_VX_TRANS(vasubu_vx)
+
+GEN_FIXED_VV_TRANS(vsmul_vv)
+GEN_FIXED_VX_TRANS(vsmul_vx)
+
+GEN_FIXED_VV_TRANS(vssrl_vv)
+GEN_FIXED_VV_TRANS(vssra_vv)
+GEN_FIXED_VX_TRANS(vssrl_vx)
+GEN_FIXED_VX_TRANS(vssra_vx)
+GEN_FIXED_VI_TRANS(vssrl_vi, vssrl_vx, a->rs1)
+GEN_FIXED_VI_TRANS(vssra_vi, vssra_vx, a->rs1)
+
+static bool opivv_narrow_shift_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                     gen_helper_opivv *fn);
+static bool opivx_narrow_shift_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                     gen_helper_opivx *fn);
+static bool opivi_narrow_shift_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                     target_long scalar_value,
+                                     gen_helper_opivx *fn);
+
+#define GEN_FIXED_NARROW_WV_TRANS(NAME)                             \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivv * const fns[3] = {                       \
+        gen_helper_##NAME##_b,                                       \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+    };                                                               \
+                                                                     \
+    if (ctx->sew >= MO_64) {                                         \
+        return false;                                                \
+    }                                                                \
+    return opivv_narrow_shift_trans(ctx, a, fns[ctx->sew]);          \
+}
+
+#define GEN_FIXED_NARROW_WX_TRANS(NAME)                             \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivx * const fns[3] = {                       \
+        gen_helper_##NAME##_b,                                       \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+    };                                                               \
+                                                                     \
+    if (ctx->sew >= MO_64) {                                         \
+        return false;                                                \
+    }                                                                \
+    return opivx_narrow_shift_trans(ctx, a, fns[ctx->sew]);          \
+}
+
+#define GEN_FIXED_NARROW_WI_TRANS(NAME, HELPER)                     \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivx * const fns[3] = {                       \
+        gen_helper_##HELPER##_b,                                     \
+        gen_helper_##HELPER##_h,                                     \
+        gen_helper_##HELPER##_w,                                     \
+    };                                                               \
+                                                                     \
+    if (ctx->sew >= MO_64) {                                         \
+        return false;                                                \
+    }                                                                \
+    return opivi_narrow_shift_trans(ctx, a, a->rs1, fns[ctx->sew]);  \
+}
+
+GEN_FIXED_NARROW_WV_TRANS(vnclipu_wv)
+GEN_FIXED_NARROW_WV_TRANS(vnclip_wv)
+GEN_FIXED_NARROW_WX_TRANS(vnclipu_wx)
+GEN_FIXED_NARROW_WX_TRANS(vnclip_wx)
+GEN_FIXED_NARROW_WI_TRANS(vnclipu_wi, vnclipu_wx)
+GEN_FIXED_NARROW_WI_TRANS(vnclip_wi, vnclip_wx)
+
+#define GEN_REDUCTION_TRANS(NAME)                                      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)          \
+{                                                                      \
+    static gen_helper_opivv * const fns[4] = {                         \
+        gen_helper_##NAME##_b,                                         \
+        gen_helper_##NAME##_h,                                         \
+        gen_helper_##NAME##_w,                                         \
+        gen_helper_##NAME##_d,                                         \
+    };                                                                 \
+                                                                       \
+    return opivv_reduction_trans(ctx, a, fns[ctx->sew], false);        \
+}
+
+#define GEN_WIDEN_REDUCTION_TRANS(NAME)                                \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)          \
+{                                                                      \
+    static gen_helper_opivv * const fns[3] = {                         \
+        gen_helper_##NAME##_b,                                         \
+        gen_helper_##NAME##_h,                                         \
+        gen_helper_##NAME##_w,                                         \
+    };                                                                 \
+                                                                       \
+    if (ctx->sew >= MO_64) {                                           \
+        return false;                                                  \
+    }                                                                  \
+    return opivv_reduction_trans(ctx, a, fns[ctx->sew], true);         \
+}
+
+GEN_REDUCTION_TRANS(vredsum_vs)
+GEN_REDUCTION_TRANS(vredand_vs)
+GEN_REDUCTION_TRANS(vredor_vs)
+GEN_REDUCTION_TRANS(vredxor_vs)
+GEN_REDUCTION_TRANS(vredminu_vs)
+GEN_REDUCTION_TRANS(vredmin_vs)
+GEN_REDUCTION_TRANS(vredmaxu_vs)
+GEN_REDUCTION_TRANS(vredmax_vs)
+GEN_WIDEN_REDUCTION_TRANS(vwredsumu_vs)
+GEN_WIDEN_REDUCTION_TRANS(vwredsum_vs)
+
+#define GEN_FP_REDUCTION_TRANS(NAME)                                 \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivv * const fns[4] = {                       \
+        NULL,                                                        \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+        gen_helper_##NAME##_d,                                       \
+    };                                                               \
+                                                                     \
+    return opfvv_reduction_trans(ctx, a, fns[ctx->sew], false);      \
+}
+
+#define GEN_FP_WIDEN_REDUCTION_TRANS(NAME)                           \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivv * const fns[4] = {                       \
+        NULL,                                                        \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+        NULL,                                                        \
+    };                                                               \
+                                                                     \
+    return opfvv_reduction_trans(ctx, a, fns[ctx->sew], true);       \
+}
+
+GEN_FP_REDUCTION_TRANS(vfredusum_vs)
+GEN_FP_REDUCTION_TRANS(vfredosum_vs)
+GEN_FP_REDUCTION_TRANS(vfredmin_vs)
+GEN_FP_REDUCTION_TRANS(vfredmax_vs)
+GEN_FP_WIDEN_REDUCTION_TRANS(vfwredusum_vs)
+GEN_FP_WIDEN_REDUCTION_TRANS(vfwredosum_vs)
+
+#define GEN_MAC_VV_TRANS(NAME)                                       \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivv * const fns[4] = {                       \
+        gen_helper_##NAME##_b,                                       \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+        gen_helper_##NAME##_d,                                       \
+    };                                                               \
+                                                                     \
+    return opivv_trans(ctx, a, fns[ctx->sew]);                       \
+}
+
+GEN_MAC_VV_TRANS(vmacc_vv)
+GEN_MAC_VV_TRANS(vnmsac_vv)
+GEN_MAC_VV_TRANS(vmadd_vv)
+GEN_MAC_VV_TRANS(vnmsub_vv)
+
+static bool trans_vsll_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vsll_vv_b,
+        gen_helper_vsll_vv_h,
+        gen_helper_vsll_vv_w,
+        gen_helper_vsll_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsrl_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vsrl_vv_b,
+        gen_helper_vsrl_vv_h,
+        gen_helper_vsrl_vv_w,
+        gen_helper_vsrl_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsra_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vsra_vv_b,
+        gen_helper_vsra_vv_h,
+        gen_helper_vsra_vv_w,
+        gen_helper_vsra_vv_d,
+    };
+
+    return opivv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vadd_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vadd_vx_b,
+        gen_helper_vadd_vx_h,
+        gen_helper_vadd_vx_w,
+        gen_helper_vadd_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vadd_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vadd_vx_b,
+        gen_helper_vadd_vx_h,
+        gen_helper_vadd_vx_w,
+        gen_helper_vadd_vx_d,
+    };
+
+    return opivi_trans(ctx, a, sextract32(a->rs1, 0, 5), fns[ctx->sew]);
+}
+
+static bool trans_vsub_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsub_vx_b,
+        gen_helper_vsub_vx_h,
+        gen_helper_vsub_vx_w,
+        gen_helper_vsub_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vrsub_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vrsub_vx_b,
+        gen_helper_vrsub_vx_h,
+        gen_helper_vrsub_vx_w,
+        gen_helper_vrsub_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vrsub_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vrsub_vx_b,
+        gen_helper_vrsub_vx_h,
+        gen_helper_vrsub_vx_w,
+        gen_helper_vrsub_vx_d,
+    };
+
+    return opivi_trans(ctx, a, sextract32(a->rs1, 0, 5), fns[ctx->sew]);
+}
+
+static bool trans_vslideup_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vslideup_vx_b,
+        gen_helper_vslideup_vx_h,
+        gen_helper_vslideup_vx_w,
+        gen_helper_vslideup_vx_d,
+    };
+
+    return opivx_slide_trans(ctx, a, false, 0, fns[ctx->sew], true);
+}
+
+static bool trans_vslideup_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vslideup_vx_b,
+        gen_helper_vslideup_vx_h,
+        gen_helper_vslideup_vx_w,
+        gen_helper_vslideup_vx_d,
+    };
+
+    return opivx_slide_trans(ctx, a, true, a->rs1, fns[ctx->sew], true);
+}
+
+static bool trans_vslide1up_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vslide1up_vx_b,
+        gen_helper_vslide1up_vx_h,
+        gen_helper_vslide1up_vx_w,
+        gen_helper_vslide1up_vx_d,
+    };
+
+    return opivx_slide_trans(ctx, a, false, 0, fns[ctx->sew], true);
+}
+
+static bool trans_vslidedown_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vslidedown_vx_b,
+        gen_helper_vslidedown_vx_h,
+        gen_helper_vslidedown_vx_w,
+        gen_helper_vslidedown_vx_d,
+    };
+
+    return opivx_slide_trans(ctx, a, false, 0, fns[ctx->sew], false);
+}
+
+static bool trans_vslidedown_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vslidedown_vx_b,
+        gen_helper_vslidedown_vx_h,
+        gen_helper_vslidedown_vx_w,
+        gen_helper_vslidedown_vx_d,
+    };
+
+    return opivx_slide_trans(ctx, a, true, a->rs1, fns[ctx->sew], false);
+}
+
+static bool trans_vslide1down_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vslide1down_vx_b,
+        gen_helper_vslide1down_vx_h,
+        gen_helper_vslide1down_vx_w,
+        gen_helper_vslide1down_vx_d,
+    };
+
+    return opivx_slide_trans(ctx, a, false, 0, fns[ctx->sew], false);
+}
+
+static bool vext_check_gather_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return require_rvv_data(ctx) &&
+           require_align(a->rd, ctx->lmul) &&
+           require_align(a->rs1, ctx->lmul) &&
+           require_align(a->rs2, ctx->lmul) &&
+           a->rd != a->rs2 &&
+           a->rd != a->rs1 &&
+           require_vm(a->vm, a->rd);
+}
+
+static bool vext_check_gatherei16_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    int8_t emul = MO_16 - ctx->sew + ctx->lmul;
+    int8_t rd_size = 1 << MAX(ctx->lmul, 0);
+
+    return require_rvv_data(ctx) &&
+           emul >= -3 &&
+           emul <= 3 &&
+           require_align(a->rd, ctx->lmul) &&
+           require_align(a->rs1, emul) &&
+           require_align(a->rs2, ctx->lmul) &&
+           a->rd != a->rs2 &&
+           a->rd != a->rs1 &&
+           !is_overlapped(a->rd, rd_size, a->rs1, 1 << MAX(emul, 0)) &&
+           !is_overlapped(a->rd, rd_size,
+                          a->rs2, 1 << MAX(ctx->lmul, 0)) &&
+           require_vm(a->vm, a->rd);
+}
+
+static bool opivv_gather_trans(DisasContext *ctx, arg_rvv_arith *a,
+                               gen_helper_opivv *fn, bool ei16)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL ||
+        !(ei16 ? vext_check_gatherei16_vv(ctx, a) :
+                 vext_check_gather_vv(ctx, a))) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vrgather_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vrgather_vv_b,
+        gen_helper_vrgather_vv_h,
+        gen_helper_vrgather_vv_w,
+        gen_helper_vrgather_vv_d,
+    };
+
+    return opivv_gather_trans(ctx, a, fns[ctx->sew], false);
+}
+
+static bool trans_vrgatherei16_vv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vrgatherei16_vv_b,
+        gen_helper_vrgatherei16_vv_h,
+        gen_helper_vrgatherei16_vv_w,
+        gen_helper_vrgatherei16_vv_d,
+    };
+
+    return opivv_gather_trans(ctx, a, fns[ctx->sew], true);
+}
+
+static bool vext_check_gather_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return require_rvv_data(ctx) &&
+           require_align(a->rd, ctx->lmul) &&
+           require_align(a->rs2, ctx->lmul) &&
+           a->rd != a->rs2 &&
+           require_vm(a->vm, a->rd);
+}
+
+static bool opivx_gather_trans(DisasContext *ctx, arg_rvv_arith *a,
+                               bool is_imm, target_long imm,
+                               gen_helper_opivx *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !vext_check_gather_vx(ctx, a)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vrgather_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vrgather_vx_b,
+        gen_helper_vrgather_vx_h,
+        gen_helper_vrgather_vx_w,
+        gen_helper_vrgather_vx_d,
+    };
+
+    return opivx_gather_trans(ctx, a, false, 0, fns[ctx->sew]);
+}
+
+static bool trans_vrgather_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vrgather_vx_b,
+        gen_helper_vrgather_vx_h,
+        gen_helper_vrgather_vx_w,
+        gen_helper_vrgather_vx_d,
+    };
+
+    return opivx_gather_trans(ctx, a, true, a->rs1, fns[ctx->sew]);
+}
+
+static bool trans_vcompress_vm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vcompress_vm_b,
+        gen_helper_vcompress_vm_h,
+        gen_helper_vcompress_vm_w,
+        gen_helper_vcompress_vm_d,
+    };
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    TCGLabel *call;
+    uint32_t data = 0;
+    int8_t rd_size = 1 << MAX(ctx->lmul, 0);
+
+    if (!require_rvv_data(ctx) ||
+        !require_align(a->rd, ctx->lmul) ||
+        !require_align(a->rs2, ctx->lmul) ||
+        a->rd == a->rs2 ||
+        is_overlapped(a->rd, rd_size, a->rs1, 1)) {
+        return false;
+    }
+
+    call = gen_new_label(tcg_ctx);
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_NE, cpu_vstart, 0, call);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    gen_set_label(tcg_ctx, call);
+
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fns[ctx->sew](tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool vmv_whole_trans(DisasContext *ctx, arg_rvv_arith *a,
+                            uint32_t len)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t vlenb = ctx->vlen >> 3;
+    uint32_t maxsz = (ctx->vlen >> 3) * len;
+
+    if (!require_rvv(ctx) ||
+        !QEMU_IS_ALIGNED(a->rd, len) ||
+        !QEMU_IS_ALIGNED(a->rs2, len)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, maxsz, over);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = tcg_const_i32(tcg_ctx, simd_desc(vlenb, maxsz, 0));
+
+    gen_helper_vmvr_v(tcg_ctx, dest, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vmv1r_v(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return vmv_whole_trans(ctx, a, 1);
+}
+
+static bool trans_vmv2r_v(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return vmv_whole_trans(ctx, a, 2);
+}
+
+static bool trans_vmv4r_v(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return vmv_whole_trans(ctx, a, 4);
+}
+
+static bool trans_vmv8r_v(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return vmv_whole_trans(ctx, a, 8);
+}
+
+static bool trans_vadc_vvm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vadc_vvm_b,
+        gen_helper_vadc_vvm_h,
+        gen_helper_vadc_vvm_w,
+        gen_helper_vadc_vvm_d,
+    };
+
+    return opivv_vadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vadc_vxm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vadc_vxm_b,
+        gen_helper_vadc_vxm_h,
+        gen_helper_vadc_vxm_w,
+        gen_helper_vadc_vxm_d,
+    };
+
+    return opivx_vadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vadc_vim(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vadc_vxm_b,
+        gen_helper_vadc_vxm_h,
+        gen_helper_vadc_vxm_w,
+        gen_helper_vadc_vxm_d,
+    };
+
+    return opivi_vadc_trans(ctx, a, sextract32(a->rs1, 0, 5),
+                            fns[ctx->sew]);
+}
+
+static bool trans_vmadc_vvm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmadc_vvm_b,
+        gen_helper_vmadc_vvm_h,
+        gen_helper_vmadc_vvm_w,
+        gen_helper_vmadc_vvm_d,
+    };
+
+    return opivv_vmadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmadc_vxm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmadc_vxm_b,
+        gen_helper_vmadc_vxm_h,
+        gen_helper_vmadc_vxm_w,
+        gen_helper_vmadc_vxm_d,
+    };
+
+    return opivx_vmadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmadc_vim(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmadc_vxm_b,
+        gen_helper_vmadc_vxm_h,
+        gen_helper_vmadc_vxm_w,
+        gen_helper_vmadc_vxm_d,
+    };
+
+    return opivi_vmadc_trans(ctx, a, sextract32(a->rs1, 0, 5),
+                             fns[ctx->sew]);
+}
+
+static bool trans_vsbc_vvm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vsbc_vvm_b,
+        gen_helper_vsbc_vvm_h,
+        gen_helper_vsbc_vvm_w,
+        gen_helper_vsbc_vvm_d,
+    };
+
+    return opivv_vadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsbc_vxm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsbc_vxm_b,
+        gen_helper_vsbc_vxm_h,
+        gen_helper_vsbc_vxm_w,
+        gen_helper_vsbc_vxm_d,
+    };
+
+    return opivx_vadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmsbc_vvm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmsbc_vvm_b,
+        gen_helper_vmsbc_vvm_h,
+        gen_helper_vmsbc_vvm_w,
+        gen_helper_vmsbc_vvm_d,
+    };
+
+    return opivv_vmadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmsbc_vxm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmsbc_vxm_b,
+        gen_helper_vmsbc_vxm_h,
+        gen_helper_vmsbc_vxm_w,
+        gen_helper_vmsbc_vxm_d,
+    };
+
+    return opivx_vmadc_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vand_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vand_vx_b,
+        gen_helper_vand_vx_h,
+        gen_helper_vand_vx_w,
+        gen_helper_vand_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vand_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vand_vx_b,
+        gen_helper_vand_vx_h,
+        gen_helper_vand_vx_w,
+        gen_helper_vand_vx_d,
+    };
+
+    return opivi_trans(ctx, a, sextract32(a->rs1, 0, 5), fns[ctx->sew]);
+}
+
+static bool trans_vor_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vor_vx_b,
+        gen_helper_vor_vx_h,
+        gen_helper_vor_vx_w,
+        gen_helper_vor_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vor_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vor_vx_b,
+        gen_helper_vor_vx_h,
+        gen_helper_vor_vx_w,
+        gen_helper_vor_vx_d,
+    };
+
+    return opivi_trans(ctx, a, sextract32(a->rs1, 0, 5), fns[ctx->sew]);
+}
+
+static bool trans_vxor_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vxor_vx_b,
+        gen_helper_vxor_vx_h,
+        gen_helper_vxor_vx_w,
+        gen_helper_vxor_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vxor_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vxor_vx_b,
+        gen_helper_vxor_vx_h,
+        gen_helper_vxor_vx_w,
+        gen_helper_vxor_vx_d,
+    };
+
+    return opivi_trans(ctx, a, sextract32(a->rs1, 0, 5), fns[ctx->sew]);
+}
+
+static bool trans_vminu_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vminu_vx_b,
+        gen_helper_vminu_vx_h,
+        gen_helper_vminu_vx_w,
+        gen_helper_vminu_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmin_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmin_vx_b,
+        gen_helper_vmin_vx_h,
+        gen_helper_vmin_vx_w,
+        gen_helper_vmin_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmaxu_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmaxu_vx_b,
+        gen_helper_vmaxu_vx_h,
+        gen_helper_vmaxu_vx_w,
+        gen_helper_vmaxu_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmax_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmax_vx_b,
+        gen_helper_vmax_vx_h,
+        gen_helper_vmax_vx_w,
+        gen_helper_vmax_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmul_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmul_vx_b,
+        gen_helper_vmul_vx_h,
+        gen_helper_vmul_vx_w,
+        gen_helper_vmul_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmulh_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmulh_vx_b,
+        gen_helper_vmulh_vx_h,
+        gen_helper_vmulh_vx_w,
+        gen_helper_vmulh_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmulhu_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmulhu_vx_b,
+        gen_helper_vmulhu_vx_h,
+        gen_helper_vmulhu_vx_w,
+        gen_helper_vmulhu_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmulhsu_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmulhsu_vx_b,
+        gen_helper_vmulhsu_vx_h,
+        gen_helper_vmulhsu_vx_w,
+        gen_helper_vmulhsu_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vdivu_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vdivu_vx_b,
+        gen_helper_vdivu_vx_h,
+        gen_helper_vdivu_vx_w,
+        gen_helper_vdivu_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vdiv_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vdiv_vx_b,
+        gen_helper_vdiv_vx_h,
+        gen_helper_vdiv_vx_w,
+        gen_helper_vdiv_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vremu_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vremu_vx_b,
+        gen_helper_vremu_vx_h,
+        gen_helper_vremu_vx_w,
+        gen_helper_vremu_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vrem_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vrem_vx_b,
+        gen_helper_vrem_vx_h,
+        gen_helper_vrem_vx_w,
+        gen_helper_vrem_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+#define GEN_MAC_VX_TRANS(NAME)                                       \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    static gen_helper_opivx * const fns[4] = {                       \
+        gen_helper_##NAME##_b,                                       \
+        gen_helper_##NAME##_h,                                       \
+        gen_helper_##NAME##_w,                                       \
+        gen_helper_##NAME##_d,                                       \
+    };                                                               \
+                                                                     \
+    return opivx_trans(ctx, a, fns[ctx->sew]);                       \
+}
+
+GEN_MAC_VX_TRANS(vmacc_vx)
+GEN_MAC_VX_TRANS(vnmsac_vx)
+GEN_MAC_VX_TRANS(vmadd_vx)
+GEN_MAC_VX_TRANS(vnmsub_vx)
+
+static bool trans_vsll_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsll_vx_b,
+        gen_helper_vsll_vx_h,
+        gen_helper_vsll_vx_w,
+        gen_helper_vsll_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsll_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsll_vx_b,
+        gen_helper_vsll_vx_h,
+        gen_helper_vsll_vx_w,
+        gen_helper_vsll_vx_d,
+    };
+
+    return opivi_trans(ctx, a, a->rs1, fns[ctx->sew]);
+}
+
+static bool trans_vsrl_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsrl_vx_b,
+        gen_helper_vsrl_vx_h,
+        gen_helper_vsrl_vx_w,
+        gen_helper_vsrl_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsrl_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsrl_vx_b,
+        gen_helper_vsrl_vx_h,
+        gen_helper_vsrl_vx_w,
+        gen_helper_vsrl_vx_d,
+    };
+
+    return opivi_trans(ctx, a, a->rs1, fns[ctx->sew]);
+}
+
+static bool trans_vsra_vx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsra_vx_b,
+        gen_helper_vsra_vx_h,
+        gen_helper_vsra_vx_w,
+        gen_helper_vsra_vx_d,
+    };
+
+    return opivx_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsra_vi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vsra_vx_b,
+        gen_helper_vsra_vx_h,
+        gen_helper_vsra_vx_w,
+        gen_helper_vsra_vx_d,
+    };
+
+    return opivi_trans(ctx, a, a->rs1, fns[ctx->sew]);
+}
+
+static bool opivv_narrow_shift_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                     gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_sds(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_narrow_shift_trans_common(DisasContext *ctx,
+                                            arg_rvv_arith *a,
+                                            bool is_imm,
+                                            target_long imm,
+                                            gen_helper_opivx *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !vext_check_sd(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_narrow_shift_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                     gen_helper_opivx *fn)
+{
+    return opivx_narrow_shift_trans_common(ctx, a, false, 0, fn);
+}
+
+static bool opivi_narrow_shift_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                     target_long scalar_value,
+                                     gen_helper_opivx *fn)
+{
+    return opivx_narrow_shift_trans_common(ctx, a, true, scalar_value, fn);
+}
+
+static bool trans_vnsrl_wv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[3] = {
+        gen_helper_vnsrl_wv_b,
+        gen_helper_vnsrl_wv_h,
+        gen_helper_vnsrl_wv_w,
+    };
+
+    if (ctx->sew >= MO_64) {
+        return false;
+    }
+    return opivv_narrow_shift_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vnsrl_wx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[3] = {
+        gen_helper_vnsrl_wx_b,
+        gen_helper_vnsrl_wx_h,
+        gen_helper_vnsrl_wx_w,
+    };
+
+    if (ctx->sew >= MO_64) {
+        return false;
+    }
+    return opivx_narrow_shift_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vnsrl_wi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[3] = {
+        gen_helper_vnsrl_wx_b,
+        gen_helper_vnsrl_wx_h,
+        gen_helper_vnsrl_wx_w,
+    };
+
+    if (ctx->sew >= MO_64) {
+        return false;
+    }
+    return opivi_narrow_shift_trans(ctx, a, a->rs1, fns[ctx->sew]);
+}
+
+static bool trans_vnsra_wv(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[3] = {
+        gen_helper_vnsra_wv_b,
+        gen_helper_vnsra_wv_h,
+        gen_helper_vnsra_wv_w,
+    };
+
+    if (ctx->sew >= MO_64) {
+        return false;
+    }
+    return opivv_narrow_shift_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vnsra_wx(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[3] = {
+        gen_helper_vnsra_wx_b,
+        gen_helper_vnsra_wx_h,
+        gen_helper_vnsra_wx_w,
+    };
+
+    if (ctx->sew >= MO_64) {
+        return false;
+    }
+    return opivx_narrow_shift_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vnsra_wi(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[3] = {
+        gen_helper_vnsra_wx_b,
+        gen_helper_vnsra_wx_h,
+        gen_helper_vnsra_wx_w,
+    };
+
+    if (ctx->sew >= MO_64) {
+        return false;
+    }
+    return opivi_narrow_shift_trans(ctx, a, a->rs1, fns[ctx->sew]);
+}
+
+static bool int_ext_check(DisasContext *ctx, arg_rvv_arith *a, uint8_t div)
+{
+    uint8_t from = (ctx->sew + 3) - div;
+
+    return require_rvv_data(ctx) &&
+           from >= 3 && from <= 8 &&
+           a->rd != a->rs2 &&
+           require_align(a->rd, ctx->lmul) &&
+           require_align(a->rs2, ctx->lmul - div) &&
+           require_vm(a->vm, a->rd) &&
+           require_noover(a->rd, ctx->lmul, a->rs2, ctx->lmul - div);
+}
+
+static bool int_ext_trans(DisasContext *ctx, arg_rvv_arith *a,
+                          gen_helper_opivm *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vzext_vf2(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivm * const fns[4] = {
+        NULL,
+        gen_helper_vzext_vf2_h,
+        gen_helper_vzext_vf2_w,
+        gen_helper_vzext_vf2_d,
+    };
+
+    if (!int_ext_check(ctx, a, 1)) {
+        return false;
+    }
+    return int_ext_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vzext_vf4(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivm * const fns[4] = {
+        NULL,
+        NULL,
+        gen_helper_vzext_vf4_w,
+        gen_helper_vzext_vf4_d,
+    };
+
+    if (!int_ext_check(ctx, a, 2)) {
+        return false;
+    }
+    return int_ext_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vzext_vf8(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivm * const fns[4] = {
+        NULL,
+        NULL,
+        NULL,
+        gen_helper_vzext_vf8_d,
+    };
+
+    if (!int_ext_check(ctx, a, 3)) {
+        return false;
+    }
+    return int_ext_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsext_vf2(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivm * const fns[4] = {
+        NULL,
+        gen_helper_vsext_vf2_h,
+        gen_helper_vsext_vf2_w,
+        gen_helper_vsext_vf2_d,
+    };
+
+    if (!int_ext_check(ctx, a, 1)) {
+        return false;
+    }
+    return int_ext_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsext_vf4(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivm * const fns[4] = {
+        NULL,
+        NULL,
+        gen_helper_vsext_vf4_w,
+        gen_helper_vsext_vf4_d,
+    };
+
+    if (!int_ext_check(ctx, a, 2)) {
+        return false;
+    }
+    return int_ext_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vsext_vf8(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivm * const fns[4] = {
+        NULL,
+        NULL,
+        NULL,
+        gen_helper_vsext_vf8_d,
+    };
+
+    if (!int_ext_check(ctx, a, 3)) {
+        return false;
+    }
+    return int_ext_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool vmerge_vv_trans(DisasContext *ctx, arg_rvv_arith *a,
+                            gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) || a->vm != 0 || a->rd == 0 ||
+        !vext_check_sss(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool vmerge_vx_trans_common(DisasContext *ctx, arg_rvv_arith *a,
+                                   bool is_imm, target_long imm,
+                                   gen_helper_opivx *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) || a->vm != 0 || a->rd == 0 ||
+        !vext_check_ss(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vmerge_vvm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivv * const fns[4] = {
+        gen_helper_vmerge_vvm_b,
+        gen_helper_vmerge_vvm_h,
+        gen_helper_vmerge_vvm_w,
+        gen_helper_vmerge_vvm_d,
+    };
+
+    return vmerge_vv_trans(ctx, a, fns[ctx->sew]);
+}
+
+static bool trans_vmerge_vxm(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmerge_vxm_b,
+        gen_helper_vmerge_vxm_h,
+        gen_helper_vmerge_vxm_w,
+        gen_helper_vmerge_vxm_d,
+    };
+
+    return vmerge_vx_trans_common(ctx, a, false, 0, fns[ctx->sew]);
+}
+
+static bool trans_vmerge_vim(DisasContext *ctx, arg_rvv_arith *a)
+{
+    static gen_helper_opivx * const fns[4] = {
+        gen_helper_vmerge_vxm_b,
+        gen_helper_vmerge_vxm_h,
+        gen_helper_vmerge_vxm_w,
+        gen_helper_vmerge_vxm_d,
+    };
+
+    return vmerge_vx_trans_common(ctx, a, true, sextract32(a->rs1, 0, 5),
+                                  fns[ctx->sew]);
+}
+
+static bool opivv_cmp_trans(DisasContext *ctx, arg_rvv_arith *a,
+                            gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) || a->vm != 1 ||
+        !vext_check_mss(ctx, a->rd, a->rs1, a->rs2)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_cmp_trans_common(DisasContext *ctx, arg_rvv_arith *a,
+                                   bool is_imm, target_long imm,
+                                   gen_helper_opivx *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) || a->vm != 1 ||
+        !vext_check_ms(ctx, a->rd, a->rs2)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    if (is_imm) {
+        scalar = tcg_const_tl(tcg_ctx, imm);
+    } else {
+        scalar = tcg_temp_new(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar, a->rs1);
+    }
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opivx_cmp_trans(DisasContext *ctx, arg_rvv_arith *a,
+                            gen_helper_opivx *fn)
+{
+    return opivx_cmp_trans_common(ctx, a, false, 0, fn);
+}
+
+static bool opivi_cmp_trans(DisasContext *ctx, arg_rvv_arith *a,
+                            target_long scalar_value, gen_helper_opivx *fn)
+{
+    return opivx_cmp_trans_common(ctx, a, true, scalar_value, fn);
+}
+
+#define GEN_CMP_VV_TRANS(NAME)                                      \
+static bool trans_##NAME##_vv(DisasContext *ctx, arg_rvv_arith *a)  \
+{                                                                   \
+    static gen_helper_opivv * const fns[4] = {                      \
+        gen_helper_##NAME##_vv_b,                                   \
+        gen_helper_##NAME##_vv_h,                                   \
+        gen_helper_##NAME##_vv_w,                                   \
+        gen_helper_##NAME##_vv_d,                                   \
+    };                                                              \
+                                                                    \
+    return opivv_cmp_trans(ctx, a, fns[ctx->sew]);                  \
+}
+
+#define GEN_CMP_VX_TRANS(NAME)                                      \
+static bool trans_##NAME##_vx(DisasContext *ctx, arg_rvv_arith *a)  \
+{                                                                   \
+    static gen_helper_opivx * const fns[4] = {                      \
+        gen_helper_##NAME##_vx_b,                                   \
+        gen_helper_##NAME##_vx_h,                                   \
+        gen_helper_##NAME##_vx_w,                                   \
+        gen_helper_##NAME##_vx_d,                                   \
+    };                                                              \
+                                                                    \
+    return opivx_cmp_trans(ctx, a, fns[ctx->sew]);                  \
+}
+
+#define GEN_CMP_VI_TRANS(NAME)                                      \
+static bool trans_##NAME##_vi(DisasContext *ctx, arg_rvv_arith *a)  \
+{                                                                   \
+    static gen_helper_opivx * const fns[4] = {                      \
+        gen_helper_##NAME##_vx_b,                                   \
+        gen_helper_##NAME##_vx_h,                                   \
+        gen_helper_##NAME##_vx_w,                                   \
+        gen_helper_##NAME##_vx_d,                                   \
+    };                                                              \
+                                                                    \
+    return opivi_cmp_trans(ctx, a, sextract32(a->rs1, 0, 5),        \
+                           fns[ctx->sew]);                         \
+}
+
+GEN_CMP_VV_TRANS(vmseq)
+GEN_CMP_VV_TRANS(vmsne)
+GEN_CMP_VV_TRANS(vmsltu)
+GEN_CMP_VV_TRANS(vmslt)
+GEN_CMP_VV_TRANS(vmsleu)
+GEN_CMP_VV_TRANS(vmsle)
+GEN_CMP_VX_TRANS(vmseq)
+GEN_CMP_VX_TRANS(vmsne)
+GEN_CMP_VX_TRANS(vmsltu)
+GEN_CMP_VX_TRANS(vmslt)
+GEN_CMP_VX_TRANS(vmsleu)
+GEN_CMP_VX_TRANS(vmsle)
+GEN_CMP_VX_TRANS(vmsgtu)
+GEN_CMP_VX_TRANS(vmsgt)
+GEN_CMP_VI_TRANS(vmseq)
+GEN_CMP_VI_TRANS(vmsne)
+GEN_CMP_VI_TRANS(vmsleu)
+GEN_CMP_VI_TRANS(vmsle)
+GEN_CMP_VI_TRANS(vmsgtu)
+GEN_CMP_VI_TRANS(vmsgt)
+
+static bool mask_mm_trans(DisasContext *ctx, arg_rvv_arith *a,
+                          gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+#define GEN_MM_TRANS(NAME)                                           \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    return mask_mm_trans(ctx, a, gen_helper_##NAME);                 \
+}
+
+GEN_MM_TRANS(vmand_mm)
+GEN_MM_TRANS(vmnand_mm)
+GEN_MM_TRANS(vmandn_mm)
+GEN_MM_TRANS(vmxor_mm)
+GEN_MM_TRANS(vmor_mm)
+GEN_MM_TRANS(vmnor_mm)
+GEN_MM_TRANS(vmorn_mm)
+GEN_MM_TRANS(vmxnor_mm)
+
+static bool mask_scalar_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              gen_helper_mscalar *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv dest;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+
+    dest = tcg_temp_new(tcg_ctx);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src2, tcg_ctx->cpu_env, desc);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+
+    tcg_temp_free(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+    return true;
+}
+
+static bool trans_vcpop_m(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return mask_scalar_trans(ctx, a, gen_helper_vcpop_m);
+}
+
+static bool trans_vfirst_m(DisasContext *ctx, arg_rvv_arith *a)
+{
+    return mask_scalar_trans(ctx, a, gen_helper_vfirst_m);
+}
+
+static bool mask_m_trans(DisasContext *ctx, arg_rvv_arith *a,
+                         gen_helper_opivm *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (!require_rvv_data(ctx) ||
+        !require_vm(a->vm, a->rd) ||
+        a->rd == a->rs2) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    return true;
+}
+
+#define GEN_M_TRANS(NAME)                                            \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)        \
+{                                                                    \
+    return mask_m_trans(ctx, a, gen_helper_##NAME);                  \
+}
+
+GEN_M_TRANS(vmsbf_m)
+GEN_M_TRANS(vmsif_m)
+GEN_M_TRANS(vmsof_m)
+
+static bool trans_viota_m(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+    static gen_helper_opivm * const fns[4] = {
+        gen_helper_viota_m_b,
+        gen_helper_viota_m_h,
+        gen_helper_viota_m_w,
+        gen_helper_viota_m_d,
+    };
+
+    if (!require_rvv_data(ctx) ||
+        !require_vm(a->vm, a->rd) ||
+        !require_align(a->rd, ctx->lmul) ||
+        is_overlapped(a->rd, 1 << MAX(ctx->lmul, 0), a->rs2, 1)) {
+        return false;
+    }
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fns[ctx->sew](tcg_ctx, dest, mask, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    return true;
+}
+
+static bool trans_vid_v(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+    static gen_helper_vid_v * const fns[4] = {
+        gen_helper_vid_v_b,
+        gen_helper_vid_v_h,
+        gen_helper_vid_v_w,
+        gen_helper_vid_v_d,
+    };
+
+    if (!require_rvv_data(ctx) ||
+        !require_align(a->rd, ctx->lmul) ||
+        !require_vm(a->vm, a->rd)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    desc = gen_rvv_desc(ctx, data);
+
+    fns[ctx->sew](tcg_ctx, dest, mask, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool vmv_vx_trans(DisasContext *ctx, int rd, int rs1,
+                         bool is_imm, target_long imm)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv scalar_tl;
+    TCGv_i64 scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    static gen_helper_vmv_vx * const fns[4] = {
+        gen_helper_vmv_v_x_b,
+        gen_helper_vmv_v_x_h,
+        gen_helper_vmv_v_x_w,
+        gen_helper_vmv_v_x_d,
+    };
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    if (is_imm) {
+        scalar_tl = NULL;
+        scalar = tcg_const_i64(tcg_ctx, imm);
+    } else {
+        scalar_tl = tcg_temp_new(tcg_ctx);
+        scalar = tcg_temp_new_i64(tcg_ctx);
+        gen_get_gpr(tcg_ctx, scalar_tl, rs1);
+        tcg_gen_ext_tl_i64(tcg_ctx, scalar, scalar_tl);
+    }
+
+    dest = gen_vreg_ptr(ctx, rd);
+    desc = gen_rvv_desc(ctx, data);
+
+    fns[ctx->sew](tcg_ctx, dest, scalar, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+    if (!is_imm) {
+        tcg_temp_free(tcg_ctx, scalar_tl);
+    }
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vmv_v_v(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr src1;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    static gen_helper_vmv_vv * const fns[4] = {
+        gen_helper_vmv_v_v_b,
+        gen_helper_vmv_v_v_h,
+        gen_helper_vmv_v_v_w,
+        gen_helper_vmv_v_v_d,
+    };
+
+    if (!require_rvv_data(ctx) || a->vm != 1 || a->rs2 != 0 ||
+        !vext_check_sss(ctx, a->rd, a->rs1, 0, 1)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    desc = gen_rvv_desc(ctx, data);
+
+    fns[ctx->sew](tcg_ctx, dest, src1, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vmv_v_x(DisasContext *ctx, arg_rvv_arith *a)
+{
+    if (!require_rvv_data(ctx) || a->vm != 1 || a->rs2 != 0 ||
+        !vext_check_ss(ctx, a->rd, 0, 1)) {
+        return false;
+    }
+
+    return vmv_vx_trans(ctx, a->rd, a->rs1, false, 0);
+}
+
+static bool trans_vmv_v_i(DisasContext *ctx, arg_rvv_arith *a)
+{
+    if (!require_rvv_data(ctx) || a->vm != 1 || a->rs2 != 0 ||
+        !vext_check_ss(ctx, a->rd, 0, 1)) {
+        return false;
+    }
+
+    return vmv_vx_trans(ctx, a->rd, 0, true, sextract32(a->rs1, 0, 5));
+}
+
+static void load_element(DisasContext *ctx, TCGv_i64 dest, TCGv_ptr base,
+                         int ofs, int sew, bool sign)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    switch (sew) {
+    case MO_8:
+        if (sign) {
+            tcg_gen_ld8s_i64(tcg_ctx, dest, base, ofs);
+        } else {
+            tcg_gen_ld8u_i64(tcg_ctx, dest, base, ofs);
+        }
+        break;
+    case MO_16:
+        if (sign) {
+            tcg_gen_ld16s_i64(tcg_ctx, dest, base, ofs);
+        } else {
+            tcg_gen_ld16u_i64(tcg_ctx, dest, base, ofs);
+        }
+        break;
+    case MO_32:
+        if (sign) {
+            tcg_gen_ld32s_i64(tcg_ctx, dest, base, ofs);
+        } else {
+            tcg_gen_ld32u_i64(tcg_ctx, dest, base, ofs);
+        }
+        break;
+    case MO_64:
+        tcg_gen_ld_i64(tcg_ctx, dest, base, ofs);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static uint32_t endian_ofs(DisasContext *ctx, int reg, int idx)
+{
+#if HOST_BIG_ENDIAN
+    return vreg_ofs(ctx, reg) + ((idx ^ (7 >> ctx->sew)) << ctx->sew);
+#else
+    return vreg_ofs(ctx, reg) + (idx << ctx->sew);
+#endif
+}
+
+static void vec_element_loadi(DisasContext *ctx, TCGv_i64 dest,
+                              int reg, int idx, bool sign)
+{
+    load_element(ctx, dest, ctx->uc->tcg_ctx->cpu_env,
+                 endian_ofs(ctx, reg, idx), ctx->sew, sign);
+}
+
+static void store_element(DisasContext *ctx, TCGv_i64 val, TCGv_ptr base,
+                          int ofs, int sew)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    switch (sew) {
+    case MO_8:
+        tcg_gen_st8_i64(tcg_ctx, val, base, ofs);
+        break;
+    case MO_16:
+        tcg_gen_st16_i64(tcg_ctx, val, base, ofs);
+        break;
+    case MO_32:
+        tcg_gen_st32_i64(tcg_ctx, val, base, ofs);
+        break;
+    case MO_64:
+        tcg_gen_st_i64(tcg_ctx, val, base, ofs);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static void vec_element_storei(DisasContext *ctx, int reg,
+                               int idx, TCGv_i64 val)
+{
+    store_element(ctx, val, ctx->uc->tcg_ctx->cpu_env,
+                  endian_ofs(ctx, reg, idx), ctx->sew);
+}
+
+static bool trans_vmv_x_s(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 elem;
+    TCGv dest;
+
+    if (!require_rvv_data(ctx) || a->vm != 1 || a->rs1 != 0) {
+        return false;
+    }
+
+    elem = tcg_temp_new_i64(tcg_ctx);
+    dest = tcg_temp_new(tcg_ctx);
+    vec_element_loadi(ctx, elem, a->rs2, 0, true);
+    tcg_gen_trunc_i64_tl(tcg_ctx, dest, elem);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+    tcg_temp_free(tcg_ctx, dest);
+    tcg_temp_free_i64(tcg_ctx, elem);
+    return true;
+}
+
+static bool trans_vmv_s_x(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv scalar_tl;
+    TCGv_i64 scalar;
+
+    if (!require_rvv_data(ctx) || a->vm != 1 || a->rs2 != 0) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    scalar_tl = tcg_temp_new(tcg_ctx);
+    scalar = tcg_temp_new_i64(tcg_ctx);
+    gen_get_gpr(tcg_ctx, scalar_tl, a->rs1);
+    tcg_gen_ext_tl_i64(tcg_ctx, scalar, scalar_tl);
+    vec_element_storei(ctx, a->rd, 0, scalar);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+    tcg_temp_free(tcg_ctx, scalar_tl);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static void rvv_nanbox_scalar(DisasContext *ctx, TCGv_i64 dest, TCGv_i64 src)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    switch (ctx->sew) {
+    case MO_16:
+        gen_check_nanbox_h(tcg_ctx, dest, src);
+        break;
+    case MO_32:
+        gen_check_nanbox_s(tcg_ctx, dest, src);
+        break;
+    case MO_64:
+        tcg_gen_mov_i64(tcg_ctx, dest, src);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static bool trans_vfmv_v_f(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_i64 scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    static gen_helper_vmv_vx * const fns[4] = {
+        NULL,
+        gen_helper_vmv_v_x_h,
+        gen_helper_vmv_v_x_w,
+        gen_helper_vmv_v_x_d,
+    };
+
+    if (ctx->sew > MO_64 || fns[ctx->sew] == NULL ||
+        !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        a->rs2 != 0 || a->vm != 1 || !require_align(a->rd, ctx->lmul)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    scalar = tcg_temp_new_i64(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    rvv_nanbox_scalar(ctx, scalar, tcg_ctx->cpu_fpr[a->rs1]);
+    fns[ctx->sew](tcg_ctx, dest, scalar, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool trans_vfmv_f_s(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    unsigned int ofs;
+    unsigned int len;
+
+    if (!require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        a->rs1 != 0 || a->vm != 1) {
+        return false;
+    }
+
+    gen_set_rm(ctx, 7);
+    vec_element_loadi(ctx, tcg_ctx->cpu_fpr[a->rd], a->rs2, 0, false);
+
+    ofs = 8 << ctx->sew;
+    len = 64 - ofs;
+    if (len != 0) {
+        TCGv_i64 ones = tcg_const_i64(tcg_ctx, UINT64_MAX);
+
+        tcg_gen_deposit_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd],
+                            tcg_ctx->cpu_fpr[a->rd], ones, ofs, len);
+        tcg_temp_free_i64(tcg_ctx, ones);
+    }
+
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_vfmv_s_f(DisasContext *ctx, arg_rvv_arith *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_i64 scalar;
+
+    if (!require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        a->rs2 != 0 || a->vm != 1) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    scalar = tcg_temp_new_i64(tcg_ctx);
+    rvv_nanbox_scalar(ctx, scalar, tcg_ctx->cpu_fpr[a->rs1]);
+    vec_element_storei(ctx, a->rd, 0, scalar);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvv_trans(DisasContext *ctx, arg_rvv_arith *a,
+                        gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        !vext_check_sss(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvf_trans(DisasContext *ctx, arg_rvv_arith *a,
+                        gen_helper_opfvf *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i64 scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        !vext_check_ss(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    scalar = tcg_temp_new_i64(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    rvv_nanbox_scalar(ctx, scalar, tcg_ctx->cpu_fpr[a->rs1]);
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvf_slide_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              gen_helper_opfvf *fn, bool slide_up)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i64 scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        !vext_check_slide(ctx, a->rd, a->rs2, a->vm, slide_up)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    scalar = tcg_temp_new_i64(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    rvv_nanbox_scalar(ctx, scalar, tcg_ctx->cpu_fpr[a->rs1]);
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvv_widen_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              gen_helper_opivv *fn, bool wide_vs2)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_scale_rvf(ctx) ||
+        !require_scale_zve32f(ctx) || !require_scale_zve64f(ctx)) {
+        return false;
+    }
+    if (wide_vs2) {
+        if (!vext_check_dds(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+            return false;
+        }
+    } else if (!vext_check_dss(ctx, a->rd, a->rs1, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvf_widen_trans(DisasContext *ctx, arg_rvv_arith *a,
+                              gen_helper_opfvf *fn, bool wide_vs2)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i64 scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_scale_rvf(ctx) ||
+        !require_scale_zve32f(ctx) || !require_scale_zve64f(ctx)) {
+        return false;
+    }
+    if (wide_vs2) {
+        if (!vext_check_dd(ctx, a->rd, a->rs2, a->vm)) {
+            return false;
+        }
+    } else if (!vext_check_ds(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    scalar = tcg_temp_new_i64(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    rvv_nanbox_scalar(ctx, scalar, tcg_ctx->cpu_fpr[a->rs1]);
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvv_cmp_trans(DisasContext *ctx, arg_rvv_arith *a,
+                            gen_helper_opivv *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src1;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        !vext_check_mss(ctx, a->rd, a->rs1, a->rs2)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src1 = gen_vreg_ptr(ctx, a->rs1);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src1, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src1);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfvf_cmp_trans(DisasContext *ctx, arg_rvv_arith *a,
+                            gen_helper_opfvf *fn)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i64 scalar;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        !vext_check_ms(ctx, a->rd, a->rs2)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, 7);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VTA_ALL_1S, ctx->rvv_ta_all_1s, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    scalar = tcg_temp_new_i64(tcg_ctx);
+    desc = gen_rvv_desc(ctx, data);
+
+    rvv_nanbox_scalar(ctx, scalar, tcg_ctx->cpu_fpr[a->rs1]);
+    fn(tcg_ctx, dest, mask, scalar, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i64(tcg_ctx, scalar);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfv_trans(DisasContext *ctx, arg_rvv_arith *a,
+                       gen_helper_opivm *fn, int rm)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx) || !require_rvf(ctx) ||
+        !require_zve32f(ctx) || !require_zve64f(ctx) ||
+        !vext_check_ss(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, rm);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfv_widen_cvt_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                 gen_helper_opivm *fn,
+                                 bool fp_src, bool fp_dest, int rm)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx)) {
+        return false;
+    }
+    if (fp_src) {
+        if (!require_rvf(ctx) || !require_zve32f(ctx) ||
+            !require_zve64f(ctx)) {
+            return false;
+        }
+    }
+    if (fp_dest) {
+        if (!require_scale_rvf(ctx) || !require_scale_zve32f(ctx) ||
+            !require_scale_zve64f(ctx)) {
+            return false;
+        }
+    }
+    if (fp_src && fp_dest && ctx->sew == MO_8) {
+        return false;
+    }
+    if (!vext_check_ds(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, rm);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+static bool opfv_narrow_cvt_trans(DisasContext *ctx, arg_rvv_arith *a,
+                                  gen_helper_opivm *fn,
+                                  bool fp_src, bool fp_dest, int rm)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGLabel *over;
+    TCGv_ptr dest;
+    TCGv_ptr mask;
+    TCGv_ptr src2;
+    TCGv_i32 desc;
+    uint32_t data = 0;
+
+    if (fn == NULL || !require_rvv_data(ctx)) {
+        return false;
+    }
+    if (fp_src) {
+        if (!require_scale_rvf(ctx) || !require_scale_zve32f(ctx) ||
+            !require_scale_zve64f(ctx)) {
+            return false;
+        }
+    }
+    if (fp_dest) {
+        if (!require_rvf(ctx) || !require_zve32f(ctx) ||
+            !require_zve64f(ctx)) {
+            return false;
+        }
+    }
+    if (fp_src && fp_dest && ctx->sew == MO_8) {
+        return false;
+    }
+    if (!vext_check_sd(ctx, a->rd, a->rs2, a->vm)) {
+        return false;
+    }
+
+    over = gen_new_label(tcg_ctx);
+    gen_set_rm(ctx, rm);
+    tcg_gen_brcondi_tl(tcg_ctx, TCG_COND_EQ, cpu_vl, 0, over);
+    tcg_gen_brcond_tl(tcg_ctx, TCG_COND_GEU, cpu_vstart, cpu_vl, over);
+
+    FIELD_DP32(data, VDATA, VM, a->vm, data);
+    FIELD_DP32(data, VDATA, LMUL, ctx->lmul, data);
+    FIELD_DP32(data, VDATA, VTA, ctx->vta, data);
+    FIELD_DP32(data, VDATA, VMA, ctx->vma, data);
+
+    dest = gen_vreg_ptr(ctx, a->rd);
+    mask = gen_vreg_ptr(ctx, 0);
+    src2 = gen_vreg_ptr(ctx, a->rs2);
+    desc = gen_rvv_desc(ctx, data);
+
+    fn(tcg_ctx, dest, mask, src2, tcg_ctx->cpu_env, desc);
+
+    tcg_temp_free_ptr(tcg_ctx, dest);
+    tcg_temp_free_ptr(tcg_ctx, mask);
+    tcg_temp_free_ptr(tcg_ctx, src2);
+    tcg_temp_free_i32(tcg_ctx, desc);
+
+    mark_vs_dirty(ctx);
+    gen_set_label(tcg_ctx, over);
+    return true;
+}
+
+#define GEN_OPFVV_TRANS(NAME)                                      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)      \
+{                                                                  \
+    static gen_helper_opivv * const fns[4] = {                     \
+        NULL,                                                      \
+        gen_helper_##NAME##_h,                                     \
+        gen_helper_##NAME##_w,                                     \
+        gen_helper_##NAME##_d,                                     \
+    };                                                             \
+                                                                   \
+    if (ctx->sew > MO_64) {                                        \
+        return false;                                              \
+    }                                                              \
+    return opfvv_trans(ctx, a, fns[ctx->sew]);                     \
+}
+
+#define GEN_OPFVF_TRANS(NAME)                                      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)      \
+{                                                                  \
+    static gen_helper_opfvf * const fns[4] = {                     \
+        NULL,                                                      \
+        gen_helper_##NAME##_h,                                     \
+        gen_helper_##NAME##_w,                                     \
+        gen_helper_##NAME##_d,                                     \
+    };                                                             \
+                                                                   \
+    if (ctx->sew > MO_64) {                                        \
+        return false;                                              \
+    }                                                              \
+    return opfvf_trans(ctx, a, fns[ctx->sew]);                     \
+}
+
+#define GEN_OPFVF_SLIDE_TRANS(NAME, SLIDE_UP)                       \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)       \
+{                                                                   \
+    static gen_helper_opfvf * const fns[4] = {                      \
+        NULL,                                                       \
+        gen_helper_##NAME##_h,                                      \
+        gen_helper_##NAME##_w,                                      \
+        gen_helper_##NAME##_d,                                      \
+    };                                                              \
+                                                                    \
+    if (ctx->sew > MO_64) {                                         \
+        return false;                                               \
+    }                                                               \
+    return opfvf_slide_trans(ctx, a, fns[ctx->sew], SLIDE_UP);      \
+}
+
+#define GEN_OPFVV_WIDEN_TRANS(NAME, WIDE_VS2)                       \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)       \
+{                                                                   \
+    static gen_helper_opivv * const fns[4] = {                      \
+        NULL,                                                       \
+        gen_helper_##NAME##_h,                                      \
+        gen_helper_##NAME##_w,                                      \
+        NULL,                                                       \
+    };                                                              \
+                                                                    \
+    return opfvv_widen_trans(ctx, a, fns[ctx->sew], WIDE_VS2);      \
+}
+
+#define GEN_OPFVF_WIDEN_TRANS(NAME, WIDE_VS2)                       \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)       \
+{                                                                   \
+    static gen_helper_opfvf * const fns[4] = {                      \
+        NULL,                                                       \
+        gen_helper_##NAME##_h,                                      \
+        gen_helper_##NAME##_w,                                      \
+        NULL,                                                       \
+    };                                                              \
+                                                                    \
+    return opfvf_widen_trans(ctx, a, fns[ctx->sew], WIDE_VS2);      \
+}
+
+#define GEN_OPFVV_CMP_TRANS(NAME)                                  \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)      \
+{                                                                  \
+    static gen_helper_opivv * const fns[4] = {                     \
+        NULL,                                                      \
+        gen_helper_##NAME##_h,                                     \
+        gen_helper_##NAME##_w,                                     \
+        gen_helper_##NAME##_d,                                     \
+    };                                                             \
+                                                                   \
+    if (ctx->sew > MO_64) {                                        \
+        return false;                                              \
+    }                                                              \
+    return opfvv_cmp_trans(ctx, a, fns[ctx->sew]);                 \
+}
+
+#define GEN_OPFVF_CMP_TRANS(NAME)                                  \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)      \
+{                                                                  \
+    static gen_helper_opfvf * const fns[4] = {                     \
+        NULL,                                                      \
+        gen_helper_##NAME##_h,                                     \
+        gen_helper_##NAME##_w,                                     \
+        gen_helper_##NAME##_d,                                     \
+    };                                                             \
+                                                                   \
+    if (ctx->sew > MO_64) {                                        \
+        return false;                                              \
+    }                                                              \
+    return opfvf_cmp_trans(ctx, a, fns[ctx->sew]);                 \
+}
+
+#define GEN_OPFV_TRANS(NAME)                                             \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)            \
+{                                                                        \
+    static gen_helper_opivm * const fns[4] = {                           \
+        NULL,                                                            \
+        gen_helper_##NAME##_h,                                           \
+        gen_helper_##NAME##_w,                                           \
+        gen_helper_##NAME##_d,                                           \
+    };                                                                   \
+                                                                         \
+    if (ctx->sew > MO_64) {                                              \
+        return false;                                                    \
+    }                                                                    \
+    return opfv_trans(ctx, a, fns[ctx->sew], RISCV_FRM_DYN);             \
+}
+
+#define GEN_OPFV_CVT_TRANS(NAME, HELPER, RM)                             \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)            \
+{                                                                        \
+    static gen_helper_opivm * const fns[4] = {                           \
+        NULL,                                                            \
+        gen_helper_##HELPER##_h,                                         \
+        gen_helper_##HELPER##_w,                                         \
+        gen_helper_##HELPER##_d,                                         \
+    };                                                                   \
+                                                                         \
+    if (ctx->sew > MO_64) {                                              \
+        return false;                                                    \
+    }                                                                    \
+    return opfv_trans(ctx, a, fns[ctx->sew], RM);                        \
+}
+
+#define GEN_OPFV_WIDEN_CVT_TRANS(NAME, HELPER, FP_SRC, FP_DEST, RM)      \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)            \
+{                                                                        \
+    static gen_helper_opivm * const fns[4] = {                           \
+        gen_helper_##HELPER##_b,                                         \
+        gen_helper_##HELPER##_h,                                         \
+        gen_helper_##HELPER##_w,                                         \
+        NULL,                                                            \
+    };                                                                   \
+                                                                         \
+    return opfv_widen_cvt_trans(ctx, a, fns[ctx->sew], FP_SRC,           \
+                                FP_DEST, RM);                            \
+}
+
+#define GEN_OPFV_WIDEN_CVT_TRANS_HW(NAME, HELPER, FP_SRC, FP_DEST, RM)   \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)            \
+{                                                                        \
+    static gen_helper_opivm * const fns[4] = {                           \
+        NULL,                                                            \
+        gen_helper_##HELPER##_h,                                         \
+        gen_helper_##HELPER##_w,                                         \
+        NULL,                                                            \
+    };                                                                   \
+                                                                         \
+    return opfv_widen_cvt_trans(ctx, a, fns[ctx->sew], FP_SRC,           \
+                                FP_DEST, RM);                            \
+}
+
+#define GEN_OPFV_NARROW_CVT_TRANS(NAME, HELPER, FP_SRC, FP_DEST, RM)     \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)            \
+{                                                                        \
+    static gen_helper_opivm * const fns[4] = {                           \
+        gen_helper_##HELPER##_b,                                         \
+        gen_helper_##HELPER##_h,                                         \
+        gen_helper_##HELPER##_w,                                         \
+        NULL,                                                            \
+    };                                                                   \
+                                                                         \
+    return opfv_narrow_cvt_trans(ctx, a, fns[ctx->sew], FP_SRC,          \
+                                 FP_DEST, RM);                           \
+}
+
+#define GEN_OPFV_NARROW_CVT_TRANS_HW(NAME, HELPER, FP_SRC, FP_DEST, RM)  \
+static bool trans_##NAME(DisasContext *ctx, arg_rvv_arith *a)            \
+{                                                                        \
+    static gen_helper_opivm * const fns[4] = {                           \
+        NULL,                                                            \
+        gen_helper_##HELPER##_h,                                         \
+        gen_helper_##HELPER##_w,                                         \
+        NULL,                                                            \
+    };                                                                   \
+                                                                         \
+    return opfv_narrow_cvt_trans(ctx, a, fns[ctx->sew], FP_SRC,          \
+                                 FP_DEST, RM);                           \
+}
+
+GEN_OPFVV_TRANS(vfadd_vv)
+GEN_OPFVV_TRANS(vfsub_vv)
+GEN_OPFVV_TRANS(vfmul_vv)
+GEN_OPFVV_TRANS(vfdiv_vv)
+GEN_OPFVV_TRANS(vfmin_vv)
+GEN_OPFVV_TRANS(vfmax_vv)
+GEN_OPFVV_TRANS(vfsgnj_vv)
+GEN_OPFVV_TRANS(vfsgnjn_vv)
+GEN_OPFVV_TRANS(vfsgnjx_vv)
+GEN_OPFVF_TRANS(vfadd_vf)
+GEN_OPFVF_TRANS(vfsub_vf)
+GEN_OPFVF_TRANS(vfrsub_vf)
+GEN_OPFVF_TRANS(vfmul_vf)
+GEN_OPFVF_TRANS(vfdiv_vf)
+GEN_OPFVF_TRANS(vfrdiv_vf)
+GEN_OPFVF_TRANS(vfmin_vf)
+GEN_OPFVF_TRANS(vfmax_vf)
+GEN_OPFVF_TRANS(vfsgnj_vf)
+GEN_OPFVF_TRANS(vfsgnjn_vf)
+GEN_OPFVF_TRANS(vfsgnjx_vf)
+GEN_OPFVV_WIDEN_TRANS(vfwadd_vv, false)
+GEN_OPFVF_WIDEN_TRANS(vfwadd_vf, false)
+GEN_OPFVV_WIDEN_TRANS(vfwsub_vv, false)
+GEN_OPFVF_WIDEN_TRANS(vfwsub_vf, false)
+GEN_OPFVV_WIDEN_TRANS(vfwadd_wv, true)
+GEN_OPFVF_WIDEN_TRANS(vfwadd_wf, true)
+GEN_OPFVV_WIDEN_TRANS(vfwsub_wv, true)
+GEN_OPFVF_WIDEN_TRANS(vfwsub_wf, true)
+GEN_OPFVV_WIDEN_TRANS(vfwmul_vv, false)
+GEN_OPFVF_WIDEN_TRANS(vfwmul_vf, false)
+GEN_OPFVV_TRANS(vfmacc_vv)
+GEN_OPFVF_TRANS(vfmacc_vf)
+GEN_OPFVV_TRANS(vfnmacc_vv)
+GEN_OPFVF_TRANS(vfnmacc_vf)
+GEN_OPFVV_TRANS(vfmsac_vv)
+GEN_OPFVF_TRANS(vfmsac_vf)
+GEN_OPFVV_TRANS(vfnmsac_vv)
+GEN_OPFVF_TRANS(vfnmsac_vf)
+GEN_OPFVV_TRANS(vfmadd_vv)
+GEN_OPFVF_TRANS(vfmadd_vf)
+GEN_OPFVV_TRANS(vfnmadd_vv)
+GEN_OPFVF_TRANS(vfnmadd_vf)
+GEN_OPFVV_TRANS(vfmsub_vv)
+GEN_OPFVF_TRANS(vfmsub_vf)
+GEN_OPFVV_TRANS(vfnmsub_vv)
+GEN_OPFVF_TRANS(vfnmsub_vf)
+GEN_OPFVV_WIDEN_TRANS(vfwmacc_vv, false)
+GEN_OPFVF_WIDEN_TRANS(vfwmacc_vf, false)
+GEN_OPFVV_WIDEN_TRANS(vfwnmacc_vv, false)
+GEN_OPFVF_WIDEN_TRANS(vfwnmacc_vf, false)
+GEN_OPFVV_WIDEN_TRANS(vfwmsac_vv, false)
+GEN_OPFVF_WIDEN_TRANS(vfwmsac_vf, false)
+GEN_OPFVV_WIDEN_TRANS(vfwnmsac_vv, false)
+GEN_OPFVF_WIDEN_TRANS(vfwnmsac_vf, false)
+GEN_OPFVV_CMP_TRANS(vmfeq_vv)
+GEN_OPFVV_CMP_TRANS(vmfne_vv)
+GEN_OPFVV_CMP_TRANS(vmflt_vv)
+GEN_OPFVV_CMP_TRANS(vmfle_vv)
+GEN_OPFVF_CMP_TRANS(vmfeq_vf)
+GEN_OPFVF_CMP_TRANS(vmfne_vf)
+GEN_OPFVF_CMP_TRANS(vmflt_vf)
+GEN_OPFVF_CMP_TRANS(vmfle_vf)
+GEN_OPFVF_CMP_TRANS(vmfgt_vf)
+GEN_OPFVF_CMP_TRANS(vmfge_vf)
+GEN_OPFV_TRANS(vfsqrt_v)
+GEN_OPFV_TRANS(vfrsqrt7_v)
+GEN_OPFV_TRANS(vfrec7_v)
+GEN_OPFV_CVT_TRANS(vfcvt_xu_f_v, vfcvt_xu_f_v, RISCV_FRM_DYN)
+GEN_OPFV_CVT_TRANS(vfcvt_x_f_v, vfcvt_x_f_v, RISCV_FRM_DYN)
+GEN_OPFV_CVT_TRANS(vfcvt_f_xu_v, vfcvt_f_xu_v, RISCV_FRM_DYN)
+GEN_OPFV_CVT_TRANS(vfcvt_f_x_v, vfcvt_f_x_v, RISCV_FRM_DYN)
+GEN_OPFV_CVT_TRANS(vfcvt_rtz_xu_f_v, vfcvt_xu_f_v, RISCV_FRM_RTZ)
+GEN_OPFV_CVT_TRANS(vfcvt_rtz_x_f_v, vfcvt_x_f_v, RISCV_FRM_RTZ)
+GEN_OPFV_WIDEN_CVT_TRANS_HW(vfwcvt_xu_f_v, vfwcvt_xu_f_v, true, false,
+                            RISCV_FRM_DYN)
+GEN_OPFV_WIDEN_CVT_TRANS_HW(vfwcvt_x_f_v, vfwcvt_x_f_v, true, false,
+                            RISCV_FRM_DYN)
+GEN_OPFV_WIDEN_CVT_TRANS_HW(vfwcvt_rtz_xu_f_v, vfwcvt_xu_f_v, true, false,
+                            RISCV_FRM_RTZ)
+GEN_OPFV_WIDEN_CVT_TRANS_HW(vfwcvt_rtz_x_f_v, vfwcvt_x_f_v, true, false,
+                            RISCV_FRM_RTZ)
+GEN_OPFV_WIDEN_CVT_TRANS(vfwcvt_f_xu_v, vfwcvt_f_xu_v, false, true,
+                         RISCV_FRM_DYN)
+GEN_OPFV_WIDEN_CVT_TRANS(vfwcvt_f_x_v, vfwcvt_f_x_v, false, true,
+                         RISCV_FRM_DYN)
+GEN_OPFV_WIDEN_CVT_TRANS_HW(vfwcvt_f_f_v, vfwcvt_f_f_v, true, true,
+                            RISCV_FRM_DYN)
+GEN_OPFV_NARROW_CVT_TRANS(vfncvt_xu_f_w, vfncvt_xu_f_w, true, false,
+                          RISCV_FRM_DYN)
+GEN_OPFV_NARROW_CVT_TRANS(vfncvt_x_f_w, vfncvt_x_f_w, true, false,
+                          RISCV_FRM_DYN)
+GEN_OPFV_NARROW_CVT_TRANS(vfncvt_rtz_xu_f_w, vfncvt_xu_f_w, true, false,
+                          RISCV_FRM_RTZ)
+GEN_OPFV_NARROW_CVT_TRANS(vfncvt_rtz_x_f_w, vfncvt_x_f_w, true, false,
+                          RISCV_FRM_RTZ)
+GEN_OPFV_NARROW_CVT_TRANS_HW(vfncvt_f_xu_w, vfncvt_f_xu_w, false, true,
+                             RISCV_FRM_DYN)
+GEN_OPFV_NARROW_CVT_TRANS_HW(vfncvt_f_x_w, vfncvt_f_x_w, false, true,
+                             RISCV_FRM_DYN)
+GEN_OPFV_NARROW_CVT_TRANS_HW(vfncvt_f_f_w, vfncvt_f_f_w, true, true,
+                             RISCV_FRM_DYN)
+GEN_OPFV_NARROW_CVT_TRANS_HW(vfncvt_rod_f_f_w, vfncvt_f_f_w, true, true,
+                             RISCV_FRM_ROD)
+GEN_OPFV_TRANS(vfclass_v)
+GEN_OPFVF_TRANS(vfmerge_vfm)
+GEN_OPFVF_SLIDE_TRANS(vfslide1up_vf, true)
+GEN_OPFVF_SLIDE_TRANS(vfslide1down_vf, false)
+
+static bool decode_rvv_config(DisasContext *ctx, uint32_t insn)
+{
+    if ((insn & 0x0000707f) != 0x00007057) {
+        return false;
+    }
+
+    if ((insn & 0xfe00707f) == 0x80007057) {
+        arg_vsetvl a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vsetvl(ctx, &a);
+    }
+
+    if ((insn & 0xc000707f) == 0xc0007057) {
+        arg_vsetivli a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .zimm = extract32(insn, 20, 10),
+        };
+
+        return trans_vsetivli(ctx, &a);
+    }
+
+    if ((insn & 0x8000707f) == 0x00007057) {
+        arg_vsetvli a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .zimm = extract32(insn, 20, 11),
+        };
+
+        return trans_vsetvli(ctx, &a);
+    }
+
+    return false;
+}
+
+static bool decode_rvv_data(DisasContext *ctx, uint32_t insn)
+{
+    static const struct {
+        uint32_t value;
+        bool (*trans)(DisasContext *, arg_rvv_ldst *);
+    } whole[] = {
+        { 0x02800007, trans_vl1re8_v },
+        { 0x02805007, trans_vl1re16_v },
+        { 0x02806007, trans_vl1re32_v },
+        { 0x02807007, trans_vl1re64_v },
+        { 0x22800007, trans_vl2re8_v },
+        { 0x22805007, trans_vl2re16_v },
+        { 0x22806007, trans_vl2re32_v },
+        { 0x22807007, trans_vl2re64_v },
+        { 0x62800007, trans_vl4re8_v },
+        { 0x62805007, trans_vl4re16_v },
+        { 0x62806007, trans_vl4re32_v },
+        { 0x62807007, trans_vl4re64_v },
+        { 0xe2800007, trans_vl8re8_v },
+        { 0xe2805007, trans_vl8re16_v },
+        { 0xe2806007, trans_vl8re32_v },
+        { 0xe2807007, trans_vl8re64_v },
+        { 0x02800027, trans_vs1r_v },
+        { 0x22800027, trans_vs2r_v },
+        { 0x62800027, trans_vs4r_v },
+        { 0xe2800027, trans_vs8r_v },
+    };
+    size_t i;
+
+    for (i = 0; i < ARRAY_SIZE(whole); i++) {
+        if ((insn & 0xfff0707f) == whole[i].value) {
+            arg_rvv_ldst a = {
+                .rd = extract32(insn, 7, 5),
+                .rs1 = extract32(insn, 15, 5),
+            };
+
+            return whole[i].trans(ctx, &a);
+        }
+    }
+
+    {
+        static const struct {
+            uint32_t value;
+            bool (*trans)(DisasContext *, arg_rvv_arith *);
+        } fp_arith[] = {
+            { 0x00001057, trans_vfadd_vv },
+            { 0x00005057, trans_vfadd_vf },
+            { 0x08001057, trans_vfsub_vv },
+            { 0x08005057, trans_vfsub_vf },
+            { 0x10001057, trans_vfmin_vv },
+            { 0x10005057, trans_vfmin_vf },
+            { 0x18001057, trans_vfmax_vv },
+            { 0x18005057, trans_vfmax_vf },
+            { 0x20001057, trans_vfsgnj_vv },
+            { 0x20005057, trans_vfsgnj_vf },
+            { 0x24001057, trans_vfsgnjn_vv },
+            { 0x24005057, trans_vfsgnjn_vf },
+            { 0x28001057, trans_vfsgnjx_vv },
+            { 0x28005057, trans_vfsgnjx_vf },
+            { 0x38005057, trans_vfslide1up_vf },
+            { 0x3c005057, trans_vfslide1down_vf },
+            { 0x60001057, trans_vmfeq_vv },
+            { 0x60005057, trans_vmfeq_vf },
+            { 0x64001057, trans_vmfle_vv },
+            { 0x64005057, trans_vmfle_vf },
+            { 0x6c001057, trans_vmflt_vv },
+            { 0x6c005057, trans_vmflt_vf },
+            { 0x70001057, trans_vmfne_vv },
+            { 0x70005057, trans_vmfne_vf },
+            { 0x74005057, trans_vmfgt_vf },
+            { 0x7c005057, trans_vmfge_vf },
+            { 0x80001057, trans_vfdiv_vv },
+            { 0x80005057, trans_vfdiv_vf },
+            { 0x84005057, trans_vfrdiv_vf },
+            { 0x90001057, trans_vfmul_vv },
+            { 0x90005057, trans_vfmul_vf },
+            { 0x9c005057, trans_vfrsub_vf },
+            { 0xa0001057, trans_vfmadd_vv },
+            { 0xa0005057, trans_vfmadd_vf },
+            { 0xa4001057, trans_vfnmadd_vv },
+            { 0xa4005057, trans_vfnmadd_vf },
+            { 0xa8001057, trans_vfmsub_vv },
+            { 0xa8005057, trans_vfmsub_vf },
+            { 0xac001057, trans_vfnmsub_vv },
+            { 0xac005057, trans_vfnmsub_vf },
+            { 0xb0001057, trans_vfmacc_vv },
+            { 0xb0005057, trans_vfmacc_vf },
+            { 0xb4001057, trans_vfnmacc_vv },
+            { 0xb4005057, trans_vfnmacc_vf },
+            { 0xb8001057, trans_vfmsac_vv },
+            { 0xb8005057, trans_vfmsac_vf },
+            { 0xbc001057, trans_vfnmsac_vv },
+            { 0xbc005057, trans_vfnmsac_vf },
+            { 0xc0001057, trans_vfwadd_vv },
+            { 0xc0005057, trans_vfwadd_vf },
+            { 0xc8001057, trans_vfwsub_vv },
+            { 0xc8005057, trans_vfwsub_vf },
+            { 0xd0001057, trans_vfwadd_wv },
+            { 0xd0005057, trans_vfwadd_wf },
+            { 0xd8001057, trans_vfwsub_wv },
+            { 0xd8005057, trans_vfwsub_wf },
+            { 0xe0001057, trans_vfwmul_vv },
+            { 0xe0005057, trans_vfwmul_vf },
+            { 0xf0001057, trans_vfwmacc_vv },
+            { 0xf0005057, trans_vfwmacc_vf },
+            { 0xf4001057, trans_vfwnmacc_vv },
+            { 0xf4005057, trans_vfwnmacc_vf },
+            { 0xf8001057, trans_vfwmsac_vv },
+            { 0xf8005057, trans_vfwmsac_vf },
+            { 0xfc001057, trans_vfwnmsac_vv },
+            { 0xfc005057, trans_vfwnmsac_vf },
+        };
+
+        for (i = 0; i < ARRAY_SIZE(fp_arith); i++) {
+            if ((insn & 0xfc00707f) == fp_arith[i].value) {
+                arg_rvv_arith a = {
+                    .rd = extract32(insn, 7, 5),
+                    .rs1 = extract32(insn, 15, 5),
+                    .rs2 = extract32(insn, 20, 5),
+                    .vm = extract32(insn, 25, 1),
+                };
+
+                return fp_arith[i].trans(ctx, &a);
+            }
+        }
+    }
+
+    {
+        static const struct {
+            uint32_t value;
+            bool (*trans)(DisasContext *, arg_rvv_arith *);
+        } fp_red[] = {
+            { 0x04001057, trans_vfredusum_vs },
+            { 0x0c001057, trans_vfredosum_vs },
+            { 0x14001057, trans_vfredmin_vs },
+            { 0x1c001057, trans_vfredmax_vs },
+            { 0xc4001057, trans_vfwredusum_vs },
+            { 0xcc001057, trans_vfwredosum_vs },
+        };
+
+        for (i = 0; i < ARRAY_SIZE(fp_red); i++) {
+            if ((insn & 0xfc00707f) == fp_red[i].value) {
+                arg_rvv_arith a = {
+                    .rd = extract32(insn, 7, 5),
+                    .rs1 = extract32(insn, 15, 5),
+                    .rs2 = extract32(insn, 20, 5),
+                    .vm = extract32(insn, 25, 1),
+                };
+
+                return fp_red[i].trans(ctx, &a);
+            }
+        }
+    }
+
+    {
+        static const struct {
+            uint32_t value;
+            bool (*trans)(DisasContext *, arg_rvv_arith *);
+        } fp_unary[] = {
+            { 0x48001057, trans_vfcvt_xu_f_v },
+            { 0x48009057, trans_vfcvt_x_f_v },
+            { 0x48011057, trans_vfcvt_f_xu_v },
+            { 0x48019057, trans_vfcvt_f_x_v },
+            { 0x48031057, trans_vfcvt_rtz_xu_f_v },
+            { 0x48039057, trans_vfcvt_rtz_x_f_v },
+            { 0x48041057, trans_vfwcvt_xu_f_v },
+            { 0x48049057, trans_vfwcvt_x_f_v },
+            { 0x48051057, trans_vfwcvt_f_xu_v },
+            { 0x48059057, trans_vfwcvt_f_x_v },
+            { 0x48061057, trans_vfwcvt_f_f_v },
+            { 0x48071057, trans_vfwcvt_rtz_xu_f_v },
+            { 0x48079057, trans_vfwcvt_rtz_x_f_v },
+            { 0x48081057, trans_vfncvt_xu_f_w },
+            { 0x48089057, trans_vfncvt_x_f_w },
+            { 0x48091057, trans_vfncvt_f_xu_w },
+            { 0x48099057, trans_vfncvt_f_x_w },
+            { 0x480a1057, trans_vfncvt_f_f_w },
+            { 0x480a9057, trans_vfncvt_rod_f_f_w },
+            { 0x480b1057, trans_vfncvt_rtz_xu_f_w },
+            { 0x480b9057, trans_vfncvt_rtz_x_f_w },
+            { 0x4c001057, trans_vfsqrt_v },
+            { 0x4c021057, trans_vfrsqrt7_v },
+            { 0x4c029057, trans_vfrec7_v },
+        };
+
+        for (i = 0; i < ARRAY_SIZE(fp_unary); i++) {
+            if ((insn & 0xfc0ff07f) == fp_unary[i].value) {
+                arg_rvv_arith a = {
+                    .rd = extract32(insn, 7, 5),
+                    .rs2 = extract32(insn, 20, 5),
+                    .vm = extract32(insn, 25, 1),
+                };
+
+                return fp_unary[i].trans(ctx, &a);
+            }
+        }
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x4c081057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vfclass_v(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x5c005057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 0,
+        };
+
+        return trans_vfmerge_vfm(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x5e005057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vfmv_v_f(ctx, &a);
+    }
+
+    if ((insn & 0xfe0ff07f) == 0x42001057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vfmv_f_s(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x42005057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vfmv_s_f(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x02b00007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+        };
+
+        return trans_vlm_v(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x02b00027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+        };
+
+        return trans_vsm_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x01000007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle8ff_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x01005007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle16ff_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x01006007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle32ff_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x01007007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle64ff_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00000007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle8_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00005007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle16_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00006007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle32_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00007007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vle64_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00000027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vse8_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00005027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vse16_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00006027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vse32_v(ctx, &a);
+    }
+
+    if ((insn & 0x1df0707f) == 0x00007027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vse64_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08000007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlse8_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08005007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlse16_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08006007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlse32_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08007007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlse64_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08000027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsse8_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08005027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsse16_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08006027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsse32_v(ctx, &a);
+    }
+
+    if ((insn & 0x1c00707f) == 0x08007027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsse64_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04000007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlxei8_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04005007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlxei16_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04006007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlxei32_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04007007) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vlxei64_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04000027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsxei8_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04005027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsxei16_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04006027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsxei32_v(ctx, &a);
+    }
+
+    if ((insn & 0x1400707f) == 0x04007027) {
+        arg_rvv_ldst a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .nf = extract32(insn, 29, 3) + 1,
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsxei64_v(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x5e000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmv_v_v(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x5e004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmv_v_x(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x5e003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmv_v_i(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x5c000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmerge_vvm(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x5c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmerge_vxm(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x5c003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmerge_vim(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x60002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmandn_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x64002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmand_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x68002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmor_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x6c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmxor_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x70002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmorn_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x74002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmnand_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x78002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmnor_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x7c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+        };
+
+        return trans_vmxnor_mm(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x40082057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vcpop_m(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x4008a057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vfirst_m(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x5000a057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsbf_m(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x5001a057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsif_m(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x50012057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsof_m(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x50082057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_viota_m(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x5008a057 && extract32(insn, 20, 5) == 0) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vid_v(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x00002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredsum_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x04002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredand_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x08002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredor_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x0c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredxor_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x10002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredminu_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x14002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredmin_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x18002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredmaxu_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x1c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vredmax_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc0000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwredsumu_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc4000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwredsum_vs(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x80000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsaddu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x84000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsadd_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x88000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssubu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x8c000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssub_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x80004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsaddu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x84004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsadd_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x88004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssubu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x8c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssub_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x80003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsaddu_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x84003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsadd_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x20002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vaaddu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x24002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vaadd_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x28002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vasubu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x2c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vasub_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x20006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vaaddu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x24006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vaadd_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x28006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vasubu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x2c006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vasub_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x9c000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsmul_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x9c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsmul_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa8000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssrl_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xac000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssra_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa8004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssrl_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xac004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssra_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa8003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssrl_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xac003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vssra_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb8000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnclipu_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xbc000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnclip_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb8004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnclipu_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xbc004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnclip_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb8003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnclipu_wi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xbc003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnclip_wi(ctx, &a);
+    }
+
+    if ((insn & 0xfe0ff07f) == 0x42002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmv_x_s(ctx, &a);
+    }
+
+    if ((insn & 0xfff0707f) == 0x42006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmv_s_x(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x00000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vadd_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x00004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vadd_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x00003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vadd_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x08000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsub_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x08004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsub_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x0c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrsub_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x0c003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrsub_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x38004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vslideup_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x38003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vslideup_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x38006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vslide1up_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x3c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vslidedown_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x3c003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vslidedown_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x3c006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vslide1down_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x30000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrgather_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x38000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrgatherei16_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x30004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrgather_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x30003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrgather_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x5c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vcompress_vm(ctx, &a);
+    }
+
+    if ((insn & 0xfe0ff07f) == 0x9e003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = 0,
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vmv1r_v(ctx, &a);
+    }
+
+    if ((insn & 0xfe0ff07f) == 0x9e00b057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = 1,
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vmv2r_v(ctx, &a);
+    }
+
+    if ((insn & 0xfe0ff07f) == 0x9e01b057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = 3,
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vmv4r_v(ctx, &a);
+    }
+
+    if ((insn & 0xfe0ff07f) == 0x9e03b057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = 7,
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vmv8r_v(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x40000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vadc_vvm(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x40004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vadc_vxm(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x40003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vadc_vim(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x44000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmadc_vvm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x44004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmadc_vxm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x44003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmadc_vim(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x48000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vsbc_vvm(ctx, &a);
+    }
+
+    if ((insn & 0xfe00707f) == 0x48004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = 1,
+        };
+
+        return trans_vsbc_vxm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x4c000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsbc_vvm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x4c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsbc_vxm(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x10000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vminu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x10004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vminu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x14000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmin_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x14004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmin_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x18000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmaxu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x18004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmaxu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x1c000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmax_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x1c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmax_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc0002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwaddu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc0006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwaddu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc4002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwadd_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc4006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwadd_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc8002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsubu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xc8006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsubu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xcc002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsub_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xcc006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsub_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xd0002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwaddu_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xd0006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwaddu_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xd4002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwadd_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xd4006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwadd_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xd8002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsubu_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xd8006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsubu_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xdc002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsub_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xdc006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwsub_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa4002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmadd_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa4006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmadd_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xac002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnmsub_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xac006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnmsub_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb4002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmacc_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb4006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmacc_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xbc002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnmsac_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xbc006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnmsac_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xe0002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmulu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xe0006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmulu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xe8002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmulsu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xe8006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmulsu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xec002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmul_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xec006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmul_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xf0002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmaccu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xf0006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmaccu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xf4002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmacc_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xf4006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmacc_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xf8006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmaccus_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xfc002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmaccsu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xfc006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vwmaccsu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x80002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vdivu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x80006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vdivu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x84002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vdiv_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x84006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vdiv_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x88002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vremu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x88006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vremu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x8c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrem_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x8c006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vrem_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x94002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmul_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x94006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmul_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x9c002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmulh_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x9c006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmulh_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x90002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmulhu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x90006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmulhu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x98002057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmulhsu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x98006057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmulhsu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x94000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsll_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x94004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsll_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x94003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsll_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa0000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsrl_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa0004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsrl_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa0003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsrl_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa4000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsra_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa4004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsra_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xa4003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsra_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb0000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnsrl_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb0004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnsrl_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb0003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnsrl_wi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb4000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnsra_wv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb4004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnsra_wx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0xb4003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vnsra_wi(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x48032057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vzext_vf2(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x48022057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vzext_vf4(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x48012057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vzext_vf8(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x4803a057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsext_vf2(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x4802a057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsext_vf4(ctx, &a);
+    }
+
+    if ((insn & 0xfc0ff07f) == 0x4801a057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vsext_vf8(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x24000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vand_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x24004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vand_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x24003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vand_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x28000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vor_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x28004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vor_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x28003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vor_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x2c000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vxor_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x2c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vxor_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x2c003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vxor_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x60000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmseq_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x60004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmseq_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x60003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmseq_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x64000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsne_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x64004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsne_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x64003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsne_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x68000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsltu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x68004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsltu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x6c000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmslt_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x6c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmslt_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x70000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsleu_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x70004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsleu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x70003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsleu_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x74000057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsle_vv(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x74004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsle_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x74003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsle_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x78004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsgtu_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x78003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsgtu_vi(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x7c004057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsgt_vx(ctx, &a);
+    }
+
+    if ((insn & 0xfc00707f) == 0x7c003057) {
+        arg_rvv_arith a = {
+            .rd = extract32(insn, 7, 5),
+            .rs1 = extract32(insn, 15, 5),
+            .rs2 = extract32(insn, 20, 5),
+            .vm = extract32(insn, 25, 1),
+        };
+
+        return trans_vmsgt_vi(ctx, &a);
+    }
+
+    return false;
+}
+
+static bool decode_rvv(DisasContext *ctx, uint32_t insn)
+{
+    return decode_rvv_config(ctx, insn) || decode_rvv_data(ctx, insn);
+}
diff --git a/qemu/target/riscv/insn_trans/trans_rvzfh.inc.c b/qemu/target/riscv/insn_trans/trans_rvzfh.inc.c
new file mode 100644
index 0000000000..a6c26e3f70
--- /dev/null
+++ b/qemu/target/riscv/insn_trans/trans_rvzfh.inc.c
@@ -0,0 +1,579 @@
+/*
+ * RISC-V translation routines for the RV64Zfh Standard Extension.
+ *
+ * Copyright (c) 2020 Chih-Min Chao, chihmin.chao@sifive.com
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define RISCV_NANBOX16_MASK UINT64_C(0xffffffffffff0000)
+
+#define REQUIRE_ZFH(ctx) do { \
+    if (!ctx->ext_zfh) { \
+        return false; \
+    } \
+} while (0)
+
+#define REQUIRE_ZFH_OR_ZFHMIN(ctx) do { \
+    if (!ctx->ext_zfh && !ctx->ext_zfhmin) { \
+        return false; \
+    } \
+} while (0)
+
+static void gen_nanbox_h(TCGContext *tcg_ctx, TCGv_i64 ret, TCGv_i64 value)
+{
+    tcg_gen_ori_i64(tcg_ctx, ret, value, RISCV_NANBOX16_MASK);
+}
+
+static void gen_check_nanbox_h(TCGContext *tcg_ctx, TCGv_i64 ret,
+                               TCGv_i64 value)
+{
+    TCGv_i64 boxed = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 mask = tcg_const_i64(tcg_ctx, RISCV_NANBOX16_MASK);
+    TCGv_i64 qnan = tcg_const_i64(tcg_ctx, 0x7e00u);
+
+    tcg_gen_andi_i64(tcg_ctx, boxed, value, RISCV_NANBOX16_MASK);
+    tcg_gen_movcond_i64(tcg_ctx, TCG_COND_EQ, ret, boxed, mask, value, qnan);
+    tcg_gen_andi_i64(tcg_ctx, ret, ret, UINT16_MAX);
+
+    tcg_temp_free_i64(tcg_ctx, qnan);
+    tcg_temp_free_i64(tcg_ctx, mask);
+    tcg_temp_free_i64(tcg_ctx, boxed);
+}
+
+static bool trans_flh(DisasContext *ctx, arg_flh *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, t0, a->rs1);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+    tcg_gen_addi_tl(tcg_ctx, t0, t0, a->imm);
+
+    tcg_gen_qemu_ld_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], t0, ctx->mem_idx,
+                        MO_TEUW);
+    gen_nanbox_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rd]);
+
+    tcg_temp_free(tcg_ctx, t0);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fsh(DisasContext *ctx, arg_fsh *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, t0, a->rs1);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+    tcg_gen_addi_tl(tcg_ctx, t0, t0, a->imm);
+
+    tcg_gen_qemu_st_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rs2], t0, ctx->mem_idx,
+                        MO_TEUW);
+
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fmadd_h(DisasContext *ctx, arg_fmadd_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fmadd_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                       tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2],
+                       tcg_ctx->cpu_fpr[a->rs3]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fmsub_h(DisasContext *ctx, arg_fmsub_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fmsub_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                       tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2],
+                       tcg_ctx->cpu_fpr[a->rs3]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fnmsub_h(DisasContext *ctx, arg_fnmsub_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fnmsub_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2],
+                        tcg_ctx->cpu_fpr[a->rs3]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fnmadd_h(DisasContext *ctx, arg_fnmadd_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fnmadd_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2],
+                        tcg_ctx->cpu_fpr[a->rs3]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fadd_h(DisasContext *ctx, arg_fadd_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fadd_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                      tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fsub_h(DisasContext *ctx, arg_fsub_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fsub_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                      tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fmul_h(DisasContext *ctx, arg_fmul_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fmul_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                      tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fdiv_h(DisasContext *ctx, arg_fdiv_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fdiv_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                      tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fsqrt_h(DisasContext *ctx, arg_fsqrt_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fsqrt_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                       tcg_ctx->cpu_fpr[a->rs1]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fsgnj_h(DisasContext *ctx, arg_fsgnj_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 frs1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 frs2 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 sign = tcg_temp_new_i64(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_check_nanbox_h(tcg_ctx, frs1, tcg_ctx->cpu_fpr[a->rs1]);
+    gen_check_nanbox_h(tcg_ctx, frs2, tcg_ctx->cpu_fpr[a->rs2]);
+    tcg_gen_andi_i64(tcg_ctx, sign, frs2, 0x8000);
+    tcg_gen_andi_i64(tcg_ctx, frs1, frs1, ~UINT64_C(0x8000));
+    tcg_gen_or_i64(tcg_ctx, frs1, frs1, sign);
+    gen_nanbox_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], frs1);
+
+    tcg_temp_free_i64(tcg_ctx, sign);
+    tcg_temp_free_i64(tcg_ctx, frs2);
+    tcg_temp_free_i64(tcg_ctx, frs1);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fsgnjn_h(DisasContext *ctx, arg_fsgnjn_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 frs1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 frs2 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 sign = tcg_temp_new_i64(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_check_nanbox_h(tcg_ctx, frs1, tcg_ctx->cpu_fpr[a->rs1]);
+    gen_check_nanbox_h(tcg_ctx, frs2, tcg_ctx->cpu_fpr[a->rs2]);
+    tcg_gen_andi_i64(tcg_ctx, sign, frs2, 0x8000);
+    tcg_gen_xori_i64(tcg_ctx, sign, sign, 0x8000);
+    tcg_gen_andi_i64(tcg_ctx, frs1, frs1, ~UINT64_C(0x8000));
+    tcg_gen_or_i64(tcg_ctx, frs1, frs1, sign);
+    gen_nanbox_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], frs1);
+
+    tcg_temp_free_i64(tcg_ctx, sign);
+    tcg_temp_free_i64(tcg_ctx, frs2);
+    tcg_temp_free_i64(tcg_ctx, frs1);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fsgnjx_h(DisasContext *ctx, arg_fsgnjx_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv_i64 frs1 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 frs2 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 sign = tcg_temp_new_i64(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_check_nanbox_h(tcg_ctx, frs1, tcg_ctx->cpu_fpr[a->rs1]);
+    gen_check_nanbox_h(tcg_ctx, frs2, tcg_ctx->cpu_fpr[a->rs2]);
+    tcg_gen_xor_i64(tcg_ctx, sign, frs1, frs2);
+    tcg_gen_andi_i64(tcg_ctx, sign, sign, 0x8000);
+    tcg_gen_andi_i64(tcg_ctx, frs1, frs1, ~UINT64_C(0x8000));
+    tcg_gen_or_i64(tcg_ctx, frs1, frs1, sign);
+    gen_nanbox_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], frs1);
+
+    tcg_temp_free_i64(tcg_ctx, sign);
+    tcg_temp_free_i64(tcg_ctx, frs2);
+    tcg_temp_free_i64(tcg_ctx, frs1);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fmin_h(DisasContext *ctx, arg_fmin_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_helper_fmin_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                      tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fmax_h(DisasContext *ctx, arg_fmax_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_helper_fmax_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                      tcg_ctx->cpu_fpr[a->rs1], tcg_ctx->cpu_fpr[a->rs2]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fcvt_s_h(DisasContext *ctx, arg_fcvt_s_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_s_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fcvt_d_h(DisasContext *ctx, arg_fcvt_d_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+    REQUIRE_EXT(ctx, RVD);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_d_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fcvt_h_s(DisasContext *ctx, arg_fcvt_h_s *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_h_s(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_fcvt_h_d(DisasContext *ctx, arg_fcvt_h_d *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+    REQUIRE_EXT(ctx, RVD);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_h_d(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1]);
+    mark_fs_dirty(ctx);
+    return true;
+}
+
+static bool trans_feq_h(DisasContext *ctx, arg_feq_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_helper_feq_h(tcg_ctx, t0, tcg_ctx->cpu_env, tcg_ctx->cpu_fpr[a->rs1],
+                     tcg_ctx->cpu_fpr[a->rs2]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_flt_h(DisasContext *ctx, arg_flt_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_helper_flt_h(tcg_ctx, t0, tcg_ctx->cpu_env, tcg_ctx->cpu_fpr[a->rs1],
+                     tcg_ctx->cpu_fpr[a->rs2]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fle_h(DisasContext *ctx, arg_fle_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_helper_fle_h(tcg_ctx, t0, tcg_ctx->cpu_env, tcg_ctx->cpu_fpr[a->rs1],
+                     tcg_ctx->cpu_fpr[a->rs2]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fclass_h(DisasContext *ctx, arg_fclass_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_helper_fclass_h(tcg_ctx, t0, tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fcvt_w_h(DisasContext *ctx, arg_fcvt_w_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_w_h(tcg_ctx, t0, tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fcvt_wu_h(DisasContext *ctx, arg_fcvt_wu_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_wu_h(tcg_ctx, t0, tcg_ctx->cpu_env,
+                         tcg_ctx->cpu_fpr[a->rs1]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fcvt_h_w(DisasContext *ctx, arg_fcvt_h_w *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, t0, a->rs1);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_h_w(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        t0);
+    mark_fs_dirty(ctx);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fcvt_h_wu(DisasContext *ctx, arg_fcvt_h_wu *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, t0, a->rs1);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_h_wu(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                         t0);
+    mark_fs_dirty(ctx);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fmv_x_h(DisasContext *ctx, arg_fmv_x_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+#if defined(TARGET_RISCV64)
+    tcg_gen_ext16s_tl(tcg_ctx, t0, tcg_ctx->cpu_fpr[a->rs1]);
+#else
+    tcg_gen_extrl_i64_i32(tcg_ctx, t0, tcg_ctx->cpu_fpr[a->rs1]);
+    tcg_gen_ext16s_tl(tcg_ctx, t0, t0);
+#endif
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fmv_h_x(DisasContext *ctx, arg_fmv_h_x *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, t0, a->rs1);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH_OR_ZFHMIN(ctx);
+#if defined(TARGET_RISCV64)
+    tcg_gen_ext16u_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], t0);
+#else
+    tcg_gen_extu_i32_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], t0);
+    tcg_gen_andi_i64(tcg_ctx, tcg_ctx->cpu_fpr[a->rd],
+                     tcg_ctx->cpu_fpr[a->rd], UINT16_MAX);
+#endif
+    gen_nanbox_h(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_fpr[a->rd]);
+    mark_fs_dirty(ctx);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+#ifdef TARGET_RISCV64
+static bool trans_fcvt_l_h(DisasContext *ctx, arg_fcvt_l_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_l_h(tcg_ctx, t0, tcg_ctx->cpu_env,
+                        tcg_ctx->cpu_fpr[a->rs1]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fcvt_lu_h(DisasContext *ctx, arg_fcvt_lu_h *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_lu_h(tcg_ctx, t0, tcg_ctx->cpu_env,
+                         tcg_ctx->cpu_fpr[a->rs1]);
+    gen_set_gpr(tcg_ctx, a->rd, t0);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fcvt_h_l(DisasContext *ctx, arg_fcvt_h_l *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, t0, a->rs1);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_h_l(tcg_ctx, tcg_ctx->cpu_fpr[a->rd], tcg_ctx->cpu_env,
+                        t0);
+    mark_fs_dirty(ctx);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+
+static bool trans_fcvt_h_lu(DisasContext *ctx, arg_fcvt_h_lu *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv t0 = tcg_temp_new(tcg_ctx);
+    gen_get_gpr(tcg_ctx, t0, a->rs1);
+
+    REQUIRE_FPU;
+    REQUIRE_ZFH(ctx);
+    gen_set_rm(ctx, a->rm);
+    gen_helper_fcvt_h_lu(tcg_ctx, tcg_ctx->cpu_fpr[a->rd],
+                         tcg_ctx->cpu_env, t0);
+    mark_fs_dirty(ctx);
+    tcg_temp_free(tcg_ctx, t0);
+    return true;
+}
+#endif
diff --git a/qemu/target/riscv/insn_trans/trans_svinval.inc.c b/qemu/target/riscv/insn_trans/trans_svinval.inc.c
new file mode 100644
index 0000000000..8d21f52be1
--- /dev/null
+++ b/qemu/target/riscv/insn_trans/trans_svinval.inc.c
@@ -0,0 +1,47 @@
+static bool trans_sinval_vma(DisasContext *ctx, arg_sfence_vma *a)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+
+    if (!ctx->ext_svinval || !has_ext(ctx, RVS)) {
+        return false;
+    }
+
+    gen_helper_tlb_flush(tcg_ctx, tcg_ctx->cpu_env);
+    return true;
+}
+
+static bool trans_sfence_w_inval(DisasContext *ctx, arg_sfence_w_inval *a)
+{
+    if (!ctx->ext_svinval || !has_ext(ctx, RVS)) {
+        return false;
+    }
+
+    return true;
+}
+
+static bool trans_sfence_inval_ir(DisasContext *ctx, arg_sfence_inval_ir *a)
+{
+    if (!ctx->ext_svinval || !has_ext(ctx, RVS)) {
+        return false;
+    }
+
+    return true;
+}
+
+static bool trans_hinval_vvma(DisasContext *ctx, arg_hinval_vvma *a)
+{
+    if (!ctx->ext_svinval || !has_ext(ctx, RVH)) {
+        return false;
+    }
+
+    return trans_hfence_bvma(ctx, a);
+}
+
+static bool trans_hinval_gvma(DisasContext *ctx, arg_hinval_gvma *a)
+{
+    if (!ctx->ext_svinval || !has_ext(ctx, RVH)) {
+        return false;
+    }
+
+    return trans_hfence_gvma(ctx, a);
+}
diff --git a/qemu/target/riscv/insn_trans/trans_xventanacondops.inc.c b/qemu/target/riscv/insn_trans/trans_xventanacondops.inc.c
new file mode 100644
index 0000000000..5446edef71
--- /dev/null
+++ b/qemu/target/riscv/insn_trans/trans_xventanacondops.inc.c
@@ -0,0 +1,59 @@
+/*
+ * RISC-V translation routines for the XVentanaCondOps extension.
+ *
+ * Copyright (c) 2021-2022 VRULL GmbH.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+static bool gen_vt_condmask(DisasContext *ctx, arg_r *a, TCGCond cond)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv dest;
+    TCGv source1;
+    TCGv source2;
+    TCGv zero;
+
+    if (!ctx->ext_xventanacondops) {
+        return false;
+    }
+    if (a->rd == 0) {
+        return true;
+    }
+
+    dest = tcg_temp_new(tcg_ctx);
+    source1 = tcg_temp_new(tcg_ctx);
+    source2 = tcg_temp_new(tcg_ctx);
+    zero = tcg_const_tl(tcg_ctx, 0);
+
+    gen_get_gpr(tcg_ctx, source1, a->rs1);
+    gen_get_gpr(tcg_ctx, source2, a->rs2);
+    tcg_gen_movcond_tl(tcg_ctx, cond, dest, source2, zero, source1, zero);
+    gen_set_gpr(tcg_ctx, a->rd, dest);
+
+    tcg_temp_free(tcg_ctx, dest);
+    tcg_temp_free(tcg_ctx, source1);
+    tcg_temp_free(tcg_ctx, source2);
+    tcg_temp_free(tcg_ctx, zero);
+    return true;
+}
+
+static bool trans_vt_maskc(DisasContext *ctx, arg_vt_maskc *a)
+{
+    return gen_vt_condmask(ctx, a, TCG_COND_NE);
+}
+
+static bool trans_vt_maskcn(DisasContext *ctx, arg_vt_maskcn *a)
+{
+    return gen_vt_condmask(ctx, a, TCG_COND_EQ);
+}
diff --git a/qemu/target/riscv/instmap.h b/qemu/target/riscv/instmap.h
index 40b6d2b64d..cebb9e0f82 100644
--- a/qemu/target/riscv/instmap.h
+++ b/qemu/target/riscv/instmap.h
@@ -181,6 +181,7 @@ enum {
     OPC_RISC_CSRRW       = OPC_RISC_SYSTEM | (0x1 << 12),
     OPC_RISC_CSRRS       = OPC_RISC_SYSTEM | (0x2 << 12),
     OPC_RISC_CSRRC       = OPC_RISC_SYSTEM | (0x3 << 12),
+    OPC_RISC_HLVHSV      = OPC_RISC_SYSTEM | (0x4 << 12),
     OPC_RISC_CSRRWI      = OPC_RISC_SYSTEM | (0x5 << 12),
     OPC_RISC_CSRRSI      = OPC_RISC_SYSTEM | (0x6 << 12),
     OPC_RISC_CSRRCI      = OPC_RISC_SYSTEM | (0x7 << 12),
@@ -310,12 +311,20 @@ enum {
                            | (extract32(inst, 12, 8) << 12) \
                            | (sextract64(inst, 31, 1) << 20))
 
+#define GET_FUNCT3(inst) extract32(inst, 12, 3)
+#define GET_FUNCT7(inst) extract32(inst, 25, 7)
 #define GET_RM(inst)   extract32(inst, 12, 3)
 #define GET_RS3(inst)  extract32(inst, 27, 5)
 #define GET_RS1(inst)  extract32(inst, 15, 5)
 #define GET_RS2(inst)  extract32(inst, 20, 5)
 #define GET_RD(inst)   extract32(inst, 7, 5)
 #define GET_IMM(inst)  sextract64(inst, 20, 12)
+#define SET_RS1(inst, val)  deposit32(inst, 15, 5, val)
+#define SET_RS2(inst, val)  deposit32(inst, 20, 5, val)
+#define SET_RD(inst, val)   deposit32(inst, 7, 5, val)
+#define SET_I_IMM(inst, val)  deposit32(inst, 20, 12, val)
+#define SET_S_IMM(inst, val)  \
+    deposit32(deposit32(inst, 7, 5, val), 25, 7, (val) >> 5)
 
 /* RVC decoding macros */
 #define GET_C_IMM(inst)             (extract32(inst, 2, 5) \
@@ -346,6 +355,8 @@ enum {
                                     | (extract32(inst, 5, 1) << 6))
 #define GET_C_LD_IMM(inst)          ((extract16(inst, 10, 3) << 3) \
                                     | (extract16(inst, 5, 2) << 6))
+#define GET_C_SW_IMM(inst)          GET_C_LW_IMM(inst)
+#define GET_C_SD_IMM(inst)          GET_C_LD_IMM(inst)
 #define GET_C_J_IMM(inst)           ((extract32(inst, 3, 3) << 1) \
                                     | (extract32(inst, 11, 1) << 4) \
                                     | (extract32(inst, 2, 1) << 5) \
@@ -366,4 +377,34 @@ enum {
 #define GET_C_RS1S(inst)            (8 + extract16(inst, 7, 3))
 #define GET_C_RS2S(inst)            (8 + extract16(inst, 2, 3))
 
+#define GET_C_FUNC(inst)            extract32(inst, 13, 3)
+#define GET_C_OP(inst)              extract32(inst, 0, 2)
+
+enum {
+    OPC_RISC_C_OP_QUAD0 = 0x0,
+    OPC_RISC_C_OP_QUAD1 = 0x1,
+    OPC_RISC_C_OP_QUAD2 = 0x2
+};
+
+enum {
+    OPC_RISC_C_FUNC_ADDI4SPN = 0x0,
+    OPC_RISC_C_FUNC_FLD_LQ = 0x1,
+    OPC_RISC_C_FUNC_LW = 0x2,
+    OPC_RISC_C_FUNC_FLW_LD = 0x3,
+    OPC_RISC_C_FUNC_FSD_SQ = 0x5,
+    OPC_RISC_C_FUNC_SW = 0x6,
+    OPC_RISC_C_FUNC_FSW_SD = 0x7
+};
+
+enum {
+    OPC_RISC_C_FUNC_SLLI_SLLI64 = 0x0,
+    OPC_RISC_C_FUNC_FLDSP_LQSP = 0x1,
+    OPC_RISC_C_FUNC_LWSP = 0x2,
+    OPC_RISC_C_FUNC_FLWSP_LDSP = 0x3,
+    OPC_RISC_C_FUNC_JR_MV_EBREAK_JALR_ADD = 0x4,
+    OPC_RISC_C_FUNC_FSDSP_SQSP = 0x5,
+    OPC_RISC_C_FUNC_SWSP = 0x6,
+    OPC_RISC_C_FUNC_FSWSP_SDSP = 0x7
+};
+
 #endif
diff --git a/qemu/target/riscv/op_helper.c b/qemu/target/riscv/op_helper.c
index 5afb2ce881..e4fadef90b 100644
--- a/qemu/target/riscv/op_helper.c
+++ b/qemu/target/riscv/op_helper.c
@@ -21,6 +21,7 @@
 #include "qemu/log.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
+#include "exec/cpu_ldst.h"
 #include "exec/helper-proto.h"
 
 /* Exceptions processing helpers */
@@ -52,6 +53,11 @@ target_ulong helper_csrrs(CPURISCVState *env, target_ulong src,
         target_ulong csr, target_ulong rs1_pass)
 {
     target_ulong val = 0;
+
+    if (csr == CSR_SEED && !rs1_pass) {
+        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
+    }
+
     if (riscv_csrrw(env, csr, &val, -1, rs1_pass ? src : 0) < 0) {
         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
     }
@@ -62,6 +68,11 @@ target_ulong helper_csrrc(CPURISCVState *env, target_ulong src,
         target_ulong csr, target_ulong rs1_pass)
 {
     target_ulong val = 0;
+
+    if (csr == CSR_SEED && !rs1_pass) {
+        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
+    }
+
     if (riscv_csrrw(env, csr, &val, 0, rs1_pass ? src : 0) < 0) {
         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
     }
@@ -86,6 +97,11 @@ target_ulong helper_sret(CPURISCVState *env, target_ulong cpu_pc_deb)
         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
     }
 
+    if (riscv_has_ext(env, RVH) && riscv_cpu_virt_enabled(env) &&
+        get_field(env->hstatus, HSTATUS_VTSR)) {
+        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
+    }
+
     mstatus = env->mstatus;
 
     if (riscv_has_ext(env, RVH) && !riscv_cpu_virt_enabled(env)) {
@@ -95,12 +111,8 @@ target_ulong helper_sret(CPURISCVState *env, target_ulong cpu_pc_deb)
         prev_priv = get_field(mstatus, MSTATUS_SPP);
         prev_virt = get_field(hstatus, HSTATUS_SPV);
 
-        hstatus = set_field(hstatus, HSTATUS_SPV,
-                                 get_field(hstatus, HSTATUS_SP2V));
-        mstatus = set_field(mstatus, MSTATUS_SPP,
-                            get_field(hstatus, HSTATUS_SP2P));
-        hstatus = set_field(hstatus, HSTATUS_SP2V, 0);
-        hstatus = set_field(hstatus, HSTATUS_SP2P, 0);
+        hstatus = set_field(hstatus, HSTATUS_SPV, 0);
+        mstatus = set_field(mstatus, MSTATUS_SPP, 0);
         mstatus = set_field(mstatus, SSTATUS_SIE,
                             get_field(mstatus, SSTATUS_SPIE));
         mstatus = set_field(mstatus, SSTATUS_SPIE, 1);
@@ -176,11 +188,12 @@ void helper_wfi(CPURISCVState *env)
     bool prv_u = env->priv == PRV_U;
     bool prv_s = env->priv == PRV_S;
 
-    if (((prv_s || (!rvs && prv_u)) && get_field(env->mstatus, MSTATUS_TW)) ||
-        (rvs && prv_u && !riscv_cpu_virt_enabled(env))) {
-        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
-    } else if (riscv_cpu_virt_enabled(env) && (prv_u ||
+    if (riscv_cpu_virt_enabled(env) && (prv_u ||
         (prv_s && get_field(env->hstatus, HSTATUS_VTW)))) {
+        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
+    } else if (((prv_s || (!rvs && prv_u)) &&
+                get_field(env->mstatus, MSTATUS_TW)) ||
+               (rvs && prv_u && !riscv_cpu_virt_enabled(env))) {
         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
     } else {
         cs->halted = 1;
@@ -197,11 +210,55 @@ void helper_tlb_flush(CPURISCVState *env)
          env->priv_ver >= PRIV_VERSION_1_10_0 &&
          get_field(env->mstatus, MSTATUS_TVM))) {
         riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
+    } else if (riscv_has_ext(env, RVH) && riscv_cpu_virt_enabled(env) &&
+               get_field(env->hstatus, HSTATUS_VTVM)) {
+        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
     } else {
         tlb_flush(cs);
     }
 }
 
+void helper_hyp_tlb_flush(CPURISCVState *env)
+{
+    CPUState *cs = env_cpu(env);
+
+    if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
+        riscv_raise_exception(env, RISCV_EXCP_VIRT_INSTRUCTION_FAULT, GETPC());
+    }
+
+    if (env->priv == PRV_M ||
+        (env->priv == PRV_S && !riscv_cpu_virt_enabled(env))) {
+        tlb_flush(cs);
+        return;
+    }
+
+    riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
+}
+
+void helper_hyp_gvma_tlb_flush(CPURISCVState *env)
+{
+    if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env) &&
+        get_field(env->mstatus, MSTATUS_TVM)) {
+        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
+    }
+
+    helper_hyp_tlb_flush(env);
+}
+
+target_ulong helper_hyp_hlvx_hu(CPURISCVState *env, target_ulong address)
+{
+    int mmu_idx = cpu_mmu_index(env, true) | TB_FLAGS_PRIV_HYP_ACCESS_MASK;
+
+    return cpu_lduw_mmuidx_ra(env, address, mmu_idx, GETPC());
+}
+
+target_ulong helper_hyp_hlvx_wu(CPURISCVState *env, target_ulong address)
+{
+    int mmu_idx = cpu_mmu_index(env, true) | TB_FLAGS_PRIV_HYP_ACCESS_MASK;
+
+    return cpu_ldl_mmuidx_ra(env, address, mmu_idx, GETPC());
+}
+
 void helper_uc_riscv_exit(CPURISCVState *env)
 {
     CPUState *cs = env_cpu(env);
@@ -209,4 +266,4 @@ void helper_uc_riscv_exit(CPURISCVState *env)
     cs->exception_index = EXCP_HLT;
     cs->halted = 1;
     cpu_loop_exit(cs);
-}
\ No newline at end of file
+}
diff --git a/qemu/target/riscv/pmp.c b/qemu/target/riscv/pmp.c
index 888b99c8d9..390e537f42 100644
--- a/qemu/target/riscv/pmp.c
+++ b/qemu/target/riscv/pmp.c
@@ -19,10 +19,6 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-/*
- * PMP (Physical Memory Protection) is as-of-yet unused and needs testing.
- */
-
 #include "qemu/osdep.h"
 #include "qemu/log.h"
 #include "cpu.h"
@@ -169,7 +165,7 @@ static void pmp_update_rule(CPURISCVState *env, uint32_t pmp_index)
 
     case PMP_AMATCH_NA4:
         sa = this_addr << 2; /* shift up from [xx:0] to [xx+2:2] */
-        ea = (this_addr + 4u) - 1u;
+        ea = (sa + 4u) - 1u;
         break;
 
     case PMP_AMATCH_NAPOT:
diff --git a/qemu/target/riscv/riscv32/decode_insn32.inc.c b/qemu/target/riscv/riscv32/decode_insn32.inc.c
index c4c25de13b..2f0ef4491f 100644
--- a/qemu/target/riscv/riscv32/decode_insn32.inc.c
+++ b/qemu/target/riscv/riscv32/decode_insn32.inc.c
@@ -95,6 +95,13 @@ typedef struct {
     int shamt;
 } arg_shift;
 
+typedef struct {
+    int rd;
+    int rs1;
+    int rs2;
+    int shamt;
+} arg_k_aes;
+
 typedef struct {
     int imm;
     int rd;
@@ -120,6 +127,20 @@ typedef arg_decode_insn3214 arg_sfence_vma;
 static bool trans_sfence_vma(DisasContext *ctx, arg_sfence_vma *a);
 typedef arg_decode_insn3215 arg_sfence_vm;
 static bool trans_sfence_vm(DisasContext *ctx, arg_sfence_vm *a);
+typedef arg_decode_insn3214 arg_sinval_vma;
+static bool trans_sinval_vma(DisasContext *ctx, arg_sinval_vma *a);
+typedef arg_empty arg_sfence_w_inval;
+static bool trans_sfence_w_inval(DisasContext *ctx, arg_sfence_w_inval *a);
+typedef arg_empty arg_sfence_inval_ir;
+static bool trans_sfence_inval_ir(DisasContext *ctx, arg_sfence_inval_ir *a);
+typedef arg_decode_insn3214 arg_hinval_vvma;
+static bool trans_hinval_vvma(DisasContext *ctx, arg_hinval_vvma *a);
+typedef arg_decode_insn3214 arg_hinval_gvma;
+static bool trans_hinval_gvma(DisasContext *ctx, arg_hinval_gvma *a);
+typedef arg_r arg_vt_maskc;
+static bool trans_vt_maskc(DisasContext *ctx, arg_vt_maskc *a);
+typedef arg_r arg_vt_maskcn;
+static bool trans_vt_maskcn(DisasContext *ctx, arg_vt_maskcn *a);
 typedef arg_u arg_lui;
 static bool trans_lui(DisasContext *ctx, arg_lui *a);
 typedef arg_u arg_auipc;
@@ -174,6 +195,48 @@ typedef arg_shift arg_srli;
 static bool trans_srli(DisasContext *ctx, arg_srli *a);
 typedef arg_shift arg_srai;
 static bool trans_srai(DisasContext *ctx, arg_srai *a);
+typedef arg_shift arg_rori;
+static bool trans_rori(DisasContext *ctx, arg_rori *a);
+typedef arg_decode_insn3213 arg_clz;
+static bool trans_clz(DisasContext *ctx, arg_clz *a);
+typedef arg_decode_insn3213 arg_ctz;
+static bool trans_ctz(DisasContext *ctx, arg_ctz *a);
+typedef arg_decode_insn3213 arg_cpop;
+static bool trans_cpop(DisasContext *ctx, arg_cpop *a);
+typedef arg_decode_insn3213 arg_orc_b;
+static bool trans_orc_b(DisasContext *ctx, arg_orc_b *a);
+typedef arg_decode_insn3213 arg_rev8_32;
+static bool trans_rev8_32(DisasContext *ctx, arg_rev8_32 *a);
+typedef arg_decode_insn3213 arg_brev8;
+static bool trans_brev8(DisasContext *ctx, arg_brev8 *a);
+typedef arg_decode_insn3213 arg_unzip;
+static bool trans_unzip(DisasContext *ctx, arg_unzip *a);
+typedef arg_decode_insn3213 arg_zip;
+static bool trans_zip(DisasContext *ctx, arg_zip *a);
+typedef arg_decode_insn3213 arg_sha256sum0;
+static bool trans_sha256sum0(DisasContext *ctx, arg_sha256sum0 *a);
+typedef arg_decode_insn3213 arg_sha256sum1;
+static bool trans_sha256sum1(DisasContext *ctx, arg_sha256sum1 *a);
+typedef arg_decode_insn3213 arg_sha256sig0;
+static bool trans_sha256sig0(DisasContext *ctx, arg_sha256sig0 *a);
+typedef arg_decode_insn3213 arg_sha256sig1;
+static bool trans_sha256sig1(DisasContext *ctx, arg_sha256sig1 *a);
+typedef arg_decode_insn3213 arg_sm3p0;
+static bool trans_sm3p0(DisasContext *ctx, arg_sm3p0 *a);
+typedef arg_decode_insn3213 arg_sm3p1;
+static bool trans_sm3p1(DisasContext *ctx, arg_sm3p1 *a);
+typedef arg_decode_insn3213 arg_sext_b;
+static bool trans_sext_b(DisasContext *ctx, arg_sext_b *a);
+typedef arg_decode_insn3213 arg_sext_h;
+static bool trans_sext_h(DisasContext *ctx, arg_sext_h *a);
+typedef arg_shift arg_bseti;
+static bool trans_bseti(DisasContext *ctx, arg_bseti *a);
+typedef arg_shift arg_bclri;
+static bool trans_bclri(DisasContext *ctx, arg_bclri *a);
+typedef arg_shift arg_binvi;
+static bool trans_binvi(DisasContext *ctx, arg_binvi *a);
+typedef arg_shift arg_bexti;
+static bool trans_bexti(DisasContext *ctx, arg_bexti *a);
 typedef arg_r arg_add;
 static bool trans_add(DisasContext *ctx, arg_add *a);
 typedef arg_r arg_sub;
@@ -194,6 +257,80 @@ typedef arg_r arg_or;
 static bool trans_or(DisasContext *ctx, arg_or *a);
 typedef arg_r arg_and;
 static bool trans_and(DisasContext *ctx, arg_and *a);
+typedef arg_r arg_sh1add;
+static bool trans_sh1add(DisasContext *ctx, arg_sh1add *a);
+typedef arg_r arg_sh2add;
+static bool trans_sh2add(DisasContext *ctx, arg_sh2add *a);
+typedef arg_r arg_sh3add;
+static bool trans_sh3add(DisasContext *ctx, arg_sh3add *a);
+typedef arg_r arg_clmul;
+static bool trans_clmul(DisasContext *ctx, arg_clmul *a);
+typedef arg_r arg_clmulh;
+static bool trans_clmulh(DisasContext *ctx, arg_clmulh *a);
+typedef arg_r arg_clmulr;
+static bool trans_clmulr(DisasContext *ctx, arg_clmulr *a);
+typedef arg_r arg_pack;
+static bool trans_pack(DisasContext *ctx, arg_pack *a);
+typedef arg_r arg_packh;
+static bool trans_packh(DisasContext *ctx, arg_packh *a);
+typedef arg_r arg_xperm4;
+static bool trans_xperm4(DisasContext *ctx, arg_xperm4 *a);
+typedef arg_r arg_xperm8;
+static bool trans_xperm8(DisasContext *ctx, arg_xperm8 *a);
+typedef arg_k_aes arg_aes32esmi;
+static bool trans_aes32esmi(DisasContext *ctx, arg_aes32esmi *a);
+typedef arg_k_aes arg_aes32esi;
+static bool trans_aes32esi(DisasContext *ctx, arg_aes32esi *a);
+typedef arg_k_aes arg_aes32dsmi;
+static bool trans_aes32dsmi(DisasContext *ctx, arg_aes32dsmi *a);
+typedef arg_k_aes arg_aes32dsi;
+static bool trans_aes32dsi(DisasContext *ctx, arg_aes32dsi *a);
+typedef arg_k_aes arg_sm4ed;
+static bool trans_sm4ed(DisasContext *ctx, arg_sm4ed *a);
+typedef arg_k_aes arg_sm4ks;
+static bool trans_sm4ks(DisasContext *ctx, arg_sm4ks *a);
+typedef arg_r arg_sha512sum0r;
+static bool trans_sha512sum0r(DisasContext *ctx, arg_sha512sum0r *a);
+typedef arg_r arg_sha512sum1r;
+static bool trans_sha512sum1r(DisasContext *ctx, arg_sha512sum1r *a);
+typedef arg_r arg_sha512sig0l;
+static bool trans_sha512sig0l(DisasContext *ctx, arg_sha512sig0l *a);
+typedef arg_r arg_sha512sig0h;
+static bool trans_sha512sig0h(DisasContext *ctx, arg_sha512sig0h *a);
+typedef arg_r arg_sha512sig1l;
+static bool trans_sha512sig1l(DisasContext *ctx, arg_sha512sig1l *a);
+typedef arg_r arg_sha512sig1h;
+static bool trans_sha512sig1h(DisasContext *ctx, arg_sha512sig1h *a);
+typedef arg_r arg_andn;
+static bool trans_andn(DisasContext *ctx, arg_andn *a);
+typedef arg_r arg_rol;
+static bool trans_rol(DisasContext *ctx, arg_rol *a);
+typedef arg_r arg_ror;
+static bool trans_ror(DisasContext *ctx, arg_ror *a);
+typedef arg_r arg_xnor;
+static bool trans_xnor(DisasContext *ctx, arg_xnor *a);
+typedef arg_r arg_min;
+static bool trans_min(DisasContext *ctx, arg_min *a);
+typedef arg_r arg_max;
+static bool trans_max(DisasContext *ctx, arg_max *a);
+typedef arg_r arg_minu;
+static bool trans_minu(DisasContext *ctx, arg_minu *a);
+typedef arg_r arg_maxu;
+static bool trans_maxu(DisasContext *ctx, arg_maxu *a);
+typedef arg_r arg_orn;
+static bool trans_orn(DisasContext *ctx, arg_orn *a);
+typedef arg_decode_insn3213 arg_zext_h_32;
+static bool trans_zext_h_32(DisasContext *ctx, arg_zext_h_32 *a);
+typedef arg_r arg_bset;
+static bool trans_bset(DisasContext *ctx, arg_bset *a);
+typedef arg_r arg_bclr;
+static bool trans_bclr(DisasContext *ctx, arg_bclr *a);
+typedef arg_r arg_binv;
+static bool trans_binv(DisasContext *ctx, arg_binv *a);
+typedef arg_r arg_bext;
+static bool trans_bext(DisasContext *ctx, arg_bext *a);
+typedef arg_empty arg_pause;
+static bool trans_pause(DisasContext *ctx, arg_pause *a);
 typedef arg_decode_insn3216 arg_fence;
 static bool trans_fence(DisasContext *ctx, arg_fence *a);
 typedef arg_empty arg_fence_i;
@@ -250,56 +387,110 @@ typedef arg_atomic arg_amomaxu_w;
 static bool trans_amomaxu_w(DisasContext *ctx, arg_amomaxu_w *a);
 typedef arg_i arg_flw;
 static bool trans_flw(DisasContext *ctx, arg_flw *a);
+typedef arg_i arg_flh;
+static bool trans_flh(DisasContext *ctx, arg_flh *a);
 typedef arg_s arg_fsw;
 static bool trans_fsw(DisasContext *ctx, arg_fsw *a);
+typedef arg_s arg_fsh;
+static bool trans_fsh(DisasContext *ctx, arg_fsh *a);
 typedef arg_decode_insn3210 arg_fmadd_s;
 static bool trans_fmadd_s(DisasContext *ctx, arg_fmadd_s *a);
+typedef arg_decode_insn3210 arg_fmadd_h;
+static bool trans_fmadd_h(DisasContext *ctx, arg_fmadd_h *a);
 typedef arg_decode_insn3210 arg_fmsub_s;
 static bool trans_fmsub_s(DisasContext *ctx, arg_fmsub_s *a);
+typedef arg_decode_insn3210 arg_fmsub_h;
+static bool trans_fmsub_h(DisasContext *ctx, arg_fmsub_h *a);
 typedef arg_decode_insn3210 arg_fnmsub_s;
 static bool trans_fnmsub_s(DisasContext *ctx, arg_fnmsub_s *a);
+typedef arg_decode_insn3210 arg_fnmsub_h;
+static bool trans_fnmsub_h(DisasContext *ctx, arg_fnmsub_h *a);
 typedef arg_decode_insn3210 arg_fnmadd_s;
 static bool trans_fnmadd_s(DisasContext *ctx, arg_fnmadd_s *a);
+typedef arg_decode_insn3210 arg_fnmadd_h;
+static bool trans_fnmadd_h(DisasContext *ctx, arg_fnmadd_h *a);
 typedef arg_decode_insn3211 arg_fadd_s;
 static bool trans_fadd_s(DisasContext *ctx, arg_fadd_s *a);
+typedef arg_decode_insn3211 arg_fadd_h;
+static bool trans_fadd_h(DisasContext *ctx, arg_fadd_h *a);
 typedef arg_decode_insn3211 arg_fsub_s;
 static bool trans_fsub_s(DisasContext *ctx, arg_fsub_s *a);
+typedef arg_decode_insn3211 arg_fsub_h;
+static bool trans_fsub_h(DisasContext *ctx, arg_fsub_h *a);
 typedef arg_decode_insn3211 arg_fmul_s;
 static bool trans_fmul_s(DisasContext *ctx, arg_fmul_s *a);
+typedef arg_decode_insn3211 arg_fmul_h;
+static bool trans_fmul_h(DisasContext *ctx, arg_fmul_h *a);
 typedef arg_decode_insn3211 arg_fdiv_s;
 static bool trans_fdiv_s(DisasContext *ctx, arg_fdiv_s *a);
+typedef arg_decode_insn3211 arg_fdiv_h;
+static bool trans_fdiv_h(DisasContext *ctx, arg_fdiv_h *a);
 typedef arg_decode_insn3212 arg_fsqrt_s;
 static bool trans_fsqrt_s(DisasContext *ctx, arg_fsqrt_s *a);
+typedef arg_decode_insn3212 arg_fsqrt_h;
+static bool trans_fsqrt_h(DisasContext *ctx, arg_fsqrt_h *a);
 typedef arg_r arg_fsgnj_s;
 static bool trans_fsgnj_s(DisasContext *ctx, arg_fsgnj_s *a);
+typedef arg_r arg_fsgnj_h;
+static bool trans_fsgnj_h(DisasContext *ctx, arg_fsgnj_h *a);
 typedef arg_r arg_fsgnjn_s;
 static bool trans_fsgnjn_s(DisasContext *ctx, arg_fsgnjn_s *a);
+typedef arg_r arg_fsgnjn_h;
+static bool trans_fsgnjn_h(DisasContext *ctx, arg_fsgnjn_h *a);
 typedef arg_r arg_fsgnjx_s;
 static bool trans_fsgnjx_s(DisasContext *ctx, arg_fsgnjx_s *a);
+typedef arg_r arg_fsgnjx_h;
+static bool trans_fsgnjx_h(DisasContext *ctx, arg_fsgnjx_h *a);
 typedef arg_r arg_fmin_s;
 static bool trans_fmin_s(DisasContext *ctx, arg_fmin_s *a);
+typedef arg_r arg_fmin_h;
+static bool trans_fmin_h(DisasContext *ctx, arg_fmin_h *a);
 typedef arg_r arg_fmax_s;
 static bool trans_fmax_s(DisasContext *ctx, arg_fmax_s *a);
+typedef arg_r arg_fmax_h;
+static bool trans_fmax_h(DisasContext *ctx, arg_fmax_h *a);
 typedef arg_decode_insn3212 arg_fcvt_w_s;
 static bool trans_fcvt_w_s(DisasContext *ctx, arg_fcvt_w_s *a);
+typedef arg_decode_insn3212 arg_fcvt_w_h;
+static bool trans_fcvt_w_h(DisasContext *ctx, arg_fcvt_w_h *a);
 typedef arg_decode_insn3212 arg_fcvt_wu_s;
 static bool trans_fcvt_wu_s(DisasContext *ctx, arg_fcvt_wu_s *a);
+typedef arg_decode_insn3212 arg_fcvt_wu_h;
+static bool trans_fcvt_wu_h(DisasContext *ctx, arg_fcvt_wu_h *a);
 typedef arg_decode_insn3213 arg_fmv_x_w;
 static bool trans_fmv_x_w(DisasContext *ctx, arg_fmv_x_w *a);
+typedef arg_decode_insn3213 arg_fmv_x_h;
+static bool trans_fmv_x_h(DisasContext *ctx, arg_fmv_x_h *a);
 typedef arg_r arg_feq_s;
 static bool trans_feq_s(DisasContext *ctx, arg_feq_s *a);
+typedef arg_r arg_feq_h;
+static bool trans_feq_h(DisasContext *ctx, arg_feq_h *a);
 typedef arg_r arg_flt_s;
 static bool trans_flt_s(DisasContext *ctx, arg_flt_s *a);
+typedef arg_r arg_flt_h;
+static bool trans_flt_h(DisasContext *ctx, arg_flt_h *a);
 typedef arg_r arg_fle_s;
 static bool trans_fle_s(DisasContext *ctx, arg_fle_s *a);
+typedef arg_r arg_fle_h;
+static bool trans_fle_h(DisasContext *ctx, arg_fle_h *a);
 typedef arg_decode_insn3213 arg_fclass_s;
 static bool trans_fclass_s(DisasContext *ctx, arg_fclass_s *a);
+typedef arg_decode_insn3213 arg_fclass_h;
+static bool trans_fclass_h(DisasContext *ctx, arg_fclass_h *a);
 typedef arg_decode_insn3212 arg_fcvt_s_w;
 static bool trans_fcvt_s_w(DisasContext *ctx, arg_fcvt_s_w *a);
+typedef arg_decode_insn3212 arg_fcvt_s_h;
+static bool trans_fcvt_s_h(DisasContext *ctx, arg_fcvt_s_h *a);
+typedef arg_decode_insn3212 arg_fcvt_h_w;
+static bool trans_fcvt_h_w(DisasContext *ctx, arg_fcvt_h_w *a);
 typedef arg_decode_insn3212 arg_fcvt_s_wu;
 static bool trans_fcvt_s_wu(DisasContext *ctx, arg_fcvt_s_wu *a);
+typedef arg_decode_insn3212 arg_fcvt_h_wu;
+static bool trans_fcvt_h_wu(DisasContext *ctx, arg_fcvt_h_wu *a);
 typedef arg_decode_insn3213 arg_fmv_w_x;
 static bool trans_fmv_w_x(DisasContext *ctx, arg_fmv_w_x *a);
+typedef arg_decode_insn3213 arg_fmv_h_x;
+static bool trans_fmv_h_x(DisasContext *ctx, arg_fmv_h_x *a);
 typedef arg_i arg_fld;
 static bool trans_fld(DisasContext *ctx, arg_fld *a);
 typedef arg_s arg_fsd;
@@ -334,8 +525,14 @@ typedef arg_r arg_fmax_d;
 static bool trans_fmax_d(DisasContext *ctx, arg_fmax_d *a);
 typedef arg_decode_insn3212 arg_fcvt_s_d;
 static bool trans_fcvt_s_d(DisasContext *ctx, arg_fcvt_s_d *a);
+typedef arg_decode_insn3212 arg_fcvt_h_s;
+static bool trans_fcvt_h_s(DisasContext *ctx, arg_fcvt_h_s *a);
 typedef arg_decode_insn3212 arg_fcvt_d_s;
 static bool trans_fcvt_d_s(DisasContext *ctx, arg_fcvt_d_s *a);
+typedef arg_decode_insn3212 arg_fcvt_d_h;
+static bool trans_fcvt_d_h(DisasContext *ctx, arg_fcvt_d_h *a);
+typedef arg_decode_insn3212 arg_fcvt_h_d;
+static bool trans_fcvt_h_d(DisasContext *ctx, arg_fcvt_h_d *a);
 typedef arg_r arg_feq_d;
 static bool trans_feq_d(DisasContext *ctx, arg_feq_d *a);
 typedef arg_r arg_flt_d;
@@ -482,6 +679,31 @@ static void decode_insn32_extract_sh(DisasContext *ctx, arg_shift *a, uint32_t i
     a->rd = extract32(insn, 7, 5);
 }
 
+static void decode_insn32_extract_zbs_sh(DisasContext *ctx, arg_shift *a,
+                                         uint32_t insn)
+{
+    a->shamt = extract32(insn, 20, 6);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
+static void decode_insn32_extract_zbb_sh(DisasContext *ctx, arg_shift *a,
+                                         uint32_t insn)
+{
+    a->shamt = extract32(insn, 20, 6);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
+static void decode_insn32_extract_k_aes(DisasContext *ctx, arg_k_aes *a,
+                                        uint32_t insn)
+{
+    a->shamt = ex_shift_3(ctx, extract32(insn, 30, 2));
+    a->rs2 = extract32(insn, 20, 5);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
 static void decode_insn32_extract_u(DisasContext *ctx, arg_u *a, uint32_t insn)
 {
     a->imm = ex_shift_12(ctx, sextract32(insn, 12, 20));
@@ -504,6 +726,7 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         arg_empty f_empty;
         arg_i f_i;
         arg_j f_j;
+        arg_k_aes f_k_aes;
         arg_r f_r;
         arg_s f_s;
         arg_shift f_shift;
@@ -546,6 +769,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         /* ........ ........ ........ .0000111 */
         decode_insn32_extract_i(ctx, &u.f_i, insn);
         switch ((insn >> 12) & 0x7) {
+        case 0x1:
+            if (trans_flh(ctx, &u.f_i)) return true;
+            return false;
         case 0x2:
             /* ........ ........ .010.... .0000111 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:156 */
@@ -562,6 +788,11 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         /* ........ ........ ........ .0001111 */
         switch ((insn >> 12) & 0x7) {
         case 0x0:
+            switch (insn & 0xffffffff) {
+            case 0x0100000f:
+                if (trans_pause(ctx, &u.f_empty)) return true;
+                return false;
+            }
             /* ........ ........ .000.... .0001111 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:123 */
             decode_insn32_extract_decode_insn32_Fmt_19(ctx, &u.f_decode_insn3216, insn);
@@ -586,13 +817,76 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x1:
             /* ........ ........ .001.... .0010011 */
-            decode_insn32_extract_sh(ctx, &u.f_shift, insn);
-            switch ((insn >> 30) & 0x3) {
-            case 0x0:
-                /* 00...... ........ .001.... .0010011 */
+            switch (insn & 0xfc007000) {
+            case 0x00001000:
+                /* 000000.. ........ .001.... .0010011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:110 */
+                decode_insn32_extract_sh(ctx, &u.f_shift, insn);
                 if (trans_slli(ctx, &u.f_shift)) return true;
                 return false;
+            case 0x28001000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_bseti(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x48001000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_bclri(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x68001000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_binvi(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x08001000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0xf:
+                    if (trans_zip(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
+            case 0x60001000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x0:
+                    if (trans_clz(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x1:
+                    if (trans_ctz(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x2:
+                    if (trans_cpop(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x4:
+                    if (trans_sext_b(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x5:
+                    if (trans_sext_h(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
+            case 0x10001000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x0:
+                    if (trans_sha256sum0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x1:
+                    if (trans_sha256sum1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x2:
+                    if (trans_sha256sig0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x3:
+                    if (trans_sha256sig1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x8:
+                    if (trans_sm3p0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x9:
+                    if (trans_sm3p1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
             }
             return false;
         case 0x2:
@@ -615,18 +909,54 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x5:
             /* ........ ........ .101.... .0010011 */
-            decode_insn32_extract_sh(ctx, &u.f_shift, insn);
-            switch ((insn >> 30) & 0x3) {
-            case 0x0:
-                /* 00...... ........ .101.... .0010011 */
+            switch (insn & 0xfc007000) {
+            case 0x00005000:
+                /* 000000.. ........ .101.... .0010011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:111 */
+                decode_insn32_extract_sh(ctx, &u.f_shift, insn);
                 if (trans_srli(ctx, &u.f_shift)) return true;
                 return false;
-            case 0x1:
-                /* 01...... ........ .101.... .0010011 */
+            case 0x40005000:
+                /* 010000.. ........ .101.... .0010011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:112 */
+                decode_insn32_extract_sh(ctx, &u.f_shift, insn);
                 if (trans_srai(ctx, &u.f_shift)) return true;
                 return false;
+            case 0x48005000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_bexti(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x08005000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0xf:
+                    if (trans_unzip(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
+            case 0x28005000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x7:
+                    if (trans_orc_b(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
+            case 0x60005000:
+                decode_insn32_extract_zbb_sh(ctx, &u.f_shift, insn);
+                if (trans_rori(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x68005000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x7:
+                    if (trans_brev8(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x18:
+                    if (trans_rev8_32(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
             }
             return false;
         case 0x6:
@@ -674,6 +1004,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         /* ........ ........ ........ .0100111 */
         decode_insn32_extract_s(ctx, &u.f_s, insn);
         switch ((insn >> 12) & 0x7) {
+        case 0x1:
+            if (trans_fsh(ctx, &u.f_s)) return true;
+            return false;
         case 0x2:
             /* ........ ........ .010.... .0100111 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:157 */
@@ -765,6 +1098,32 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
     case 0x00000033:
         /* ........ ........ ........ .0110011 */
         decode_insn32_extract_r(ctx, &u.f_r, insn);
+        switch (insn & 0x3e007000) {
+        case 0x22000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_aes32esi(ctx, &u.f_k_aes)) return true;
+            return false;
+        case 0x26000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_aes32esmi(ctx, &u.f_k_aes)) return true;
+            return false;
+        case 0x2a000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_aes32dsi(ctx, &u.f_k_aes)) return true;
+            return false;
+        case 0x2e000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_aes32dsmi(ctx, &u.f_k_aes)) return true;
+            return false;
+        case 0x30000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_sm4ed(ctx, &u.f_k_aes)) return true;
+            return false;
+        case 0x34000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_sm4ks(ctx, &u.f_k_aes)) return true;
+            return false;
+        }
         switch (insn & 0xfe007000) {
         case 0x00000000:
             /* 0000000. ........ .000.... .0110011 */
@@ -806,6 +1165,56 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:122 */
             if (trans_and(ctx, &u.f_r)) return true;
             return false;
+        case 0x28001000:
+            if (trans_bset(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20002000:
+            if (trans_sh1add(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20004000:
+            if (trans_sh2add(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20006000:
+            if (trans_sh3add(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a001000:
+            if (trans_clmul(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a002000:
+            if (trans_clmulr(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a003000:
+            if (trans_clmulh(ctx, &u.f_r)) return true;
+            return false;
+        case 0x60001000:
+            if (trans_rol(ctx, &u.f_r)) return true;
+            return false;
+        case 0x60005000:
+            if (trans_ror(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a004000:
+            if (trans_min(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a005000:
+            if (trans_minu(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a006000:
+            if (trans_max(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a007000:
+            if (trans_maxu(ctx, &u.f_r)) return true;
+            return false;
+        case 0x08004000:
+            decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+            if (((insn >> 20) & 0x1f) == 0 &&
+                trans_zext_h_32(ctx, &u.f_decode_insn3213)) {
+                return true;
+            }
+            if (trans_pack(ctx, &u.f_r)) return true;
+            return false;
+        case 0x08007000:
+            if (trans_packh(ctx, &u.f_r)) return true;
+            return false;
         case 0x02000000:
             /* 0000001. ........ .000.... .0110011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:133 */
@@ -851,11 +1260,53 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:114 */
             if (trans_sub(ctx, &u.f_r)) return true;
             return false;
+        case 0x40004000:
+            if (trans_xnor(ctx, &u.f_r)) return true;
+            return false;
         case 0x40005000:
             /* 0100000. ........ .101.... .0110011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:120 */
             if (trans_sra(ctx, &u.f_r)) return true;
             return false;
+        case 0x40006000:
+            if (trans_orn(ctx, &u.f_r)) return true;
+            return false;
+        case 0x40007000:
+            if (trans_andn(ctx, &u.f_r)) return true;
+            return false;
+        case 0x48001000:
+            if (trans_bclr(ctx, &u.f_r)) return true;
+            return false;
+        case 0x48005000:
+            if (trans_bext(ctx, &u.f_r)) return true;
+            return false;
+        case 0x68001000:
+            if (trans_binv(ctx, &u.f_r)) return true;
+            return false;
+        case 0x50000000:
+            if (trans_sha512sum0r(ctx, &u.f_r)) return true;
+            return false;
+        case 0x52000000:
+            if (trans_sha512sum1r(ctx, &u.f_r)) return true;
+            return false;
+        case 0x54000000:
+            if (trans_sha512sig0l(ctx, &u.f_r)) return true;
+            return false;
+        case 0x5c000000:
+            if (trans_sha512sig0h(ctx, &u.f_r)) return true;
+            return false;
+        case 0x56000000:
+            if (trans_sha512sig1l(ctx, &u.f_r)) return true;
+            return false;
+        case 0x5e000000:
+            if (trans_sha512sig1h(ctx, &u.f_r)) return true;
+            return false;
+        case 0x28002000:
+            if (trans_xperm4(ctx, &u.f_r)) return true;
+            return false;
+        case 0x28004000:
+            if (trans_xperm8(ctx, &u.f_r)) return true;
+            return false;
         }
         return false;
     case 0x00000037:
@@ -878,6 +1329,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:186 */
             if (trans_fmadd_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fmadd_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x00000047:
@@ -894,6 +1348,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:187 */
             if (trans_fmsub_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fmsub_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x0000004b:
@@ -910,6 +1367,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:188 */
             if (trans_fnmsub_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fnmsub_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x0000004f:
@@ -926,6 +1386,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:189 */
             if (trans_fnmadd_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fnmadd_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x00000053:
@@ -943,6 +1406,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fadd_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0x2:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fadd_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0x4:
             /* 0000100. ........ ........ .1010011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:163 */
@@ -955,6 +1422,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fsub_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0x6:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fsub_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0x8:
             /* 0001000. ........ ........ .1010011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:164 */
@@ -967,6 +1438,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fmul_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0xa:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fmul_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0xc:
             /* 0001100. ........ ........ .1010011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:165 */
@@ -979,6 +1454,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fdiv_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0xe:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fdiv_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0x10:
             /* 0010000. ........ ........ .1010011 */
             decode_insn32_extract_r(ctx, &u.f_r, insn);
@@ -1021,6 +1500,20 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x12:
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            switch ((insn >> 12) & 0x7) {
+            case 0x0:
+                if (trans_fsgnj_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x1:
+                if (trans_fsgnjn_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x2:
+                if (trans_fsgnjx_h(ctx, &u.f_r)) return true;
+                return false;
+            }
+            return false;
         case 0x14:
             /* 0010100. ........ ........ .1010011 */
             decode_insn32_extract_r(ctx, &u.f_r, insn);
@@ -1053,6 +1546,17 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x16:
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            switch ((insn >> 12) & 0x7) {
+            case 0x0:
+                if (trans_fmin_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x1:
+                if (trans_fmax_h(ctx, &u.f_r)) return true;
+                return false;
+            }
+            return false;
         case 0x20:
             /* 0100000. ........ ........ .1010011 */
             decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
@@ -1062,6 +1566,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:200 */
                 if (trans_fcvt_s_d(ctx, &u.f_decode_insn3212)) return true;
                 return false;
+            case 0x2:
+                if (trans_fcvt_s_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
             }
             return false;
         case 0x21:
@@ -1073,6 +1580,20 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:201 */
                 if (trans_fcvt_d_s(ctx, &u.f_decode_insn3212)) return true;
                 return false;
+            case 0x2:
+                if (trans_fcvt_d_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
+        case 0x22:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fcvt_h_s(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x1:
+                if (trans_fcvt_h_d(ctx, &u.f_decode_insn3212)) return true;
+                return false;
             }
             return false;
         case 0x2c:
@@ -1097,6 +1618,14 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x2e:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fsqrt_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
         case 0x50:
             /* 1010000. ........ ........ .1010011 */
             decode_insn32_extract_r(ctx, &u.f_r, insn);
@@ -1139,6 +1668,20 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x52:
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            switch ((insn >> 12) & 0x7) {
+            case 0x0:
+                if (trans_fle_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x1:
+                if (trans_flt_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x2:
+                if (trans_feq_h(ctx, &u.f_r)) return true;
+                return false;
+            }
+            return false;
         case 0x60:
             /* 1100000. ........ ........ .1010011 */
             decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
@@ -1171,6 +1714,17 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x62:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fcvt_w_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x1:
+                if (trans_fcvt_wu_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
         case 0x68:
             /* 1101000. ........ ........ .1010011 */
             decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
@@ -1203,6 +1757,17 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x6a:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fcvt_h_w(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x1:
+                if (trans_fcvt_h_wu(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
         case 0x70:
             /* 1110000. ........ ........ .1010011 */
             decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
@@ -1230,6 +1795,17 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x72:
+            decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+            switch (insn & 0x01f07000) {
+            case 0x00000000:
+                if (trans_fmv_x_h(ctx, &u.f_decode_insn3213)) return true;
+                return false;
+            case 0x00001000:
+                if (trans_fclass_h(ctx, &u.f_decode_insn3213)) return true;
+                return false;
+            }
+            return false;
         case 0x78:
             /* 1111000. ........ ........ .1010011 */
             decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
@@ -1241,6 +1817,14 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x7a:
+            decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+            switch (insn & 0x01f07000) {
+            case 0x00000000:
+                if (trans_fmv_h_x(ctx, &u.f_decode_insn3213)) return true;
+                return false;
+            }
+            return false;
         }
         return false;
     case 0x00000063:
@@ -1362,12 +1946,31 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 decode_insn32_extract_sfence_vma(ctx, &u.f_decode_insn3214, insn);
                 if (trans_sfence_vma(ctx, &u.f_decode_insn3214)) return true;
                 return false;
+            case 0x16000000:
+                decode_insn32_extract_sfence_vma(ctx, &u.f_decode_insn3214, insn);
+                if (trans_sinval_vma(ctx, &u.f_decode_insn3214)) return true;
+                return false;
+            case 0x18000000:
+                decode_insn32_extract_decode_insn32_Fmt_18(ctx, &u.f_empty, insn);
+                switch ((insn >> 15) & 0x3ff) {
+                case 0x0:
+                    if (trans_sfence_w_inval(ctx, &u.f_empty)) return true;
+                    return false;
+                case 0x20:
+                    if (trans_sfence_inval_ir(ctx, &u.f_empty)) return true;
+                    return false;
+                }
+                return false;
             case 0x22000000:
                 /* 0010001. ........ .0000000 01110011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:81 */
                 decode_insn32_extract_hfence_bvma(ctx, &u.f_decode_insn3214, insn);
                 if (trans_hfence_bvma(ctx, &u.f_decode_insn3214)) return true;
                 return false;
+            case 0x26000000:
+                decode_insn32_extract_hfence_bvma(ctx, &u.f_decode_insn3214, insn);
+                if (trans_hinval_vvma(ctx, &u.f_decode_insn3214)) return true;
+                return false;
             case 0x30000000:
                 /* 0011000. ........ .0000000 01110011 */
                 decode_insn32_extract_decode_insn32_Fmt_18(ctx, &u.f_empty, insn);
@@ -1385,6 +1988,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 decode_insn32_extract_hfence_gvma(ctx, &u.f_decode_insn3214, insn);
                 if (trans_hfence_gvma(ctx, &u.f_decode_insn3214)) return true;
                 return false;
+            case 0x66000000:
+                decode_insn32_extract_hfence_gvma(ctx, &u.f_decode_insn3214, insn);
+                if (trans_hinval_gvma(ctx, &u.f_decode_insn3214)) return true;
+                return false;
             }
             return false;
         case 0x1:
@@ -1425,6 +2032,21 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             return false;
         }
         return false;
+    case 0x0000007b:
+        /* ....... ........ ........ .1111011 */
+        if ((insn & 0xfe00707f) == 0x0000607b) {
+            /* 0000000. ........ .110.... .1111011 */
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            if (trans_vt_maskc(ctx, &u.f_r)) return true;
+            return false;
+        }
+        if ((insn & 0xfe00707f) == 0x0000707b) {
+            /* 0000000. ........ .111.... .1111011 */
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            if (trans_vt_maskcn(ctx, &u.f_r)) return true;
+            return false;
+        }
+        return false;
     }
     return false;
 }
diff --git a/qemu/target/riscv/riscv64/decode_insn32.inc.c b/qemu/target/riscv/riscv64/decode_insn32.inc.c
index b5d7896091..65e0df9988 100644
--- a/qemu/target/riscv/riscv64/decode_insn32.inc.c
+++ b/qemu/target/riscv/riscv64/decode_insn32.inc.c
@@ -95,6 +95,13 @@ typedef struct {
     int shamt;
 } arg_shift;
 
+typedef struct {
+    int rd;
+    int rs1;
+    int rs2;
+    int shamt;
+} arg_k_aes;
+
 typedef struct {
     int imm;
     int rd;
@@ -120,6 +127,20 @@ typedef arg_decode_insn3214 arg_sfence_vma;
 static bool trans_sfence_vma(DisasContext *ctx, arg_sfence_vma *a);
 typedef arg_decode_insn3215 arg_sfence_vm;
 static bool trans_sfence_vm(DisasContext *ctx, arg_sfence_vm *a);
+typedef arg_decode_insn3214 arg_sinval_vma;
+static bool trans_sinval_vma(DisasContext *ctx, arg_sinval_vma *a);
+typedef arg_empty arg_sfence_w_inval;
+static bool trans_sfence_w_inval(DisasContext *ctx, arg_sfence_w_inval *a);
+typedef arg_empty arg_sfence_inval_ir;
+static bool trans_sfence_inval_ir(DisasContext *ctx, arg_sfence_inval_ir *a);
+typedef arg_decode_insn3214 arg_hinval_vvma;
+static bool trans_hinval_vvma(DisasContext *ctx, arg_hinval_vvma *a);
+typedef arg_decode_insn3214 arg_hinval_gvma;
+static bool trans_hinval_gvma(DisasContext *ctx, arg_hinval_gvma *a);
+typedef arg_r arg_vt_maskc;
+static bool trans_vt_maskc(DisasContext *ctx, arg_vt_maskc *a);
+typedef arg_r arg_vt_maskcn;
+static bool trans_vt_maskcn(DisasContext *ctx, arg_vt_maskcn *a);
 typedef arg_u arg_lui;
 static bool trans_lui(DisasContext *ctx, arg_lui *a);
 typedef arg_u arg_auipc;
@@ -174,6 +195,70 @@ typedef arg_shift arg_srli;
 static bool trans_srli(DisasContext *ctx, arg_srli *a);
 typedef arg_shift arg_srai;
 static bool trans_srai(DisasContext *ctx, arg_srai *a);
+typedef arg_shift arg_rori;
+static bool trans_rori(DisasContext *ctx, arg_rori *a);
+typedef arg_decode_insn3213 arg_clz;
+static bool trans_clz(DisasContext *ctx, arg_clz *a);
+typedef arg_decode_insn3213 arg_ctz;
+static bool trans_ctz(DisasContext *ctx, arg_ctz *a);
+typedef arg_decode_insn3213 arg_cpop;
+static bool trans_cpop(DisasContext *ctx, arg_cpop *a);
+typedef arg_decode_insn3213 arg_orc_b;
+static bool trans_orc_b(DisasContext *ctx, arg_orc_b *a);
+typedef arg_decode_insn3213 arg_rev8_64;
+static bool trans_rev8_64(DisasContext *ctx, arg_rev8_64 *a);
+typedef arg_decode_insn3213 arg_brev8;
+static bool trans_brev8(DisasContext *ctx, arg_brev8 *a);
+typedef arg_r arg_aes64es;
+static bool trans_aes64es(DisasContext *ctx, arg_aes64es *a);
+typedef arg_r arg_aes64esm;
+static bool trans_aes64esm(DisasContext *ctx, arg_aes64esm *a);
+typedef arg_r arg_aes64ds;
+static bool trans_aes64ds(DisasContext *ctx, arg_aes64ds *a);
+typedef arg_r arg_aes64dsm;
+static bool trans_aes64dsm(DisasContext *ctx, arg_aes64dsm *a);
+typedef arg_r arg_aes64ks2;
+static bool trans_aes64ks2(DisasContext *ctx, arg_aes64ks2 *a);
+typedef arg_i arg_aes64ks1i;
+static bool trans_aes64ks1i(DisasContext *ctx, arg_aes64ks1i *a);
+typedef arg_decode_insn3213 arg_aes64im;
+static bool trans_aes64im(DisasContext *ctx, arg_aes64im *a);
+typedef arg_k_aes arg_sm4ed;
+static bool trans_sm4ed(DisasContext *ctx, arg_sm4ed *a);
+typedef arg_k_aes arg_sm4ks;
+static bool trans_sm4ks(DisasContext *ctx, arg_sm4ks *a);
+typedef arg_decode_insn3213 arg_sha256sum0;
+static bool trans_sha256sum0(DisasContext *ctx, arg_sha256sum0 *a);
+typedef arg_decode_insn3213 arg_sha256sum1;
+static bool trans_sha256sum1(DisasContext *ctx, arg_sha256sum1 *a);
+typedef arg_decode_insn3213 arg_sha256sig0;
+static bool trans_sha256sig0(DisasContext *ctx, arg_sha256sig0 *a);
+typedef arg_decode_insn3213 arg_sha256sig1;
+static bool trans_sha256sig1(DisasContext *ctx, arg_sha256sig1 *a);
+typedef arg_decode_insn3213 arg_sha512sum0;
+static bool trans_sha512sum0(DisasContext *ctx, arg_sha512sum0 *a);
+typedef arg_decode_insn3213 arg_sha512sum1;
+static bool trans_sha512sum1(DisasContext *ctx, arg_sha512sum1 *a);
+typedef arg_decode_insn3213 arg_sha512sig0;
+static bool trans_sha512sig0(DisasContext *ctx, arg_sha512sig0 *a);
+typedef arg_decode_insn3213 arg_sha512sig1;
+static bool trans_sha512sig1(DisasContext *ctx, arg_sha512sig1 *a);
+typedef arg_decode_insn3213 arg_sm3p0;
+static bool trans_sm3p0(DisasContext *ctx, arg_sm3p0 *a);
+typedef arg_decode_insn3213 arg_sm3p1;
+static bool trans_sm3p1(DisasContext *ctx, arg_sm3p1 *a);
+typedef arg_decode_insn3213 arg_sext_b;
+static bool trans_sext_b(DisasContext *ctx, arg_sext_b *a);
+typedef arg_decode_insn3213 arg_sext_h;
+static bool trans_sext_h(DisasContext *ctx, arg_sext_h *a);
+typedef arg_shift arg_bseti;
+static bool trans_bseti(DisasContext *ctx, arg_bseti *a);
+typedef arg_shift arg_bclri;
+static bool trans_bclri(DisasContext *ctx, arg_bclri *a);
+typedef arg_shift arg_binvi;
+static bool trans_binvi(DisasContext *ctx, arg_binvi *a);
+typedef arg_shift arg_bexti;
+static bool trans_bexti(DisasContext *ctx, arg_bexti *a);
 typedef arg_r arg_add;
 static bool trans_add(DisasContext *ctx, arg_add *a);
 typedef arg_r arg_sub;
@@ -194,6 +279,54 @@ typedef arg_r arg_or;
 static bool trans_or(DisasContext *ctx, arg_or *a);
 typedef arg_r arg_and;
 static bool trans_and(DisasContext *ctx, arg_and *a);
+typedef arg_r arg_sh1add;
+static bool trans_sh1add(DisasContext *ctx, arg_sh1add *a);
+typedef arg_r arg_sh2add;
+static bool trans_sh2add(DisasContext *ctx, arg_sh2add *a);
+typedef arg_r arg_sh3add;
+static bool trans_sh3add(DisasContext *ctx, arg_sh3add *a);
+typedef arg_r arg_clmul;
+static bool trans_clmul(DisasContext *ctx, arg_clmul *a);
+typedef arg_r arg_clmulh;
+static bool trans_clmulh(DisasContext *ctx, arg_clmulh *a);
+typedef arg_r arg_clmulr;
+static bool trans_clmulr(DisasContext *ctx, arg_clmulr *a);
+typedef arg_r arg_pack;
+static bool trans_pack(DisasContext *ctx, arg_pack *a);
+typedef arg_r arg_packh;
+static bool trans_packh(DisasContext *ctx, arg_packh *a);
+typedef arg_r arg_xperm4;
+static bool trans_xperm4(DisasContext *ctx, arg_xperm4 *a);
+typedef arg_r arg_xperm8;
+static bool trans_xperm8(DisasContext *ctx, arg_xperm8 *a);
+typedef arg_r arg_andn;
+static bool trans_andn(DisasContext *ctx, arg_andn *a);
+typedef arg_r arg_rol;
+static bool trans_rol(DisasContext *ctx, arg_rol *a);
+typedef arg_r arg_ror;
+static bool trans_ror(DisasContext *ctx, arg_ror *a);
+typedef arg_r arg_xnor;
+static bool trans_xnor(DisasContext *ctx, arg_xnor *a);
+typedef arg_r arg_min;
+static bool trans_min(DisasContext *ctx, arg_min *a);
+typedef arg_r arg_max;
+static bool trans_max(DisasContext *ctx, arg_max *a);
+typedef arg_r arg_minu;
+static bool trans_minu(DisasContext *ctx, arg_minu *a);
+typedef arg_r arg_maxu;
+static bool trans_maxu(DisasContext *ctx, arg_maxu *a);
+typedef arg_r arg_orn;
+static bool trans_orn(DisasContext *ctx, arg_orn *a);
+typedef arg_r arg_bset;
+static bool trans_bset(DisasContext *ctx, arg_bset *a);
+typedef arg_r arg_bclr;
+static bool trans_bclr(DisasContext *ctx, arg_bclr *a);
+typedef arg_r arg_binv;
+static bool trans_binv(DisasContext *ctx, arg_binv *a);
+typedef arg_r arg_bext;
+static bool trans_bext(DisasContext *ctx, arg_bext *a);
+typedef arg_empty arg_pause;
+static bool trans_pause(DisasContext *ctx, arg_pause *a);
 typedef arg_decode_insn3216 arg_fence;
 static bool trans_fence(DisasContext *ctx, arg_fence *a);
 typedef arg_empty arg_fence_i;
@@ -250,56 +383,110 @@ typedef arg_atomic arg_amomaxu_w;
 static bool trans_amomaxu_w(DisasContext *ctx, arg_amomaxu_w *a);
 typedef arg_i arg_flw;
 static bool trans_flw(DisasContext *ctx, arg_flw *a);
+typedef arg_i arg_flh;
+static bool trans_flh(DisasContext *ctx, arg_flh *a);
 typedef arg_s arg_fsw;
 static bool trans_fsw(DisasContext *ctx, arg_fsw *a);
+typedef arg_s arg_fsh;
+static bool trans_fsh(DisasContext *ctx, arg_fsh *a);
 typedef arg_decode_insn3210 arg_fmadd_s;
 static bool trans_fmadd_s(DisasContext *ctx, arg_fmadd_s *a);
+typedef arg_decode_insn3210 arg_fmadd_h;
+static bool trans_fmadd_h(DisasContext *ctx, arg_fmadd_h *a);
 typedef arg_decode_insn3210 arg_fmsub_s;
 static bool trans_fmsub_s(DisasContext *ctx, arg_fmsub_s *a);
+typedef arg_decode_insn3210 arg_fmsub_h;
+static bool trans_fmsub_h(DisasContext *ctx, arg_fmsub_h *a);
 typedef arg_decode_insn3210 arg_fnmsub_s;
 static bool trans_fnmsub_s(DisasContext *ctx, arg_fnmsub_s *a);
+typedef arg_decode_insn3210 arg_fnmsub_h;
+static bool trans_fnmsub_h(DisasContext *ctx, arg_fnmsub_h *a);
 typedef arg_decode_insn3210 arg_fnmadd_s;
 static bool trans_fnmadd_s(DisasContext *ctx, arg_fnmadd_s *a);
+typedef arg_decode_insn3210 arg_fnmadd_h;
+static bool trans_fnmadd_h(DisasContext *ctx, arg_fnmadd_h *a);
 typedef arg_decode_insn3211 arg_fadd_s;
 static bool trans_fadd_s(DisasContext *ctx, arg_fadd_s *a);
+typedef arg_decode_insn3211 arg_fadd_h;
+static bool trans_fadd_h(DisasContext *ctx, arg_fadd_h *a);
 typedef arg_decode_insn3211 arg_fsub_s;
 static bool trans_fsub_s(DisasContext *ctx, arg_fsub_s *a);
+typedef arg_decode_insn3211 arg_fsub_h;
+static bool trans_fsub_h(DisasContext *ctx, arg_fsub_h *a);
 typedef arg_decode_insn3211 arg_fmul_s;
 static bool trans_fmul_s(DisasContext *ctx, arg_fmul_s *a);
+typedef arg_decode_insn3211 arg_fmul_h;
+static bool trans_fmul_h(DisasContext *ctx, arg_fmul_h *a);
 typedef arg_decode_insn3211 arg_fdiv_s;
 static bool trans_fdiv_s(DisasContext *ctx, arg_fdiv_s *a);
+typedef arg_decode_insn3211 arg_fdiv_h;
+static bool trans_fdiv_h(DisasContext *ctx, arg_fdiv_h *a);
 typedef arg_decode_insn3212 arg_fsqrt_s;
 static bool trans_fsqrt_s(DisasContext *ctx, arg_fsqrt_s *a);
+typedef arg_decode_insn3212 arg_fsqrt_h;
+static bool trans_fsqrt_h(DisasContext *ctx, arg_fsqrt_h *a);
 typedef arg_r arg_fsgnj_s;
 static bool trans_fsgnj_s(DisasContext *ctx, arg_fsgnj_s *a);
+typedef arg_r arg_fsgnj_h;
+static bool trans_fsgnj_h(DisasContext *ctx, arg_fsgnj_h *a);
 typedef arg_r arg_fsgnjn_s;
 static bool trans_fsgnjn_s(DisasContext *ctx, arg_fsgnjn_s *a);
+typedef arg_r arg_fsgnjn_h;
+static bool trans_fsgnjn_h(DisasContext *ctx, arg_fsgnjn_h *a);
 typedef arg_r arg_fsgnjx_s;
 static bool trans_fsgnjx_s(DisasContext *ctx, arg_fsgnjx_s *a);
+typedef arg_r arg_fsgnjx_h;
+static bool trans_fsgnjx_h(DisasContext *ctx, arg_fsgnjx_h *a);
 typedef arg_r arg_fmin_s;
 static bool trans_fmin_s(DisasContext *ctx, arg_fmin_s *a);
+typedef arg_r arg_fmin_h;
+static bool trans_fmin_h(DisasContext *ctx, arg_fmin_h *a);
 typedef arg_r arg_fmax_s;
 static bool trans_fmax_s(DisasContext *ctx, arg_fmax_s *a);
+typedef arg_r arg_fmax_h;
+static bool trans_fmax_h(DisasContext *ctx, arg_fmax_h *a);
 typedef arg_decode_insn3212 arg_fcvt_w_s;
 static bool trans_fcvt_w_s(DisasContext *ctx, arg_fcvt_w_s *a);
+typedef arg_decode_insn3212 arg_fcvt_w_h;
+static bool trans_fcvt_w_h(DisasContext *ctx, arg_fcvt_w_h *a);
 typedef arg_decode_insn3212 arg_fcvt_wu_s;
 static bool trans_fcvt_wu_s(DisasContext *ctx, arg_fcvt_wu_s *a);
+typedef arg_decode_insn3212 arg_fcvt_wu_h;
+static bool trans_fcvt_wu_h(DisasContext *ctx, arg_fcvt_wu_h *a);
 typedef arg_decode_insn3213 arg_fmv_x_w;
 static bool trans_fmv_x_w(DisasContext *ctx, arg_fmv_x_w *a);
+typedef arg_decode_insn3213 arg_fmv_x_h;
+static bool trans_fmv_x_h(DisasContext *ctx, arg_fmv_x_h *a);
 typedef arg_r arg_feq_s;
 static bool trans_feq_s(DisasContext *ctx, arg_feq_s *a);
+typedef arg_r arg_feq_h;
+static bool trans_feq_h(DisasContext *ctx, arg_feq_h *a);
 typedef arg_r arg_flt_s;
 static bool trans_flt_s(DisasContext *ctx, arg_flt_s *a);
+typedef arg_r arg_flt_h;
+static bool trans_flt_h(DisasContext *ctx, arg_flt_h *a);
 typedef arg_r arg_fle_s;
 static bool trans_fle_s(DisasContext *ctx, arg_fle_s *a);
+typedef arg_r arg_fle_h;
+static bool trans_fle_h(DisasContext *ctx, arg_fle_h *a);
 typedef arg_decode_insn3213 arg_fclass_s;
 static bool trans_fclass_s(DisasContext *ctx, arg_fclass_s *a);
+typedef arg_decode_insn3213 arg_fclass_h;
+static bool trans_fclass_h(DisasContext *ctx, arg_fclass_h *a);
 typedef arg_decode_insn3212 arg_fcvt_s_w;
 static bool trans_fcvt_s_w(DisasContext *ctx, arg_fcvt_s_w *a);
+typedef arg_decode_insn3212 arg_fcvt_s_h;
+static bool trans_fcvt_s_h(DisasContext *ctx, arg_fcvt_s_h *a);
+typedef arg_decode_insn3212 arg_fcvt_h_w;
+static bool trans_fcvt_h_w(DisasContext *ctx, arg_fcvt_h_w *a);
 typedef arg_decode_insn3212 arg_fcvt_s_wu;
 static bool trans_fcvt_s_wu(DisasContext *ctx, arg_fcvt_s_wu *a);
+typedef arg_decode_insn3212 arg_fcvt_h_wu;
+static bool trans_fcvt_h_wu(DisasContext *ctx, arg_fcvt_h_wu *a);
 typedef arg_decode_insn3213 arg_fmv_w_x;
 static bool trans_fmv_w_x(DisasContext *ctx, arg_fmv_w_x *a);
+typedef arg_decode_insn3213 arg_fmv_h_x;
+static bool trans_fmv_h_x(DisasContext *ctx, arg_fmv_h_x *a);
 typedef arg_i arg_fld;
 static bool trans_fld(DisasContext *ctx, arg_fld *a);
 typedef arg_s arg_fsd;
@@ -334,8 +521,14 @@ typedef arg_r arg_fmax_d;
 static bool trans_fmax_d(DisasContext *ctx, arg_fmax_d *a);
 typedef arg_decode_insn3212 arg_fcvt_s_d;
 static bool trans_fcvt_s_d(DisasContext *ctx, arg_fcvt_s_d *a);
+typedef arg_decode_insn3212 arg_fcvt_h_s;
+static bool trans_fcvt_h_s(DisasContext *ctx, arg_fcvt_h_s *a);
 typedef arg_decode_insn3212 arg_fcvt_d_s;
 static bool trans_fcvt_d_s(DisasContext *ctx, arg_fcvt_d_s *a);
+typedef arg_decode_insn3212 arg_fcvt_d_h;
+static bool trans_fcvt_d_h(DisasContext *ctx, arg_fcvt_d_h *a);
+typedef arg_decode_insn3212 arg_fcvt_h_d;
+static bool trans_fcvt_h_d(DisasContext *ctx, arg_fcvt_h_d *a);
 typedef arg_r arg_feq_d;
 static bool trans_feq_d(DisasContext *ctx, arg_feq_d *a);
 typedef arg_r arg_flt_d;
@@ -366,6 +559,16 @@ typedef arg_shift arg_srliw;
 static bool trans_srliw(DisasContext *ctx, arg_srliw *a);
 typedef arg_shift arg_sraiw;
 static bool trans_sraiw(DisasContext *ctx, arg_sraiw *a);
+typedef arg_shift arg_slli_uw;
+static bool trans_slli_uw(DisasContext *ctx, arg_slli_uw *a);
+typedef arg_shift arg_roriw;
+static bool trans_roriw(DisasContext *ctx, arg_roriw *a);
+typedef arg_decode_insn3213 arg_clzw;
+static bool trans_clzw(DisasContext *ctx, arg_clzw *a);
+typedef arg_decode_insn3213 arg_ctzw;
+static bool trans_ctzw(DisasContext *ctx, arg_ctzw *a);
+typedef arg_decode_insn3213 arg_cpopw;
+static bool trans_cpopw(DisasContext *ctx, arg_cpopw *a);
 typedef arg_r arg_addw;
 static bool trans_addw(DisasContext *ctx, arg_addw *a);
 typedef arg_r arg_subw;
@@ -376,6 +579,22 @@ typedef arg_r arg_srlw;
 static bool trans_srlw(DisasContext *ctx, arg_srlw *a);
 typedef arg_r arg_sraw;
 static bool trans_sraw(DisasContext *ctx, arg_sraw *a);
+typedef arg_r arg_add_uw;
+static bool trans_add_uw(DisasContext *ctx, arg_add_uw *a);
+typedef arg_r arg_sh1add_uw;
+static bool trans_sh1add_uw(DisasContext *ctx, arg_sh1add_uw *a);
+typedef arg_r arg_sh2add_uw;
+static bool trans_sh2add_uw(DisasContext *ctx, arg_sh2add_uw *a);
+typedef arg_r arg_sh3add_uw;
+static bool trans_sh3add_uw(DisasContext *ctx, arg_sh3add_uw *a);
+typedef arg_r arg_packw;
+static bool trans_packw(DisasContext *ctx, arg_packw *a);
+typedef arg_r arg_rolw;
+static bool trans_rolw(DisasContext *ctx, arg_rolw *a);
+typedef arg_r arg_rorw;
+static bool trans_rorw(DisasContext *ctx, arg_rorw *a);
+typedef arg_decode_insn3213 arg_zext_h_64;
+static bool trans_zext_h_64(DisasContext *ctx, arg_zext_h_64 *a);
 typedef arg_r arg_mulw;
 static bool trans_mulw(DisasContext *ctx, arg_mulw *a);
 typedef arg_r arg_divw;
@@ -410,12 +629,20 @@ typedef arg_atomic arg_amomaxu_d;
 static bool trans_amomaxu_d(DisasContext *ctx, arg_amomaxu_d *a);
 typedef arg_decode_insn3212 arg_fcvt_l_s;
 static bool trans_fcvt_l_s(DisasContext *ctx, arg_fcvt_l_s *a);
+typedef arg_decode_insn3212 arg_fcvt_l_h;
+static bool trans_fcvt_l_h(DisasContext *ctx, arg_fcvt_l_h *a);
 typedef arg_decode_insn3212 arg_fcvt_lu_s;
 static bool trans_fcvt_lu_s(DisasContext *ctx, arg_fcvt_lu_s *a);
+typedef arg_decode_insn3212 arg_fcvt_lu_h;
+static bool trans_fcvt_lu_h(DisasContext *ctx, arg_fcvt_lu_h *a);
 typedef arg_decode_insn3212 arg_fcvt_s_l;
 static bool trans_fcvt_s_l(DisasContext *ctx, arg_fcvt_s_l *a);
+typedef arg_decode_insn3212 arg_fcvt_h_l;
+static bool trans_fcvt_h_l(DisasContext *ctx, arg_fcvt_h_l *a);
 typedef arg_decode_insn3212 arg_fcvt_s_lu;
 static bool trans_fcvt_s_lu(DisasContext *ctx, arg_fcvt_s_lu *a);
+typedef arg_decode_insn3212 arg_fcvt_h_lu;
+static bool trans_fcvt_h_lu(DisasContext *ctx, arg_fcvt_h_lu *a);
 typedef arg_decode_insn3212 arg_fcvt_l_d;
 static bool trans_fcvt_l_d(DisasContext *ctx, arg_fcvt_l_d *a);
 typedef arg_decode_insn3212 arg_fcvt_lu_d;
@@ -558,6 +785,30 @@ static void decode_insn32_extract_sh(DisasContext *ctx, arg_shift *a, uint32_t i
     a->rd = extract32(insn, 7, 5);
 }
 
+static void decode_insn32_extract_zbs_sh(DisasContext *ctx, arg_shift *a,
+                                         uint32_t insn)
+{
+    a->shamt = extract32(insn, 20, 6);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
+static void decode_insn32_extract_zbb_sh(DisasContext *ctx, arg_shift *a,
+                                         uint32_t insn)
+{
+    a->shamt = extract32(insn, 20, 6);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
+static void decode_insn32_extract_zba_sh7(DisasContext *ctx, arg_shift *a,
+                                          uint32_t insn)
+{
+    a->shamt = extract32(insn, 20, 7);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
 static void decode_insn32_extract_sh5(DisasContext *ctx, arg_shift *a, uint32_t insn)
 {
     a->shamt = extract32(insn, 20, 5);
@@ -565,6 +816,23 @@ static void decode_insn32_extract_sh5(DisasContext *ctx, arg_shift *a, uint32_t
     a->rd = extract32(insn, 7, 5);
 }
 
+static void decode_insn32_extract_k_aes(DisasContext *ctx, arg_k_aes *a,
+                                        uint32_t insn)
+{
+    a->shamt = ex_shift_3(ctx, extract32(insn, 30, 2));
+    a->rs2 = extract32(insn, 20, 5);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
+static void decode_insn32_extract_i_aes(DisasContext *ctx, arg_i *a,
+                                        uint32_t insn)
+{
+    a->imm = extract32(insn, 20, 4);
+    a->rs1 = extract32(insn, 15, 5);
+    a->rd = extract32(insn, 7, 5);
+}
+
 static void decode_insn32_extract_u(DisasContext *ctx, arg_u *a, uint32_t insn)
 {
     a->imm = ex_shift_12(ctx, sextract32(insn, 12, 20));
@@ -587,6 +855,7 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         arg_empty f_empty;
         arg_i f_i;
         arg_j f_j;
+        arg_k_aes f_k_aes;
         arg_r f_r;
         arg_s f_s;
         arg_shift f_shift;
@@ -639,6 +908,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         /* ........ ........ ........ .0000111 */
         decode_insn32_extract_i(ctx, &u.f_i, insn);
         switch ((insn >> 12) & 0x7) {
+        case 0x1:
+            if (trans_flh(ctx, &u.f_i)) return true;
+            return false;
         case 0x2:
             /* ........ ........ .010.... .0000111 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:156 */
@@ -655,6 +927,11 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         /* ........ ........ ........ .0001111 */
         switch ((insn >> 12) & 0x7) {
         case 0x0:
+            switch (insn & 0xffffffff) {
+            case 0x0100000f:
+                if (trans_pause(ctx, &u.f_empty)) return true;
+                return false;
+            }
             /* ........ ........ .000.... .0001111 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:123 */
             decode_insn32_extract_decode_insn32_Fmt_19(ctx, &u.f_decode_insn3216, insn);
@@ -679,13 +956,93 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x1:
             /* ........ ........ .001.... .0010011 */
-            decode_insn32_extract_sh(ctx, &u.f_shift, insn);
-            switch ((insn >> 30) & 0x3) {
-            case 0x0:
-                /* 00...... ........ .001.... .0010011 */
+            switch (insn & 0xfc007000) {
+            case 0x00001000:
+                /* 000000.. ........ .001.... .0010011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:110 */
+                decode_insn32_extract_sh(ctx, &u.f_shift, insn);
                 if (trans_slli(ctx, &u.f_shift)) return true;
                 return false;
+            case 0x28001000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_bseti(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x48001000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_bclri(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x68001000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_binvi(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x30001000:
+                switch ((insn >> 20) & 0x1f) {
+                case 0x0:
+                    decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                    if (trans_aes64im(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                default:
+                    if (((insn >> 20) & 0x10) != 0) {
+                        decode_insn32_extract_i_aes(ctx, &u.f_i, insn);
+                        if (trans_aes64ks1i(ctx, &u.f_i)) return true;
+                    }
+                    return false;
+                }
+            case 0x60001000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x0:
+                    if (trans_clz(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x1:
+                    if (trans_ctz(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x2:
+                    if (trans_cpop(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x4:
+                    if (trans_sext_b(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x5:
+                    if (trans_sext_h(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
+            case 0x10001000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x0:
+                    if (trans_sha256sum0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x1:
+                    if (trans_sha256sum1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x2:
+                    if (trans_sha256sig0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x3:
+                    if (trans_sha256sig1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x4:
+                    if (trans_sha512sum0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x5:
+                    if (trans_sha512sum1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x6:
+                    if (trans_sha512sig0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x7:
+                    if (trans_sha512sig1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x8:
+                    if (trans_sm3p0(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x9:
+                    if (trans_sm3p1(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
             }
             return false;
         case 0x2:
@@ -708,18 +1065,46 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x5:
             /* ........ ........ .101.... .0010011 */
-            decode_insn32_extract_sh(ctx, &u.f_shift, insn);
-            switch ((insn >> 30) & 0x3) {
-            case 0x0:
-                /* 00...... ........ .101.... .0010011 */
+            switch (insn & 0xfc007000) {
+            case 0x00005000:
+                /* 000000.. ........ .101.... .0010011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:111 */
+                decode_insn32_extract_sh(ctx, &u.f_shift, insn);
                 if (trans_srli(ctx, &u.f_shift)) return true;
                 return false;
-            case 0x1:
-                /* 01...... ........ .101.... .0010011 */
+            case 0x40005000:
+                /* 010000.. ........ .101.... .0010011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:112 */
+                decode_insn32_extract_sh(ctx, &u.f_shift, insn);
                 if (trans_srai(ctx, &u.f_shift)) return true;
                 return false;
+            case 0x48005000:
+                decode_insn32_extract_zbs_sh(ctx, &u.f_shift, insn);
+                if (trans_bexti(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x28005000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x7:
+                    if (trans_orc_b(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
+            case 0x60005000:
+                decode_insn32_extract_zbb_sh(ctx, &u.f_shift, insn);
+                if (trans_rori(ctx, &u.f_shift)) return true;
+                return false;
+            case 0x68005000:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x3f) {
+                case 0x7:
+                    if (trans_brev8(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x38:
+                    if (trans_rev8_64(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
             }
             return false;
         case 0x6:
@@ -753,6 +1138,12 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             return false;
         case 0x1:
             /* ........ ........ .001.... .0011011 */
+            if ((insn & 0xf8007000) == 0x08001000) {
+                decode_insn32_extract_zba_sh7(ctx, &u.f_shift, insn);
+                if (trans_slli_uw(ctx, &u.f_shift)) return true;
+                return false;
+            }
+
             decode_insn32_extract_sh5(ctx, &u.f_shift, insn);
             switch ((insn >> 25) & 0x7f) {
             case 0x0:
@@ -760,6 +1151,20 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32-64.decode:31 */
                 if (trans_slliw(ctx, &u.f_shift)) return true;
                 return false;
+            case 0x30:
+                decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+                switch ((insn >> 20) & 0x1f) {
+                case 0x0:
+                    if (trans_clzw(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x1:
+                    if (trans_ctzw(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                case 0x2:
+                    if (trans_cpopw(ctx, &u.f_decode_insn3213)) return true;
+                    return false;
+                }
+                return false;
             }
             return false;
         case 0x5:
@@ -776,6 +1181,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32-64.decode:33 */
                 if (trans_sraiw(ctx, &u.f_shift)) return true;
                 return false;
+            case 0x30:
+                if (trans_roriw(ctx, &u.f_shift)) return true;
+                return false;
             }
             return false;
         }
@@ -810,6 +1218,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
         /* ........ ........ ........ .0100111 */
         decode_insn32_extract_s(ctx, &u.f_s, insn);
         switch ((insn >> 12) & 0x7) {
+        case 0x1:
+            if (trans_fsh(ctx, &u.f_s)) return true;
+            return false;
         case 0x2:
             /* ........ ........ .010.... .0100111 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:157 */
@@ -972,6 +1383,16 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
     case 0x00000033:
         /* ........ ........ ........ .0110011 */
         decode_insn32_extract_r(ctx, &u.f_r, insn);
+        switch (insn & 0x3e007000) {
+        case 0x30000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_sm4ed(ctx, &u.f_k_aes)) return true;
+            return false;
+        case 0x34000000:
+            decode_insn32_extract_k_aes(ctx, &u.f_k_aes, insn);
+            if (trans_sm4ks(ctx, &u.f_k_aes)) return true;
+            return false;
+        }
         switch (insn & 0xfe007000) {
         case 0x00000000:
             /* 0000000. ........ .000.... .0110011 */
@@ -1013,6 +1434,66 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:122 */
             if (trans_and(ctx, &u.f_r)) return true;
             return false;
+        case 0x28001000:
+            if (trans_bset(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20002000:
+            if (trans_sh1add(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20004000:
+            if (trans_sh2add(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20006000:
+            if (trans_sh3add(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a001000:
+            if (trans_clmul(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a002000:
+            if (trans_clmulr(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a003000:
+            if (trans_clmulh(ctx, &u.f_r)) return true;
+            return false;
+        case 0x08004000:
+            if (trans_pack(ctx, &u.f_r)) return true;
+            return false;
+        case 0x08007000:
+            if (trans_packh(ctx, &u.f_r)) return true;
+            return false;
+        case 0x60001000:
+            if (trans_rol(ctx, &u.f_r)) return true;
+            return false;
+        case 0x60005000:
+            if (trans_ror(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a004000:
+            if (trans_min(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a005000:
+            if (trans_minu(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a006000:
+            if (trans_max(ctx, &u.f_r)) return true;
+            return false;
+        case 0x0a007000:
+            if (trans_maxu(ctx, &u.f_r)) return true;
+            return false;
+        case 0x32000000:
+            if (trans_aes64es(ctx, &u.f_r)) return true;
+            return false;
+        case 0x36000000:
+            if (trans_aes64esm(ctx, &u.f_r)) return true;
+            return false;
+        case 0x3a000000:
+            if (trans_aes64ds(ctx, &u.f_r)) return true;
+            return false;
+        case 0x3e000000:
+            if (trans_aes64dsm(ctx, &u.f_r)) return true;
+            return false;
+        case 0x7e000000:
+            if (trans_aes64ks2(ctx, &u.f_r)) return true;
+            return false;
         case 0x02000000:
             /* 0000001. ........ .000.... .0110011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:133 */
@@ -1058,11 +1539,35 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:114 */
             if (trans_sub(ctx, &u.f_r)) return true;
             return false;
+        case 0x40004000:
+            if (trans_xnor(ctx, &u.f_r)) return true;
+            return false;
         case 0x40005000:
             /* 0100000. ........ .101.... .0110011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:120 */
             if (trans_sra(ctx, &u.f_r)) return true;
             return false;
+        case 0x40006000:
+            if (trans_orn(ctx, &u.f_r)) return true;
+            return false;
+        case 0x40007000:
+            if (trans_andn(ctx, &u.f_r)) return true;
+            return false;
+        case 0x48001000:
+            if (trans_bclr(ctx, &u.f_r)) return true;
+            return false;
+        case 0x48005000:
+            if (trans_bext(ctx, &u.f_r)) return true;
+            return false;
+        case 0x68001000:
+            if (trans_binv(ctx, &u.f_r)) return true;
+            return false;
+        case 0x28002000:
+            if (trans_xperm4(ctx, &u.f_r)) return true;
+            return false;
+        case 0x28004000:
+            if (trans_xperm8(ctx, &u.f_r)) return true;
+            return false;
         }
         return false;
     case 0x00000037:
@@ -1090,6 +1595,32 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32-64.decode:37 */
             if (trans_srlw(ctx, &u.f_r)) return true;
             return false;
+        case 0x08000000:
+            if (trans_add_uw(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20002000:
+            if (trans_sh1add_uw(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20004000:
+            if (trans_sh2add_uw(ctx, &u.f_r)) return true;
+            return false;
+        case 0x20006000:
+            if (trans_sh3add_uw(ctx, &u.f_r)) return true;
+            return false;
+        case 0x60001000:
+            if (trans_rolw(ctx, &u.f_r)) return true;
+            return false;
+        case 0x60005000:
+            if (trans_rorw(ctx, &u.f_r)) return true;
+            return false;
+        case 0x08004000:
+            decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+            if (((insn >> 20) & 0x1f) == 0 &&
+                trans_zext_h_64(ctx, &u.f_decode_insn3213)) {
+                return true;
+            }
+            if (trans_packw(ctx, &u.f_r)) return true;
+            return false;
         case 0x02000000:
             /* 0000001. ........ .000.... .0111011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32-64.decode:41 */
@@ -1141,6 +1672,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:186 */
             if (trans_fmadd_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fmadd_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x00000047:
@@ -1157,6 +1691,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:187 */
             if (trans_fmsub_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fmsub_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x0000004b:
@@ -1173,6 +1710,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:188 */
             if (trans_fnmsub_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fnmsub_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x0000004f:
@@ -1189,6 +1729,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:189 */
             if (trans_fnmadd_d(ctx, &u.f_decode_insn3210)) return true;
             return false;
+        case 0x2:
+            if (trans_fnmadd_h(ctx, &u.f_decode_insn3210)) return true;
+            return false;
         }
         return false;
     case 0x00000053:
@@ -1206,6 +1749,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fadd_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0x2:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fadd_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0x4:
             /* 0000100. ........ ........ .1010011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:163 */
@@ -1218,6 +1765,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fsub_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0x6:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fsub_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0x8:
             /* 0001000. ........ ........ .1010011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:164 */
@@ -1230,6 +1781,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fmul_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0xa:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fmul_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0xc:
             /* 0001100. ........ ........ .1010011 */
             /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:165 */
@@ -1242,6 +1797,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
             if (trans_fdiv_d(ctx, &u.f_decode_insn3211)) return true;
             return false;
+        case 0xe:
+            decode_insn32_extract_r_rm(ctx, &u.f_decode_insn3211, insn);
+            if (trans_fdiv_h(ctx, &u.f_decode_insn3211)) return true;
+            return false;
         case 0x10:
             /* 0010000. ........ ........ .1010011 */
             decode_insn32_extract_r(ctx, &u.f_r, insn);
@@ -1284,6 +1843,20 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x12:
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            switch ((insn >> 12) & 0x7) {
+            case 0x0:
+                if (trans_fsgnj_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x1:
+                if (trans_fsgnjn_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x2:
+                if (trans_fsgnjx_h(ctx, &u.f_r)) return true;
+                return false;
+            }
+            return false;
         case 0x14:
             /* 0010100. ........ ........ .1010011 */
             decode_insn32_extract_r(ctx, &u.f_r, insn);
@@ -1316,6 +1889,17 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x16:
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            switch ((insn >> 12) & 0x7) {
+            case 0x0:
+                if (trans_fmin_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x1:
+                if (trans_fmax_h(ctx, &u.f_r)) return true;
+                return false;
+            }
+            return false;
         case 0x20:
             /* 0100000. ........ ........ .1010011 */
             decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
@@ -1325,6 +1909,9 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:200 */
                 if (trans_fcvt_s_d(ctx, &u.f_decode_insn3212)) return true;
                 return false;
+            case 0x2:
+                if (trans_fcvt_s_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
             }
             return false;
         case 0x21:
@@ -1336,6 +1923,20 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:201 */
                 if (trans_fcvt_d_s(ctx, &u.f_decode_insn3212)) return true;
                 return false;
+            case 0x2:
+                if (trans_fcvt_d_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
+        case 0x22:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fcvt_h_s(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x1:
+                if (trans_fcvt_h_d(ctx, &u.f_decode_insn3212)) return true;
+                return false;
             }
             return false;
         case 0x2c:
@@ -1360,6 +1961,14 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x2e:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fsqrt_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
         case 0x50:
             /* 1010000. ........ ........ .1010011 */
             decode_insn32_extract_r(ctx, &u.f_r, insn);
@@ -1402,6 +2011,20 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x52:
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            switch ((insn >> 12) & 0x7) {
+            case 0x0:
+                if (trans_fle_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x1:
+                if (trans_flt_h(ctx, &u.f_r)) return true;
+                return false;
+            case 0x2:
+                if (trans_feq_h(ctx, &u.f_r)) return true;
+                return false;
+            }
+            return false;
         case 0x60:
             /* 1100000. ........ ........ .1010011 */
             decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
@@ -1454,6 +2077,23 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x62:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fcvt_w_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x1:
+                if (trans_fcvt_wu_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x2:
+                if (trans_fcvt_l_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x3:
+                if (trans_fcvt_lu_h(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
         case 0x68:
             /* 1101000. ........ ........ .1010011 */
             decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
@@ -1506,6 +2146,23 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x6a:
+            decode_insn32_extract_r2_rm(ctx, &u.f_decode_insn3212, insn);
+            switch ((insn >> 20) & 0x1f) {
+            case 0x0:
+                if (trans_fcvt_h_w(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x1:
+                if (trans_fcvt_h_wu(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x2:
+                if (trans_fcvt_h_l(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            case 0x3:
+                if (trans_fcvt_h_lu(ctx, &u.f_decode_insn3212)) return true;
+                return false;
+            }
+            return false;
         case 0x70:
             /* 1110000. ........ ........ .1010011 */
             decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
@@ -1538,6 +2195,17 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x72:
+            decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+            switch (insn & 0x01f07000) {
+            case 0x00000000:
+                if (trans_fmv_x_h(ctx, &u.f_decode_insn3213)) return true;
+                return false;
+            case 0x00001000:
+                if (trans_fclass_h(ctx, &u.f_decode_insn3213)) return true;
+                return false;
+            }
+            return false;
         case 0x78:
             /* 1111000. ........ ........ .1010011 */
             decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
@@ -1560,6 +2228,14 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 return false;
             }
             return false;
+        case 0x7a:
+            decode_insn32_extract_r2(ctx, &u.f_decode_insn3213, insn);
+            switch (insn & 0x01f07000) {
+            case 0x00000000:
+                if (trans_fmv_h_x(ctx, &u.f_decode_insn3213)) return true;
+                return false;
+            }
+            return false;
         }
         return false;
     case 0x00000063:
@@ -1681,12 +2357,31 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 decode_insn32_extract_sfence_vma(ctx, &u.f_decode_insn3214, insn);
                 if (trans_sfence_vma(ctx, &u.f_decode_insn3214)) return true;
                 return false;
+            case 0x16000000:
+                decode_insn32_extract_sfence_vma(ctx, &u.f_decode_insn3214, insn);
+                if (trans_sinval_vma(ctx, &u.f_decode_insn3214)) return true;
+                return false;
+            case 0x18000000:
+                decode_insn32_extract_decode_insn32_Fmt_18(ctx, &u.f_empty, insn);
+                switch ((insn >> 15) & 0x3ff) {
+                case 0x0:
+                    if (trans_sfence_w_inval(ctx, &u.f_empty)) return true;
+                    return false;
+                case 0x20:
+                    if (trans_sfence_inval_ir(ctx, &u.f_empty)) return true;
+                    return false;
+                }
+                return false;
             case 0x22000000:
                 /* 0010001. ........ .0000000 01110011 */
                 /* /home/me/projects/unicorn2/qemu-5.0.0-build/target/riscv/insn32.decode:81 */
                 decode_insn32_extract_hfence_bvma(ctx, &u.f_decode_insn3214, insn);
                 if (trans_hfence_bvma(ctx, &u.f_decode_insn3214)) return true;
                 return false;
+            case 0x26000000:
+                decode_insn32_extract_hfence_bvma(ctx, &u.f_decode_insn3214, insn);
+                if (trans_hinval_vvma(ctx, &u.f_decode_insn3214)) return true;
+                return false;
             case 0x30000000:
                 /* 0011000. ........ .0000000 01110011 */
                 decode_insn32_extract_decode_insn32_Fmt_18(ctx, &u.f_empty, insn);
@@ -1704,6 +2399,10 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
                 decode_insn32_extract_hfence_gvma(ctx, &u.f_decode_insn3214, insn);
                 if (trans_hfence_gvma(ctx, &u.f_decode_insn3214)) return true;
                 return false;
+            case 0x66000000:
+                decode_insn32_extract_hfence_gvma(ctx, &u.f_decode_insn3214, insn);
+                if (trans_hinval_gvma(ctx, &u.f_decode_insn3214)) return true;
+                return false;
             }
             return false;
         case 0x1:
@@ -1744,6 +2443,21 @@ static bool decode_insn32(DisasContext *ctx, uint32_t insn)
             return false;
         }
         return false;
+    case 0x0000007b:
+        /* ....... ........ ........ .1111011 */
+        if ((insn & 0xfe00707f) == 0x0000607b) {
+            /* 0000000. ........ .110.... .1111011 */
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            if (trans_vt_maskc(ctx, &u.f_r)) return true;
+            return false;
+        }
+        if ((insn & 0xfe00707f) == 0x0000707b) {
+            /* 0000000. ........ .111.... .1111011 */
+            decode_insn32_extract_r(ctx, &u.f_r, insn);
+            if (trans_vt_maskcn(ctx, &u.f_r)) return true;
+            return false;
+        }
+        return false;
     }
     return false;
 }
diff --git a/qemu/target/riscv/translate.c b/qemu/target/riscv/translate.c
index 6aedcb86bc..30a69177f9 100644
--- a/qemu/target/riscv/translate.c
+++ b/qemu/target/riscv/translate.c
@@ -26,6 +26,7 @@
 #include "exec/helper-gen.h"
 
 #include "exec/translator.h"
+#include "tcg/tcg-gvec-desc.h"
 
 #include "instmap.h"
 
@@ -34,6 +35,10 @@
 
 #include "exec/gen-icount.h"
 
+#define RISCV_FRM_RTZ 1
+#define RISCV_FRM_DYN 7
+#define RISCV_FRM_ROD 8
+
 /*
  * Unicorn: Special disas state for exiting in the middle of tb.
  */
@@ -46,9 +51,12 @@ typedef struct DisasContext {
     target_ulong priv_ver;
     bool virt_enabled;
     uint32_t opcode;
+    TCGOp *insn_start;
     uint32_t mstatus_fs;
+    uint32_t mstatus_vs;
     uint32_t misa;
     uint32_t mem_idx;
+    bool hlsx;
     /* Remember the rounding mode encoded in the previous fp instruction,
        which we have already installed into env->fp_status.  Or -1 for
        no previous fp instruction.  Note that we exit the TB when writing
@@ -56,6 +64,35 @@ typedef struct DisasContext {
        to reset this known value.  */
     int frm;
     bool ext_ifencei;
+    bool ext_zihintpause;
+    bool ext_zba;
+    bool ext_zbb;
+    bool ext_zbc;
+    bool ext_zbkb;
+    bool ext_zbkc;
+    bool ext_zbkx;
+    bool ext_zbs;
+    bool ext_zfh;
+    bool ext_zfhmin;
+    bool ext_zknd;
+    bool ext_zkne;
+    bool ext_zknh;
+    bool ext_zksed;
+    bool ext_zksh;
+    bool ext_svinval;
+    bool ext_xventanacondops;
+    bool ext_zmmul;
+    bool ext_zve32f;
+    bool ext_zve64f;
+    bool vill;
+    int8_t lmul;
+    uint8_t sew;
+    uint8_t vta;
+    uint8_t vma;
+    bool rvv_ta_all_1s;
+    bool rvv_ma_all_1s;
+    uint16_t vlen;
+    uint16_t elen;
 
     // Unicorn
     struct uc_struct *uc;
@@ -77,6 +114,9 @@ static const int tcg_memop_lookup[8] = {
 };
 #endif
 
+static TCGv cpu_vl;
+static TCGv cpu_vstart;
+
 #ifdef TARGET_RISCV64
 #define CASE_OP_32_64(X) case X: case glue(X, W)
 #else
@@ -88,6 +128,12 @@ static inline bool has_ext(DisasContext *ctx, uint32_t ext)
     return ctx->misa & ext;
 }
 
+static void decode_save_opc(DisasContext *ctx)
+{
+    tcg_set_insn_start_param(ctx->insn_start, 1, ctx->opcode);
+    ctx->insn_start = NULL;
+}
+
 static void generate_exception(DisasContext *ctx, int excp)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -99,6 +145,16 @@ static void generate_exception(DisasContext *ctx, int excp)
     ctx->base.is_jmp = DISAS_NORETURN;
 }
 
+static void gen_store_binst(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv opcode = tcg_const_tl(tcg_ctx, ctx->opcode);
+
+    tcg_gen_st_tl(tcg_ctx, opcode, tcg_ctx->cpu_env,
+                  offsetof(CPURISCVState, bins));
+    tcg_temp_free(tcg_ctx, opcode);
+}
+
 static void generate_exception_mbadaddr(DisasContext *ctx, int excp)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -144,9 +200,16 @@ static void lookup_and_goto_ptr(DisasContext *ctx)
 
 static void gen_exception_illegal(DisasContext *ctx)
 {
+    gen_store_binst(ctx);
     generate_exception(ctx, RISCV_EXCP_ILLEGAL_INST);
 }
 
+static void gen_exception_virtual_instruction(DisasContext *ctx)
+{
+    gen_store_binst(ctx);
+    generate_exception(ctx, RISCV_EXCP_VIRT_INSTRUCTION_FAULT);
+}
+
 static void gen_exception_inst_addr_mis(DisasContext *ctx)
 {
     generate_exception_mbadaddr(ctx, RISCV_EXCP_INST_ADDR_MIS);
@@ -421,6 +484,34 @@ static void mark_fs_dirty(DisasContext *ctx)
     tcg_temp_free(tcg_ctx, tmp);
 }
 
+static void mark_vs_dirty(DisasContext *ctx)
+{
+    TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
+    TCGv tmp;
+
+    if (ctx->mstatus_vs == MSTATUS_VS) {
+        return;
+    }
+
+    ctx->mstatus_vs = MSTATUS_VS;
+
+    tmp = tcg_temp_new(tcg_ctx);
+    tcg_gen_ld_tl(tcg_ctx, tmp, tcg_ctx->cpu_env,
+                  offsetof(CPURISCVState, mstatus));
+    tcg_gen_ori_tl(tcg_ctx, tmp, tmp, MSTATUS_VS | MSTATUS_SD);
+    tcg_gen_st_tl(tcg_ctx, tmp, tcg_ctx->cpu_env,
+                  offsetof(CPURISCVState, mstatus));
+
+    if (ctx->virt_enabled) {
+        tcg_gen_ld_tl(tcg_ctx, tmp, tcg_ctx->cpu_env,
+                      offsetof(CPURISCVState, mstatus_hs));
+        tcg_gen_ori_tl(tcg_ctx, tmp, tmp, MSTATUS_VS | MSTATUS_SD);
+        tcg_gen_st_tl(tcg_ctx, tmp, tcg_ctx->cpu_env,
+                      offsetof(CPURISCVState, mstatus_hs));
+    }
+    tcg_temp_free(tcg_ctx, tmp);
+}
+
 #if !defined(TARGET_RISCV64)
 static void gen_fp_load(DisasContext *ctx, uint32_t opc, int rd,
         int rs1, target_long imm)
@@ -509,6 +600,10 @@ static void gen_set_rm(DisasContext *ctx, int rm)
         return;
     }
     ctx->frm = rm;
+    if (rm == RISCV_FRM_ROD) {
+        gen_helper_set_rod_rounding_mode(tcg_ctx, tcg_ctx->cpu_env);
+        return;
+    }
     t0 = tcg_const_i32(tcg_ctx, rm);
     gen_helper_set_rounding_mode(tcg_ctx, tcg_ctx->cpu_env, t0);
     tcg_temp_free_i32(tcg_ctx, t0);
@@ -733,7 +828,14 @@ static bool gen_shift(DisasContext *ctx, arg_r *a,
 #include "insn_trans/trans_rva.inc.c"
 #include "insn_trans/trans_rvf.inc.c"
 #include "insn_trans/trans_rvd.inc.c"
+#include "insn_trans/trans_rvzfh.inc.c"
+#include "insn_trans/trans_rvb.inc.c"
+#include "insn_trans/trans_rvk.inc.c"
+#include "insn_trans/trans_rvv.inc.c"
+#include "insn_trans/trans_rvh.inc.c"
 #include "insn_trans/trans_privileged.inc.c"
+#include "insn_trans/trans_svinval.inc.c"
+#include "insn_trans/trans_xventanacondops.inc.c"
 
 /* Include the auto-generated decoder for 16 bit insn */
 #ifdef TARGET_RISCV32
@@ -746,8 +848,11 @@ static void decode_opc(CPURISCVState *env, DisasContext *ctx, uint16_t opcode)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
 
+    ctx->opcode = opcode;
+
     /* check for compressed insn */
     if (extract16(opcode, 0, 2) != 3) {
+        decode_save_opc(ctx);
         if (!has_ext(ctx, RVC)) {
             gen_exception_illegal(ctx);
         } else {
@@ -767,8 +872,12 @@ static void decode_opc(CPURISCVState *env, DisasContext *ctx, uint16_t opcode)
         uint32_t opcode32 = opcode;
         opcode32 = deposit32(opcode32, 16, 16,
                              translator_lduw(tcg_ctx, env, ctx->base.pc_next + 2));
+        ctx->opcode = opcode32;
+        decode_save_opc(ctx);
         ctx->pc_succ_insn = ctx->base.pc_next + 4;
-        if (!decode_insn32(ctx, opcode32)) {
+        if (!decode_rvh(ctx, opcode32) &&
+            !decode_rvv(ctx, opcode32) &&
+            !decode_insn32(ctx, opcode32)) {
             gen_exception_illegal(ctx);
         }
     }
@@ -786,27 +895,55 @@ static void riscv_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs)
     ctx->pc_succ_insn = ctx->base.pc_first;
     ctx->mem_idx = ctx->base.tb->flags & TB_FLAGS_MMU_MASK;
     ctx->mstatus_fs = ctx->base.tb->flags & TB_FLAGS_MSTATUS_FS;
+    ctx->mstatus_vs = ctx->base.tb->flags & TB_FLAGS_MSTATUS_VS;
     ctx->priv_ver = env->priv_ver;
 
     if (riscv_has_ext(env, RVH)) {
         ctx->virt_enabled = riscv_cpu_virt_enabled(env);
-        if (env->priv_ver == PRV_M &&
+        ctx->hlsx = FIELD_EX32(ctx->base.tb->flags, TB_FLAGS, HLSX);
+        if (env->priv == PRV_M &&
             get_field(env->mstatus, MSTATUS_MPRV) &&
             MSTATUS_MPV_ISSET(env)) {
             ctx->virt_enabled = true;
-        } else if (env->priv == PRV_S &&
-                   !riscv_cpu_virt_enabled(env) &&
-                   get_field(env->hstatus, HSTATUS_SPRV) &&
-                   get_field(env->hstatus, HSTATUS_SPV)) {
-            ctx->virt_enabled = true;
         }
     } else {
         ctx->virt_enabled = false;
+        ctx->hlsx = false;
     }
 
     ctx->misa = env->misa;
     ctx->frm = -1;  /* unknown rounding mode */
     ctx->ext_ifencei = cpu->cfg.ext_ifencei;
+    ctx->ext_zihintpause = cpu->cfg.ext_zihintpause;
+    ctx->ext_zba = cpu->cfg.ext_zba;
+    ctx->ext_zbb = cpu->cfg.ext_zbb;
+    ctx->ext_zbc = cpu->cfg.ext_zbc;
+    ctx->ext_zbkb = cpu->cfg.ext_zbkb;
+    ctx->ext_zbkc = cpu->cfg.ext_zbkc;
+    ctx->ext_zbkx = cpu->cfg.ext_zbkx;
+    ctx->ext_zbs = cpu->cfg.ext_zbs;
+    ctx->ext_zfh = cpu->cfg.ext_zfh;
+    ctx->ext_zfhmin = cpu->cfg.ext_zfhmin;
+    ctx->ext_zknd = cpu->cfg.ext_zknd;
+    ctx->ext_zkne = cpu->cfg.ext_zkne;
+    ctx->ext_zknh = cpu->cfg.ext_zknh;
+    ctx->ext_zksed = cpu->cfg.ext_zksed;
+    ctx->ext_zksh = cpu->cfg.ext_zksh;
+    ctx->ext_svinval = cpu->cfg.ext_svinval;
+    ctx->ext_xventanacondops = cpu->cfg.ext_xventanacondops;
+    ctx->ext_zmmul = cpu->cfg.ext_zmmul;
+    ctx->ext_zve32f = cpu->cfg.ext_zve32f;
+    ctx->ext_zve64f = cpu->cfg.ext_zve64f;
+    ctx->vill = FIELD_EX32(ctx->base.tb->flags, TB_FLAGS, VILL);
+    ctx->lmul = sextract32(FIELD_EX32(ctx->base.tb->flags, TB_FLAGS, LMUL),
+                           0, 3);
+    ctx->sew = FIELD_EX32(ctx->base.tb->flags, TB_FLAGS, SEW);
+    ctx->vta = FIELD_EX32(ctx->base.tb->flags, TB_FLAGS, VTA);
+    ctx->vma = FIELD_EX32(ctx->base.tb->flags, TB_FLAGS, VMA);
+    ctx->rvv_ta_all_1s = cpu->cfg.rvv_ta_all_1s;
+    ctx->rvv_ma_all_1s = cpu->cfg.rvv_ma_all_1s;
+    ctx->vlen = cpu->cfg.vlen;
+    ctx->elen = cpu->cfg.elen;
 }
 
 static void riscv_tr_tb_start(DisasContextBase *db, CPUState *cpu)
@@ -818,7 +955,8 @@ static void riscv_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu)
     DisasContext *ctx = container_of(dcbase, DisasContext, base);
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
 
-    tcg_gen_insn_start(tcg_ctx, ctx->base.pc_next);
+    tcg_gen_insn_start(tcg_ctx, ctx->base.pc_next, 0);
+    ctx->insn_start = tcg_last_op(tcg_ctx);
 }
 
 static bool riscv_tr_breakpoint_check(DisasContextBase *dcbase, CPUState *cpu,
@@ -966,6 +1104,11 @@ void riscv_translate_init(struct uc_struct *uc)
             offsetof(CPURISCVState, fpr[i]), riscv_fpr_regnames[i]);
     }
 
+    cpu_vl = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env,
+                                offsetof(CPURISCVState, vl), "vl");
+    cpu_vstart = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env,
+                                    offsetof(CPURISCVState, vstart),
+                                    "vstart");
     tcg_ctx->cpu_pc = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env, offsetof(CPURISCVState, pc), "pc");
     tcg_ctx->load_res = tcg_global_mem_new(tcg_ctx, tcg_ctx->cpu_env, offsetof(CPURISCVState, load_res),
                              "load_res");
diff --git a/qemu/target/riscv/unicorn.c b/qemu/target/riscv/unicorn.c
index 80c282580c..acf4f1375b 100644
--- a/qemu/target/riscv/unicorn.c
+++ b/qemu/target/riscv/unicorn.c
@@ -9,6 +9,7 @@
 #include "cpu_bits.h"
 #include <unicorn/riscv.h>
 #include "unicorn.h"
+#include <string.h>
 
 static int csrno_map[] = {
     CSR_USTATUS,       CSR_UIE,           CSR_UTVEC,         CSR_USCRATCH,
@@ -136,6 +137,112 @@ static uc_err reg_write_priv(CPURISCVState *env, target_ulong value)
     return UC_ERR_OK;
 }
 
+static uc_err riscv_read_target_ulong(target_ulong val, void *value,
+                                      size_t *size)
+{
+#ifdef TARGET_RISCV64
+    if (unlikely(*size < sizeof(uint64_t))) {
+        return UC_ERR_OVERFLOW;
+    }
+    *size = sizeof(uint64_t);
+    *(uint64_t *)value = val;
+#else
+    if (unlikely(*size < sizeof(uint32_t))) {
+        return UC_ERR_OVERFLOW;
+    }
+    *size = sizeof(uint32_t);
+    *(uint32_t *)value = val;
+#endif
+
+    return UC_ERR_OK;
+}
+
+static uc_err riscv_get_target_ulong(const void *value, size_t *size,
+                                     target_ulong *val)
+{
+#ifdef TARGET_RISCV64
+    if (unlikely(*size < sizeof(uint64_t))) {
+        return UC_ERR_OVERFLOW;
+    }
+    *size = sizeof(uint64_t);
+    *val = *(const uint64_t *)value;
+#else
+    if (unlikely(*size < sizeof(uint32_t))) {
+        return UC_ERR_OVERFLOW;
+    }
+    *size = sizeof(uint32_t);
+    *val = *(const uint32_t *)value;
+#endif
+
+    return UC_ERR_OK;
+}
+
+static target_ulong riscv_get_vtype(CPURISCVState *env)
+{
+    target_ulong vill = env->vill ?
+        ((target_ulong)1 << (TARGET_LONG_BITS - 1)) : 0;
+
+    return vill | env->vtype;
+}
+
+static void riscv_mark_vector_dirty(CPURISCVState *env)
+{
+    env->mstatus |= MSTATUS_VS | MSTATUS_SD;
+}
+
+static size_t riscv_vlenb(CPURISCVState *env)
+{
+    return env_archcpu(env)->cfg.vlen >> 3;
+}
+
+static uint8_t *riscv_vreg_ptr(CPURISCVState *env, unsigned int regid)
+{
+    return (uint8_t *)env->vreg +
+           (regid - UC_RISCV_REG_V0) * riscv_vlenb(env);
+}
+
+static void riscv_write_vl(CPURISCVState *env, target_ulong value)
+{
+    target_ulong vlmax = 0;
+
+    if (!env->vill) {
+        vlmax = vext_get_vlmax(env_archcpu(env), env->vtype);
+    }
+    env->vl = value <= vlmax ? value : vlmax;
+    riscv_mark_vector_dirty(env);
+}
+
+static void riscv_write_vtype(CPURISCVState *env, target_ulong value)
+{
+    RISCVCPU *cpu = env_archcpu(env);
+    uint64_t lmul = FIELD_EX64(value, VTYPE, VLMUL);
+    uint16_t sew = 8 << FIELD_EX64(value, VTYPE, VSEW);
+    uint8_t ediv = FIELD_EX64(value, VTYPE, VEDIV);
+    bool vill = (value >> (TARGET_LONG_BITS - 1)) & 1;
+    target_ulong reserved;
+
+    reserved = value & MAKE_64BIT_MASK(R_VTYPE_RESERVED_SHIFT,
+                                       TARGET_LONG_BITS - 1 -
+                                       R_VTYPE_RESERVED_SHIFT);
+    if (lmul & 4) {
+        if (lmul == 4 || cpu->cfg.elen >> (8 - lmul) < sew) {
+            vill = true;
+        }
+    }
+
+    if (sew > cpu->cfg.elen || vill || ediv != 0 || reserved != 0) {
+        env->vill = true;
+        env->vtype = 0;
+        env->vl = 0;
+        env->vstart = 0;
+    } else {
+        env->vill = false;
+        env->vtype = value;
+        riscv_write_vl(env, env->vl);
+    }
+    riscv_mark_vector_dirty(env);
+}
+
 DEFAULT_VISIBILITY
 uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
                 size_t *size)
@@ -167,6 +274,11 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
         CHECK_REG_TYPE(uint32_t);
         *(uint32_t *)value = (uint32_t)val;
 #endif
+    } else if (regid >= UC_RISCV_REG_V0 &&
+               regid <= UC_RISCV_REG_V31) {
+        CHECK_REG_TYPE(uc_riscv_vreg);
+        memset(value, 0, sizeof(uc_riscv_vreg));
+        memcpy(value, riscv_vreg_ptr(env, regid), riscv_vlenb(env));
     } else {
         switch (regid) {
         default:
@@ -194,6 +306,30 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
             *(uint32_t *)value = priv_value;
 #endif
             break;
+        case UC_RISCV_REG_VSTART:
+            ret = riscv_read_target_ulong(env->vstart, value, size);
+            break;
+        case UC_RISCV_REG_VXSAT:
+            ret = riscv_read_target_ulong(env->vxsat, value, size);
+            break;
+        case UC_RISCV_REG_VXRM:
+            ret = riscv_read_target_ulong(env->vxrm, value, size);
+            break;
+        case UC_RISCV_REG_VCSR:
+            ret = riscv_read_target_ulong(
+                (env->vxrm << VCSR_VXRM_SHIFT) |
+                (env->vxsat << VCSR_VXSAT_SHIFT), value, size);
+            break;
+        case UC_RISCV_REG_VL:
+            ret = riscv_read_target_ulong(env->vl, value, size);
+            break;
+        case UC_RISCV_REG_VTYPE:
+            ret = riscv_read_target_ulong(riscv_get_vtype(env), value, size);
+            break;
+        case UC_RISCV_REG_VLENB:
+            ret = riscv_read_target_ulong(env_archcpu(env)->cfg.vlen >> 3,
+                                          value, size);
+            break;
         }
     }
 
@@ -231,6 +367,11 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
         CHECK_REG_TYPE(uint32_t);
         riscv_csrrw(env, csrno, &val, *(uint32_t *)value, -1);
 #endif
+    } else if (regid >= UC_RISCV_REG_V0 &&
+               regid <= UC_RISCV_REG_V31) {
+        CHECK_REG_TYPE(uc_riscv_vreg);
+        memcpy(riscv_vreg_ptr(env, regid), value, riscv_vlenb(env));
+        riscv_mark_vector_dirty(env);
     } else {
         switch (regid) {
         default:
@@ -256,6 +397,62 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             val = *(uint32_t *)value;
 #endif
             ret = reg_write_priv(env, (target_ulong)val);
+            break;
+        case UC_RISCV_REG_VSTART:;
+            target_ulong vstart;
+            ret = riscv_get_target_ulong(value, size, &vstart);
+            if (ret == UC_ERR_OK) {
+                RISCVCPU *cpu = env_archcpu(env);
+                env->vstart = vstart & (cpu->cfg.vlen - 1);
+                riscv_mark_vector_dirty(env);
+            }
+            break;
+        case UC_RISCV_REG_VXSAT:;
+            target_ulong vxsat;
+            ret = riscv_get_target_ulong(value, size, &vxsat);
+            if (ret == UC_ERR_OK) {
+                env->vxsat = vxsat & (VCSR_VXSAT >> VCSR_VXSAT_SHIFT);
+                riscv_mark_vector_dirty(env);
+            }
+            break;
+        case UC_RISCV_REG_VXRM:;
+            target_ulong vxrm;
+            ret = riscv_get_target_ulong(value, size, &vxrm);
+            if (ret == UC_ERR_OK) {
+                env->vxrm = vxrm & (VCSR_VXRM >> VCSR_VXRM_SHIFT);
+                riscv_mark_vector_dirty(env);
+            }
+            break;
+        case UC_RISCV_REG_VCSR:;
+            target_ulong vcsr;
+            ret = riscv_get_target_ulong(value, size, &vcsr);
+            if (ret == UC_ERR_OK) {
+                env->vxrm = (vcsr & VCSR_VXRM) >> VCSR_VXRM_SHIFT;
+                env->vxsat = (vcsr & VCSR_VXSAT) >> VCSR_VXSAT_SHIFT;
+                riscv_mark_vector_dirty(env);
+            }
+            break;
+        case UC_RISCV_REG_VL:;
+            target_ulong vl;
+            ret = riscv_get_target_ulong(value, size, &vl);
+            if (ret == UC_ERR_OK) {
+                riscv_write_vl(env, vl);
+            }
+            break;
+        case UC_RISCV_REG_VTYPE:;
+            target_ulong vtype;
+            ret = riscv_get_target_ulong(value, size, &vtype);
+            if (ret == UC_ERR_OK) {
+                riscv_write_vtype(env, vtype);
+            }
+            break;
+        case UC_RISCV_REG_VLENB:;
+            target_ulong ignored;
+            ret = riscv_get_target_ulong(value, size, &ignored);
+            if (ret == UC_ERR_OK) {
+                return UC_ERR_ARG;
+            }
+            return ret;
         }
     }
 
diff --git a/qemu/target/riscv/vector_helper.c b/qemu/target/riscv/vector_helper.c
new file mode 100644
index 0000000000..56d07da707
--- /dev/null
+++ b/qemu/target/riscv/vector_helper.c
@@ -0,0 +1,5145 @@
+/*
+ * RISC-V Vector Extension Helpers for QEMU.
+ *
+ * Copyright (c) 2020 T-Head Semiconductor Co., Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
+#include "exec/cpu_ldst.h"
+#include "exec/helper-proto.h"
+#include "fpu/softfloat.h"
+#include "tcg/tcg-gvec-desc.h"
+
+target_ulong HELPER(vsetvl)(CPURISCVState *env, target_ulong s1,
+                            target_ulong s2)
+{
+    int vlmax;
+    int vl;
+    RISCVCPU *cpu = env_archcpu(env);
+    uint64_t lmul = FIELD_EX64(s2, VTYPE, VLMUL);
+    uint16_t sew = 8 << FIELD_EX64(s2, VTYPE, VSEW);
+    uint8_t ediv = FIELD_EX64(s2, VTYPE, VEDIV);
+    bool vill = (s2 >> (TARGET_LONG_BITS - 1)) & 1;
+    target_ulong reserved;
+
+    reserved = s2 & MAKE_64BIT_MASK(R_VTYPE_RESERVED_SHIFT,
+                                    TARGET_LONG_BITS - 1 -
+                                    R_VTYPE_RESERVED_SHIFT);
+    if (lmul & 4) {
+        if (lmul == 4 || cpu->cfg.elen >> (8 - lmul) < sew) {
+            vill = true;
+        }
+    }
+
+    if (sew > cpu->cfg.elen || vill || ediv != 0 || reserved != 0) {
+        env->vill = true;
+        env->vtype = 0;
+        env->vl = 0;
+        env->vstart = 0;
+        return 0;
+    }
+
+    vlmax = vext_get_vlmax(cpu, s2);
+    if (s1 <= vlmax) {
+        vl = s1;
+    } else {
+        vl = vlmax;
+    }
+
+    env->vl = vl;
+    env->vtype = s2;
+    env->vstart = 0;
+    env->vill = false;
+    return vl;
+}
+
+#if HOST_BIG_ENDIAN
+#define H1(x) ((x) ^ 7)
+#define H2(x) ((x) ^ 3)
+#define H4(x) ((x) ^ 1)
+#define H8(x) (x)
+#else
+#define H1(x) (x)
+#define H2(x) (x)
+#define H4(x) (x)
+#define H8(x) (x)
+#endif
+
+static inline uint32_t vext_nf(uint32_t desc)
+{
+    return FIELD_EX32(simd_data(desc), VDATA, NF);
+}
+
+static inline uint32_t vext_vm(uint32_t desc)
+{
+    return FIELD_EX32(simd_data(desc), VDATA, VM);
+}
+
+static inline int32_t vext_lmul(uint32_t desc)
+{
+    return sextract32(FIELD_EX32(simd_data(desc), VDATA, LMUL), 0, 3);
+}
+
+static inline uint32_t vext_vta(uint32_t desc)
+{
+    return FIELD_EX32(simd_data(desc), VDATA, VTA);
+}
+
+static inline uint32_t vext_vta_all_1s(uint32_t desc)
+{
+    return FIELD_EX32(simd_data(desc), VDATA, VTA_ALL_1S);
+}
+
+static inline uint32_t vext_vma(uint32_t desc)
+{
+    return FIELD_EX32(simd_data(desc), VDATA, VMA);
+}
+
+static inline uint32_t vext_log2_esz(uint32_t esz)
+{
+    switch (esz) {
+    case 1:
+        return 0;
+    case 2:
+        return 1;
+    case 4:
+        return 2;
+    default:
+        return 3;
+    }
+}
+
+static inline uint32_t vext_max_elems(uint32_t desc, uint32_t log2_esz)
+{
+    uint32_t vlenb = simd_maxsz(desc);
+    int scale = vext_lmul(desc) - log2_esz;
+
+    return scale < 0 ? vlenb >> -scale : vlenb << scale;
+}
+
+static inline uint32_t vext_get_total_elems(CPURISCVState *env,
+                                            uint32_t desc, uint32_t esz)
+{
+    uint32_t vlenb = simd_maxsz(desc);
+    uint32_t sew = FIELD_EX64(env->vtype, VTYPE, VSEW);
+    int8_t emul = vext_log2_esz(esz) - sew + vext_lmul(desc);
+
+    if (emul < 0) {
+        emul = 0;
+    }
+    return (vlenb << emul) / esz;
+}
+
+static void vext_set_elems_1s(void *base, uint32_t is_agnostic,
+                              uint32_t cnt, uint32_t tot)
+{
+    uint8_t *ptr = base;
+
+    if (!is_agnostic || tot <= cnt) {
+        return;
+    }
+    memset(ptr + cnt, -1, tot - cnt);
+}
+
+static inline int vext_elem_mask(void *v0, int index)
+{
+    int idx = index / 64;
+    int pos = index % 64;
+
+    return (((uint64_t *)v0)[idx] >> pos) & 1;
+}
+
+static inline void vext_set_elem_mask(void *v0, int index, uint8_t value)
+{
+    int idx = index / 64;
+    int pos = index % 64;
+    uint64_t old = ((uint64_t *)v0)[idx];
+
+    ((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value);
+}
+
+static void probe_pages(CPURISCVState *env, target_ulong addr,
+                        target_ulong len, uintptr_t ra,
+                        MMUAccessType access_type)
+{
+    target_ulong pagelen = (target_ulong)0 - (addr | TARGET_PAGE_MASK);
+    target_ulong curlen = MIN(pagelen, len);
+
+    probe_access(env, addr, curlen, access_type,
+                 cpu_mmu_index(env, false), ra);
+    if (len > curlen) {
+        addr += curlen;
+        curlen = len - curlen;
+        probe_access(env, addr, curlen, access_type,
+                     cpu_mmu_index(env, false), ra);
+    }
+}
+
+typedef void vext_ldst_elem_fn(CPURISCVState *env, target_ulong addr,
+                               uint32_t idx, void *vd, uintptr_t retaddr);
+
+static void lde_b(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint8_t *cur = (uint8_t *)vd + H1(idx);
+
+    *cur = cpu_ldub_data_ra(env, addr, retaddr);
+}
+
+static void lde_h(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint16_t *cur = (uint16_t *)vd + H2(idx);
+
+    *cur = cpu_lduw_data_ra(env, addr, retaddr);
+}
+
+static void lde_w(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint32_t *cur = (uint32_t *)vd + H4(idx);
+
+    *cur = cpu_ldl_data_ra(env, addr, retaddr);
+}
+
+static void lde_d(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint64_t *cur = (uint64_t *)vd + H8(idx);
+
+    *cur = cpu_ldq_data_ra(env, addr, retaddr);
+}
+
+static void ste_b(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint8_t data = *((uint8_t *)vd + H1(idx));
+
+    cpu_stb_data_ra(env, addr, data, retaddr);
+}
+
+static void ste_h(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint16_t data = *((uint16_t *)vd + H2(idx));
+
+    cpu_stw_data_ra(env, addr, data, retaddr);
+}
+
+static void ste_w(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint32_t data = *((uint32_t *)vd + H4(idx));
+
+    cpu_stl_data_ra(env, addr, data, retaddr);
+}
+
+static void ste_d(CPURISCVState *env, target_ulong addr, uint32_t idx,
+                  void *vd, uintptr_t retaddr)
+{
+    uint64_t data = *((uint64_t *)vd + H8(idx));
+
+    cpu_stq_data_ra(env, addr, data, retaddr);
+}
+
+static void vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env,
+                         uint32_t desc, vext_ldst_elem_fn *ldst_elem,
+                         uint32_t log2_esz, uint32_t evl, uintptr_t ra)
+{
+    uint32_t i;
+    uint32_t k;
+    uint32_t nf = vext_nf(desc);
+    uint32_t max_elems = vext_max_elems(desc, log2_esz);
+    uint32_t esz = 1 << log2_esz;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+
+    for (i = env->vstart; i < evl; i++, env->vstart++) {
+        for (k = 0; k < nf; k++) {
+            target_ulong addr = base + ((i * nf + k) << log2_esz);
+
+            ldst_elem(env, addr, i + k * max_elems, vd, ra);
+        }
+    }
+    env->vstart = 0;
+
+    for (k = 0; k < nf; k++) {
+        vext_set_elems_1s(vd, vta, (k * max_elems + evl) * esz,
+                          (k * max_elems + max_elems) * esz);
+    }
+    if ((nf * max_elems) % total_elems != 0) {
+        uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3;
+        uint32_t registers_used =
+            ((nf * max_elems) * esz + (vlenb - 1)) / vlenb;
+
+        vext_set_elems_1s(vd, vta, (nf * max_elems) * esz,
+                          registers_used * vlenb);
+    }
+}
+
+static void vext_ldst_stride(void *vd, void *v0, target_ulong base,
+                             target_ulong stride, CPURISCVState *env,
+                             uint32_t desc, vext_ldst_elem_fn *ldst_elem,
+                             uint32_t log2_esz, uintptr_t ra)
+{
+    uint32_t i;
+    uint32_t k;
+    uint32_t nf = vext_nf(desc);
+    uint32_t max_elems = vext_max_elems(desc, log2_esz);
+    uint32_t esz = 1 << log2_esz;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < env->vl; i++, env->vstart++) {
+        for (k = 0; k < nf; k++) {
+            if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+                vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
+                                  (i + k * max_elems + 1) * esz);
+                continue;
+            }
+            ldst_elem(env, base + stride * i + (k << log2_esz),
+                      i + k * max_elems, vd, ra);
+        }
+    }
+    env->vstart = 0;
+
+    for (k = 0; k < nf; k++) {
+        vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz,
+                          (k * max_elems + max_elems) * esz);
+    }
+    if ((nf * max_elems) % total_elems != 0) {
+        uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3;
+        uint32_t registers_used =
+            ((nf * max_elems) * esz + (vlenb - 1)) / vlenb;
+
+        vext_set_elems_1s(vd, vta, (nf * max_elems) * esz,
+                          registers_used * vlenb);
+    }
+}
+
+#define GEN_VEXT_LD_US(NAME, LOAD_FN, LOG2_ESZ)                      \
+void HELPER(NAME)(void *vd, void *v0, target_ulong base,             \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    if (vext_vm(desc)) {                                             \
+        vext_ldst_us(vd, base, env, desc, LOAD_FN, LOG2_ESZ,         \
+                     env->vl, GETPC());                             \
+    } else {                                                         \
+        uint32_t stride = vext_nf(desc) << LOG2_ESZ;                 \
+                                                                     \
+        vext_ldst_stride(vd, v0, base, stride, env, desc, LOAD_FN,   \
+                         LOG2_ESZ, GETPC());                        \
+    }                                                                \
+}
+
+GEN_VEXT_LD_US(vle8_v, lde_b, 0)
+GEN_VEXT_LD_US(vle16_v, lde_h, 1)
+GEN_VEXT_LD_US(vle32_v, lde_w, 2)
+GEN_VEXT_LD_US(vle64_v, lde_d, 3)
+
+#define GEN_VEXT_ST_US(NAME, STORE_FN, LOG2_ESZ)                     \
+void HELPER(NAME)(void *vd, void *v0, target_ulong base,             \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    if (vext_vm(desc)) {                                             \
+        vext_ldst_us(vd, base, env, desc, STORE_FN, LOG2_ESZ,        \
+                     env->vl, GETPC());                             \
+    } else {                                                         \
+        uint32_t stride = vext_nf(desc) << LOG2_ESZ;                 \
+                                                                     \
+        vext_ldst_stride(vd, v0, base, stride, env, desc, STORE_FN,  \
+                         LOG2_ESZ, GETPC());                        \
+    }                                                                \
+}
+
+GEN_VEXT_ST_US(vse8_v, ste_b, 0)
+GEN_VEXT_ST_US(vse16_v, ste_h, 1)
+GEN_VEXT_ST_US(vse32_v, ste_w, 2)
+GEN_VEXT_ST_US(vse64_v, ste_d, 3)
+
+#define GEN_VEXT_LD_STRIDE(NAME, LOAD_FN, LOG2_ESZ)                 \
+void HELPER(NAME)(void *vd, void *v0, target_ulong base,            \
+                  target_ulong stride, CPURISCVState *env,          \
+                  uint32_t desc)                                    \
+{                                                                   \
+    vext_ldst_stride(vd, v0, base, stride, env, desc, LOAD_FN,      \
+                     LOG2_ESZ, GETPC());                           \
+}
+
+GEN_VEXT_LD_STRIDE(vlse8_v, lde_b, 0)
+GEN_VEXT_LD_STRIDE(vlse16_v, lde_h, 1)
+GEN_VEXT_LD_STRIDE(vlse32_v, lde_w, 2)
+GEN_VEXT_LD_STRIDE(vlse64_v, lde_d, 3)
+
+#define GEN_VEXT_ST_STRIDE(NAME, STORE_FN, LOG2_ESZ)                \
+void HELPER(NAME)(void *vd, void *v0, target_ulong base,            \
+                  target_ulong stride, CPURISCVState *env,          \
+                  uint32_t desc)                                    \
+{                                                                   \
+    vext_ldst_stride(vd, v0, base, stride, env, desc, STORE_FN,     \
+                     LOG2_ESZ, GETPC());                           \
+}
+
+GEN_VEXT_ST_STRIDE(vsse8_v, ste_b, 0)
+GEN_VEXT_ST_STRIDE(vsse16_v, ste_h, 1)
+GEN_VEXT_ST_STRIDE(vsse32_v, ste_w, 2)
+GEN_VEXT_ST_STRIDE(vsse64_v, ste_d, 3)
+
+typedef target_ulong vext_get_index_addr(target_ulong base, uint32_t idx,
+                                         void *vs2);
+
+#define GEN_VEXT_GET_INDEX_ADDR(NAME, ETYPE, H)                    \
+static target_ulong NAME(target_ulong base, uint32_t idx, void *vs2) \
+{                                                                  \
+    return base + *((ETYPE *)vs2 + H(idx));                        \
+}
+
+GEN_VEXT_GET_INDEX_ADDR(idx_b, uint8_t, H1)
+GEN_VEXT_GET_INDEX_ADDR(idx_h, uint16_t, H2)
+GEN_VEXT_GET_INDEX_ADDR(idx_w, uint32_t, H4)
+GEN_VEXT_GET_INDEX_ADDR(idx_d, uint64_t, H8)
+
+static void vext_ldst_index(void *vd, void *v0, target_ulong base,
+                            void *vs2, CPURISCVState *env, uint32_t desc,
+                            vext_get_index_addr *get_index_addr,
+                            vext_ldst_elem_fn *ldst_elem,
+                            uint32_t log2_esz, uintptr_t ra)
+{
+    uint32_t i;
+    uint32_t k;
+    uint32_t nf = vext_nf(desc);
+    uint32_t max_elems = vext_max_elems(desc, log2_esz);
+    uint32_t esz = 1 << log2_esz;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < env->vl; i++, env->vstart++) {
+        for (k = 0; k < nf; k++) {
+            if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+                vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
+                                  (i + k * max_elems + 1) * esz);
+                continue;
+            }
+            ldst_elem(env, get_index_addr(base, i, vs2) + (k << log2_esz),
+                      i + k * max_elems, vd, ra);
+        }
+    }
+    env->vstart = 0;
+
+    for (k = 0; k < nf; k++) {
+        vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz,
+                          (k * max_elems + max_elems) * esz);
+    }
+    if ((nf * max_elems) % total_elems != 0) {
+        uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3;
+        uint32_t registers_used =
+            ((nf * max_elems) * esz + (vlenb - 1)) / vlenb;
+
+        vext_set_elems_1s(vd, vta, (nf * max_elems) * esz,
+                          registers_used * vlenb);
+    }
+}
+
+#define GEN_VEXT_LD_INDEX(NAME, INDEX_FN, LOAD_FN, LOG2_ESZ)        \
+void HELPER(NAME)(void *vd, void *v0, target_ulong base,            \
+                  void *vs2, CPURISCVState *env, uint32_t desc)     \
+{                                                                   \
+    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,         \
+                    LOAD_FN, LOG2_ESZ, GETPC());                   \
+}
+
+GEN_VEXT_LD_INDEX(vlxei8_8_v, idx_b, lde_b, 0)
+GEN_VEXT_LD_INDEX(vlxei8_16_v, idx_b, lde_h, 1)
+GEN_VEXT_LD_INDEX(vlxei8_32_v, idx_b, lde_w, 2)
+GEN_VEXT_LD_INDEX(vlxei8_64_v, idx_b, lde_d, 3)
+GEN_VEXT_LD_INDEX(vlxei16_8_v, idx_h, lde_b, 0)
+GEN_VEXT_LD_INDEX(vlxei16_16_v, idx_h, lde_h, 1)
+GEN_VEXT_LD_INDEX(vlxei16_32_v, idx_h, lde_w, 2)
+GEN_VEXT_LD_INDEX(vlxei16_64_v, idx_h, lde_d, 3)
+GEN_VEXT_LD_INDEX(vlxei32_8_v, idx_w, lde_b, 0)
+GEN_VEXT_LD_INDEX(vlxei32_16_v, idx_w, lde_h, 1)
+GEN_VEXT_LD_INDEX(vlxei32_32_v, idx_w, lde_w, 2)
+GEN_VEXT_LD_INDEX(vlxei32_64_v, idx_w, lde_d, 3)
+GEN_VEXT_LD_INDEX(vlxei64_8_v, idx_d, lde_b, 0)
+GEN_VEXT_LD_INDEX(vlxei64_16_v, idx_d, lde_h, 1)
+GEN_VEXT_LD_INDEX(vlxei64_32_v, idx_d, lde_w, 2)
+GEN_VEXT_LD_INDEX(vlxei64_64_v, idx_d, lde_d, 3)
+
+#define GEN_VEXT_ST_INDEX(NAME, INDEX_FN, STORE_FN, LOG2_ESZ)       \
+void HELPER(NAME)(void *vd, void *v0, target_ulong base,            \
+                  void *vs2, CPURISCVState *env, uint32_t desc)     \
+{                                                                   \
+    vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN,         \
+                    STORE_FN, LOG2_ESZ, GETPC());                  \
+}
+
+GEN_VEXT_ST_INDEX(vsxei8_8_v, idx_b, ste_b, 0)
+GEN_VEXT_ST_INDEX(vsxei8_16_v, idx_b, ste_h, 1)
+GEN_VEXT_ST_INDEX(vsxei8_32_v, idx_b, ste_w, 2)
+GEN_VEXT_ST_INDEX(vsxei8_64_v, idx_b, ste_d, 3)
+GEN_VEXT_ST_INDEX(vsxei16_8_v, idx_h, ste_b, 0)
+GEN_VEXT_ST_INDEX(vsxei16_16_v, idx_h, ste_h, 1)
+GEN_VEXT_ST_INDEX(vsxei16_32_v, idx_h, ste_w, 2)
+GEN_VEXT_ST_INDEX(vsxei16_64_v, idx_h, ste_d, 3)
+GEN_VEXT_ST_INDEX(vsxei32_8_v, idx_w, ste_b, 0)
+GEN_VEXT_ST_INDEX(vsxei32_16_v, idx_w, ste_h, 1)
+GEN_VEXT_ST_INDEX(vsxei32_32_v, idx_w, ste_w, 2)
+GEN_VEXT_ST_INDEX(vsxei32_64_v, idx_w, ste_d, 3)
+GEN_VEXT_ST_INDEX(vsxei64_8_v, idx_d, ste_b, 0)
+GEN_VEXT_ST_INDEX(vsxei64_16_v, idx_d, ste_h, 1)
+GEN_VEXT_ST_INDEX(vsxei64_32_v, idx_d, ste_w, 2)
+GEN_VEXT_ST_INDEX(vsxei64_64_v, idx_d, ste_d, 3)
+
+static void vext_ldff(void *vd, void *v0, target_ulong base,
+                      CPURISCVState *env, uint32_t desc,
+                      vext_ldst_elem_fn *ldst_elem, uint32_t log2_esz,
+                      uintptr_t ra)
+{
+    void *host;
+    uint32_t i;
+    uint32_t k;
+    uint32_t vl = 0;
+    uint32_t nf = vext_nf(desc);
+    uint32_t max_elems = vext_max_elems(desc, log2_esz);
+    uint32_t esz = 1 << log2_esz;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+    target_ulong addr;
+    target_ulong offset;
+    target_ulong remain;
+
+    for (i = env->vstart; i < env->vl; i++) {
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+            continue;
+        }
+        addr = base + i * (nf << log2_esz);
+        if (i == 0) {
+            probe_pages(env, addr, nf << log2_esz, ra, MMU_DATA_LOAD);
+        } else {
+            remain = nf << log2_esz;
+            while (remain > 0) {
+                offset = (target_ulong)0 - (addr | TARGET_PAGE_MASK);
+                host = tlb_vaddr_to_host(env, addr, MMU_DATA_LOAD,
+                                         cpu_mmu_index(env, false));
+                if (host) {
+                    probe_pages(env, addr, offset, ra, MMU_DATA_LOAD);
+                } else {
+                    vl = i;
+                    goto probe_success;
+                }
+                if (remain <= offset) {
+                    break;
+                }
+                remain -= offset;
+                addr += offset;
+            }
+        }
+    }
+
+probe_success:
+    if (vl != 0) {
+        env->vl = vl;
+    }
+    for (i = env->vstart; i < env->vl; i++, env->vstart++) {
+        for (k = 0; k < nf; k++) {
+            if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+                vext_set_elems_1s(vd, vma, (i + k * max_elems) * esz,
+                                  (i + k * max_elems + 1) * esz);
+                continue;
+            }
+            ldst_elem(env, base + ((i * nf + k) << log2_esz),
+                      i + k * max_elems, vd, ra);
+        }
+    }
+    env->vstart = 0;
+
+    for (k = 0; k < nf; k++) {
+        vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz,
+                          (k * max_elems + max_elems) * esz);
+    }
+    if ((nf * max_elems) % total_elems != 0) {
+        uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3;
+        uint32_t registers_used =
+            ((nf * max_elems) * esz + (vlenb - 1)) / vlenb;
+
+        vext_set_elems_1s(vd, vta, (nf * max_elems) * esz,
+                          registers_used * vlenb);
+    }
+}
+
+#define GEN_VEXT_LDFF(NAME, LOAD_FN, LOG2_ESZ)                       \
+void HELPER(NAME)(void *vd, void *v0, target_ulong base,             \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    vext_ldff(vd, v0, base, env, desc, LOAD_FN, LOG2_ESZ, GETPC());  \
+}
+
+GEN_VEXT_LDFF(vle8ff_v, lde_b, 0)
+GEN_VEXT_LDFF(vle16ff_v, lde_h, 1)
+GEN_VEXT_LDFF(vle32ff_v, lde_w, 2)
+GEN_VEXT_LDFF(vle64ff_v, lde_d, 3)
+
+static void vext_ldst_whole(void *vd, target_ulong base,
+                            CPURISCVState *env, uint32_t desc,
+                            vext_ldst_elem_fn *ldst_elem,
+                            uint32_t log2_esz, uintptr_t ra)
+{
+    uint32_t i;
+    uint32_t k;
+    uint32_t off;
+    uint32_t pos;
+    uint32_t nf = vext_nf(desc);
+    uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3;
+    uint32_t max_elems = vlenb >> log2_esz;
+    target_ulong addr;
+
+    k = env->vstart / max_elems;
+    off = env->vstart % max_elems;
+
+    if (off) {
+        for (pos = off; pos < max_elems; pos++, env->vstart++) {
+            addr = base + ((pos + k * max_elems) << log2_esz);
+            ldst_elem(env, addr, pos + k * max_elems, vd, ra);
+        }
+        k++;
+    }
+
+    for (; k < nf; k++) {
+        for (i = 0; i < max_elems; i++, env->vstart++) {
+            addr = base + ((i + k * max_elems) << log2_esz);
+            ldst_elem(env, addr, i + k * max_elems, vd, ra);
+        }
+    }
+
+    env->vstart = 0;
+}
+
+#define GEN_VEXT_LD_WHOLE(NAME, LOAD_FN, LOG2_ESZ)                   \
+void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env,  \
+                  uint32_t desc)                                     \
+{                                                                    \
+    vext_ldst_whole(vd, base, env, desc, LOAD_FN, LOG2_ESZ,          \
+                    GETPC());                                        \
+}
+
+GEN_VEXT_LD_WHOLE(vl1re8_v, lde_b, 0)
+GEN_VEXT_LD_WHOLE(vl1re16_v, lde_h, 1)
+GEN_VEXT_LD_WHOLE(vl1re32_v, lde_w, 2)
+GEN_VEXT_LD_WHOLE(vl1re64_v, lde_d, 3)
+GEN_VEXT_LD_WHOLE(vl2re8_v, lde_b, 0)
+GEN_VEXT_LD_WHOLE(vl2re16_v, lde_h, 1)
+GEN_VEXT_LD_WHOLE(vl2re32_v, lde_w, 2)
+GEN_VEXT_LD_WHOLE(vl2re64_v, lde_d, 3)
+GEN_VEXT_LD_WHOLE(vl4re8_v, lde_b, 0)
+GEN_VEXT_LD_WHOLE(vl4re16_v, lde_h, 1)
+GEN_VEXT_LD_WHOLE(vl4re32_v, lde_w, 2)
+GEN_VEXT_LD_WHOLE(vl4re64_v, lde_d, 3)
+GEN_VEXT_LD_WHOLE(vl8re8_v, lde_b, 0)
+GEN_VEXT_LD_WHOLE(vl8re16_v, lde_h, 1)
+GEN_VEXT_LD_WHOLE(vl8re32_v, lde_w, 2)
+GEN_VEXT_LD_WHOLE(vl8re64_v, lde_d, 3)
+
+#define GEN_VEXT_ST_WHOLE(NAME)                                      \
+void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env,  \
+                  uint32_t desc)                                     \
+{                                                                    \
+    vext_ldst_whole(vd, base, env, desc, ste_b, 0, GETPC());         \
+}
+
+GEN_VEXT_ST_WHOLE(vs1r_v)
+GEN_VEXT_ST_WHOLE(vs2r_v)
+GEN_VEXT_ST_WHOLE(vs4r_v)
+GEN_VEXT_ST_WHOLE(vs8r_v)
+
+void HELPER(vlm_v)(void *vd, void *v0, target_ulong base,
+                   CPURISCVState *env, uint32_t desc)
+{
+    uint32_t evl = (env->vl + 7) >> 3;
+
+    vext_ldst_us(vd, base, env, desc, lde_b, 0, evl, GETPC());
+}
+
+void HELPER(vsm_v)(void *vd, void *v0, target_ulong base,
+                   CPURISCVState *env, uint32_t desc)
+{
+    uint32_t evl = (env->vl + 7) >> 3;
+
+    vext_ldst_us(vd, base, env, desc, ste_b, 0, evl, GETPC());
+}
+
+typedef void opivv2_fn(void *vd, void *vs1, void *vs2, int i);
+
+#define DO_ADD(N, M) ((N) + (M))
+#define DO_SUB(N, M) ((N) - (M))
+#define DO_RSUB(N, M) ((M) - (N))
+#define DO_AND(N, M) ((N) & (M))
+#define DO_OR(N, M)  ((N) | (M))
+#define DO_XOR(N, M) ((N) ^ (M))
+#define DO_MIN(N, M) ((N) < (M) ? (N) : (M))
+#define DO_MAX(N, M) ((N) > (M) ? (N) : (M))
+#define DO_SLL(N, M) ((N) << (M))
+#define DO_SRL(N, M) ((N) >> (M))
+
+#define OPIVV2(NAME, TD, T1, T2, HD, HS1, HS2, OP) \
+static void do_##NAME(void *vd, void *vs1,         \
+                      void *vs2, int i)            \
+{                                                  \
+    T1 s1 = *((T1 *)vs1 + HS1(i));                 \
+    T2 s2 = *((T2 *)vs2 + HS2(i));                 \
+                                                   \
+    *((TD *)vd + HD(i)) = OP(s2, s1);              \
+}
+
+OPIVV2(vadd_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_ADD)
+OPIVV2(vadd_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_ADD)
+OPIVV2(vadd_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_ADD)
+OPIVV2(vadd_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_ADD)
+OPIVV2(vsub_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_SUB)
+OPIVV2(vsub_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_SUB)
+OPIVV2(vsub_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_SUB)
+OPIVV2(vsub_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_SUB)
+OPIVV2(vand_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_AND)
+OPIVV2(vand_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_AND)
+OPIVV2(vand_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_AND)
+OPIVV2(vand_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_AND)
+OPIVV2(vor_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_OR)
+OPIVV2(vor_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_OR)
+OPIVV2(vor_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_OR)
+OPIVV2(vor_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_OR)
+OPIVV2(vxor_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_XOR)
+OPIVV2(vxor_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_XOR)
+OPIVV2(vxor_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_XOR)
+OPIVV2(vxor_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_XOR)
+OPIVV2(vminu_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_MIN)
+OPIVV2(vminu_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_MIN)
+OPIVV2(vminu_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_MIN)
+OPIVV2(vminu_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_MIN)
+OPIVV2(vmin_vv_b, int8_t, int8_t, int8_t, H1, H1, H1, DO_MIN)
+OPIVV2(vmin_vv_h, int16_t, int16_t, int16_t, H2, H2, H2, DO_MIN)
+OPIVV2(vmin_vv_w, int32_t, int32_t, int32_t, H4, H4, H4, DO_MIN)
+OPIVV2(vmin_vv_d, int64_t, int64_t, int64_t, H8, H8, H8, DO_MIN)
+OPIVV2(vmaxu_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_MAX)
+OPIVV2(vmaxu_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_MAX)
+OPIVV2(vmaxu_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_MAX)
+OPIVV2(vmaxu_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_MAX)
+OPIVV2(vmax_vv_b, int8_t, int8_t, int8_t, H1, H1, H1, DO_MAX)
+OPIVV2(vmax_vv_h, int16_t, int16_t, int16_t, H2, H2, H2, DO_MAX)
+OPIVV2(vmax_vv_w, int32_t, int32_t, int32_t, H4, H4, H4, DO_MAX)
+OPIVV2(vmax_vv_d, int64_t, int64_t, int64_t, H8, H8, H8, DO_MAX)
+
+#define RVVCALL(MACRO, NAME, ...) MACRO(NAME, __VA_ARGS__)
+
+#define OP_SSS_B int8_t, int8_t, int8_t, int8_t, int8_t
+#define OP_SSS_H int16_t, int16_t, int16_t, int16_t, int16_t
+#define OP_SSS_W int32_t, int32_t, int32_t, int32_t, int32_t
+#define OP_SSS_D int64_t, int64_t, int64_t, int64_t, int64_t
+#define OP_UUU_B uint8_t, uint8_t, uint8_t, uint8_t, uint8_t
+#define OP_UUU_H uint16_t, uint16_t, uint16_t, uint16_t, uint16_t
+#define OP_UUU_W uint32_t, uint32_t, uint32_t, uint32_t, uint32_t
+#define OP_UUU_D uint64_t, uint64_t, uint64_t, uint64_t, uint64_t
+#define OP_UU_H uint16_t, uint16_t, uint16_t
+#define OP_UU_W uint32_t, uint32_t, uint32_t
+#define OP_UU_D uint64_t, uint64_t, uint64_t
+#define WOP_UU_B uint16_t, uint8_t, uint8_t
+#define WOP_UU_H uint32_t, uint16_t, uint16_t
+#define WOP_UU_W uint64_t, uint32_t, uint32_t
+#define NOP_UU_B uint8_t, uint16_t, uint32_t
+#define NOP_UU_H uint16_t, uint32_t, uint32_t
+#define NOP_UU_W uint32_t, uint64_t, uint64_t
+
+#define OPIVV2_WIDE(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
+static void do_##NAME(void *vd, void *vs1, void *vs2, int i)      \
+{                                                                 \
+    TX1 s1 = *((T1 *)vs1 + HS1(i));                               \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                               \
+                                                                  \
+    *((TD *)vd + HD(i)) = OP(s2, s1);                             \
+}
+
+#define WOP_UUU_B uint16_t, uint8_t, uint8_t, uint16_t, uint16_t
+#define WOP_UUU_H uint32_t, uint16_t, uint16_t, uint32_t, uint32_t
+#define WOP_UUU_W uint64_t, uint32_t, uint32_t, uint64_t, uint64_t
+#define WOP_SSS_B int16_t, int8_t, int8_t, int16_t, int16_t
+#define WOP_SSS_H int32_t, int16_t, int16_t, int32_t, int32_t
+#define WOP_SSS_W int64_t, int32_t, int32_t, int64_t, int64_t
+#define WOP_SSU_B int16_t, int8_t, uint8_t, int16_t, uint16_t
+#define WOP_SSU_H int32_t, int16_t, uint16_t, int32_t, uint32_t
+#define WOP_SSU_W int64_t, int32_t, uint32_t, int64_t, uint64_t
+#define WOP_SUS_B int16_t, uint8_t, int8_t, uint16_t, int16_t
+#define WOP_SUS_H int32_t, uint16_t, int16_t, uint32_t, int32_t
+#define WOP_SUS_W int64_t, uint32_t, int32_t, uint64_t, int64_t
+#define WOP_WUUU_B uint16_t, uint8_t, uint16_t, uint16_t, uint16_t
+#define WOP_WUUU_H uint32_t, uint16_t, uint32_t, uint32_t, uint32_t
+#define WOP_WUUU_W uint64_t, uint32_t, uint64_t, uint64_t, uint64_t
+#define WOP_WSSS_B int16_t, int8_t, int16_t, int16_t, int16_t
+#define WOP_WSSS_H int32_t, int16_t, int32_t, int32_t, int32_t
+#define WOP_WSSS_W int64_t, int32_t, int64_t, int64_t, int64_t
+#define NOP_SSS_B int8_t, int8_t, int16_t, int8_t, int16_t
+#define NOP_SSS_H int16_t, int16_t, int32_t, int16_t, int32_t
+#define NOP_SSS_W int32_t, int32_t, int64_t, int32_t, int64_t
+#define NOP_UUU_B uint8_t, uint8_t, uint16_t, uint8_t, uint16_t
+#define NOP_UUU_H uint16_t, uint16_t, uint32_t, uint16_t, uint32_t
+#define NOP_UUU_W uint32_t, uint32_t, uint64_t, uint32_t, uint64_t
+
+RVVCALL(OPIVV2_WIDE, vwaddu_vv_b, WOP_UUU_B, H2, H1, H1, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwaddu_vv_h, WOP_UUU_H, H4, H2, H2, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwaddu_vv_w, WOP_UUU_W, H8, H4, H4, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwsubu_vv_b, WOP_UUU_B, H2, H1, H1, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsubu_vv_h, WOP_UUU_H, H4, H2, H2, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsubu_vv_w, WOP_UUU_W, H8, H4, H4, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwadd_vv_b, WOP_SSS_B, H2, H1, H1, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwadd_vv_h, WOP_SSS_H, H4, H2, H2, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwadd_vv_w, WOP_SSS_W, H8, H4, H4, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwsub_vv_b, WOP_SSS_B, H2, H1, H1, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsub_vv_h, WOP_SSS_H, H4, H2, H2, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsub_vv_w, WOP_SSS_W, H8, H4, H4, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwaddu_wv_b, WOP_WUUU_B, H2, H1, H1, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwaddu_wv_h, WOP_WUUU_H, H4, H2, H2, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwaddu_wv_w, WOP_WUUU_W, H8, H4, H4, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwsubu_wv_b, WOP_WUUU_B, H2, H1, H1, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsubu_wv_h, WOP_WUUU_H, H4, H2, H2, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsubu_wv_w, WOP_WUUU_W, H8, H4, H4, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwadd_wv_b, WOP_WSSS_B, H2, H1, H1, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwadd_wv_h, WOP_WSSS_H, H4, H2, H2, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwadd_wv_w, WOP_WSSS_W, H8, H4, H4, DO_ADD)
+RVVCALL(OPIVV2_WIDE, vwsub_wv_b, WOP_WSSS_B, H2, H1, H1, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsub_wv_h, WOP_WSSS_H, H4, H2, H2, DO_SUB)
+RVVCALL(OPIVV2_WIDE, vwsub_wv_w, WOP_WSSS_W, H8, H4, H4, DO_SUB)
+
+#define DO_MUL(N, M) ((N) * (M))
+
+RVVCALL(OPIVV2_WIDE, vwmul_vv_b, WOP_SSS_B, H2, H1, H1, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmul_vv_h, WOP_SSS_H, H4, H2, H2, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmul_vv_w, WOP_SSS_W, H8, H4, H4, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmulu_vv_b, WOP_UUU_B, H2, H1, H1, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmulu_vv_h, WOP_UUU_H, H4, H2, H2, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmulu_vv_w, WOP_UUU_W, H8, H4, H4, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmulsu_vv_b, WOP_SUS_B, H2, H1, H1, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmulsu_vv_h, WOP_SUS_H, H4, H2, H2, DO_MUL)
+RVVCALL(OPIVV2_WIDE, vwmulsu_vv_w, WOP_SUS_W, H8, H4, H4, DO_MUL)
+
+#define OPIVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
+static void do_##NAME(void *vd, void *vs1, void *vs2, int i)  \
+{                                                             \
+    TX1 s1 = *((T1 *)vs1 + HS1(i));                           \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                           \
+    TD d = *((TD *)vd + HD(i));                               \
+                                                              \
+    *((TD *)vd + HD(i)) = OP(s2, s1, d);                      \
+}
+
+#define DO_MACC(N, M, D) (((M) * (N)) + (D))
+#define DO_NMSAC(N, M, D) (-((M) * (N)) + (D))
+#define DO_MADD(N, M, D) (((M) * (D)) + (N))
+#define DO_NMSUB(N, M, D) (-((M) * (D)) + (N))
+
+RVVCALL(OPIVV3, vmacc_vv_b, OP_SSS_B, H1, H1, H1, DO_MACC)
+RVVCALL(OPIVV3, vmacc_vv_h, OP_SSS_H, H2, H2, H2, DO_MACC)
+RVVCALL(OPIVV3, vmacc_vv_w, OP_SSS_W, H4, H4, H4, DO_MACC)
+RVVCALL(OPIVV3, vmacc_vv_d, OP_SSS_D, H8, H8, H8, DO_MACC)
+RVVCALL(OPIVV3, vnmsac_vv_b, OP_SSS_B, H1, H1, H1, DO_NMSAC)
+RVVCALL(OPIVV3, vnmsac_vv_h, OP_SSS_H, H2, H2, H2, DO_NMSAC)
+RVVCALL(OPIVV3, vnmsac_vv_w, OP_SSS_W, H4, H4, H4, DO_NMSAC)
+RVVCALL(OPIVV3, vnmsac_vv_d, OP_SSS_D, H8, H8, H8, DO_NMSAC)
+RVVCALL(OPIVV3, vmadd_vv_b, OP_SSS_B, H1, H1, H1, DO_MADD)
+RVVCALL(OPIVV3, vmadd_vv_h, OP_SSS_H, H2, H2, H2, DO_MADD)
+RVVCALL(OPIVV3, vmadd_vv_w, OP_SSS_W, H4, H4, H4, DO_MADD)
+RVVCALL(OPIVV3, vmadd_vv_d, OP_SSS_D, H8, H8, H8, DO_MADD)
+RVVCALL(OPIVV3, vnmsub_vv_b, OP_SSS_B, H1, H1, H1, DO_NMSUB)
+RVVCALL(OPIVV3, vnmsub_vv_h, OP_SSS_H, H2, H2, H2, DO_NMSUB)
+RVVCALL(OPIVV3, vnmsub_vv_w, OP_SSS_W, H4, H4, H4, DO_NMSUB)
+RVVCALL(OPIVV3, vnmsub_vv_d, OP_SSS_D, H8, H8, H8, DO_NMSUB)
+RVVCALL(OPIVV3, vwmaccu_vv_b, WOP_UUU_B, H2, H1, H1, DO_MACC)
+RVVCALL(OPIVV3, vwmaccu_vv_h, WOP_UUU_H, H4, H2, H2, DO_MACC)
+RVVCALL(OPIVV3, vwmaccu_vv_w, WOP_UUU_W, H8, H4, H4, DO_MACC)
+RVVCALL(OPIVV3, vwmacc_vv_b, WOP_SSS_B, H2, H1, H1, DO_MACC)
+RVVCALL(OPIVV3, vwmacc_vv_h, WOP_SSS_H, H4, H2, H2, DO_MACC)
+RVVCALL(OPIVV3, vwmacc_vv_w, WOP_SSS_W, H8, H4, H4, DO_MACC)
+RVVCALL(OPIVV3, vwmaccsu_vv_b, WOP_SSU_B, H2, H1, H1, DO_MACC)
+RVVCALL(OPIVV3, vwmaccsu_vv_h, WOP_SSU_H, H4, H2, H2, DO_MACC)
+RVVCALL(OPIVV3, vwmaccsu_vv_w, WOP_SSU_W, H8, H4, H4, DO_MACC)
+
+OPIVV2(vmul_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, DO_MUL)
+OPIVV2(vmul_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, DO_MUL)
+OPIVV2(vmul_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, DO_MUL)
+OPIVV2(vmul_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, DO_MUL)
+
+static int8_t do_mulh_b(int8_t s2, int8_t s1)
+{
+    return (int16_t)s2 * (int16_t)s1 >> 8;
+}
+
+static int16_t do_mulh_h(int16_t s2, int16_t s1)
+{
+    return (int32_t)s2 * (int32_t)s1 >> 16;
+}
+
+static int32_t do_mulh_w(int32_t s2, int32_t s1)
+{
+    return (int64_t)s2 * (int64_t)s1 >> 32;
+}
+
+static int64_t do_mulh_d(int64_t s2, int64_t s1)
+{
+    uint64_t hi_64;
+    uint64_t lo_64;
+
+    muls64(&lo_64, &hi_64, s1, s2);
+    return hi_64;
+}
+
+static uint8_t do_mulhu_b(uint8_t s2, uint8_t s1)
+{
+    return (uint16_t)s2 * (uint16_t)s1 >> 8;
+}
+
+static uint16_t do_mulhu_h(uint16_t s2, uint16_t s1)
+{
+    return (uint32_t)s2 * (uint32_t)s1 >> 16;
+}
+
+static uint32_t do_mulhu_w(uint32_t s2, uint32_t s1)
+{
+    return (uint64_t)s2 * (uint64_t)s1 >> 32;
+}
+
+static uint64_t do_mulhu_d(uint64_t s2, uint64_t s1)
+{
+    uint64_t hi_64;
+    uint64_t lo_64;
+
+    mulu64(&lo_64, &hi_64, s2, s1);
+    return hi_64;
+}
+
+static int8_t do_mulhsu_b(int8_t s2, uint8_t s1)
+{
+    return (int16_t)s2 * (uint16_t)s1 >> 8;
+}
+
+static int16_t do_mulhsu_h(int16_t s2, uint16_t s1)
+{
+    return (int32_t)s2 * (uint32_t)s1 >> 16;
+}
+
+static int32_t do_mulhsu_w(int32_t s2, uint32_t s1)
+{
+    return (int64_t)s2 * (uint64_t)s1 >> 32;
+}
+
+static int64_t do_mulhsu_d(int64_t s2, uint64_t s1)
+{
+    uint64_t hi_64;
+    uint64_t lo_64;
+
+    mulu64(&lo_64, &hi_64, s2, s1);
+    hi_64 -= s2 < 0 ? s1 : 0;
+    return hi_64;
+}
+
+OPIVV2(vmulh_vv_b, int8_t, int8_t, int8_t,
+       H1, H1, H1, do_mulh_b)
+OPIVV2(vmulh_vv_h, int16_t, int16_t, int16_t,
+       H2, H2, H2, do_mulh_h)
+OPIVV2(vmulh_vv_w, int32_t, int32_t, int32_t,
+       H4, H4, H4, do_mulh_w)
+OPIVV2(vmulh_vv_d, int64_t, int64_t, int64_t,
+       H8, H8, H8, do_mulh_d)
+OPIVV2(vmulhu_vv_b, uint8_t, uint8_t, uint8_t,
+       H1, H1, H1, do_mulhu_b)
+OPIVV2(vmulhu_vv_h, uint16_t, uint16_t, uint16_t,
+       H2, H2, H2, do_mulhu_h)
+OPIVV2(vmulhu_vv_w, uint32_t, uint32_t, uint32_t,
+       H4, H4, H4, do_mulhu_w)
+OPIVV2(vmulhu_vv_d, uint64_t, uint64_t, uint64_t,
+       H8, H8, H8, do_mulhu_d)
+OPIVV2(vmulhsu_vv_b, int8_t, uint8_t, int8_t,
+       H1, H1, H1, do_mulhsu_b)
+OPIVV2(vmulhsu_vv_h, int16_t, uint16_t, int16_t,
+       H2, H2, H2, do_mulhsu_h)
+OPIVV2(vmulhsu_vv_w, int32_t, uint32_t, int32_t,
+       H4, H4, H4, do_mulhsu_w)
+OPIVV2(vmulhsu_vv_d, int64_t, uint64_t, int64_t,
+       H8, H8, H8, do_mulhsu_d)
+
+static uint8_t do_divu_b(uint8_t s2, uint8_t s1)
+{
+    return s1 == 0 ? (uint8_t)-1 : s2 / s1;
+}
+
+static uint16_t do_divu_h(uint16_t s2, uint16_t s1)
+{
+    return s1 == 0 ? (uint16_t)-1 : s2 / s1;
+}
+
+static uint32_t do_divu_w(uint32_t s2, uint32_t s1)
+{
+    return s1 == 0 ? (uint32_t)-1 : s2 / s1;
+}
+
+static uint64_t do_divu_d(uint64_t s2, uint64_t s1)
+{
+    return s1 == 0 ? (uint64_t)-1 : s2 / s1;
+}
+
+static int8_t do_div_b(int8_t s2, int8_t s1)
+{
+    if (s1 == 0) {
+        return -1;
+    }
+    if (s2 == INT8_MIN && s1 == -1) {
+        return s2;
+    }
+    return s2 / s1;
+}
+
+static int16_t do_div_h(int16_t s2, int16_t s1)
+{
+    if (s1 == 0) {
+        return -1;
+    }
+    if (s2 == INT16_MIN && s1 == -1) {
+        return s2;
+    }
+    return s2 / s1;
+}
+
+static int32_t do_div_w(int32_t s2, int32_t s1)
+{
+    if (s1 == 0) {
+        return -1;
+    }
+    if (s2 == INT32_MIN && s1 == -1) {
+        return s2;
+    }
+    return s2 / s1;
+}
+
+static int64_t do_div_d(int64_t s2, int64_t s1)
+{
+    if (s1 == 0) {
+        return -1;
+    }
+    if (s2 == INT64_MIN && s1 == -1) {
+        return s2;
+    }
+    return s2 / s1;
+}
+
+static uint8_t do_remu_b(uint8_t s2, uint8_t s1)
+{
+    return s1 == 0 ? s2 : s2 % s1;
+}
+
+static uint16_t do_remu_h(uint16_t s2, uint16_t s1)
+{
+    return s1 == 0 ? s2 : s2 % s1;
+}
+
+static uint32_t do_remu_w(uint32_t s2, uint32_t s1)
+{
+    return s1 == 0 ? s2 : s2 % s1;
+}
+
+static uint64_t do_remu_d(uint64_t s2, uint64_t s1)
+{
+    return s1 == 0 ? s2 : s2 % s1;
+}
+
+static int8_t do_rem_b(int8_t s2, int8_t s1)
+{
+    if (s1 == 0) {
+        return s2;
+    }
+    if (s2 == INT8_MIN && s1 == -1) {
+        return 0;
+    }
+    return s2 % s1;
+}
+
+static int16_t do_rem_h(int16_t s2, int16_t s1)
+{
+    if (s1 == 0) {
+        return s2;
+    }
+    if (s2 == INT16_MIN && s1 == -1) {
+        return 0;
+    }
+    return s2 % s1;
+}
+
+static int32_t do_rem_w(int32_t s2, int32_t s1)
+{
+    if (s1 == 0) {
+        return s2;
+    }
+    if (s2 == INT32_MIN && s1 == -1) {
+        return 0;
+    }
+    return s2 % s1;
+}
+
+static int64_t do_rem_d(int64_t s2, int64_t s1)
+{
+    if (s1 == 0) {
+        return s2;
+    }
+    if (s2 == INT64_MIN && s1 == -1) {
+        return 0;
+    }
+    return s2 % s1;
+}
+
+OPIVV2(vdivu_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, do_divu_b)
+OPIVV2(vdivu_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, do_divu_h)
+OPIVV2(vdivu_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, do_divu_w)
+OPIVV2(vdivu_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, do_divu_d)
+OPIVV2(vdiv_vv_b, int8_t, int8_t, int8_t, H1, H1, H1, do_div_b)
+OPIVV2(vdiv_vv_h, int16_t, int16_t, int16_t, H2, H2, H2, do_div_h)
+OPIVV2(vdiv_vv_w, int32_t, int32_t, int32_t, H4, H4, H4, do_div_w)
+OPIVV2(vdiv_vv_d, int64_t, int64_t, int64_t, H8, H8, H8, do_div_d)
+OPIVV2(vremu_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1, do_remu_b)
+OPIVV2(vremu_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2, do_remu_h)
+OPIVV2(vremu_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4, do_remu_w)
+OPIVV2(vremu_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8, do_remu_d)
+OPIVV2(vrem_vv_b, int8_t, int8_t, int8_t, H1, H1, H1, do_rem_b)
+OPIVV2(vrem_vv_h, int16_t, int16_t, int16_t, H2, H2, H2, do_rem_h)
+OPIVV2(vrem_vv_w, int32_t, int32_t, int32_t, H4, H4, H4, do_rem_w)
+OPIVV2(vrem_vv_d, int64_t, int64_t, int64_t, H8, H8, H8, do_rem_d)
+
+#define OPIVV2_SHIFT(NAME, TD, T1, T2, HD, HS1, HS2, OP, MASK) \
+static void do_##NAME(void *vd, void *vs1,                    \
+                      void *vs2, int i)                       \
+{                                                             \
+    T1 s1 = *((T1 *)vs1 + HS1(i));                            \
+    T2 s2 = *((T2 *)vs2 + HS2(i));                            \
+                                                              \
+    *((TD *)vd + HD(i)) = (TD)OP(s2, s1 & MASK);              \
+}
+
+OPIVV2_SHIFT(vsll_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1,
+             DO_SLL, 0x7)
+OPIVV2_SHIFT(vsll_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2,
+             DO_SLL, 0xf)
+OPIVV2_SHIFT(vsll_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4,
+             DO_SLL, 0x1f)
+OPIVV2_SHIFT(vsll_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8,
+             DO_SLL, 0x3f)
+OPIVV2_SHIFT(vsrl_vv_b, uint8_t, uint8_t, uint8_t, H1, H1, H1,
+             DO_SRL, 0x7)
+OPIVV2_SHIFT(vsrl_vv_h, uint16_t, uint16_t, uint16_t, H2, H2, H2,
+             DO_SRL, 0xf)
+OPIVV2_SHIFT(vsrl_vv_w, uint32_t, uint32_t, uint32_t, H4, H4, H4,
+             DO_SRL, 0x1f)
+OPIVV2_SHIFT(vsrl_vv_d, uint64_t, uint64_t, uint64_t, H8, H8, H8,
+             DO_SRL, 0x3f)
+OPIVV2_SHIFT(vsra_vv_b, uint8_t, uint8_t, int8_t, H1, H1, H1,
+             DO_SRL, 0x7)
+OPIVV2_SHIFT(vsra_vv_h, uint16_t, uint16_t, int16_t, H2, H2, H2,
+             DO_SRL, 0xf)
+OPIVV2_SHIFT(vsra_vv_w, uint32_t, uint32_t, int32_t, H4, H4, H4,
+             DO_SRL, 0x1f)
+OPIVV2_SHIFT(vsra_vv_d, uint64_t, uint64_t, int64_t, H8, H8, H8,
+             DO_SRL, 0x3f)
+
+static void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2,
+                       CPURISCVState *env, uint32_t desc,
+                       opivv2_fn *fn, uint32_t esz)
+{
+    uint32_t i;
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, vs1, vs2, i);
+    }
+    env->vstart = 0;
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_VV(NAME, ESZ)                              \
+void HELPER(NAME)(void *vd, void *v0, void *vs1,            \
+                  void *vs2, CPURISCVState *env,            \
+                  uint32_t desc)                            \
+{                                                           \
+    do_vext_vv(vd, v0, vs1, vs2, env, desc, do_##NAME, ESZ);\
+}
+
+GEN_VEXT_VV(vadd_vv_b, 1)
+GEN_VEXT_VV(vadd_vv_h, 2)
+GEN_VEXT_VV(vadd_vv_w, 4)
+GEN_VEXT_VV(vadd_vv_d, 8)
+GEN_VEXT_VV(vsub_vv_b, 1)
+GEN_VEXT_VV(vsub_vv_h, 2)
+GEN_VEXT_VV(vsub_vv_w, 4)
+GEN_VEXT_VV(vsub_vv_d, 8)
+GEN_VEXT_VV(vand_vv_b, 1)
+GEN_VEXT_VV(vand_vv_h, 2)
+GEN_VEXT_VV(vand_vv_w, 4)
+GEN_VEXT_VV(vand_vv_d, 8)
+GEN_VEXT_VV(vor_vv_b, 1)
+GEN_VEXT_VV(vor_vv_h, 2)
+GEN_VEXT_VV(vor_vv_w, 4)
+GEN_VEXT_VV(vor_vv_d, 8)
+GEN_VEXT_VV(vxor_vv_b, 1)
+GEN_VEXT_VV(vxor_vv_h, 2)
+GEN_VEXT_VV(vxor_vv_w, 4)
+GEN_VEXT_VV(vxor_vv_d, 8)
+GEN_VEXT_VV(vminu_vv_b, 1)
+GEN_VEXT_VV(vminu_vv_h, 2)
+GEN_VEXT_VV(vminu_vv_w, 4)
+GEN_VEXT_VV(vminu_vv_d, 8)
+GEN_VEXT_VV(vmin_vv_b, 1)
+GEN_VEXT_VV(vmin_vv_h, 2)
+GEN_VEXT_VV(vmin_vv_w, 4)
+GEN_VEXT_VV(vmin_vv_d, 8)
+GEN_VEXT_VV(vmaxu_vv_b, 1)
+GEN_VEXT_VV(vmaxu_vv_h, 2)
+GEN_VEXT_VV(vmaxu_vv_w, 4)
+GEN_VEXT_VV(vmaxu_vv_d, 8)
+GEN_VEXT_VV(vmax_vv_b, 1)
+GEN_VEXT_VV(vmax_vv_h, 2)
+GEN_VEXT_VV(vmax_vv_w, 4)
+GEN_VEXT_VV(vmax_vv_d, 8)
+GEN_VEXT_VV(vwaddu_vv_b, 2)
+GEN_VEXT_VV(vwaddu_vv_h, 4)
+GEN_VEXT_VV(vwaddu_vv_w, 8)
+GEN_VEXT_VV(vwsubu_vv_b, 2)
+GEN_VEXT_VV(vwsubu_vv_h, 4)
+GEN_VEXT_VV(vwsubu_vv_w, 8)
+GEN_VEXT_VV(vwadd_vv_b, 2)
+GEN_VEXT_VV(vwadd_vv_h, 4)
+GEN_VEXT_VV(vwadd_vv_w, 8)
+GEN_VEXT_VV(vwsub_vv_b, 2)
+GEN_VEXT_VV(vwsub_vv_h, 4)
+GEN_VEXT_VV(vwsub_vv_w, 8)
+GEN_VEXT_VV(vwaddu_wv_b, 2)
+GEN_VEXT_VV(vwaddu_wv_h, 4)
+GEN_VEXT_VV(vwaddu_wv_w, 8)
+GEN_VEXT_VV(vwsubu_wv_b, 2)
+GEN_VEXT_VV(vwsubu_wv_h, 4)
+GEN_VEXT_VV(vwsubu_wv_w, 8)
+GEN_VEXT_VV(vwadd_wv_b, 2)
+GEN_VEXT_VV(vwadd_wv_h, 4)
+GEN_VEXT_VV(vwadd_wv_w, 8)
+GEN_VEXT_VV(vwsub_wv_b, 2)
+GEN_VEXT_VV(vwsub_wv_h, 4)
+GEN_VEXT_VV(vwsub_wv_w, 8)
+GEN_VEXT_VV(vwmul_vv_b, 2)
+GEN_VEXT_VV(vwmul_vv_h, 4)
+GEN_VEXT_VV(vwmul_vv_w, 8)
+GEN_VEXT_VV(vwmulu_vv_b, 2)
+GEN_VEXT_VV(vwmulu_vv_h, 4)
+GEN_VEXT_VV(vwmulu_vv_w, 8)
+GEN_VEXT_VV(vwmulsu_vv_b, 2)
+GEN_VEXT_VV(vwmulsu_vv_h, 4)
+GEN_VEXT_VV(vwmulsu_vv_w, 8)
+GEN_VEXT_VV(vmacc_vv_b, 1)
+GEN_VEXT_VV(vmacc_vv_h, 2)
+GEN_VEXT_VV(vmacc_vv_w, 4)
+GEN_VEXT_VV(vmacc_vv_d, 8)
+GEN_VEXT_VV(vnmsac_vv_b, 1)
+GEN_VEXT_VV(vnmsac_vv_h, 2)
+GEN_VEXT_VV(vnmsac_vv_w, 4)
+GEN_VEXT_VV(vnmsac_vv_d, 8)
+GEN_VEXT_VV(vmadd_vv_b, 1)
+GEN_VEXT_VV(vmadd_vv_h, 2)
+GEN_VEXT_VV(vmadd_vv_w, 4)
+GEN_VEXT_VV(vmadd_vv_d, 8)
+GEN_VEXT_VV(vnmsub_vv_b, 1)
+GEN_VEXT_VV(vnmsub_vv_h, 2)
+GEN_VEXT_VV(vnmsub_vv_w, 4)
+GEN_VEXT_VV(vnmsub_vv_d, 8)
+GEN_VEXT_VV(vwmaccu_vv_b, 2)
+GEN_VEXT_VV(vwmaccu_vv_h, 4)
+GEN_VEXT_VV(vwmaccu_vv_w, 8)
+GEN_VEXT_VV(vwmacc_vv_b, 2)
+GEN_VEXT_VV(vwmacc_vv_h, 4)
+GEN_VEXT_VV(vwmacc_vv_w, 8)
+GEN_VEXT_VV(vwmaccsu_vv_b, 2)
+GEN_VEXT_VV(vwmaccsu_vv_h, 4)
+GEN_VEXT_VV(vwmaccsu_vv_w, 8)
+GEN_VEXT_VV(vmul_vv_b, 1)
+GEN_VEXT_VV(vmul_vv_h, 2)
+GEN_VEXT_VV(vmul_vv_w, 4)
+GEN_VEXT_VV(vmul_vv_d, 8)
+GEN_VEXT_VV(vmulh_vv_b, 1)
+GEN_VEXT_VV(vmulh_vv_h, 2)
+GEN_VEXT_VV(vmulh_vv_w, 4)
+GEN_VEXT_VV(vmulh_vv_d, 8)
+GEN_VEXT_VV(vmulhu_vv_b, 1)
+GEN_VEXT_VV(vmulhu_vv_h, 2)
+GEN_VEXT_VV(vmulhu_vv_w, 4)
+GEN_VEXT_VV(vmulhu_vv_d, 8)
+GEN_VEXT_VV(vmulhsu_vv_b, 1)
+GEN_VEXT_VV(vmulhsu_vv_h, 2)
+GEN_VEXT_VV(vmulhsu_vv_w, 4)
+GEN_VEXT_VV(vmulhsu_vv_d, 8)
+GEN_VEXT_VV(vdivu_vv_b, 1)
+GEN_VEXT_VV(vdivu_vv_h, 2)
+GEN_VEXT_VV(vdivu_vv_w, 4)
+GEN_VEXT_VV(vdivu_vv_d, 8)
+GEN_VEXT_VV(vdiv_vv_b, 1)
+GEN_VEXT_VV(vdiv_vv_h, 2)
+GEN_VEXT_VV(vdiv_vv_w, 4)
+GEN_VEXT_VV(vdiv_vv_d, 8)
+GEN_VEXT_VV(vremu_vv_b, 1)
+GEN_VEXT_VV(vremu_vv_h, 2)
+GEN_VEXT_VV(vremu_vv_w, 4)
+GEN_VEXT_VV(vremu_vv_d, 8)
+GEN_VEXT_VV(vrem_vv_b, 1)
+GEN_VEXT_VV(vrem_vv_h, 2)
+GEN_VEXT_VV(vrem_vv_w, 4)
+GEN_VEXT_VV(vrem_vv_d, 8)
+GEN_VEXT_VV(vsll_vv_b, 1)
+GEN_VEXT_VV(vsll_vv_h, 2)
+GEN_VEXT_VV(vsll_vv_w, 4)
+GEN_VEXT_VV(vsll_vv_d, 8)
+GEN_VEXT_VV(vsrl_vv_b, 1)
+GEN_VEXT_VV(vsrl_vv_h, 2)
+GEN_VEXT_VV(vsrl_vv_w, 4)
+GEN_VEXT_VV(vsrl_vv_d, 8)
+GEN_VEXT_VV(vsra_vv_b, 1)
+GEN_VEXT_VV(vsra_vv_h, 2)
+GEN_VEXT_VV(vsra_vv_w, 4)
+GEN_VEXT_VV(vsra_vv_d, 8)
+
+typedef void opfvv2_fn(void *vd, void *vs1, void *vs2, int i,
+                       CPURISCVState *env);
+
+#define OPFVV2(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
+static void do_##NAME(void *vd, void *vs1, void *vs2, int i, \
+                      CPURISCVState *env)                    \
+{                                                            \
+    TX1 s1 = *((T1 *)vs1 + HS1(i));                          \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                          \
+                                                             \
+    *((TD *)vd + HD(i)) = OP(s2, s1, &env->fp_status);       \
+}
+
+static void do_vext_vv_env(void *vd, void *v0, void *vs1, void *vs2,
+                           CPURISCVState *env, uint32_t desc,
+                           opfvv2_fn *fn, uint32_t esz)
+{
+    uint32_t i;
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, vs1, vs2, i, env);
+    }
+    env->vstart = 0;
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_VV_ENV(NAME, ESZ)                                  \
+void HELPER(NAME)(void *vd, void *v0, void *vs1,                    \
+                  void *vs2, CPURISCVState *env, uint32_t desc)     \
+{                                                                   \
+    do_vext_vv_env(vd, v0, vs1, vs2, env, desc, do_##NAME, ESZ);    \
+}
+
+RVVCALL(OPFVV2, vfadd_vv_h, OP_UUU_H, H2, H2, H2, float16_add)
+RVVCALL(OPFVV2, vfadd_vv_w, OP_UUU_W, H4, H4, H4, float32_add)
+RVVCALL(OPFVV2, vfadd_vv_d, OP_UUU_D, H8, H8, H8, float64_add)
+RVVCALL(OPFVV2, vfsub_vv_h, OP_UUU_H, H2, H2, H2, float16_sub)
+RVVCALL(OPFVV2, vfsub_vv_w, OP_UUU_W, H4, H4, H4, float32_sub)
+RVVCALL(OPFVV2, vfsub_vv_d, OP_UUU_D, H8, H8, H8, float64_sub)
+RVVCALL(OPFVV2, vfmul_vv_h, OP_UUU_H, H2, H2, H2, float16_mul)
+RVVCALL(OPFVV2, vfmul_vv_w, OP_UUU_W, H4, H4, H4, float32_mul)
+RVVCALL(OPFVV2, vfmul_vv_d, OP_UUU_D, H8, H8, H8, float64_mul)
+RVVCALL(OPFVV2, vfdiv_vv_h, OP_UUU_H, H2, H2, H2, float16_div)
+RVVCALL(OPFVV2, vfdiv_vv_w, OP_UUU_W, H4, H4, H4, float32_div)
+RVVCALL(OPFVV2, vfdiv_vv_d, OP_UUU_D, H8, H8, H8, float64_div)
+RVVCALL(OPFVV2, vfmin_vv_h, OP_UUU_H, H2, H2, H2,
+        float16_minimum_number)
+RVVCALL(OPFVV2, vfmin_vv_w, OP_UUU_W, H4, H4, H4,
+        float32_minimum_number)
+RVVCALL(OPFVV2, vfmin_vv_d, OP_UUU_D, H8, H8, H8,
+        float64_minimum_number)
+RVVCALL(OPFVV2, vfmax_vv_h, OP_UUU_H, H2, H2, H2,
+        float16_maximum_number)
+RVVCALL(OPFVV2, vfmax_vv_w, OP_UUU_W, H4, H4, H4,
+        float32_maximum_number)
+RVVCALL(OPFVV2, vfmax_vv_d, OP_UUU_D, H8, H8, H8,
+        float64_maximum_number)
+
+static uint16_t fsgnj16(uint16_t a, uint16_t b, float_status *s)
+{
+    return deposit64(b, 0, 15, a);
+}
+
+static uint32_t fsgnj32(uint32_t a, uint32_t b, float_status *s)
+{
+    return deposit64(b, 0, 31, a);
+}
+
+static uint64_t fsgnj64(uint64_t a, uint64_t b, float_status *s)
+{
+    return deposit64(b, 0, 63, a);
+}
+
+RVVCALL(OPFVV2, vfsgnj_vv_h, OP_UUU_H, H2, H2, H2, fsgnj16)
+RVVCALL(OPFVV2, vfsgnj_vv_w, OP_UUU_W, H4, H4, H4, fsgnj32)
+RVVCALL(OPFVV2, vfsgnj_vv_d, OP_UUU_D, H8, H8, H8, fsgnj64)
+
+static uint16_t fsgnjn16(uint16_t a, uint16_t b, float_status *s)
+{
+    return deposit64(~b, 0, 15, a);
+}
+
+static uint32_t fsgnjn32(uint32_t a, uint32_t b, float_status *s)
+{
+    return deposit64(~b, 0, 31, a);
+}
+
+static uint64_t fsgnjn64(uint64_t a, uint64_t b, float_status *s)
+{
+    return deposit64(~b, 0, 63, a);
+}
+
+RVVCALL(OPFVV2, vfsgnjn_vv_h, OP_UUU_H, H2, H2, H2, fsgnjn16)
+RVVCALL(OPFVV2, vfsgnjn_vv_w, OP_UUU_W, H4, H4, H4, fsgnjn32)
+RVVCALL(OPFVV2, vfsgnjn_vv_d, OP_UUU_D, H8, H8, H8, fsgnjn64)
+
+static uint16_t fsgnjx16(uint16_t a, uint16_t b, float_status *s)
+{
+    return deposit64(b ^ a, 0, 15, a);
+}
+
+static uint32_t fsgnjx32(uint32_t a, uint32_t b, float_status *s)
+{
+    return deposit64(b ^ a, 0, 31, a);
+}
+
+static uint64_t fsgnjx64(uint64_t a, uint64_t b, float_status *s)
+{
+    return deposit64(b ^ a, 0, 63, a);
+}
+
+RVVCALL(OPFVV2, vfsgnjx_vv_h, OP_UUU_H, H2, H2, H2, fsgnjx16)
+RVVCALL(OPFVV2, vfsgnjx_vv_w, OP_UUU_W, H4, H4, H4, fsgnjx32)
+RVVCALL(OPFVV2, vfsgnjx_vv_d, OP_UUU_D, H8, H8, H8, fsgnjx64)
+GEN_VEXT_VV_ENV(vfadd_vv_h, 2)
+GEN_VEXT_VV_ENV(vfadd_vv_w, 4)
+GEN_VEXT_VV_ENV(vfadd_vv_d, 8)
+GEN_VEXT_VV_ENV(vfsub_vv_h, 2)
+GEN_VEXT_VV_ENV(vfsub_vv_w, 4)
+GEN_VEXT_VV_ENV(vfsub_vv_d, 8)
+GEN_VEXT_VV_ENV(vfmul_vv_h, 2)
+GEN_VEXT_VV_ENV(vfmul_vv_w, 4)
+GEN_VEXT_VV_ENV(vfmul_vv_d, 8)
+GEN_VEXT_VV_ENV(vfdiv_vv_h, 2)
+GEN_VEXT_VV_ENV(vfdiv_vv_w, 4)
+GEN_VEXT_VV_ENV(vfdiv_vv_d, 8)
+GEN_VEXT_VV_ENV(vfmin_vv_h, 2)
+GEN_VEXT_VV_ENV(vfmin_vv_w, 4)
+GEN_VEXT_VV_ENV(vfmin_vv_d, 8)
+GEN_VEXT_VV_ENV(vfmax_vv_h, 2)
+GEN_VEXT_VV_ENV(vfmax_vv_w, 4)
+GEN_VEXT_VV_ENV(vfmax_vv_d, 8)
+GEN_VEXT_VV_ENV(vfsgnj_vv_h, 2)
+GEN_VEXT_VV_ENV(vfsgnj_vv_w, 4)
+GEN_VEXT_VV_ENV(vfsgnj_vv_d, 8)
+GEN_VEXT_VV_ENV(vfsgnjn_vv_h, 2)
+GEN_VEXT_VV_ENV(vfsgnjn_vv_w, 4)
+GEN_VEXT_VV_ENV(vfsgnjn_vv_d, 8)
+GEN_VEXT_VV_ENV(vfsgnjx_vv_h, 2)
+GEN_VEXT_VV_ENV(vfsgnjx_vv_w, 4)
+GEN_VEXT_VV_ENV(vfsgnjx_vv_d, 8)
+
+typedef void opivx2_fn(void *vd, target_ulong s1, void *vs2, int i);
+
+#define OPIVX2(NAME, TD, T1, T2, HD, HS2, OP)      \
+static void do_##NAME(void *vd, target_ulong s1,   \
+                      void *vs2, int i)            \
+{                                                  \
+    T2 s2 = *((T2 *)vs2 + HS2(i));                 \
+                                                   \
+    *((TD *)vd + HD(i)) = OP(s2, (T1)(target_long)s1); \
+}
+
+OPIVX2(vadd_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_ADD)
+OPIVX2(vadd_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_ADD)
+OPIVX2(vadd_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_ADD)
+OPIVX2(vadd_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_ADD)
+OPIVX2(vsub_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_SUB)
+OPIVX2(vsub_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_SUB)
+OPIVX2(vsub_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_SUB)
+OPIVX2(vsub_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_SUB)
+OPIVX2(vrsub_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_RSUB)
+OPIVX2(vrsub_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_RSUB)
+OPIVX2(vrsub_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_RSUB)
+OPIVX2(vrsub_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_RSUB)
+OPIVX2(vand_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_AND)
+OPIVX2(vand_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_AND)
+OPIVX2(vand_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_AND)
+OPIVX2(vand_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_AND)
+OPIVX2(vor_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_OR)
+OPIVX2(vor_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_OR)
+OPIVX2(vor_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_OR)
+OPIVX2(vor_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_OR)
+OPIVX2(vxor_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_XOR)
+OPIVX2(vxor_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_XOR)
+OPIVX2(vxor_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_XOR)
+OPIVX2(vxor_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_XOR)
+OPIVX2(vminu_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_MIN)
+OPIVX2(vminu_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_MIN)
+OPIVX2(vminu_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_MIN)
+OPIVX2(vminu_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_MIN)
+OPIVX2(vmin_vx_b, int8_t, int8_t, int8_t, H1, H1, DO_MIN)
+OPIVX2(vmin_vx_h, int16_t, int16_t, int16_t, H2, H2, DO_MIN)
+OPIVX2(vmin_vx_w, int32_t, int32_t, int32_t, H4, H4, DO_MIN)
+OPIVX2(vmin_vx_d, int64_t, int64_t, int64_t, H8, H8, DO_MIN)
+OPIVX2(vmaxu_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_MAX)
+OPIVX2(vmaxu_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_MAX)
+OPIVX2(vmaxu_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_MAX)
+OPIVX2(vmaxu_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_MAX)
+OPIVX2(vmax_vx_b, int8_t, int8_t, int8_t, H1, H1, DO_MAX)
+OPIVX2(vmax_vx_h, int16_t, int16_t, int16_t, H2, H2, DO_MAX)
+OPIVX2(vmax_vx_w, int32_t, int32_t, int32_t, H4, H4, DO_MAX)
+OPIVX2(vmax_vx_d, int64_t, int64_t, int64_t, H8, H8, DO_MAX)
+OPIVX2(vmul_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, DO_MUL)
+OPIVX2(vmul_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, DO_MUL)
+OPIVX2(vmul_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, DO_MUL)
+OPIVX2(vmul_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, DO_MUL)
+OPIVX2(vmulh_vx_b, int8_t, int8_t, int8_t, H1, H1, do_mulh_b)
+OPIVX2(vmulh_vx_h, int16_t, int16_t, int16_t, H2, H2, do_mulh_h)
+OPIVX2(vmulh_vx_w, int32_t, int32_t, int32_t, H4, H4, do_mulh_w)
+OPIVX2(vmulh_vx_d, int64_t, int64_t, int64_t, H8, H8, do_mulh_d)
+OPIVX2(vmulhu_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, do_mulhu_b)
+OPIVX2(vmulhu_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, do_mulhu_h)
+OPIVX2(vmulhu_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, do_mulhu_w)
+OPIVX2(vmulhu_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, do_mulhu_d)
+OPIVX2(vmulhsu_vx_b, int8_t, uint8_t, int8_t, H1, H1, do_mulhsu_b)
+OPIVX2(vmulhsu_vx_h, int16_t, uint16_t, int16_t, H2, H2, do_mulhsu_h)
+OPIVX2(vmulhsu_vx_w, int32_t, uint32_t, int32_t, H4, H4, do_mulhsu_w)
+OPIVX2(vmulhsu_vx_d, int64_t, uint64_t, int64_t, H8, H8, do_mulhsu_d)
+OPIVX2(vdivu_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, do_divu_b)
+OPIVX2(vdivu_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, do_divu_h)
+OPIVX2(vdivu_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, do_divu_w)
+OPIVX2(vdivu_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, do_divu_d)
+OPIVX2(vdiv_vx_b, int8_t, int8_t, int8_t, H1, H1, do_div_b)
+OPIVX2(vdiv_vx_h, int16_t, int16_t, int16_t, H2, H2, do_div_h)
+OPIVX2(vdiv_vx_w, int32_t, int32_t, int32_t, H4, H4, do_div_w)
+OPIVX2(vdiv_vx_d, int64_t, int64_t, int64_t, H8, H8, do_div_d)
+OPIVX2(vremu_vx_b, uint8_t, uint8_t, uint8_t, H1, H1, do_remu_b)
+OPIVX2(vremu_vx_h, uint16_t, uint16_t, uint16_t, H2, H2, do_remu_h)
+OPIVX2(vremu_vx_w, uint32_t, uint32_t, uint32_t, H4, H4, do_remu_w)
+OPIVX2(vremu_vx_d, uint64_t, uint64_t, uint64_t, H8, H8, do_remu_d)
+OPIVX2(vrem_vx_b, int8_t, int8_t, int8_t, H1, H1, do_rem_b)
+OPIVX2(vrem_vx_h, int16_t, int16_t, int16_t, H2, H2, do_rem_h)
+OPIVX2(vrem_vx_w, int32_t, int32_t, int32_t, H4, H4, do_rem_w)
+OPIVX2(vrem_vx_d, int64_t, int64_t, int64_t, H8, H8, do_rem_d)
+
+#define OPIVX2_WIDE(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \
+static void do_##NAME(void *vd, target_ulong s1,             \
+                      void *vs2, int i)                      \
+{                                                            \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                          \
+                                                             \
+    *((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)(target_long)s1);  \
+}
+
+RVVCALL(OPIVX2_WIDE, vwaddu_vx_b, WOP_UUU_B, H2, H1, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwaddu_vx_h, WOP_UUU_H, H4, H2, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwaddu_vx_w, WOP_UUU_W, H8, H4, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwsubu_vx_b, WOP_UUU_B, H2, H1, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsubu_vx_h, WOP_UUU_H, H4, H2, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsubu_vx_w, WOP_UUU_W, H8, H4, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwadd_vx_b, WOP_SSS_B, H2, H1, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwadd_vx_h, WOP_SSS_H, H4, H2, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwadd_vx_w, WOP_SSS_W, H8, H4, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwsub_vx_b, WOP_SSS_B, H2, H1, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsub_vx_h, WOP_SSS_H, H4, H2, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsub_vx_w, WOP_SSS_W, H8, H4, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwaddu_wx_b, WOP_WUUU_B, H2, H1, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwaddu_wx_h, WOP_WUUU_H, H4, H2, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwaddu_wx_w, WOP_WUUU_W, H8, H4, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwsubu_wx_b, WOP_WUUU_B, H2, H1, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsubu_wx_h, WOP_WUUU_H, H4, H2, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsubu_wx_w, WOP_WUUU_W, H8, H4, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwadd_wx_b, WOP_WSSS_B, H2, H1, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwadd_wx_h, WOP_WSSS_H, H4, H2, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwadd_wx_w, WOP_WSSS_W, H8, H4, DO_ADD)
+RVVCALL(OPIVX2_WIDE, vwsub_wx_b, WOP_WSSS_B, H2, H1, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsub_wx_h, WOP_WSSS_H, H4, H2, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwsub_wx_w, WOP_WSSS_W, H8, H4, DO_SUB)
+RVVCALL(OPIVX2_WIDE, vwmul_vx_b, WOP_SSS_B, H2, H1, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmul_vx_h, WOP_SSS_H, H4, H2, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmul_vx_w, WOP_SSS_W, H8, H4, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmulu_vx_b, WOP_UUU_B, H2, H1, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmulu_vx_h, WOP_UUU_H, H4, H2, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmulu_vx_w, WOP_UUU_W, H8, H4, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmulsu_vx_b, WOP_SUS_B, H2, H1, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmulsu_vx_h, WOP_SUS_H, H4, H2, DO_MUL)
+RVVCALL(OPIVX2_WIDE, vwmulsu_vx_w, WOP_SUS_W, H8, H4, DO_MUL)
+
+#define OPIVX3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP)             \
+static void do_##NAME(void *vd, target_ulong s1, void *vs2, int i)  \
+{                                                                   \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                                 \
+    TD d = *((TD *)vd + HD(i));                                     \
+                                                                    \
+    *((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)(target_long)s1, d);      \
+}
+
+RVVCALL(OPIVX3, vmacc_vx_b, OP_SSS_B, H1, H1, DO_MACC)
+RVVCALL(OPIVX3, vmacc_vx_h, OP_SSS_H, H2, H2, DO_MACC)
+RVVCALL(OPIVX3, vmacc_vx_w, OP_SSS_W, H4, H4, DO_MACC)
+RVVCALL(OPIVX3, vmacc_vx_d, OP_SSS_D, H8, H8, DO_MACC)
+RVVCALL(OPIVX3, vnmsac_vx_b, OP_SSS_B, H1, H1, DO_NMSAC)
+RVVCALL(OPIVX3, vnmsac_vx_h, OP_SSS_H, H2, H2, DO_NMSAC)
+RVVCALL(OPIVX3, vnmsac_vx_w, OP_SSS_W, H4, H4, DO_NMSAC)
+RVVCALL(OPIVX3, vnmsac_vx_d, OP_SSS_D, H8, H8, DO_NMSAC)
+RVVCALL(OPIVX3, vmadd_vx_b, OP_SSS_B, H1, H1, DO_MADD)
+RVVCALL(OPIVX3, vmadd_vx_h, OP_SSS_H, H2, H2, DO_MADD)
+RVVCALL(OPIVX3, vmadd_vx_w, OP_SSS_W, H4, H4, DO_MADD)
+RVVCALL(OPIVX3, vmadd_vx_d, OP_SSS_D, H8, H8, DO_MADD)
+RVVCALL(OPIVX3, vnmsub_vx_b, OP_SSS_B, H1, H1, DO_NMSUB)
+RVVCALL(OPIVX3, vnmsub_vx_h, OP_SSS_H, H2, H2, DO_NMSUB)
+RVVCALL(OPIVX3, vnmsub_vx_w, OP_SSS_W, H4, H4, DO_NMSUB)
+RVVCALL(OPIVX3, vnmsub_vx_d, OP_SSS_D, H8, H8, DO_NMSUB)
+RVVCALL(OPIVX3, vwmaccu_vx_b, WOP_UUU_B, H2, H1, DO_MACC)
+RVVCALL(OPIVX3, vwmaccu_vx_h, WOP_UUU_H, H4, H2, DO_MACC)
+RVVCALL(OPIVX3, vwmaccu_vx_w, WOP_UUU_W, H8, H4, DO_MACC)
+RVVCALL(OPIVX3, vwmacc_vx_b, WOP_SSS_B, H2, H1, DO_MACC)
+RVVCALL(OPIVX3, vwmacc_vx_h, WOP_SSS_H, H4, H2, DO_MACC)
+RVVCALL(OPIVX3, vwmacc_vx_w, WOP_SSS_W, H8, H4, DO_MACC)
+RVVCALL(OPIVX3, vwmaccsu_vx_b, WOP_SSU_B, H2, H1, DO_MACC)
+RVVCALL(OPIVX3, vwmaccsu_vx_h, WOP_SSU_H, H4, H2, DO_MACC)
+RVVCALL(OPIVX3, vwmaccsu_vx_w, WOP_SSU_W, H8, H4, DO_MACC)
+RVVCALL(OPIVX3, vwmaccus_vx_b, WOP_SUS_B, H2, H1, DO_MACC)
+RVVCALL(OPIVX3, vwmaccus_vx_h, WOP_SUS_H, H4, H2, DO_MACC)
+RVVCALL(OPIVX3, vwmaccus_vx_w, WOP_SUS_W, H8, H4, DO_MACC)
+
+#define OPIVX2_SHIFT(NAME, TD, T2, HD, HS2, OP, MASK) \
+static void do_##NAME(void *vd, target_ulong s1,      \
+                      void *vs2, int i)               \
+{                                                     \
+    T2 s2 = *((T2 *)vs2 + HS2(i));                    \
+                                                      \
+    *((TD *)vd + HD(i)) = (TD)OP(s2, s1 & MASK);      \
+}
+
+OPIVX2_SHIFT(vsll_vx_b, uint8_t, uint8_t, H1, H1, DO_SLL, 0x7)
+OPIVX2_SHIFT(vsll_vx_h, uint16_t, uint16_t, H2, H2, DO_SLL, 0xf)
+OPIVX2_SHIFT(vsll_vx_w, uint32_t, uint32_t, H4, H4, DO_SLL, 0x1f)
+OPIVX2_SHIFT(vsll_vx_d, uint64_t, uint64_t, H8, H8, DO_SLL, 0x3f)
+OPIVX2_SHIFT(vsrl_vx_b, uint8_t, uint8_t, H1, H1, DO_SRL, 0x7)
+OPIVX2_SHIFT(vsrl_vx_h, uint16_t, uint16_t, H2, H2, DO_SRL, 0xf)
+OPIVX2_SHIFT(vsrl_vx_w, uint32_t, uint32_t, H4, H4, DO_SRL, 0x1f)
+OPIVX2_SHIFT(vsrl_vx_d, uint64_t, uint64_t, H8, H8, DO_SRL, 0x3f)
+OPIVX2_SHIFT(vsra_vx_b, int8_t, int8_t, H1, H1, DO_SRL, 0x7)
+OPIVX2_SHIFT(vsra_vx_h, int16_t, int16_t, H2, H2, DO_SRL, 0xf)
+OPIVX2_SHIFT(vsra_vx_w, int32_t, int32_t, H4, H4, DO_SRL, 0x1f)
+OPIVX2_SHIFT(vsra_vx_d, int64_t, int64_t, H8, H8, DO_SRL, 0x3f)
+
+#define OPIVV2_NSHIFT(NAME, TD, T1, T2, HD, HS1, HS2, OP, MASK) \
+static void do_##NAME(void *vd, void *vs1,                      \
+                      void *vs2, int i)                         \
+{                                                               \
+    T1 s1 = *((T1 *)vs1 + HS1(i));                              \
+    T2 s2 = *((T2 *)vs2 + HS2(i));                              \
+                                                                \
+    *((TD *)vd + HD(i)) = (TD)OP(s2, s1 & MASK);                \
+}
+
+OPIVV2_NSHIFT(vnsrl_wv_b, uint8_t, uint8_t, uint16_t,
+              H1, H1, H2, DO_SRL, 0xf)
+OPIVV2_NSHIFT(vnsrl_wv_h, uint16_t, uint16_t, uint32_t,
+              H2, H2, H4, DO_SRL, 0x1f)
+OPIVV2_NSHIFT(vnsrl_wv_w, uint32_t, uint32_t, uint64_t,
+              H4, H4, H8, DO_SRL, 0x3f)
+OPIVV2_NSHIFT(vnsra_wv_b, uint8_t, uint8_t, int16_t,
+              H1, H1, H2, DO_SRL, 0xf)
+OPIVV2_NSHIFT(vnsra_wv_h, uint16_t, uint16_t, int32_t,
+              H2, H2, H4, DO_SRL, 0x1f)
+OPIVV2_NSHIFT(vnsra_wv_w, uint32_t, uint32_t, int64_t,
+              H4, H4, H8, DO_SRL, 0x3f)
+
+#define OPIVX2_NSHIFT(NAME, TD, T2, HD, HS2, OP, MASK) \
+static void do_##NAME(void *vd, target_ulong s1,       \
+                      void *vs2, int i)                \
+{                                                      \
+    T2 s2 = *((T2 *)vs2 + HS2(i));                     \
+                                                       \
+    *((TD *)vd + HD(i)) = (TD)OP(s2, s1 & MASK);       \
+}
+
+OPIVX2_NSHIFT(vnsrl_wx_b, uint8_t, uint16_t, H1, H2, DO_SRL, 0xf)
+OPIVX2_NSHIFT(vnsrl_wx_h, uint16_t, uint32_t, H2, H4, DO_SRL, 0x1f)
+OPIVX2_NSHIFT(vnsrl_wx_w, uint32_t, uint64_t, H4, H8, DO_SRL, 0x3f)
+OPIVX2_NSHIFT(vnsra_wx_b, int8_t, int16_t, H1, H2, DO_SRL, 0xf)
+OPIVX2_NSHIFT(vnsra_wx_h, int16_t, int32_t, H2, H4, DO_SRL, 0x1f)
+OPIVX2_NSHIFT(vnsra_wx_w, int32_t, int64_t, H4, H8, DO_SRL, 0x3f)
+
+GEN_VEXT_VV(vnsrl_wv_b, 1)
+GEN_VEXT_VV(vnsrl_wv_h, 2)
+GEN_VEXT_VV(vnsrl_wv_w, 4)
+GEN_VEXT_VV(vnsra_wv_b, 1)
+GEN_VEXT_VV(vnsra_wv_h, 2)
+GEN_VEXT_VV(vnsra_wv_w, 4)
+
+static void do_vext_vx(void *vd, void *v0, target_ulong s1, void *vs2,
+                       CPURISCVState *env, uint32_t desc, opivx2_fn *fn,
+                       uint32_t esz)
+{
+    uint32_t i;
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, s1, vs2, i);
+    }
+    env->vstart = 0;
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_VX(NAME, ESZ)                              \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1,      \
+                  void *vs2, CPURISCVState *env,            \
+                  uint32_t desc)                            \
+{                                                           \
+    do_vext_vx(vd, v0, s1, vs2, env, desc, do_##NAME, ESZ); \
+}
+
+GEN_VEXT_VX(vadd_vx_b, 1)
+GEN_VEXT_VX(vadd_vx_h, 2)
+GEN_VEXT_VX(vadd_vx_w, 4)
+GEN_VEXT_VX(vadd_vx_d, 8)
+GEN_VEXT_VX(vsub_vx_b, 1)
+GEN_VEXT_VX(vsub_vx_h, 2)
+GEN_VEXT_VX(vsub_vx_w, 4)
+GEN_VEXT_VX(vsub_vx_d, 8)
+GEN_VEXT_VX(vrsub_vx_b, 1)
+GEN_VEXT_VX(vrsub_vx_h, 2)
+GEN_VEXT_VX(vrsub_vx_w, 4)
+GEN_VEXT_VX(vrsub_vx_d, 8)
+GEN_VEXT_VX(vand_vx_b, 1)
+GEN_VEXT_VX(vand_vx_h, 2)
+GEN_VEXT_VX(vand_vx_w, 4)
+GEN_VEXT_VX(vand_vx_d, 8)
+GEN_VEXT_VX(vor_vx_b, 1)
+GEN_VEXT_VX(vor_vx_h, 2)
+GEN_VEXT_VX(vor_vx_w, 4)
+GEN_VEXT_VX(vor_vx_d, 8)
+GEN_VEXT_VX(vxor_vx_b, 1)
+GEN_VEXT_VX(vxor_vx_h, 2)
+GEN_VEXT_VX(vxor_vx_w, 4)
+GEN_VEXT_VX(vxor_vx_d, 8)
+GEN_VEXT_VX(vminu_vx_b, 1)
+GEN_VEXT_VX(vminu_vx_h, 2)
+GEN_VEXT_VX(vminu_vx_w, 4)
+GEN_VEXT_VX(vminu_vx_d, 8)
+GEN_VEXT_VX(vmin_vx_b, 1)
+GEN_VEXT_VX(vmin_vx_h, 2)
+GEN_VEXT_VX(vmin_vx_w, 4)
+GEN_VEXT_VX(vmin_vx_d, 8)
+GEN_VEXT_VX(vmaxu_vx_b, 1)
+GEN_VEXT_VX(vmaxu_vx_h, 2)
+GEN_VEXT_VX(vmaxu_vx_w, 4)
+GEN_VEXT_VX(vmaxu_vx_d, 8)
+GEN_VEXT_VX(vmax_vx_b, 1)
+GEN_VEXT_VX(vmax_vx_h, 2)
+GEN_VEXT_VX(vmax_vx_w, 4)
+GEN_VEXT_VX(vmax_vx_d, 8)
+GEN_VEXT_VX(vmul_vx_b, 1)
+GEN_VEXT_VX(vmul_vx_h, 2)
+GEN_VEXT_VX(vmul_vx_w, 4)
+GEN_VEXT_VX(vmul_vx_d, 8)
+GEN_VEXT_VX(vmulh_vx_b, 1)
+GEN_VEXT_VX(vmulh_vx_h, 2)
+GEN_VEXT_VX(vmulh_vx_w, 4)
+GEN_VEXT_VX(vmulh_vx_d, 8)
+GEN_VEXT_VX(vmulhu_vx_b, 1)
+GEN_VEXT_VX(vmulhu_vx_h, 2)
+GEN_VEXT_VX(vmulhu_vx_w, 4)
+GEN_VEXT_VX(vmulhu_vx_d, 8)
+GEN_VEXT_VX(vmulhsu_vx_b, 1)
+GEN_VEXT_VX(vmulhsu_vx_h, 2)
+GEN_VEXT_VX(vmulhsu_vx_w, 4)
+GEN_VEXT_VX(vmulhsu_vx_d, 8)
+GEN_VEXT_VX(vdivu_vx_b, 1)
+GEN_VEXT_VX(vdivu_vx_h, 2)
+GEN_VEXT_VX(vdivu_vx_w, 4)
+GEN_VEXT_VX(vdivu_vx_d, 8)
+GEN_VEXT_VX(vdiv_vx_b, 1)
+GEN_VEXT_VX(vdiv_vx_h, 2)
+GEN_VEXT_VX(vdiv_vx_w, 4)
+GEN_VEXT_VX(vdiv_vx_d, 8)
+GEN_VEXT_VX(vremu_vx_b, 1)
+GEN_VEXT_VX(vremu_vx_h, 2)
+GEN_VEXT_VX(vremu_vx_w, 4)
+GEN_VEXT_VX(vremu_vx_d, 8)
+GEN_VEXT_VX(vrem_vx_b, 1)
+GEN_VEXT_VX(vrem_vx_h, 2)
+GEN_VEXT_VX(vrem_vx_w, 4)
+GEN_VEXT_VX(vrem_vx_d, 8)
+GEN_VEXT_VX(vwaddu_vx_b, 2)
+GEN_VEXT_VX(vwaddu_vx_h, 4)
+GEN_VEXT_VX(vwaddu_vx_w, 8)
+GEN_VEXT_VX(vwsubu_vx_b, 2)
+GEN_VEXT_VX(vwsubu_vx_h, 4)
+GEN_VEXT_VX(vwsubu_vx_w, 8)
+GEN_VEXT_VX(vwadd_vx_b, 2)
+GEN_VEXT_VX(vwadd_vx_h, 4)
+GEN_VEXT_VX(vwadd_vx_w, 8)
+GEN_VEXT_VX(vwsub_vx_b, 2)
+GEN_VEXT_VX(vwsub_vx_h, 4)
+GEN_VEXT_VX(vwsub_vx_w, 8)
+GEN_VEXT_VX(vwaddu_wx_b, 2)
+GEN_VEXT_VX(vwaddu_wx_h, 4)
+GEN_VEXT_VX(vwaddu_wx_w, 8)
+GEN_VEXT_VX(vwsubu_wx_b, 2)
+GEN_VEXT_VX(vwsubu_wx_h, 4)
+GEN_VEXT_VX(vwsubu_wx_w, 8)
+GEN_VEXT_VX(vwadd_wx_b, 2)
+GEN_VEXT_VX(vwadd_wx_h, 4)
+GEN_VEXT_VX(vwadd_wx_w, 8)
+GEN_VEXT_VX(vwsub_wx_b, 2)
+GEN_VEXT_VX(vwsub_wx_h, 4)
+GEN_VEXT_VX(vwsub_wx_w, 8)
+GEN_VEXT_VX(vwmul_vx_b, 2)
+GEN_VEXT_VX(vwmul_vx_h, 4)
+GEN_VEXT_VX(vwmul_vx_w, 8)
+GEN_VEXT_VX(vwmulu_vx_b, 2)
+GEN_VEXT_VX(vwmulu_vx_h, 4)
+GEN_VEXT_VX(vwmulu_vx_w, 8)
+GEN_VEXT_VX(vwmulsu_vx_b, 2)
+GEN_VEXT_VX(vwmulsu_vx_h, 4)
+GEN_VEXT_VX(vwmulsu_vx_w, 8)
+GEN_VEXT_VX(vmacc_vx_b, 1)
+GEN_VEXT_VX(vmacc_vx_h, 2)
+GEN_VEXT_VX(vmacc_vx_w, 4)
+GEN_VEXT_VX(vmacc_vx_d, 8)
+GEN_VEXT_VX(vnmsac_vx_b, 1)
+GEN_VEXT_VX(vnmsac_vx_h, 2)
+GEN_VEXT_VX(vnmsac_vx_w, 4)
+GEN_VEXT_VX(vnmsac_vx_d, 8)
+GEN_VEXT_VX(vmadd_vx_b, 1)
+GEN_VEXT_VX(vmadd_vx_h, 2)
+GEN_VEXT_VX(vmadd_vx_w, 4)
+GEN_VEXT_VX(vmadd_vx_d, 8)
+GEN_VEXT_VX(vnmsub_vx_b, 1)
+GEN_VEXT_VX(vnmsub_vx_h, 2)
+GEN_VEXT_VX(vnmsub_vx_w, 4)
+GEN_VEXT_VX(vnmsub_vx_d, 8)
+GEN_VEXT_VX(vwmaccu_vx_b, 2)
+GEN_VEXT_VX(vwmaccu_vx_h, 4)
+GEN_VEXT_VX(vwmaccu_vx_w, 8)
+GEN_VEXT_VX(vwmacc_vx_b, 2)
+GEN_VEXT_VX(vwmacc_vx_h, 4)
+GEN_VEXT_VX(vwmacc_vx_w, 8)
+GEN_VEXT_VX(vwmaccsu_vx_b, 2)
+GEN_VEXT_VX(vwmaccsu_vx_h, 4)
+GEN_VEXT_VX(vwmaccsu_vx_w, 8)
+GEN_VEXT_VX(vwmaccus_vx_b, 2)
+GEN_VEXT_VX(vwmaccus_vx_h, 4)
+GEN_VEXT_VX(vwmaccus_vx_w, 8)
+GEN_VEXT_VX(vsll_vx_b, 1)
+GEN_VEXT_VX(vsll_vx_h, 2)
+GEN_VEXT_VX(vsll_vx_w, 4)
+GEN_VEXT_VX(vsll_vx_d, 8)
+GEN_VEXT_VX(vsrl_vx_b, 1)
+GEN_VEXT_VX(vsrl_vx_h, 2)
+GEN_VEXT_VX(vsrl_vx_w, 4)
+GEN_VEXT_VX(vsrl_vx_d, 8)
+GEN_VEXT_VX(vsra_vx_b, 1)
+GEN_VEXT_VX(vsra_vx_h, 2)
+GEN_VEXT_VX(vsra_vx_w, 4)
+GEN_VEXT_VX(vsra_vx_d, 8)
+GEN_VEXT_VX(vnsrl_wx_b, 1)
+GEN_VEXT_VX(vnsrl_wx_h, 2)
+GEN_VEXT_VX(vnsrl_wx_w, 4)
+GEN_VEXT_VX(vnsra_wx_b, 1)
+GEN_VEXT_VX(vnsra_wx_h, 2)
+GEN_VEXT_VX(vnsra_wx_w, 4)
+
+typedef void opfvf2_fn(void *vd, uint64_t s1, void *vs2, int i,
+                       CPURISCVState *env);
+
+#define OPFVF2(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP)        \
+static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \
+                      CPURISCVState *env)                      \
+{                                                              \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                            \
+                                                               \
+    *((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)s1, &env->fp_status);\
+}
+
+static void do_vext_vf(void *vd, void *v0, uint64_t s1, void *vs2,
+                       CPURISCVState *env, uint32_t desc,
+                       opfvf2_fn *fn, uint32_t esz)
+{
+    uint32_t i;
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, s1, vs2, i, env);
+    }
+    env->vstart = 0;
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_VF(NAME, ESZ)                                  \
+void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2,   \
+                  CPURISCVState *env, uint32_t desc)            \
+{                                                               \
+    do_vext_vf(vd, v0, s1, vs2, env, desc, do_##NAME, ESZ);     \
+}
+
+RVVCALL(OPFVF2, vfadd_vf_h, OP_UUU_H, H2, H2, float16_add)
+RVVCALL(OPFVF2, vfadd_vf_w, OP_UUU_W, H4, H4, float32_add)
+RVVCALL(OPFVF2, vfadd_vf_d, OP_UUU_D, H8, H8, float64_add)
+RVVCALL(OPFVF2, vfsub_vf_h, OP_UUU_H, H2, H2, float16_sub)
+RVVCALL(OPFVF2, vfsub_vf_w, OP_UUU_W, H4, H4, float32_sub)
+RVVCALL(OPFVF2, vfsub_vf_d, OP_UUU_D, H8, H8, float64_sub)
+static uint16_t float16_rsub(uint16_t a, uint16_t b, float_status *s)
+{
+    return float16_sub(b, a, s);
+}
+
+static uint32_t float32_rsub(uint32_t a, uint32_t b, float_status *s)
+{
+    return float32_sub(b, a, s);
+}
+
+static uint64_t float64_rsub(uint64_t a, uint64_t b, float_status *s)
+{
+    return float64_sub(b, a, s);
+}
+
+RVVCALL(OPFVF2, vfrsub_vf_h, OP_UUU_H, H2, H2, float16_rsub)
+RVVCALL(OPFVF2, vfrsub_vf_w, OP_UUU_W, H4, H4, float32_rsub)
+RVVCALL(OPFVF2, vfrsub_vf_d, OP_UUU_D, H8, H8, float64_rsub)
+RVVCALL(OPFVF2, vfmul_vf_h, OP_UUU_H, H2, H2, float16_mul)
+RVVCALL(OPFVF2, vfmul_vf_w, OP_UUU_W, H4, H4, float32_mul)
+RVVCALL(OPFVF2, vfmul_vf_d, OP_UUU_D, H8, H8, float64_mul)
+RVVCALL(OPFVF2, vfdiv_vf_h, OP_UUU_H, H2, H2, float16_div)
+RVVCALL(OPFVF2, vfdiv_vf_w, OP_UUU_W, H4, H4, float32_div)
+RVVCALL(OPFVF2, vfdiv_vf_d, OP_UUU_D, H8, H8, float64_div)
+static uint16_t float16_rdiv(uint16_t a, uint16_t b, float_status *s)
+{
+    return float16_div(b, a, s);
+}
+
+static uint32_t float32_rdiv(uint32_t a, uint32_t b, float_status *s)
+{
+    return float32_div(b, a, s);
+}
+
+static uint64_t float64_rdiv(uint64_t a, uint64_t b, float_status *s)
+{
+    return float64_div(b, a, s);
+}
+
+RVVCALL(OPFVF2, vfrdiv_vf_h, OP_UUU_H, H2, H2, float16_rdiv)
+RVVCALL(OPFVF2, vfrdiv_vf_w, OP_UUU_W, H4, H4, float32_rdiv)
+RVVCALL(OPFVF2, vfrdiv_vf_d, OP_UUU_D, H8, H8, float64_rdiv)
+RVVCALL(OPFVF2, vfmin_vf_h, OP_UUU_H, H2, H2, float16_minimum_number)
+RVVCALL(OPFVF2, vfmin_vf_w, OP_UUU_W, H4, H4, float32_minimum_number)
+RVVCALL(OPFVF2, vfmin_vf_d, OP_UUU_D, H8, H8, float64_minimum_number)
+RVVCALL(OPFVF2, vfmax_vf_h, OP_UUU_H, H2, H2, float16_maximum_number)
+RVVCALL(OPFVF2, vfmax_vf_w, OP_UUU_W, H4, H4, float32_maximum_number)
+RVVCALL(OPFVF2, vfmax_vf_d, OP_UUU_D, H8, H8, float64_maximum_number)
+RVVCALL(OPFVF2, vfsgnj_vf_h, OP_UUU_H, H2, H2, fsgnj16)
+RVVCALL(OPFVF2, vfsgnj_vf_w, OP_UUU_W, H4, H4, fsgnj32)
+RVVCALL(OPFVF2, vfsgnj_vf_d, OP_UUU_D, H8, H8, fsgnj64)
+RVVCALL(OPFVF2, vfsgnjn_vf_h, OP_UUU_H, H2, H2, fsgnjn16)
+RVVCALL(OPFVF2, vfsgnjn_vf_w, OP_UUU_W, H4, H4, fsgnjn32)
+RVVCALL(OPFVF2, vfsgnjn_vf_d, OP_UUU_D, H8, H8, fsgnjn64)
+RVVCALL(OPFVF2, vfsgnjx_vf_h, OP_UUU_H, H2, H2, fsgnjx16)
+RVVCALL(OPFVF2, vfsgnjx_vf_w, OP_UUU_W, H4, H4, fsgnjx32)
+RVVCALL(OPFVF2, vfsgnjx_vf_d, OP_UUU_D, H8, H8, fsgnjx64)
+GEN_VEXT_VF(vfadd_vf_h, 2)
+GEN_VEXT_VF(vfadd_vf_w, 4)
+GEN_VEXT_VF(vfadd_vf_d, 8)
+GEN_VEXT_VF(vfsub_vf_h, 2)
+GEN_VEXT_VF(vfsub_vf_w, 4)
+GEN_VEXT_VF(vfsub_vf_d, 8)
+GEN_VEXT_VF(vfrsub_vf_h, 2)
+GEN_VEXT_VF(vfrsub_vf_w, 4)
+GEN_VEXT_VF(vfrsub_vf_d, 8)
+GEN_VEXT_VF(vfmul_vf_h, 2)
+GEN_VEXT_VF(vfmul_vf_w, 4)
+GEN_VEXT_VF(vfmul_vf_d, 8)
+GEN_VEXT_VF(vfdiv_vf_h, 2)
+GEN_VEXT_VF(vfdiv_vf_w, 4)
+GEN_VEXT_VF(vfdiv_vf_d, 8)
+GEN_VEXT_VF(vfrdiv_vf_h, 2)
+GEN_VEXT_VF(vfrdiv_vf_w, 4)
+GEN_VEXT_VF(vfrdiv_vf_d, 8)
+GEN_VEXT_VF(vfmin_vf_h, 2)
+GEN_VEXT_VF(vfmin_vf_w, 4)
+GEN_VEXT_VF(vfmin_vf_d, 8)
+GEN_VEXT_VF(vfmax_vf_h, 2)
+GEN_VEXT_VF(vfmax_vf_w, 4)
+GEN_VEXT_VF(vfmax_vf_d, 8)
+GEN_VEXT_VF(vfsgnj_vf_h, 2)
+GEN_VEXT_VF(vfsgnj_vf_w, 4)
+GEN_VEXT_VF(vfsgnj_vf_d, 8)
+GEN_VEXT_VF(vfsgnjn_vf_h, 2)
+GEN_VEXT_VF(vfsgnjn_vf_w, 4)
+GEN_VEXT_VF(vfsgnjn_vf_d, 8)
+GEN_VEXT_VF(vfsgnjx_vf_h, 2)
+GEN_VEXT_VF(vfsgnjx_vf_w, 4)
+GEN_VEXT_VF(vfsgnjx_vf_d, 8)
+
+static uint32_t vfwadd16(uint16_t a, uint16_t b, float_status *s)
+{
+    float32 fa = float16_to_float32(a, true, s);
+    float32 fb = float16_to_float32(b, true, s);
+
+    return float32_add(fa, fb, s);
+}
+
+static uint64_t vfwadd32(uint32_t a, uint32_t b, float_status *s)
+{
+    float64 fa = float32_to_float64(a, s);
+    float64 fb = float32_to_float64(b, s);
+
+    return float64_add(fa, fb, s);
+}
+
+RVVCALL(OPFVV2, vfwadd_vv_h, WOP_UUU_H, H4, H2, H2, vfwadd16)
+RVVCALL(OPFVV2, vfwadd_vv_w, WOP_UUU_W, H8, H4, H4, vfwadd32)
+GEN_VEXT_VV_ENV(vfwadd_vv_h, 4)
+GEN_VEXT_VV_ENV(vfwadd_vv_w, 8)
+RVVCALL(OPFVF2, vfwadd_vf_h, WOP_UUU_H, H4, H2, vfwadd16)
+RVVCALL(OPFVF2, vfwadd_vf_w, WOP_UUU_W, H8, H4, vfwadd32)
+GEN_VEXT_VF(vfwadd_vf_h, 4)
+GEN_VEXT_VF(vfwadd_vf_w, 8)
+
+static uint32_t vfwsub16(uint16_t a, uint16_t b, float_status *s)
+{
+    float32 fa = float16_to_float32(a, true, s);
+    float32 fb = float16_to_float32(b, true, s);
+
+    return float32_sub(fa, fb, s);
+}
+
+static uint64_t vfwsub32(uint32_t a, uint32_t b, float_status *s)
+{
+    float64 fa = float32_to_float64(a, s);
+    float64 fb = float32_to_float64(b, s);
+
+    return float64_sub(fa, fb, s);
+}
+
+RVVCALL(OPFVV2, vfwsub_vv_h, WOP_UUU_H, H4, H2, H2, vfwsub16)
+RVVCALL(OPFVV2, vfwsub_vv_w, WOP_UUU_W, H8, H4, H4, vfwsub32)
+GEN_VEXT_VV_ENV(vfwsub_vv_h, 4)
+GEN_VEXT_VV_ENV(vfwsub_vv_w, 8)
+RVVCALL(OPFVF2, vfwsub_vf_h, WOP_UUU_H, H4, H2, vfwsub16)
+RVVCALL(OPFVF2, vfwsub_vf_w, WOP_UUU_W, H8, H4, vfwsub32)
+GEN_VEXT_VF(vfwsub_vf_h, 4)
+GEN_VEXT_VF(vfwsub_vf_w, 8)
+
+static uint32_t vfwaddw16(uint32_t a, uint16_t b, float_status *s)
+{
+    return float32_add(a, float16_to_float32(b, true, s), s);
+}
+
+static uint64_t vfwaddw32(uint64_t a, uint32_t b, float_status *s)
+{
+    return float64_add(a, float32_to_float64(b, s), s);
+}
+
+RVVCALL(OPFVV2, vfwadd_wv_h, WOP_WUUU_H, H4, H2, H2, vfwaddw16)
+RVVCALL(OPFVV2, vfwadd_wv_w, WOP_WUUU_W, H8, H4, H4, vfwaddw32)
+GEN_VEXT_VV_ENV(vfwadd_wv_h, 4)
+GEN_VEXT_VV_ENV(vfwadd_wv_w, 8)
+RVVCALL(OPFVF2, vfwadd_wf_h, WOP_WUUU_H, H4, H2, vfwaddw16)
+RVVCALL(OPFVF2, vfwadd_wf_w, WOP_WUUU_W, H8, H4, vfwaddw32)
+GEN_VEXT_VF(vfwadd_wf_h, 4)
+GEN_VEXT_VF(vfwadd_wf_w, 8)
+
+static uint32_t vfwsubw16(uint32_t a, uint16_t b, float_status *s)
+{
+    return float32_sub(a, float16_to_float32(b, true, s), s);
+}
+
+static uint64_t vfwsubw32(uint64_t a, uint32_t b, float_status *s)
+{
+    return float64_sub(a, float32_to_float64(b, s), s);
+}
+
+RVVCALL(OPFVV2, vfwsub_wv_h, WOP_WUUU_H, H4, H2, H2, vfwsubw16)
+RVVCALL(OPFVV2, vfwsub_wv_w, WOP_WUUU_W, H8, H4, H4, vfwsubw32)
+GEN_VEXT_VV_ENV(vfwsub_wv_h, 4)
+GEN_VEXT_VV_ENV(vfwsub_wv_w, 8)
+RVVCALL(OPFVF2, vfwsub_wf_h, WOP_WUUU_H, H4, H2, vfwsubw16)
+RVVCALL(OPFVF2, vfwsub_wf_w, WOP_WUUU_W, H8, H4, vfwsubw32)
+GEN_VEXT_VF(vfwsub_wf_h, 4)
+GEN_VEXT_VF(vfwsub_wf_w, 8)
+
+static uint32_t vfwmul16(uint16_t a, uint16_t b, float_status *s)
+{
+    float32 fa = float16_to_float32(a, true, s);
+    float32 fb = float16_to_float32(b, true, s);
+
+    return float32_mul(fa, fb, s);
+}
+
+static uint64_t vfwmul32(uint32_t a, uint32_t b, float_status *s)
+{
+    float64 fa = float32_to_float64(a, s);
+    float64 fb = float32_to_float64(b, s);
+
+    return float64_mul(fa, fb, s);
+}
+
+RVVCALL(OPFVV2, vfwmul_vv_h, WOP_UUU_H, H4, H2, H2, vfwmul16)
+RVVCALL(OPFVV2, vfwmul_vv_w, WOP_UUU_W, H8, H4, H4, vfwmul32)
+GEN_VEXT_VV_ENV(vfwmul_vv_h, 4)
+GEN_VEXT_VV_ENV(vfwmul_vv_w, 8)
+RVVCALL(OPFVF2, vfwmul_vf_h, WOP_UUU_H, H4, H2, vfwmul16)
+RVVCALL(OPFVF2, vfwmul_vf_w, WOP_UUU_W, H8, H4, vfwmul32)
+GEN_VEXT_VF(vfwmul_vf_h, 4)
+GEN_VEXT_VF(vfwmul_vf_w, 8)
+
+typedef void opfvv3_fn(void *vd, void *vs1, void *vs2, int i,
+                       CPURISCVState *env);
+
+#define OPFVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP)    \
+static void do_##NAME(void *vd, void *vs1, void *vs2, int i,    \
+                      CPURISCVState *env)                       \
+{                                                               \
+    TX1 s1 = *((T1 *)vs1 + HS1(i));                             \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                             \
+    TD d = *((TD *)vd + HD(i));                                 \
+                                                                \
+    *((TD *)vd + HD(i)) = OP(s2, s1, d, &env->fp_status);       \
+}
+
+typedef void opfvf3_fn(void *vd, uint64_t s1, void *vs2, int i,
+                       CPURISCVState *env);
+
+#define OPFVF3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP)          \
+static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i,   \
+                      CPURISCVState *env)                        \
+{                                                                \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                              \
+    TD d = *((TD *)vd + HD(i));                                  \
+                                                                 \
+    *((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)s1, d, &env->fp_status);\
+}
+
+static uint16_t fmacc16(uint16_t a, uint16_t b, uint16_t d,
+                        float_status *s)
+{
+    return float16_muladd(a, b, d, 0, s);
+}
+
+static uint32_t fmacc32(uint32_t a, uint32_t b, uint32_t d,
+                        float_status *s)
+{
+    return float32_muladd(a, b, d, 0, s);
+}
+
+static uint64_t fmacc64(uint64_t a, uint64_t b, uint64_t d,
+                        float_status *s)
+{
+    return float64_muladd(a, b, d, 0, s);
+}
+
+static uint16_t fnmacc16(uint16_t a, uint16_t b, uint16_t d,
+                         float_status *s)
+{
+    return float16_muladd(a, b, d,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint32_t fnmacc32(uint32_t a, uint32_t b, uint32_t d,
+                         float_status *s)
+{
+    return float32_muladd(a, b, d,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint64_t fnmacc64(uint64_t a, uint64_t b, uint64_t d,
+                         float_status *s)
+{
+    return float64_muladd(a, b, d,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint16_t fmsac16(uint16_t a, uint16_t b, uint16_t d,
+                        float_status *s)
+{
+    return float16_muladd(a, b, d, float_muladd_negate_c, s);
+}
+
+static uint32_t fmsac32(uint32_t a, uint32_t b, uint32_t d,
+                        float_status *s)
+{
+    return float32_muladd(a, b, d, float_muladd_negate_c, s);
+}
+
+static uint64_t fmsac64(uint64_t a, uint64_t b, uint64_t d,
+                        float_status *s)
+{
+    return float64_muladd(a, b, d, float_muladd_negate_c, s);
+}
+
+static uint16_t fnmsac16(uint16_t a, uint16_t b, uint16_t d,
+                         float_status *s)
+{
+    return float16_muladd(a, b, d, float_muladd_negate_product, s);
+}
+
+static uint32_t fnmsac32(uint32_t a, uint32_t b, uint32_t d,
+                         float_status *s)
+{
+    return float32_muladd(a, b, d, float_muladd_negate_product, s);
+}
+
+static uint64_t fnmsac64(uint64_t a, uint64_t b, uint64_t d,
+                         float_status *s)
+{
+    return float64_muladd(a, b, d, float_muladd_negate_product, s);
+}
+
+static uint16_t fmadd16(uint16_t a, uint16_t b, uint16_t d,
+                        float_status *s)
+{
+    return float16_muladd(d, b, a, 0, s);
+}
+
+static uint32_t fmadd32(uint32_t a, uint32_t b, uint32_t d,
+                        float_status *s)
+{
+    return float32_muladd(d, b, a, 0, s);
+}
+
+static uint64_t fmadd64(uint64_t a, uint64_t b, uint64_t d,
+                        float_status *s)
+{
+    return float64_muladd(d, b, a, 0, s);
+}
+
+static uint16_t fnmadd16(uint16_t a, uint16_t b, uint16_t d,
+                         float_status *s)
+{
+    return float16_muladd(d, b, a,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint32_t fnmadd32(uint32_t a, uint32_t b, uint32_t d,
+                         float_status *s)
+{
+    return float32_muladd(d, b, a,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint64_t fnmadd64(uint64_t a, uint64_t b, uint64_t d,
+                         float_status *s)
+{
+    return float64_muladd(d, b, a,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint16_t fmsub16(uint16_t a, uint16_t b, uint16_t d,
+                        float_status *s)
+{
+    return float16_muladd(d, b, a, float_muladd_negate_c, s);
+}
+
+static uint32_t fmsub32(uint32_t a, uint32_t b, uint32_t d,
+                        float_status *s)
+{
+    return float32_muladd(d, b, a, float_muladd_negate_c, s);
+}
+
+static uint64_t fmsub64(uint64_t a, uint64_t b, uint64_t d,
+                        float_status *s)
+{
+    return float64_muladd(d, b, a, float_muladd_negate_c, s);
+}
+
+static uint16_t fnmsub16(uint16_t a, uint16_t b, uint16_t d,
+                         float_status *s)
+{
+    return float16_muladd(d, b, a, float_muladd_negate_product, s);
+}
+
+static uint32_t fnmsub32(uint32_t a, uint32_t b, uint32_t d,
+                         float_status *s)
+{
+    return float32_muladd(d, b, a, float_muladd_negate_product, s);
+}
+
+static uint64_t fnmsub64(uint64_t a, uint64_t b, uint64_t d,
+                         float_status *s)
+{
+    return float64_muladd(d, b, a, float_muladd_negate_product, s);
+}
+
+RVVCALL(OPFVV3, vfmacc_vv_h, OP_UUU_H, H2, H2, H2, fmacc16)
+RVVCALL(OPFVV3, vfmacc_vv_w, OP_UUU_W, H4, H4, H4, fmacc32)
+RVVCALL(OPFVV3, vfmacc_vv_d, OP_UUU_D, H8, H8, H8, fmacc64)
+RVVCALL(OPFVF3, vfmacc_vf_h, OP_UUU_H, H2, H2, fmacc16)
+RVVCALL(OPFVF3, vfmacc_vf_w, OP_UUU_W, H4, H4, fmacc32)
+RVVCALL(OPFVF3, vfmacc_vf_d, OP_UUU_D, H8, H8, fmacc64)
+RVVCALL(OPFVV3, vfnmacc_vv_h, OP_UUU_H, H2, H2, H2, fnmacc16)
+RVVCALL(OPFVV3, vfnmacc_vv_w, OP_UUU_W, H4, H4, H4, fnmacc32)
+RVVCALL(OPFVV3, vfnmacc_vv_d, OP_UUU_D, H8, H8, H8, fnmacc64)
+RVVCALL(OPFVF3, vfnmacc_vf_h, OP_UUU_H, H2, H2, fnmacc16)
+RVVCALL(OPFVF3, vfnmacc_vf_w, OP_UUU_W, H4, H4, fnmacc32)
+RVVCALL(OPFVF3, vfnmacc_vf_d, OP_UUU_D, H8, H8, fnmacc64)
+RVVCALL(OPFVV3, vfmsac_vv_h, OP_UUU_H, H2, H2, H2, fmsac16)
+RVVCALL(OPFVV3, vfmsac_vv_w, OP_UUU_W, H4, H4, H4, fmsac32)
+RVVCALL(OPFVV3, vfmsac_vv_d, OP_UUU_D, H8, H8, H8, fmsac64)
+RVVCALL(OPFVF3, vfmsac_vf_h, OP_UUU_H, H2, H2, fmsac16)
+RVVCALL(OPFVF3, vfmsac_vf_w, OP_UUU_W, H4, H4, fmsac32)
+RVVCALL(OPFVF3, vfmsac_vf_d, OP_UUU_D, H8, H8, fmsac64)
+RVVCALL(OPFVV3, vfnmsac_vv_h, OP_UUU_H, H2, H2, H2, fnmsac16)
+RVVCALL(OPFVV3, vfnmsac_vv_w, OP_UUU_W, H4, H4, H4, fnmsac32)
+RVVCALL(OPFVV3, vfnmsac_vv_d, OP_UUU_D, H8, H8, H8, fnmsac64)
+RVVCALL(OPFVF3, vfnmsac_vf_h, OP_UUU_H, H2, H2, fnmsac16)
+RVVCALL(OPFVF3, vfnmsac_vf_w, OP_UUU_W, H4, H4, fnmsac32)
+RVVCALL(OPFVF3, vfnmsac_vf_d, OP_UUU_D, H8, H8, fnmsac64)
+RVVCALL(OPFVV3, vfmadd_vv_h, OP_UUU_H, H2, H2, H2, fmadd16)
+RVVCALL(OPFVV3, vfmadd_vv_w, OP_UUU_W, H4, H4, H4, fmadd32)
+RVVCALL(OPFVV3, vfmadd_vv_d, OP_UUU_D, H8, H8, H8, fmadd64)
+RVVCALL(OPFVF3, vfmadd_vf_h, OP_UUU_H, H2, H2, fmadd16)
+RVVCALL(OPFVF3, vfmadd_vf_w, OP_UUU_W, H4, H4, fmadd32)
+RVVCALL(OPFVF3, vfmadd_vf_d, OP_UUU_D, H8, H8, fmadd64)
+RVVCALL(OPFVV3, vfnmadd_vv_h, OP_UUU_H, H2, H2, H2, fnmadd16)
+RVVCALL(OPFVV3, vfnmadd_vv_w, OP_UUU_W, H4, H4, H4, fnmadd32)
+RVVCALL(OPFVV3, vfnmadd_vv_d, OP_UUU_D, H8, H8, H8, fnmadd64)
+RVVCALL(OPFVF3, vfnmadd_vf_h, OP_UUU_H, H2, H2, fnmadd16)
+RVVCALL(OPFVF3, vfnmadd_vf_w, OP_UUU_W, H4, H4, fnmadd32)
+RVVCALL(OPFVF3, vfnmadd_vf_d, OP_UUU_D, H8, H8, fnmadd64)
+RVVCALL(OPFVV3, vfmsub_vv_h, OP_UUU_H, H2, H2, H2, fmsub16)
+RVVCALL(OPFVV3, vfmsub_vv_w, OP_UUU_W, H4, H4, H4, fmsub32)
+RVVCALL(OPFVV3, vfmsub_vv_d, OP_UUU_D, H8, H8, H8, fmsub64)
+RVVCALL(OPFVF3, vfmsub_vf_h, OP_UUU_H, H2, H2, fmsub16)
+RVVCALL(OPFVF3, vfmsub_vf_w, OP_UUU_W, H4, H4, fmsub32)
+RVVCALL(OPFVF3, vfmsub_vf_d, OP_UUU_D, H8, H8, fmsub64)
+RVVCALL(OPFVV3, vfnmsub_vv_h, OP_UUU_H, H2, H2, H2, fnmsub16)
+RVVCALL(OPFVV3, vfnmsub_vv_w, OP_UUU_W, H4, H4, H4, fnmsub32)
+RVVCALL(OPFVV3, vfnmsub_vv_d, OP_UUU_D, H8, H8, H8, fnmsub64)
+RVVCALL(OPFVF3, vfnmsub_vf_h, OP_UUU_H, H2, H2, fnmsub16)
+RVVCALL(OPFVF3, vfnmsub_vf_w, OP_UUU_W, H4, H4, fnmsub32)
+RVVCALL(OPFVF3, vfnmsub_vf_d, OP_UUU_D, H8, H8, fnmsub64)
+
+#define GEN_VEXT_FMA(NAME, ESZ)                \
+GEN_VEXT_VV_ENV(NAME##_vv_h, ESZ)              \
+GEN_VEXT_VV_ENV(NAME##_vv_w, 4)                \
+GEN_VEXT_VV_ENV(NAME##_vv_d, 8)                \
+GEN_VEXT_VF(NAME##_vf_h, ESZ)                  \
+GEN_VEXT_VF(NAME##_vf_w, 4)                    \
+GEN_VEXT_VF(NAME##_vf_d, 8)
+
+GEN_VEXT_FMA(vfmacc, 2)
+GEN_VEXT_FMA(vfnmacc, 2)
+GEN_VEXT_FMA(vfmsac, 2)
+GEN_VEXT_FMA(vfnmsac, 2)
+GEN_VEXT_FMA(vfmadd, 2)
+GEN_VEXT_FMA(vfnmadd, 2)
+GEN_VEXT_FMA(vfmsub, 2)
+GEN_VEXT_FMA(vfnmsub, 2)
+
+static uint32_t fwmacc16(uint16_t a, uint16_t b, uint32_t d,
+                         float_status *s)
+{
+    return float32_muladd(float16_to_float32(a, true, s),
+                          float16_to_float32(b, true, s), d, 0, s);
+}
+
+static uint64_t fwmacc32(uint32_t a, uint32_t b, uint64_t d,
+                         float_status *s)
+{
+    return float64_muladd(float32_to_float64(a, s),
+                          float32_to_float64(b, s), d, 0, s);
+}
+
+static uint32_t fwnmacc16(uint16_t a, uint16_t b, uint32_t d,
+                          float_status *s)
+{
+    return float32_muladd(float16_to_float32(a, true, s),
+                          float16_to_float32(b, true, s), d,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint64_t fwnmacc32(uint32_t a, uint32_t b, uint64_t d,
+                          float_status *s)
+{
+    return float64_muladd(float32_to_float64(a, s),
+                          float32_to_float64(b, s), d,
+                          float_muladd_negate_c |
+                          float_muladd_negate_product, s);
+}
+
+static uint32_t fwmsac16(uint16_t a, uint16_t b, uint32_t d,
+                         float_status *s)
+{
+    return float32_muladd(float16_to_float32(a, true, s),
+                          float16_to_float32(b, true, s), d,
+                          float_muladd_negate_c, s);
+}
+
+static uint64_t fwmsac32(uint32_t a, uint32_t b, uint64_t d,
+                         float_status *s)
+{
+    return float64_muladd(float32_to_float64(a, s),
+                          float32_to_float64(b, s), d,
+                          float_muladd_negate_c, s);
+}
+
+static uint32_t fwnmsac16(uint16_t a, uint16_t b, uint32_t d,
+                          float_status *s)
+{
+    return float32_muladd(float16_to_float32(a, true, s),
+                          float16_to_float32(b, true, s), d,
+                          float_muladd_negate_product, s);
+}
+
+static uint64_t fwnmsac32(uint32_t a, uint32_t b, uint64_t d,
+                          float_status *s)
+{
+    return float64_muladd(float32_to_float64(a, s),
+                          float32_to_float64(b, s), d,
+                          float_muladd_negate_product, s);
+}
+
+RVVCALL(OPFVV3, vfwmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwmacc16)
+RVVCALL(OPFVV3, vfwmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwmacc32)
+RVVCALL(OPFVF3, vfwmacc_vf_h, WOP_UUU_H, H4, H2, fwmacc16)
+RVVCALL(OPFVF3, vfwmacc_vf_w, WOP_UUU_W, H8, H4, fwmacc32)
+RVVCALL(OPFVV3, vfwnmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwnmacc16)
+RVVCALL(OPFVV3, vfwnmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwnmacc32)
+RVVCALL(OPFVF3, vfwnmacc_vf_h, WOP_UUU_H, H4, H2, fwnmacc16)
+RVVCALL(OPFVF3, vfwnmacc_vf_w, WOP_UUU_W, H8, H4, fwnmacc32)
+RVVCALL(OPFVV3, vfwmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwmsac16)
+RVVCALL(OPFVV3, vfwmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwmsac32)
+RVVCALL(OPFVF3, vfwmsac_vf_h, WOP_UUU_H, H4, H2, fwmsac16)
+RVVCALL(OPFVF3, vfwmsac_vf_w, WOP_UUU_W, H8, H4, fwmsac32)
+RVVCALL(OPFVV3, vfwnmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwnmsac16)
+RVVCALL(OPFVV3, vfwnmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwnmsac32)
+RVVCALL(OPFVF3, vfwnmsac_vf_h, WOP_UUU_H, H4, H2, fwnmsac16)
+RVVCALL(OPFVF3, vfwnmsac_vf_w, WOP_UUU_W, H8, H4, fwnmsac32)
+
+#define GEN_VEXT_FWMA(NAME)                  \
+GEN_VEXT_VV_ENV(NAME##_vv_h, 4)              \
+GEN_VEXT_VV_ENV(NAME##_vv_w, 8)              \
+GEN_VEXT_VF(NAME##_vf_h, 4)                  \
+GEN_VEXT_VF(NAME##_vf_w, 8)
+
+GEN_VEXT_FWMA(vfwmacc)
+GEN_VEXT_FWMA(vfwnmacc)
+GEN_VEXT_FWMA(vfwmsac)
+GEN_VEXT_FWMA(vfwnmsac)
+
+#define GEN_VEXT_FCMP_VV(NAME, ETYPE, H, DO_OP)                      \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,          \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    uint32_t vm = vext_vm(desc);                                     \
+    uint32_t vl = env->vl;                                           \
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;               \
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);                     \
+    uint32_t vma = vext_vma(desc);                                   \
+    uint32_t i;                                                      \
+                                                                     \
+    for (i = env->vstart; i < vl; i++) {                             \
+        ETYPE s1 = *((ETYPE *)vs1 + H(i));                           \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                           \
+                                                                     \
+        if (!vm && !vext_elem_mask(v0, i)) {                         \
+            if (vma) {                                               \
+                vext_set_elem_mask(vd, i, 1);                        \
+            }                                                        \
+            continue;                                                \
+        }                                                            \
+        vext_set_elem_mask(vd, i, DO_OP(s2, s1, &env->fp_status));   \
+    }                                                                \
+    env->vstart = 0;                                                 \
+    if (vta_all_1s) {                                                \
+        for (; i < total_elems; i++) {                               \
+            vext_set_elem_mask(vd, i, 1);                            \
+        }                                                            \
+    }                                                                \
+}
+
+#define GEN_VEXT_FCMP_VF(NAME, ETYPE, H, DO_OP)                      \
+void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2,        \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    uint32_t vm = vext_vm(desc);                                     \
+    uint32_t vl = env->vl;                                           \
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;               \
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);                     \
+    uint32_t vma = vext_vma(desc);                                   \
+    uint32_t i;                                                      \
+                                                                     \
+    for (i = env->vstart; i < vl; i++) {                             \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                           \
+                                                                     \
+        if (!vm && !vext_elem_mask(v0, i)) {                         \
+            if (vma) {                                               \
+                vext_set_elem_mask(vd, i, 1);                        \
+            }                                                        \
+            continue;                                                \
+        }                                                            \
+        vext_set_elem_mask(vd, i,                                   \
+                           DO_OP(s2, (ETYPE)s1, &env->fp_status));   \
+    }                                                                \
+    env->vstart = 0;                                                 \
+    if (vta_all_1s) {                                                \
+        for (; i < total_elems; i++) {                               \
+            vext_set_elem_mask(vd, i, 1);                            \
+        }                                                            \
+    }                                                                \
+}
+
+GEN_VEXT_FCMP_VV(vmfeq_vv_h, uint16_t, H2, float16_eq_quiet)
+GEN_VEXT_FCMP_VV(vmfeq_vv_w, uint32_t, H4, float32_eq_quiet)
+GEN_VEXT_FCMP_VV(vmfeq_vv_d, uint64_t, H8, float64_eq_quiet)
+GEN_VEXT_FCMP_VF(vmfeq_vf_h, uint16_t, H2, float16_eq_quiet)
+GEN_VEXT_FCMP_VF(vmfeq_vf_w, uint32_t, H4, float32_eq_quiet)
+GEN_VEXT_FCMP_VF(vmfeq_vf_d, uint64_t, H8, float64_eq_quiet)
+
+static bool vmfne16(uint16_t a, uint16_t b, float_status *s)
+{
+    FloatRelation compare = float16_compare_quiet(a, b, s);
+
+    return compare != float_relation_equal;
+}
+
+static bool vmfne32(uint32_t a, uint32_t b, float_status *s)
+{
+    FloatRelation compare = float32_compare_quiet(a, b, s);
+
+    return compare != float_relation_equal;
+}
+
+static bool vmfne64(uint64_t a, uint64_t b, float_status *s)
+{
+    FloatRelation compare = float64_compare_quiet(a, b, s);
+
+    return compare != float_relation_equal;
+}
+
+GEN_VEXT_FCMP_VV(vmfne_vv_h, uint16_t, H2, vmfne16)
+GEN_VEXT_FCMP_VV(vmfne_vv_w, uint32_t, H4, vmfne32)
+GEN_VEXT_FCMP_VV(vmfne_vv_d, uint64_t, H8, vmfne64)
+GEN_VEXT_FCMP_VF(vmfne_vf_h, uint16_t, H2, vmfne16)
+GEN_VEXT_FCMP_VF(vmfne_vf_w, uint32_t, H4, vmfne32)
+GEN_VEXT_FCMP_VF(vmfne_vf_d, uint64_t, H8, vmfne64)
+
+GEN_VEXT_FCMP_VV(vmflt_vv_h, uint16_t, H2, float16_lt)
+GEN_VEXT_FCMP_VV(vmflt_vv_w, uint32_t, H4, float32_lt)
+GEN_VEXT_FCMP_VV(vmflt_vv_d, uint64_t, H8, float64_lt)
+GEN_VEXT_FCMP_VF(vmflt_vf_h, uint16_t, H2, float16_lt)
+GEN_VEXT_FCMP_VF(vmflt_vf_w, uint32_t, H4, float32_lt)
+GEN_VEXT_FCMP_VF(vmflt_vf_d, uint64_t, H8, float64_lt)
+
+GEN_VEXT_FCMP_VV(vmfle_vv_h, uint16_t, H2, float16_le)
+GEN_VEXT_FCMP_VV(vmfle_vv_w, uint32_t, H4, float32_le)
+GEN_VEXT_FCMP_VV(vmfle_vv_d, uint64_t, H8, float64_le)
+GEN_VEXT_FCMP_VF(vmfle_vf_h, uint16_t, H2, float16_le)
+GEN_VEXT_FCMP_VF(vmfle_vf_w, uint32_t, H4, float32_le)
+GEN_VEXT_FCMP_VF(vmfle_vf_d, uint64_t, H8, float64_le)
+
+static bool vmfgt16(uint16_t a, uint16_t b, float_status *s)
+{
+    FloatRelation compare = float16_compare(a, b, s);
+
+    return compare == float_relation_greater;
+}
+
+static bool vmfgt32(uint32_t a, uint32_t b, float_status *s)
+{
+    FloatRelation compare = float32_compare(a, b, s);
+
+    return compare == float_relation_greater;
+}
+
+static bool vmfgt64(uint64_t a, uint64_t b, float_status *s)
+{
+    FloatRelation compare = float64_compare(a, b, s);
+
+    return compare == float_relation_greater;
+}
+
+GEN_VEXT_FCMP_VF(vmfgt_vf_h, uint16_t, H2, vmfgt16)
+GEN_VEXT_FCMP_VF(vmfgt_vf_w, uint32_t, H4, vmfgt32)
+GEN_VEXT_FCMP_VF(vmfgt_vf_d, uint64_t, H8, vmfgt64)
+
+static bool vmfge16(uint16_t a, uint16_t b, float_status *s)
+{
+    FloatRelation compare = float16_compare(a, b, s);
+
+    return compare == float_relation_greater ||
+           compare == float_relation_equal;
+}
+
+static bool vmfge32(uint32_t a, uint32_t b, float_status *s)
+{
+    FloatRelation compare = float32_compare(a, b, s);
+
+    return compare == float_relation_greater ||
+           compare == float_relation_equal;
+}
+
+static bool vmfge64(uint64_t a, uint64_t b, float_status *s)
+{
+    FloatRelation compare = float64_compare(a, b, s);
+
+    return compare == float_relation_greater ||
+           compare == float_relation_equal;
+}
+
+GEN_VEXT_FCMP_VF(vmfge_vf_h, uint16_t, H2, vmfge16)
+GEN_VEXT_FCMP_VF(vmfge_vf_w, uint32_t, H4, vmfge32)
+GEN_VEXT_FCMP_VF(vmfge_vf_d, uint64_t, H8, vmfge64)
+
+typedef void opivv1_fn(void *vd, void *vs2, int i);
+
+#define OPIVV1(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
+static void do_##NAME(void *vd, void *vs2, int i)             \
+{                                                             \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                           \
+                                                              \
+    *((TD *)vd + HD(i)) = OP(s2);                             \
+}
+
+static void do_vext_v(void *vd, void *v0, void *vs2,
+                      CPURISCVState *env, uint32_t desc,
+                      opivv1_fn *fn, uint32_t esz)
+{
+    uint32_t i;
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, vs2, i);
+    }
+    env->vstart = 0;
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_V(NAME, ESZ)                                      \
+void HELPER(NAME)(void *vd, void *v0, void *vs2,                   \
+                  CPURISCVState *env, uint32_t desc)               \
+{                                                                  \
+    do_vext_v(vd, v0, vs2, env, desc, do_##NAME, ESZ);             \
+}
+
+typedef void opfvv1_fn(void *vd, void *vs2, int i, CPURISCVState *env);
+
+#define OPFVV1(NAME, TD, T2, TX2, HD, HS2, OP)             \
+static void do_##NAME(void *vd, void *vs2, int i,          \
+                      CPURISCVState *env)                  \
+{                                                          \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                        \
+                                                           \
+    *((TD *)vd + HD(i)) = OP(s2, &env->fp_status);         \
+}
+
+static void do_vext_v_env(void *vd, void *v0, void *vs2,
+                          CPURISCVState *env, uint32_t desc,
+                          opfvv1_fn *fn, uint32_t esz)
+{
+    uint32_t i;
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, vs2, i, env);
+    }
+    env->vstart = 0;
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_V_ENV(NAME, ESZ)                                  \
+void HELPER(NAME)(void *vd, void *v0, void *vs2,                   \
+                  CPURISCVState *env, uint32_t desc)               \
+{                                                                  \
+    do_vext_v_env(vd, v0, vs2, env, desc, do_##NAME, ESZ);         \
+}
+
+RVVCALL(OPFVV1, vfsqrt_v_h, OP_UU_H, H2, H2, float16_sqrt)
+RVVCALL(OPFVV1, vfsqrt_v_w, OP_UU_W, H4, H4, float32_sqrt)
+RVVCALL(OPFVV1, vfsqrt_v_d, OP_UU_D, H8, H8, float64_sqrt)
+GEN_VEXT_V_ENV(vfsqrt_v_h, 2)
+GEN_VEXT_V_ENV(vfsqrt_v_w, 4)
+GEN_VEXT_V_ENV(vfsqrt_v_d, 8)
+
+static uint64_t frsqrt7(uint64_t f, int exp_size, int frac_size)
+{
+    uint64_t sign = extract64(f, frac_size + exp_size, 1);
+    uint64_t exp = extract64(f, frac_size, exp_size);
+    uint64_t frac = extract64(f, 0, frac_size);
+    const uint8_t lookup_table[] = {
+        52, 51, 50, 48, 47, 46, 44, 43,
+        42, 41, 40, 39, 38, 36, 35, 34,
+        33, 32, 31, 30, 30, 29, 28, 27,
+        26, 25, 24, 23, 23, 22, 21, 20,
+        19, 19, 18, 17, 16, 16, 15, 14,
+        14, 13, 12, 12, 11, 10, 10, 9,
+        9, 8, 7, 7, 6, 6, 5, 4,
+        4, 3, 3, 2, 2, 1, 1, 0,
+        127, 125, 123, 121, 119, 118, 116, 114,
+        113, 111, 109, 108, 106, 105, 103, 102,
+        100, 99, 97, 96, 95, 93, 92, 91,
+        90, 88, 87, 86, 85, 84, 83, 82,
+        80, 79, 78, 77, 76, 75, 74, 73,
+        72, 71, 70, 70, 69, 68, 67, 66,
+        65, 64, 63, 63, 62, 61, 60, 59,
+        59, 58, 57, 56, 56, 55, 54, 53
+    };
+    const int precision = 7;
+    int idx;
+    uint64_t out_frac;
+    uint64_t out_exp;
+    uint64_t val = 0;
+
+    if (exp == 0 && frac != 0) {
+        while (extract64(frac, frac_size - 1, 1) == 0) {
+            exp--;
+            frac <<= 1;
+        }
+        frac = (frac << 1) & MAKE_64BIT_MASK(0, frac_size);
+    }
+
+    idx = ((exp & 1) << (precision - 1)) |
+          (frac >> (frac_size - precision + 1));
+    out_frac = (uint64_t)lookup_table[idx] << (frac_size - precision);
+    out_exp = (3 * MAKE_64BIT_MASK(0, exp_size - 1) + ~exp) / 2;
+
+    val = deposit64(val, 0, frac_size, out_frac);
+    val = deposit64(val, frac_size, exp_size, out_exp);
+    val = deposit64(val, frac_size + exp_size, 1, sign);
+    return val;
+}
+
+static float16 frsqrt7_h(float16 f, float_status *s)
+{
+    int exp_size = 5;
+    int frac_size = 10;
+    bool sign = float16_is_neg(f);
+    uint64_t val;
+
+    if (float16_is_signaling_nan(f, s) ||
+        (float16_is_infinity(f) && sign) ||
+        (float16_is_normal(f) && sign) ||
+        (float16_is_zero_or_denormal(f) && !float16_is_zero(f) && sign)) {
+        s->float_exception_flags |= float_flag_invalid;
+        return float16_default_nan(s);
+    }
+    if (float16_is_quiet_nan(f, s)) {
+        return float16_default_nan(s);
+    }
+    if (float16_is_zero(f)) {
+        s->float_exception_flags |= float_flag_divbyzero;
+        return float16_set_sign(float16_infinity, sign);
+    }
+    if (float16_is_infinity(f) && !sign) {
+        return float16_set_sign(float16_zero, sign);
+    }
+
+    val = frsqrt7(f, exp_size, frac_size);
+    return make_float16(val);
+}
+
+static float32 frsqrt7_s(float32 f, float_status *s)
+{
+    int exp_size = 8;
+    int frac_size = 23;
+    bool sign = float32_is_neg(f);
+    uint64_t val;
+
+    if (float32_is_signaling_nan(f, s) ||
+        (float32_is_infinity(f) && sign) ||
+        (float32_is_normal(f) && sign) ||
+        (float32_is_zero_or_denormal(f) && !float32_is_zero(f) && sign)) {
+        s->float_exception_flags |= float_flag_invalid;
+        return float32_default_nan(s);
+    }
+    if (float32_is_quiet_nan(f, s)) {
+        return float32_default_nan(s);
+    }
+    if (float32_is_zero(f)) {
+        s->float_exception_flags |= float_flag_divbyzero;
+        return float32_set_sign(float32_infinity, sign);
+    }
+    if (float32_is_infinity(f) && !sign) {
+        return float32_set_sign(float32_zero, sign);
+    }
+
+    val = frsqrt7(f, exp_size, frac_size);
+    return make_float32(val);
+}
+
+static float64 frsqrt7_d(float64 f, float_status *s)
+{
+    int exp_size = 11;
+    int frac_size = 52;
+    bool sign = float64_is_neg(f);
+    uint64_t val;
+
+    if (float64_is_signaling_nan(f, s) ||
+        (float64_is_infinity(f) && sign) ||
+        (float64_is_normal(f) && sign) ||
+        (float64_is_zero_or_denormal(f) && !float64_is_zero(f) && sign)) {
+        s->float_exception_flags |= float_flag_invalid;
+        return float64_default_nan(s);
+    }
+    if (float64_is_quiet_nan(f, s)) {
+        return float64_default_nan(s);
+    }
+    if (float64_is_zero(f)) {
+        s->float_exception_flags |= float_flag_divbyzero;
+        return float64_set_sign(float64_infinity, sign);
+    }
+    if (float64_is_infinity(f) && !sign) {
+        return float64_set_sign(float64_zero, sign);
+    }
+
+    val = frsqrt7(f, exp_size, frac_size);
+    return make_float64(val);
+}
+
+RVVCALL(OPFVV1, vfrsqrt7_v_h, OP_UU_H, H2, H2, frsqrt7_h)
+RVVCALL(OPFVV1, vfrsqrt7_v_w, OP_UU_W, H4, H4, frsqrt7_s)
+RVVCALL(OPFVV1, vfrsqrt7_v_d, OP_UU_D, H8, H8, frsqrt7_d)
+GEN_VEXT_V_ENV(vfrsqrt7_v_h, 2)
+GEN_VEXT_V_ENV(vfrsqrt7_v_w, 4)
+GEN_VEXT_V_ENV(vfrsqrt7_v_d, 8)
+
+static uint64_t frec7(uint64_t f, int exp_size, int frac_size,
+                      float_status *s)
+{
+    uint64_t sign = extract64(f, frac_size + exp_size, 1);
+    uint64_t exp = extract64(f, frac_size, exp_size);
+    uint64_t frac = extract64(f, 0, frac_size);
+    const uint8_t lookup_table[] = {
+        127, 125, 123, 121, 119, 117, 116, 114,
+        112, 110, 109, 107, 105, 104, 102, 100,
+        99, 97, 96, 94, 93, 91, 90, 88,
+        87, 85, 84, 83, 81, 80, 79, 77,
+        76, 75, 74, 72, 71, 70, 69, 68,
+        66, 65, 64, 63, 62, 61, 60, 59,
+        58, 57, 56, 55, 54, 53, 52, 51,
+        50, 49, 48, 47, 46, 45, 44, 43,
+        42, 41, 40, 40, 39, 38, 37, 36,
+        35, 35, 34, 33, 32, 31, 31, 30,
+        29, 28, 28, 27, 26, 25, 25, 24,
+        23, 23, 22, 21, 21, 20, 19, 19,
+        18, 17, 17, 16, 15, 15, 14, 14,
+        13, 12, 12, 11, 11, 10, 9, 9,
+        8, 8, 7, 7, 6, 5, 5, 4,
+        4, 3, 3, 2, 2, 1, 1, 0
+    };
+    const int precision = 7;
+    int idx;
+    uint64_t out_frac;
+    uint64_t out_exp;
+    uint64_t val = 0;
+
+    if (exp == 0 && frac != 0) {
+        while (extract64(frac, frac_size - 1, 1) == 0) {
+            exp--;
+            frac <<= 1;
+        }
+
+        frac = (frac << 1) & MAKE_64BIT_MASK(0, frac_size);
+        if (exp != 0 && exp != UINT64_MAX) {
+            s->float_exception_flags |= float_flag_inexact |
+                                        float_flag_overflow;
+            if (s->float_rounding_mode == float_round_to_zero ||
+                (s->float_rounding_mode == float_round_down && !sign) ||
+                (s->float_rounding_mode == float_round_up && sign)) {
+                return (sign << (exp_size + frac_size)) |
+                       (MAKE_64BIT_MASK(frac_size, exp_size) - 1);
+            } else {
+                return (sign << (exp_size + frac_size)) |
+                       MAKE_64BIT_MASK(frac_size, exp_size);
+            }
+        }
+    }
+
+    idx = frac >> (frac_size - precision);
+    out_frac = (uint64_t)lookup_table[idx] << (frac_size - precision);
+    out_exp = 2 * MAKE_64BIT_MASK(0, exp_size - 1) + ~exp;
+    if (out_exp == 0 || out_exp == UINT64_MAX) {
+        out_frac = (out_frac >> 1) | MAKE_64BIT_MASK(frac_size - 1, 1);
+        if (out_exp == UINT64_MAX) {
+            out_frac >>= 1;
+            out_exp = 0;
+        }
+    }
+
+    val = deposit64(val, 0, frac_size, out_frac);
+    val = deposit64(val, frac_size, exp_size, out_exp);
+    val = deposit64(val, frac_size + exp_size, 1, sign);
+    return val;
+}
+
+static float16 frec7_h(float16 f, float_status *s)
+{
+    int exp_size = 5;
+    int frac_size = 10;
+    bool sign = float16_is_neg(f);
+    uint64_t val;
+
+    if (float16_is_infinity(f)) {
+        return float16_set_sign(float16_zero, sign);
+    }
+    if (float16_is_zero(f)) {
+        s->float_exception_flags |= float_flag_divbyzero;
+        return float16_set_sign(float16_infinity, sign);
+    }
+    if (float16_is_signaling_nan(f, s)) {
+        s->float_exception_flags |= float_flag_invalid;
+        return float16_default_nan(s);
+    }
+    if (float16_is_quiet_nan(f, s)) {
+        return float16_default_nan(s);
+    }
+
+    val = frec7(f, exp_size, frac_size, s);
+    return make_float16(val);
+}
+
+static float32 frec7_s(float32 f, float_status *s)
+{
+    int exp_size = 8;
+    int frac_size = 23;
+    bool sign = float32_is_neg(f);
+    uint64_t val;
+
+    if (float32_is_infinity(f)) {
+        return float32_set_sign(float32_zero, sign);
+    }
+    if (float32_is_zero(f)) {
+        s->float_exception_flags |= float_flag_divbyzero;
+        return float32_set_sign(float32_infinity, sign);
+    }
+    if (float32_is_signaling_nan(f, s)) {
+        s->float_exception_flags |= float_flag_invalid;
+        return float32_default_nan(s);
+    }
+    if (float32_is_quiet_nan(f, s)) {
+        return float32_default_nan(s);
+    }
+
+    val = frec7(f, exp_size, frac_size, s);
+    return make_float32(val);
+}
+
+static float64 frec7_d(float64 f, float_status *s)
+{
+    int exp_size = 11;
+    int frac_size = 52;
+    bool sign = float64_is_neg(f);
+    uint64_t val;
+
+    if (float64_is_infinity(f)) {
+        return float64_set_sign(float64_zero, sign);
+    }
+    if (float64_is_zero(f)) {
+        s->float_exception_flags |= float_flag_divbyzero;
+        return float64_set_sign(float64_infinity, sign);
+    }
+    if (float64_is_signaling_nan(f, s)) {
+        s->float_exception_flags |= float_flag_invalid;
+        return float64_default_nan(s);
+    }
+    if (float64_is_quiet_nan(f, s)) {
+        return float64_default_nan(s);
+    }
+
+    val = frec7(f, exp_size, frac_size, s);
+    return make_float64(val);
+}
+
+RVVCALL(OPFVV1, vfrec7_v_h, OP_UU_H, H2, H2, frec7_h)
+RVVCALL(OPFVV1, vfrec7_v_w, OP_UU_W, H4, H4, frec7_s)
+RVVCALL(OPFVV1, vfrec7_v_d, OP_UU_D, H8, H8, frec7_d)
+GEN_VEXT_V_ENV(vfrec7_v_h, 2)
+GEN_VEXT_V_ENV(vfrec7_v_w, 4)
+GEN_VEXT_V_ENV(vfrec7_v_d, 8)
+
+RVVCALL(OPFVV1, vfcvt_xu_f_v_h, OP_UU_H, H2, H2, float16_to_uint16)
+RVVCALL(OPFVV1, vfcvt_xu_f_v_w, OP_UU_W, H4, H4, float32_to_uint32)
+RVVCALL(OPFVV1, vfcvt_xu_f_v_d, OP_UU_D, H8, H8, float64_to_uint64)
+GEN_VEXT_V_ENV(vfcvt_xu_f_v_h, 2)
+GEN_VEXT_V_ENV(vfcvt_xu_f_v_w, 4)
+GEN_VEXT_V_ENV(vfcvt_xu_f_v_d, 8)
+
+RVVCALL(OPFVV1, vfcvt_x_f_v_h, OP_UU_H, H2, H2, float16_to_int16)
+RVVCALL(OPFVV1, vfcvt_x_f_v_w, OP_UU_W, H4, H4, float32_to_int32)
+RVVCALL(OPFVV1, vfcvt_x_f_v_d, OP_UU_D, H8, H8, float64_to_int64)
+GEN_VEXT_V_ENV(vfcvt_x_f_v_h, 2)
+GEN_VEXT_V_ENV(vfcvt_x_f_v_w, 4)
+GEN_VEXT_V_ENV(vfcvt_x_f_v_d, 8)
+
+RVVCALL(OPFVV1, vfcvt_f_xu_v_h, OP_UU_H, H2, H2, uint16_to_float16)
+RVVCALL(OPFVV1, vfcvt_f_xu_v_w, OP_UU_W, H4, H4, uint32_to_float32)
+RVVCALL(OPFVV1, vfcvt_f_xu_v_d, OP_UU_D, H8, H8, uint64_to_float64)
+GEN_VEXT_V_ENV(vfcvt_f_xu_v_h, 2)
+GEN_VEXT_V_ENV(vfcvt_f_xu_v_w, 4)
+GEN_VEXT_V_ENV(vfcvt_f_xu_v_d, 8)
+
+RVVCALL(OPFVV1, vfcvt_f_x_v_h, OP_UU_H, H2, H2, int16_to_float16)
+RVVCALL(OPFVV1, vfcvt_f_x_v_w, OP_UU_W, H4, H4, int32_to_float32)
+RVVCALL(OPFVV1, vfcvt_f_x_v_d, OP_UU_D, H8, H8, int64_to_float64)
+GEN_VEXT_V_ENV(vfcvt_f_x_v_h, 2)
+GEN_VEXT_V_ENV(vfcvt_f_x_v_w, 4)
+GEN_VEXT_V_ENV(vfcvt_f_x_v_d, 8)
+
+RVVCALL(OPFVV1, vfwcvt_xu_f_v_h, WOP_UU_H, H4, H2, float16_to_uint32)
+RVVCALL(OPFVV1, vfwcvt_xu_f_v_w, WOP_UU_W, H8, H4, float32_to_uint64)
+GEN_VEXT_V_ENV(vfwcvt_xu_f_v_h, 4)
+GEN_VEXT_V_ENV(vfwcvt_xu_f_v_w, 8)
+
+RVVCALL(OPFVV1, vfwcvt_x_f_v_h, WOP_UU_H, H4, H2, float16_to_int32)
+RVVCALL(OPFVV1, vfwcvt_x_f_v_w, WOP_UU_W, H8, H4, float32_to_int64)
+GEN_VEXT_V_ENV(vfwcvt_x_f_v_h, 4)
+GEN_VEXT_V_ENV(vfwcvt_x_f_v_w, 8)
+
+RVVCALL(OPFVV1, vfwcvt_f_xu_v_b, WOP_UU_B, H2, H1, uint8_to_float16)
+RVVCALL(OPFVV1, vfwcvt_f_xu_v_h, WOP_UU_H, H4, H2, uint16_to_float32)
+RVVCALL(OPFVV1, vfwcvt_f_xu_v_w, WOP_UU_W, H8, H4, uint32_to_float64)
+GEN_VEXT_V_ENV(vfwcvt_f_xu_v_b, 2)
+GEN_VEXT_V_ENV(vfwcvt_f_xu_v_h, 4)
+GEN_VEXT_V_ENV(vfwcvt_f_xu_v_w, 8)
+
+RVVCALL(OPFVV1, vfwcvt_f_x_v_b, WOP_UU_B, H2, H1, int8_to_float16)
+RVVCALL(OPFVV1, vfwcvt_f_x_v_h, WOP_UU_H, H4, H2, int16_to_float32)
+RVVCALL(OPFVV1, vfwcvt_f_x_v_w, WOP_UU_W, H8, H4, int32_to_float64)
+GEN_VEXT_V_ENV(vfwcvt_f_x_v_b, 2)
+GEN_VEXT_V_ENV(vfwcvt_f_x_v_h, 4)
+GEN_VEXT_V_ENV(vfwcvt_f_x_v_w, 8)
+
+static uint32_t vfwcvtffv16(uint16_t a, float_status *s)
+{
+    return float16_to_float32(a, true, s);
+}
+
+RVVCALL(OPFVV1, vfwcvt_f_f_v_h, WOP_UU_H, H4, H2, vfwcvtffv16)
+RVVCALL(OPFVV1, vfwcvt_f_f_v_w, WOP_UU_W, H8, H4, float32_to_float64)
+GEN_VEXT_V_ENV(vfwcvt_f_f_v_h, 4)
+GEN_VEXT_V_ENV(vfwcvt_f_f_v_w, 8)
+
+RVVCALL(OPFVV1, vfncvt_xu_f_w_b, NOP_UU_B, H1, H2, float16_to_uint8)
+RVVCALL(OPFVV1, vfncvt_xu_f_w_h, NOP_UU_H, H2, H4, float32_to_uint16)
+RVVCALL(OPFVV1, vfncvt_xu_f_w_w, NOP_UU_W, H4, H8, float64_to_uint32)
+GEN_VEXT_V_ENV(vfncvt_xu_f_w_b, 1)
+GEN_VEXT_V_ENV(vfncvt_xu_f_w_h, 2)
+GEN_VEXT_V_ENV(vfncvt_xu_f_w_w, 4)
+
+RVVCALL(OPFVV1, vfncvt_x_f_w_b, NOP_UU_B, H1, H2, float16_to_int8)
+RVVCALL(OPFVV1, vfncvt_x_f_w_h, NOP_UU_H, H2, H4, float32_to_int16)
+RVVCALL(OPFVV1, vfncvt_x_f_w_w, NOP_UU_W, H4, H8, float64_to_int32)
+GEN_VEXT_V_ENV(vfncvt_x_f_w_b, 1)
+GEN_VEXT_V_ENV(vfncvt_x_f_w_h, 2)
+GEN_VEXT_V_ENV(vfncvt_x_f_w_w, 4)
+
+RVVCALL(OPFVV1, vfncvt_f_xu_w_h, NOP_UU_H, H2, H4, uint32_to_float16)
+RVVCALL(OPFVV1, vfncvt_f_xu_w_w, NOP_UU_W, H4, H8, uint64_to_float32)
+GEN_VEXT_V_ENV(vfncvt_f_xu_w_h, 2)
+GEN_VEXT_V_ENV(vfncvt_f_xu_w_w, 4)
+
+RVVCALL(OPFVV1, vfncvt_f_x_w_h, NOP_UU_H, H2, H4, int32_to_float16)
+RVVCALL(OPFVV1, vfncvt_f_x_w_w, NOP_UU_W, H4, H8, int64_to_float32)
+GEN_VEXT_V_ENV(vfncvt_f_x_w_h, 2)
+GEN_VEXT_V_ENV(vfncvt_f_x_w_w, 4)
+
+static uint16_t vfncvtffv16(uint32_t a, float_status *s)
+{
+    return float32_to_float16(a, true, s);
+}
+
+RVVCALL(OPFVV1, vfncvt_f_f_w_h, NOP_UU_H, H2, H4, vfncvtffv16)
+RVVCALL(OPFVV1, vfncvt_f_f_w_w, NOP_UU_W, H4, H8, float64_to_float32)
+GEN_VEXT_V_ENV(vfncvt_f_f_w_h, 2)
+GEN_VEXT_V_ENV(vfncvt_f_f_w_w, 4)
+
+static target_ulong rvv_fclass_h(uint16_t frs1)
+{
+    float16 f = frs1;
+    bool sign = float16_is_neg(f);
+
+    if (float16_is_infinity(f)) {
+        return sign ? 1 << 0 : 1 << 7;
+    } else if (float16_is_zero(f)) {
+        return sign ? 1 << 3 : 1 << 4;
+    } else if (float16_is_zero_or_denormal(f)) {
+        return sign ? 1 << 2 : 1 << 5;
+    } else if (float16_is_any_nan(f)) {
+        float_status s = { };
+
+        return float16_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
+    } else {
+        return sign ? 1 << 1 : 1 << 6;
+    }
+}
+
+static target_ulong rvv_fclass_s(uint32_t frs1)
+{
+    float32 f = frs1;
+    bool sign = float32_is_neg(f);
+
+    if (float32_is_infinity(f)) {
+        return sign ? 1 << 0 : 1 << 7;
+    } else if (float32_is_zero(f)) {
+        return sign ? 1 << 3 : 1 << 4;
+    } else if (float32_is_zero_or_denormal(f)) {
+        return sign ? 1 << 2 : 1 << 5;
+    } else if (float32_is_any_nan(f)) {
+        float_status s = { };
+
+        return float32_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
+    } else {
+        return sign ? 1 << 1 : 1 << 6;
+    }
+}
+
+static target_ulong rvv_fclass_d(uint64_t frs1)
+{
+    float64 f = frs1;
+    bool sign = float64_is_neg(f);
+
+    if (float64_is_infinity(f)) {
+        return sign ? 1 << 0 : 1 << 7;
+    } else if (float64_is_zero(f)) {
+        return sign ? 1 << 3 : 1 << 4;
+    } else if (float64_is_zero_or_denormal(f)) {
+        return sign ? 1 << 2 : 1 << 5;
+    } else if (float64_is_any_nan(f)) {
+        float_status s = { };
+
+        return float64_is_quiet_nan(f, &s) ? 1 << 9 : 1 << 8;
+    } else {
+        return sign ? 1 << 1 : 1 << 6;
+    }
+}
+
+RVVCALL(OPIVV1, vfclass_v_h, OP_UUU_H, H2, H2, H2, rvv_fclass_h)
+RVVCALL(OPIVV1, vfclass_v_w, OP_UUU_W, H4, H4, H4, rvv_fclass_s)
+RVVCALL(OPIVV1, vfclass_v_d, OP_UUU_D, H8, H8, H8, rvv_fclass_d)
+GEN_VEXT_V(vfclass_v_h, 2)
+GEN_VEXT_V(vfclass_v_w, 4)
+GEN_VEXT_V(vfclass_v_d, 8)
+
+#define GEN_VFMERGE_VF(NAME, ETYPE, H)                       \
+void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2,\
+                  CPURISCVState *env, uint32_t desc)         \
+{                                                            \
+    uint32_t vl = env->vl;                                   \
+    uint32_t esz = sizeof(ETYPE);                            \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);\
+    uint32_t vta = vext_vta(desc);                           \
+    uint32_t i;                                              \
+                                                             \
+    for (i = env->vstart; i < vl; i++) {                     \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                   \
+                                                             \
+        *((ETYPE *)vd + H(i)) =                              \
+            (!vext_vm(desc) && !vext_elem_mask(v0, i) ?      \
+             s2 : (ETYPE)s1);                                \
+    }                                                        \
+    env->vstart = 0;                                         \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \
+}
+
+GEN_VFMERGE_VF(vfmerge_vfm_h, int16_t, H2)
+GEN_VFMERGE_VF(vfmerge_vfm_w, int32_t, H4)
+GEN_VFMERGE_VF(vfmerge_vfm_d, int64_t, H8)
+
+#define GEN_VEXT_VSLIDEUP_VX(NAME, ETYPE, H)                         \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2,    \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    uint32_t vl = env->vl;                                           \
+    uint32_t esz = sizeof(ETYPE);                                    \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);     \
+    uint32_t vta = vext_vta(desc);                                   \
+    uint32_t vma = vext_vma(desc);                                   \
+    target_ulong offset = s1;                                        \
+    target_ulong i_min = MAX(env->vstart, offset);                   \
+    target_ulong i;                                                  \
+                                                                     \
+    for (i = i_min; i < vl; i++) {                                   \
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {              \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);      \
+            continue;                                                \
+        }                                                            \
+        *((ETYPE *)vd + H(i)) = *((ETYPE *)vs2 + H(i - offset));     \
+    }                                                                \
+    env->vstart = 0;                                                 \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);         \
+}
+
+GEN_VEXT_VSLIDEUP_VX(vslideup_vx_b, uint8_t, H1)
+GEN_VEXT_VSLIDEUP_VX(vslideup_vx_h, uint16_t, H2)
+GEN_VEXT_VSLIDEUP_VX(vslideup_vx_w, uint32_t, H4)
+GEN_VEXT_VSLIDEUP_VX(vslideup_vx_d, uint64_t, H8)
+
+#define GEN_VEXT_VSLIDEDOWN_VX(NAME, ETYPE, H)                       \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2,    \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    uint32_t vlmax = vext_max_elems(desc, ctzl(sizeof(ETYPE)));      \
+    uint32_t vl = env->vl;                                           \
+    uint32_t esz = sizeof(ETYPE);                                    \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);     \
+    uint32_t vta = vext_vta(desc);                                   \
+    uint32_t vma = vext_vma(desc);                                   \
+    target_ulong i_max = MAX(MIN(s1 < vlmax ? vlmax - s1 : 0, vl),   \
+                             env->vstart);                           \
+    target_ulong i;                                                  \
+                                                                     \
+    for (i = env->vstart; i < i_max; i++) {                          \
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {              \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);      \
+            continue;                                                \
+        }                                                            \
+        *((ETYPE *)vd + H(i)) = *((ETYPE *)vs2 + H(i + s1));         \
+    }                                                                \
+                                                                     \
+    for (i = i_max; i < vl; i++) {                                   \
+        if (vext_vm(desc) || vext_elem_mask(v0, i)) {                \
+            *((ETYPE *)vd + H(i)) = 0;                               \
+        }                                                            \
+    }                                                                \
+                                                                     \
+    env->vstart = 0;                                                 \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);         \
+}
+
+GEN_VEXT_VSLIDEDOWN_VX(vslidedown_vx_b, uint8_t, H1)
+GEN_VEXT_VSLIDEDOWN_VX(vslidedown_vx_h, uint16_t, H2)
+GEN_VEXT_VSLIDEDOWN_VX(vslidedown_vx_w, uint32_t, H4)
+GEN_VEXT_VSLIDEDOWN_VX(vslidedown_vx_d, uint64_t, H8)
+
+#define GEN_VEXT_VSLIDE1UP(BITWIDTH, H)                              \
+static void vslide1up_##BITWIDTH(void *vd, void *v0, uint64_t s1,    \
+                                  void *vs2, CPURISCVState *env,     \
+                                  uint32_t desc)                     \
+{                                                                    \
+    typedef uint##BITWIDTH##_t ETYPE;                                \
+    uint32_t vl = env->vl;                                           \
+    uint32_t esz = sizeof(ETYPE);                                    \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);     \
+    uint32_t vta = vext_vta(desc);                                   \
+    uint32_t vma = vext_vma(desc);                                   \
+    uint32_t i;                                                      \
+                                                                     \
+    for (i = env->vstart; i < vl; i++) {                             \
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {              \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);      \
+            continue;                                                \
+        }                                                            \
+        if (i == 0) {                                                \
+            *((ETYPE *)vd + H(i)) = (ETYPE)s1;                       \
+        } else {                                                     \
+            *((ETYPE *)vd + H(i)) = *((ETYPE *)vs2 + H(i - 1));      \
+        }                                                            \
+    }                                                                \
+    env->vstart = 0;                                                 \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);         \
+}
+
+GEN_VEXT_VSLIDE1UP(8, H1)
+GEN_VEXT_VSLIDE1UP(16, H2)
+GEN_VEXT_VSLIDE1UP(32, H4)
+GEN_VEXT_VSLIDE1UP(64, H8)
+
+#define GEN_VEXT_VSLIDE1DOWN(BITWIDTH, H)                            \
+static void vslide1down_##BITWIDTH(void *vd, void *v0, uint64_t s1,  \
+                                    void *vs2, CPURISCVState *env,   \
+                                    uint32_t desc)                   \
+{                                                                    \
+    typedef uint##BITWIDTH##_t ETYPE;                                \
+    uint32_t vl = env->vl;                                           \
+    uint32_t esz = sizeof(ETYPE);                                    \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);     \
+    uint32_t vta = vext_vta(desc);                                   \
+    uint32_t vma = vext_vma(desc);                                   \
+    uint32_t i;                                                      \
+                                                                     \
+    for (i = env->vstart; i < vl; i++) {                             \
+        if (!vext_vm(desc) && !vext_elem_mask(v0, i)) {              \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);      \
+            continue;                                                \
+        }                                                            \
+        if (i == vl - 1) {                                           \
+            *((ETYPE *)vd + H(i)) = (ETYPE)s1;                       \
+        } else {                                                     \
+            *((ETYPE *)vd + H(i)) = *((ETYPE *)vs2 + H(i + 1));      \
+        }                                                            \
+    }                                                                \
+    env->vstart = 0;                                                 \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);         \
+}
+
+GEN_VEXT_VSLIDE1DOWN(8, H1)
+GEN_VEXT_VSLIDE1DOWN(16, H2)
+GEN_VEXT_VSLIDE1DOWN(32, H4)
+GEN_VEXT_VSLIDE1DOWN(64, H8)
+
+#define GEN_VEXT_VSLIDE1UP_VX(NAME, BITWIDTH)                        \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2,    \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    vslide1up_##BITWIDTH(vd, v0, s1, vs2, env, desc);                \
+}
+
+GEN_VEXT_VSLIDE1UP_VX(vslide1up_vx_b, 8)
+GEN_VEXT_VSLIDE1UP_VX(vslide1up_vx_h, 16)
+GEN_VEXT_VSLIDE1UP_VX(vslide1up_vx_w, 32)
+GEN_VEXT_VSLIDE1UP_VX(vslide1up_vx_d, 64)
+
+#define GEN_VEXT_VSLIDE1DOWN_VX(NAME, BITWIDTH)                      \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2,    \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    vslide1down_##BITWIDTH(vd, v0, s1, vs2, env, desc);              \
+}
+
+GEN_VEXT_VSLIDE1DOWN_VX(vslide1down_vx_b, 8)
+GEN_VEXT_VSLIDE1DOWN_VX(vslide1down_vx_h, 16)
+GEN_VEXT_VSLIDE1DOWN_VX(vslide1down_vx_w, 32)
+GEN_VEXT_VSLIDE1DOWN_VX(vslide1down_vx_d, 64)
+
+#define GEN_VEXT_VFSLIDE1UP_VF(NAME, BITWIDTH)                       \
+void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2,        \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    vslide1up_##BITWIDTH(vd, v0, s1, vs2, env, desc);                \
+}
+
+GEN_VEXT_VFSLIDE1UP_VF(vfslide1up_vf_h, 16)
+GEN_VEXT_VFSLIDE1UP_VF(vfslide1up_vf_w, 32)
+GEN_VEXT_VFSLIDE1UP_VF(vfslide1up_vf_d, 64)
+
+#define GEN_VEXT_VFSLIDE1DOWN_VF(NAME, BITWIDTH)                     \
+void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2,        \
+                  CPURISCVState *env, uint32_t desc)                 \
+{                                                                    \
+    vslide1down_##BITWIDTH(vd, v0, s1, vs2, env, desc);              \
+}
+
+GEN_VEXT_VFSLIDE1DOWN_VF(vfslide1down_vf_h, 16)
+GEN_VEXT_VFSLIDE1DOWN_VF(vfslide1down_vf_w, 32)
+GEN_VEXT_VFSLIDE1DOWN_VF(vfslide1down_vf_d, 64)
+
+#define GEN_VEXT_VRGATHER_VV(NAME, TS1, TS2, HS1, HS2)                   \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,              \
+                  CPURISCVState *env, uint32_t desc)                     \
+{                                                                        \
+    uint32_t vlmax = vext_max_elems(desc, ctzl(sizeof(TS2)));            \
+    uint32_t vm = vext_vm(desc);                                         \
+    uint32_t vl = env->vl;                                               \
+    uint32_t esz = sizeof(TS2);                                          \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);         \
+    uint32_t vta = vext_vta(desc);                                       \
+    uint32_t vma = vext_vma(desc);                                       \
+    uint64_t index;                                                      \
+    uint32_t i;                                                          \
+                                                                         \
+    for (i = env->vstart; i < vl; i++) {                                 \
+        if (!vm && !vext_elem_mask(v0, i)) {                             \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);          \
+            continue;                                                    \
+        }                                                                \
+        index = *((TS1 *)vs1 + HS1(i));                                  \
+        if (index >= vlmax) {                                            \
+            *((TS2 *)vd + HS2(i)) = 0;                                   \
+        } else {                                                         \
+            *((TS2 *)vd + HS2(i)) = *((TS2 *)vs2 + HS2(index));          \
+        }                                                                \
+    }                                                                    \
+    env->vstart = 0;                                                     \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);             \
+}
+
+GEN_VEXT_VRGATHER_VV(vrgather_vv_b, uint8_t, uint8_t, H1, H1)
+GEN_VEXT_VRGATHER_VV(vrgather_vv_h, uint16_t, uint16_t, H2, H2)
+GEN_VEXT_VRGATHER_VV(vrgather_vv_w, uint32_t, uint32_t, H4, H4)
+GEN_VEXT_VRGATHER_VV(vrgather_vv_d, uint64_t, uint64_t, H8, H8)
+
+GEN_VEXT_VRGATHER_VV(vrgatherei16_vv_b, uint16_t, uint8_t, H2, H1)
+GEN_VEXT_VRGATHER_VV(vrgatherei16_vv_h, uint16_t, uint16_t, H2, H2)
+GEN_VEXT_VRGATHER_VV(vrgatherei16_vv_w, uint16_t, uint32_t, H2, H4)
+GEN_VEXT_VRGATHER_VV(vrgatherei16_vv_d, uint16_t, uint64_t, H2, H8)
+
+#define GEN_VEXT_VRGATHER_VX(NAME, ETYPE, H)                             \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2,        \
+                  CPURISCVState *env, uint32_t desc)                     \
+{                                                                        \
+    uint32_t vlmax = vext_max_elems(desc, ctzl(sizeof(ETYPE)));          \
+    uint32_t vm = vext_vm(desc);                                         \
+    uint32_t vl = env->vl;                                               \
+    uint32_t esz = sizeof(ETYPE);                                        \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);         \
+    uint32_t vta = vext_vta(desc);                                       \
+    uint32_t vma = vext_vma(desc);                                       \
+    uint64_t index = s1;                                                 \
+    uint32_t i;                                                          \
+                                                                         \
+    for (i = env->vstart; i < vl; i++) {                                 \
+        if (!vm && !vext_elem_mask(v0, i)) {                             \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);          \
+            continue;                                                    \
+        }                                                                \
+        if (index >= vlmax) {                                            \
+            *((ETYPE *)vd + H(i)) = 0;                                   \
+        } else {                                                         \
+            *((ETYPE *)vd + H(i)) = *((ETYPE *)vs2 + H(index));          \
+        }                                                                \
+    }                                                                    \
+    env->vstart = 0;                                                     \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);             \
+}
+
+GEN_VEXT_VRGATHER_VX(vrgather_vx_b, uint8_t, H1)
+GEN_VEXT_VRGATHER_VX(vrgather_vx_h, uint16_t, H2)
+GEN_VEXT_VRGATHER_VX(vrgather_vx_w, uint32_t, H4)
+GEN_VEXT_VRGATHER_VX(vrgather_vx_d, uint64_t, H8)
+
+#define GEN_VEXT_VCOMPRESS_VM(NAME, ETYPE, H)                            \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,              \
+                  CPURISCVState *env, uint32_t desc)                     \
+{                                                                        \
+    uint32_t vl = env->vl;                                               \
+    uint32_t esz = sizeof(ETYPE);                                        \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);         \
+    uint32_t vta = vext_vta(desc);                                       \
+    uint32_t num = 0;                                                    \
+    uint32_t i;                                                          \
+                                                                         \
+    if (env->vstart != 0) {                                              \
+        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());    \
+    }                                                                    \
+                                                                         \
+    for (i = env->vstart; i < vl; i++) {                                 \
+        if (!vext_elem_mask(vs1, i)) {                                   \
+            continue;                                                    \
+        }                                                                \
+        *((ETYPE *)vd + H(num)) = *((ETYPE *)vs2 + H(i));                \
+        num++;                                                           \
+    }                                                                    \
+    env->vstart = 0;                                                     \
+    vext_set_elems_1s(vd, vta, num * esz, total_elems * esz);            \
+}
+
+GEN_VEXT_VCOMPRESS_VM(vcompress_vm_b, uint8_t, H1)
+GEN_VEXT_VCOMPRESS_VM(vcompress_vm_h, uint16_t, H2)
+GEN_VEXT_VCOMPRESS_VM(vcompress_vm_w, uint32_t, H4)
+GEN_VEXT_VCOMPRESS_VM(vcompress_vm_d, uint64_t, H8)
+
+void HELPER(vmvr_v)(void *vd, void *vs2, CPURISCVState *env, uint32_t desc)
+{
+    uint32_t maxsz = simd_maxsz(desc);
+    uint32_t sewb = 1 << FIELD_EX64(env->vtype, VTYPE, VSEW);
+    uint32_t startb = env->vstart * sewb;
+
+    if (startb >= maxsz) {
+        env->vstart = 0;
+        return;
+    }
+
+    memcpy((uint8_t *)vd + H1(startb),
+           (uint8_t *)vs2 + H1(startb),
+           maxsz - startb);
+    env->vstart = 0;
+}
+
+#define GEN_VEXT_INT_EXT(NAME, ETYPE, DTYPE, HD, HS2)                  \
+void HELPER(NAME)(void *vd, void *v0, void *vs2, CPURISCVState *env,  \
+                  uint32_t desc)                                      \
+{                                                                     \
+    uint32_t i;                                                       \
+    uint32_t vl = env->vl;                                            \
+    uint32_t vm = vext_vm(desc);                                      \
+    uint32_t esz = sizeof(ETYPE);                                     \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);      \
+    uint32_t vta = vext_vta(desc);                                    \
+    uint32_t vma = vext_vma(desc);                                    \
+                                                                      \
+    for (i = env->vstart; i < vl; i++) {                              \
+        if (!vm && !vext_elem_mask(v0, i)) {                          \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);       \
+            continue;                                                 \
+        }                                                             \
+        *((ETYPE *)vd + HD(i)) = *((DTYPE *)vs2 + HS2(i));            \
+    }                                                                 \
+    env->vstart = 0;                                                  \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);          \
+}
+
+GEN_VEXT_INT_EXT(vzext_vf2_h, uint16_t, uint8_t, H2, H1)
+GEN_VEXT_INT_EXT(vzext_vf2_w, uint32_t, uint16_t, H4, H2)
+GEN_VEXT_INT_EXT(vzext_vf2_d, uint64_t, uint32_t, H8, H4)
+GEN_VEXT_INT_EXT(vzext_vf4_w, uint32_t, uint8_t, H4, H1)
+GEN_VEXT_INT_EXT(vzext_vf4_d, uint64_t, uint16_t, H8, H2)
+GEN_VEXT_INT_EXT(vzext_vf8_d, uint64_t, uint8_t, H8, H1)
+GEN_VEXT_INT_EXT(vsext_vf2_h, int16_t, int8_t, H2, H1)
+GEN_VEXT_INT_EXT(vsext_vf2_w, int32_t, int16_t, H4, H2)
+GEN_VEXT_INT_EXT(vsext_vf2_d, int64_t, int32_t, H8, H4)
+GEN_VEXT_INT_EXT(vsext_vf4_w, int32_t, int8_t, H4, H1)
+GEN_VEXT_INT_EXT(vsext_vf4_d, int64_t, int16_t, H8, H2)
+GEN_VEXT_INT_EXT(vsext_vf8_d, int64_t, int8_t, H8, H1)
+
+#define DO_VADC(N, M, C) ((N) + (M) + (C))
+#define DO_VSBC(N, M, C) ((N) - (M) - (C))
+#define DO_MADC(N, M, C, ETYPE) \
+    ((C) ? (ETYPE)((N) + (M) + 1) <= (N) : (ETYPE)((N) + (M)) < (N))
+#define DO_MSBC(N, M, C, ETYPE) ((C) ? (N) <= (M) : (N) < (M))
+
+#define GEN_VEXT_VADC_VVM(NAME, ETYPE, H, OP)                 \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,   \
+                  CPURISCVState *env, uint32_t desc)          \
+{                                                             \
+    uint32_t i;                                               \
+    uint32_t vl = env->vl;                                    \
+    uint32_t esz = sizeof(ETYPE);                             \
+    uint32_t total_elems = vext_get_total_elems(env, desc,    \
+                                                esz);         \
+    uint32_t vta = vext_vta(desc);                            \
+                                                              \
+    for (i = env->vstart; i < vl; i++) {                      \
+        ETYPE s1 = *((ETYPE *)vs1 + H(i));                    \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                    \
+        ETYPE carry = vext_elem_mask(v0, i);                  \
+                                                              \
+        *((ETYPE *)vd + H(i)) = OP(s2, s1, carry);            \
+    }                                                         \
+    env->vstart = 0;                                          \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);  \
+}
+
+GEN_VEXT_VADC_VVM(vadc_vvm_b, uint8_t, H1, DO_VADC)
+GEN_VEXT_VADC_VVM(vadc_vvm_h, uint16_t, H2, DO_VADC)
+GEN_VEXT_VADC_VVM(vadc_vvm_w, uint32_t, H4, DO_VADC)
+GEN_VEXT_VADC_VVM(vadc_vvm_d, uint64_t, H8, DO_VADC)
+GEN_VEXT_VADC_VVM(vsbc_vvm_b, uint8_t, H1, DO_VSBC)
+GEN_VEXT_VADC_VVM(vsbc_vvm_h, uint16_t, H2, DO_VSBC)
+GEN_VEXT_VADC_VVM(vsbc_vvm_w, uint32_t, H4, DO_VSBC)
+GEN_VEXT_VADC_VVM(vsbc_vvm_d, uint64_t, H8, DO_VSBC)
+
+#define GEN_VEXT_VADC_VXM(NAME, ETYPE, H, OP)                    \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1,           \
+                  void *vs2, CPURISCVState *env, uint32_t desc)  \
+{                                                                \
+    uint32_t i;                                                  \
+    uint32_t vl = env->vl;                                       \
+    uint32_t esz = sizeof(ETYPE);                                \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
+    uint32_t vta = vext_vta(desc);                               \
+                                                                 \
+    for (i = env->vstart; i < vl; i++) {                         \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                       \
+        ETYPE carry = vext_elem_mask(v0, i);                     \
+                                                                 \
+        *((ETYPE *)vd + H(i)) =                                  \
+            OP(s2, (ETYPE)(target_long)s1, carry);               \
+    }                                                            \
+    env->vstart = 0;                                             \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);     \
+}
+
+GEN_VEXT_VADC_VXM(vadc_vxm_b, uint8_t, H1, DO_VADC)
+GEN_VEXT_VADC_VXM(vadc_vxm_h, uint16_t, H2, DO_VADC)
+GEN_VEXT_VADC_VXM(vadc_vxm_w, uint32_t, H4, DO_VADC)
+GEN_VEXT_VADC_VXM(vadc_vxm_d, uint64_t, H8, DO_VADC)
+GEN_VEXT_VADC_VXM(vsbc_vxm_b, uint8_t, H1, DO_VSBC)
+GEN_VEXT_VADC_VXM(vsbc_vxm_h, uint16_t, H2, DO_VSBC)
+GEN_VEXT_VADC_VXM(vsbc_vxm_w, uint32_t, H4, DO_VSBC)
+GEN_VEXT_VADC_VXM(vsbc_vxm_d, uint64_t, H8, DO_VSBC)
+
+#define GEN_VEXT_VMADC_VVM(NAME, ETYPE, H, OP)                \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,   \
+                  CPURISCVState *env, uint32_t desc)          \
+{                                                             \
+    uint32_t i;                                               \
+    uint32_t vl = env->vl;                                    \
+    uint32_t vm = vext_vm(desc);                              \
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;        \
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);              \
+                                                              \
+    for (i = env->vstart; i < vl; i++) {                      \
+        ETYPE s1 = *((ETYPE *)vs1 + H(i));                    \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                    \
+        ETYPE carry = !vm && vext_elem_mask(v0, i);           \
+                                                              \
+        vext_set_elem_mask(vd, i, OP(s2, s1, carry, ETYPE));  \
+    }                                                         \
+    env->vstart = 0;                                          \
+    if (vta_all_1s) {                                         \
+        for (; i < total_elems; i++) {                        \
+            vext_set_elem_mask(vd, i, 1);                     \
+        }                                                     \
+    }                                                         \
+}
+
+GEN_VEXT_VMADC_VVM(vmadc_vvm_b, uint8_t, H1, DO_MADC)
+GEN_VEXT_VMADC_VVM(vmadc_vvm_h, uint16_t, H2, DO_MADC)
+GEN_VEXT_VMADC_VVM(vmadc_vvm_w, uint32_t, H4, DO_MADC)
+GEN_VEXT_VMADC_VVM(vmadc_vvm_d, uint64_t, H8, DO_MADC)
+GEN_VEXT_VMADC_VVM(vmsbc_vvm_b, uint8_t, H1, DO_MSBC)
+GEN_VEXT_VMADC_VVM(vmsbc_vvm_h, uint16_t, H2, DO_MSBC)
+GEN_VEXT_VMADC_VVM(vmsbc_vvm_w, uint32_t, H4, DO_MSBC)
+GEN_VEXT_VMADC_VVM(vmsbc_vvm_d, uint64_t, H8, DO_MSBC)
+
+#define GEN_VEXT_VMADC_VXM(NAME, ETYPE, H, OP)                   \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1,           \
+                  void *vs2, CPURISCVState *env, uint32_t desc)  \
+{                                                                \
+    uint32_t i;                                                  \
+    uint32_t vl = env->vl;                                       \
+    uint32_t vm = vext_vm(desc);                                 \
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;           \
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);                 \
+                                                                 \
+    for (i = env->vstart; i < vl; i++) {                         \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                       \
+        ETYPE src1 = (ETYPE)(target_long)s1;                     \
+        ETYPE carry = !vm && vext_elem_mask(v0, i);              \
+                                                                 \
+        vext_set_elem_mask(vd, i, OP(s2, src1, carry, ETYPE));   \
+    }                                                            \
+    env->vstart = 0;                                             \
+    if (vta_all_1s) {                                            \
+        for (; i < total_elems; i++) {                           \
+            vext_set_elem_mask(vd, i, 1);                        \
+        }                                                        \
+    }                                                            \
+}
+
+GEN_VEXT_VMADC_VXM(vmadc_vxm_b, uint8_t, H1, DO_MADC)
+GEN_VEXT_VMADC_VXM(vmadc_vxm_h, uint16_t, H2, DO_MADC)
+GEN_VEXT_VMADC_VXM(vmadc_vxm_w, uint32_t, H4, DO_MADC)
+GEN_VEXT_VMADC_VXM(vmadc_vxm_d, uint64_t, H8, DO_MADC)
+GEN_VEXT_VMADC_VXM(vmsbc_vxm_b, uint8_t, H1, DO_MSBC)
+GEN_VEXT_VMADC_VXM(vmsbc_vxm_h, uint16_t, H2, DO_MSBC)
+GEN_VEXT_VMADC_VXM(vmsbc_vxm_w, uint32_t, H4, DO_MSBC)
+GEN_VEXT_VMADC_VXM(vmsbc_vxm_d, uint64_t, H8, DO_MSBC)
+
+#define DO_MSEQ(N, M) ((N) == (M))
+#define DO_MSNE(N, M) ((N) != (M))
+#define DO_MSLT(N, M) ((N) < (M))
+#define DO_MSLE(N, M) ((N) <= (M))
+#define DO_MSGT(N, M) ((N) > (M))
+
+#define GEN_VEXT_CMP_VV(NAME, ETYPE, H, OP)                \
+void HELPER(NAME)(void *vd, void *v0, void *vs1,           \
+                  void *vs2, CPURISCVState *env,           \
+                  uint32_t desc)                           \
+{                                                          \
+    uint32_t i;                                            \
+    uint32_t vl = env->vl;                                 \
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;     \
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);           \
+                                                           \
+    for (i = env->vstart; i < vl; i++) {                   \
+        ETYPE s1 = *((ETYPE *)vs1 + H(i));                 \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                 \
+                                                           \
+        vext_set_elem_mask(vd, i, OP(s2, s1));             \
+    }                                                      \
+    env->vstart = 0;                                       \
+    if (vta_all_1s) {                                      \
+        for (; i < total_elems; i++) {                     \
+            vext_set_elem_mask(vd, i, 1);                  \
+        }                                                  \
+    }                                                      \
+}
+
+GEN_VEXT_CMP_VV(vmseq_vv_b, uint8_t, H1, DO_MSEQ)
+GEN_VEXT_CMP_VV(vmseq_vv_h, uint16_t, H2, DO_MSEQ)
+GEN_VEXT_CMP_VV(vmseq_vv_w, uint32_t, H4, DO_MSEQ)
+GEN_VEXT_CMP_VV(vmseq_vv_d, uint64_t, H8, DO_MSEQ)
+GEN_VEXT_CMP_VV(vmsne_vv_b, uint8_t, H1, DO_MSNE)
+GEN_VEXT_CMP_VV(vmsne_vv_h, uint16_t, H2, DO_MSNE)
+GEN_VEXT_CMP_VV(vmsne_vv_w, uint32_t, H4, DO_MSNE)
+GEN_VEXT_CMP_VV(vmsne_vv_d, uint64_t, H8, DO_MSNE)
+GEN_VEXT_CMP_VV(vmsltu_vv_b, uint8_t, H1, DO_MSLT)
+GEN_VEXT_CMP_VV(vmsltu_vv_h, uint16_t, H2, DO_MSLT)
+GEN_VEXT_CMP_VV(vmsltu_vv_w, uint32_t, H4, DO_MSLT)
+GEN_VEXT_CMP_VV(vmsltu_vv_d, uint64_t, H8, DO_MSLT)
+GEN_VEXT_CMP_VV(vmslt_vv_b, int8_t, H1, DO_MSLT)
+GEN_VEXT_CMP_VV(vmslt_vv_h, int16_t, H2, DO_MSLT)
+GEN_VEXT_CMP_VV(vmslt_vv_w, int32_t, H4, DO_MSLT)
+GEN_VEXT_CMP_VV(vmslt_vv_d, int64_t, H8, DO_MSLT)
+GEN_VEXT_CMP_VV(vmsleu_vv_b, uint8_t, H1, DO_MSLE)
+GEN_VEXT_CMP_VV(vmsleu_vv_h, uint16_t, H2, DO_MSLE)
+GEN_VEXT_CMP_VV(vmsleu_vv_w, uint32_t, H4, DO_MSLE)
+GEN_VEXT_CMP_VV(vmsleu_vv_d, uint64_t, H8, DO_MSLE)
+GEN_VEXT_CMP_VV(vmsle_vv_b, int8_t, H1, DO_MSLE)
+GEN_VEXT_CMP_VV(vmsle_vv_h, int16_t, H2, DO_MSLE)
+GEN_VEXT_CMP_VV(vmsle_vv_w, int32_t, H4, DO_MSLE)
+GEN_VEXT_CMP_VV(vmsle_vv_d, int64_t, H8, DO_MSLE)
+
+#define GEN_VEXT_CMP_VX(NAME, ETYPE, H, OP)                \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1,     \
+                  void *vs2, CPURISCVState *env,           \
+                  uint32_t desc)                           \
+{                                                          \
+    uint32_t i;                                            \
+    uint32_t vl = env->vl;                                 \
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;     \
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);           \
+                                                           \
+    for (i = env->vstart; i < vl; i++) {                   \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                 \
+                                                           \
+        vext_set_elem_mask(vd, i, OP(s2, (ETYPE)(target_long)s1)); \
+    }                                                      \
+    env->vstart = 0;                                       \
+    if (vta_all_1s) {                                      \
+        for (; i < total_elems; i++) {                     \
+            vext_set_elem_mask(vd, i, 1);                  \
+        }                                                  \
+    }                                                      \
+}
+
+GEN_VEXT_CMP_VX(vmseq_vx_b, uint8_t, H1, DO_MSEQ)
+GEN_VEXT_CMP_VX(vmseq_vx_h, uint16_t, H2, DO_MSEQ)
+GEN_VEXT_CMP_VX(vmseq_vx_w, uint32_t, H4, DO_MSEQ)
+GEN_VEXT_CMP_VX(vmseq_vx_d, uint64_t, H8, DO_MSEQ)
+GEN_VEXT_CMP_VX(vmsne_vx_b, uint8_t, H1, DO_MSNE)
+GEN_VEXT_CMP_VX(vmsne_vx_h, uint16_t, H2, DO_MSNE)
+GEN_VEXT_CMP_VX(vmsne_vx_w, uint32_t, H4, DO_MSNE)
+GEN_VEXT_CMP_VX(vmsne_vx_d, uint64_t, H8, DO_MSNE)
+GEN_VEXT_CMP_VX(vmsltu_vx_b, uint8_t, H1, DO_MSLT)
+GEN_VEXT_CMP_VX(vmsltu_vx_h, uint16_t, H2, DO_MSLT)
+GEN_VEXT_CMP_VX(vmsltu_vx_w, uint32_t, H4, DO_MSLT)
+GEN_VEXT_CMP_VX(vmsltu_vx_d, uint64_t, H8, DO_MSLT)
+GEN_VEXT_CMP_VX(vmslt_vx_b, int8_t, H1, DO_MSLT)
+GEN_VEXT_CMP_VX(vmslt_vx_h, int16_t, H2, DO_MSLT)
+GEN_VEXT_CMP_VX(vmslt_vx_w, int32_t, H4, DO_MSLT)
+GEN_VEXT_CMP_VX(vmslt_vx_d, int64_t, H8, DO_MSLT)
+GEN_VEXT_CMP_VX(vmsleu_vx_b, uint8_t, H1, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsleu_vx_h, uint16_t, H2, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsleu_vx_w, uint32_t, H4, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsleu_vx_d, uint64_t, H8, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsle_vx_b, int8_t, H1, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsle_vx_h, int16_t, H2, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsle_vx_w, int32_t, H4, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsle_vx_d, int64_t, H8, DO_MSLE)
+GEN_VEXT_CMP_VX(vmsgtu_vx_b, uint8_t, H1, DO_MSGT)
+GEN_VEXT_CMP_VX(vmsgtu_vx_h, uint16_t, H2, DO_MSGT)
+GEN_VEXT_CMP_VX(vmsgtu_vx_w, uint32_t, H4, DO_MSGT)
+GEN_VEXT_CMP_VX(vmsgtu_vx_d, uint64_t, H8, DO_MSGT)
+GEN_VEXT_CMP_VX(vmsgt_vx_b, int8_t, H1, DO_MSGT)
+GEN_VEXT_CMP_VX(vmsgt_vx_h, int16_t, H2, DO_MSGT)
+GEN_VEXT_CMP_VX(vmsgt_vx_w, int32_t, H4, DO_MSGT)
+GEN_VEXT_CMP_VX(vmsgt_vx_d, int64_t, H8, DO_MSGT)
+
+typedef void opivv2_rm_fn(void *vd, void *vs1, void *vs2, int i,
+                          CPURISCVState *env, int vxrm);
+
+#define OPIVV2_RM(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
+static void do_##NAME(void *vd, void *vs1, void *vs2, int i,    \
+                      CPURISCVState *env, int vxrm)             \
+{                                                               \
+    TX1 s1 = *((T1 *)vs1 + HS1(i));                             \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                             \
+                                                                \
+    *((TD *)vd + HD(i)) = OP(env, vxrm, s2, s1);                \
+}
+
+static void vext_vv_rm_1(void *vd, void *v0, void *vs1, void *vs2,
+                         CPURISCVState *env, uint32_t vl, uint32_t vm,
+                         int vxrm, opivv2_rm_fn *fn, uint32_t vma,
+                         uint32_t esz)
+{
+    uint32_t i;
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vm && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, vs1, vs2, i, env, vxrm);
+    }
+    env->vstart = 0;
+}
+
+static void vext_vv_rm_2(void *vd, void *v0, void *vs1, void *vs2,
+                         CPURISCVState *env, uint32_t desc,
+                         opivv2_rm_fn *fn, uint32_t esz)
+{
+    uint32_t vm = vext_vm(desc);
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    switch (env->vxrm) {
+    case 0:
+        vext_vv_rm_1(vd, v0, vs1, vs2, env, vl, vm, 0, fn, vma, esz);
+        break;
+    case 1:
+        vext_vv_rm_1(vd, v0, vs1, vs2, env, vl, vm, 1, fn, vma, esz);
+        break;
+    case 2:
+        vext_vv_rm_1(vd, v0, vs1, vs2, env, vl, vm, 2, fn, vma, esz);
+        break;
+    default:
+        vext_vv_rm_1(vd, v0, vs1, vs2, env, vl, vm, 3, fn, vma, esz);
+        break;
+    }
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_VV_RM(NAME, ESZ)                               \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,     \
+                  CPURISCVState *env, uint32_t desc)            \
+{                                                               \
+    vext_vv_rm_2(vd, v0, vs1, vs2, env, desc, do_##NAME, ESZ);  \
+}
+
+static inline uint8_t saddu8(CPURISCVState *env, int vxrm, uint8_t a,
+                             uint8_t b)
+{
+    uint8_t res = a + b;
+
+    if (res < a) {
+        res = UINT8_MAX;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline uint16_t saddu16(CPURISCVState *env, int vxrm, uint16_t a,
+                               uint16_t b)
+{
+    uint16_t res = a + b;
+
+    if (res < a) {
+        res = UINT16_MAX;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline uint32_t saddu32(CPURISCVState *env, int vxrm, uint32_t a,
+                               uint32_t b)
+{
+    uint32_t res = a + b;
+
+    if (res < a) {
+        res = UINT32_MAX;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline uint64_t saddu64(CPURISCVState *env, int vxrm, uint64_t a,
+                               uint64_t b)
+{
+    uint64_t res = a + b;
+
+    if (res < a) {
+        res = UINT64_MAX;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+RVVCALL(OPIVV2_RM, vsaddu_vv_b, OP_UUU_B, H1, H1, H1, saddu8)
+RVVCALL(OPIVV2_RM, vsaddu_vv_h, OP_UUU_H, H2, H2, H2, saddu16)
+RVVCALL(OPIVV2_RM, vsaddu_vv_w, OP_UUU_W, H4, H4, H4, saddu32)
+RVVCALL(OPIVV2_RM, vsaddu_vv_d, OP_UUU_D, H8, H8, H8, saddu64)
+GEN_VEXT_VV_RM(vsaddu_vv_b, 1)
+GEN_VEXT_VV_RM(vsaddu_vv_h, 2)
+GEN_VEXT_VV_RM(vsaddu_vv_w, 4)
+GEN_VEXT_VV_RM(vsaddu_vv_d, 8)
+
+typedef void opivx2_rm_fn(void *vd, target_long s1, void *vs2, int i,
+                          CPURISCVState *env, int vxrm);
+
+#define OPIVX2_RM(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP)  \
+static void do_##NAME(void *vd, target_long s1, void *vs2,  \
+                      int i, CPURISCVState *env, int vxrm)  \
+{                                                           \
+    TX2 s2 = *((T2 *)vs2 + HS2(i));                         \
+                                                            \
+    *((TD *)vd + HD(i)) = OP(env, vxrm, s2, (TX1)(T1)s1);   \
+}
+
+static void vext_vx_rm_1(void *vd, void *v0, target_long s1, void *vs2,
+                         CPURISCVState *env, uint32_t vl, uint32_t vm,
+                         int vxrm, opivx2_rm_fn *fn, uint32_t vma,
+                         uint32_t esz)
+{
+    uint32_t i;
+
+    for (i = env->vstart; i < vl; i++) {
+        if (!vm && !vext_elem_mask(v0, i)) {
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
+            continue;
+        }
+        fn(vd, s1, vs2, i, env, vxrm);
+    }
+    env->vstart = 0;
+}
+
+static void vext_vx_rm_2(void *vd, void *v0, target_long s1, void *vs2,
+                         CPURISCVState *env, uint32_t desc,
+                         opivx2_rm_fn *fn, uint32_t esz)
+{
+    uint32_t vm = vext_vm(desc);
+    uint32_t vl = env->vl;
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);
+    uint32_t vta = vext_vta(desc);
+    uint32_t vma = vext_vma(desc);
+
+    switch (env->vxrm) {
+    case 0:
+        vext_vx_rm_1(vd, v0, s1, vs2, env, vl, vm, 0, fn, vma, esz);
+        break;
+    case 1:
+        vext_vx_rm_1(vd, v0, s1, vs2, env, vl, vm, 1, fn, vma, esz);
+        break;
+    case 2:
+        vext_vx_rm_1(vd, v0, s1, vs2, env, vl, vm, 2, fn, vma, esz);
+        break;
+    default:
+        vext_vx_rm_1(vd, v0, s1, vs2, env, vl, vm, 3, fn, vma, esz);
+        break;
+    }
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
+}
+
+#define GEN_VEXT_VX_RM(NAME, ESZ)                              \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1,         \
+                  void *vs2, CPURISCVState *env, uint32_t desc) \
+{                                                              \
+    vext_vx_rm_2(vd, v0, s1, vs2, env, desc, do_##NAME, ESZ);  \
+}
+
+RVVCALL(OPIVX2_RM, vsaddu_vx_b, OP_UUU_B, H1, H1, saddu8)
+RVVCALL(OPIVX2_RM, vsaddu_vx_h, OP_UUU_H, H2, H2, saddu16)
+RVVCALL(OPIVX2_RM, vsaddu_vx_w, OP_UUU_W, H4, H4, saddu32)
+RVVCALL(OPIVX2_RM, vsaddu_vx_d, OP_UUU_D, H8, H8, saddu64)
+GEN_VEXT_VX_RM(vsaddu_vx_b, 1)
+GEN_VEXT_VX_RM(vsaddu_vx_h, 2)
+GEN_VEXT_VX_RM(vsaddu_vx_w, 4)
+GEN_VEXT_VX_RM(vsaddu_vx_d, 8)
+
+static inline int8_t sadd8(CPURISCVState *env, int vxrm, int8_t a, int8_t b)
+{
+    int8_t res = a + b;
+
+    if ((res ^ a) & (res ^ b) & INT8_MIN) {
+        res = a > 0 ? INT8_MAX : INT8_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline int16_t sadd16(CPURISCVState *env, int vxrm, int16_t a,
+                             int16_t b)
+{
+    int16_t res = a + b;
+
+    if ((res ^ a) & (res ^ b) & INT16_MIN) {
+        res = a > 0 ? INT16_MAX : INT16_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline int32_t sadd32(CPURISCVState *env, int vxrm, int32_t a,
+                             int32_t b)
+{
+    int32_t res = a + b;
+
+    if ((res ^ a) & (res ^ b) & INT32_MIN) {
+        res = a > 0 ? INT32_MAX : INT32_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline int64_t sadd64(CPURISCVState *env, int vxrm, int64_t a,
+                             int64_t b)
+{
+    int64_t res = a + b;
+
+    if ((res ^ a) & (res ^ b) & INT64_MIN) {
+        res = a > 0 ? INT64_MAX : INT64_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+RVVCALL(OPIVV2_RM, vsadd_vv_b, OP_SSS_B, H1, H1, H1, sadd8)
+RVVCALL(OPIVV2_RM, vsadd_vv_h, OP_SSS_H, H2, H2, H2, sadd16)
+RVVCALL(OPIVV2_RM, vsadd_vv_w, OP_SSS_W, H4, H4, H4, sadd32)
+RVVCALL(OPIVV2_RM, vsadd_vv_d, OP_SSS_D, H8, H8, H8, sadd64)
+GEN_VEXT_VV_RM(vsadd_vv_b, 1)
+GEN_VEXT_VV_RM(vsadd_vv_h, 2)
+GEN_VEXT_VV_RM(vsadd_vv_w, 4)
+GEN_VEXT_VV_RM(vsadd_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vsadd_vx_b, OP_SSS_B, H1, H1, sadd8)
+RVVCALL(OPIVX2_RM, vsadd_vx_h, OP_SSS_H, H2, H2, sadd16)
+RVVCALL(OPIVX2_RM, vsadd_vx_w, OP_SSS_W, H4, H4, sadd32)
+RVVCALL(OPIVX2_RM, vsadd_vx_d, OP_SSS_D, H8, H8, sadd64)
+GEN_VEXT_VX_RM(vsadd_vx_b, 1)
+GEN_VEXT_VX_RM(vsadd_vx_h, 2)
+GEN_VEXT_VX_RM(vsadd_vx_w, 4)
+GEN_VEXT_VX_RM(vsadd_vx_d, 8)
+
+static inline uint8_t ssubu8(CPURISCVState *env, int vxrm, uint8_t a,
+                             uint8_t b)
+{
+    uint8_t res = a - b;
+
+    if (res > a) {
+        res = 0;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline uint16_t ssubu16(CPURISCVState *env, int vxrm, uint16_t a,
+                               uint16_t b)
+{
+    uint16_t res = a - b;
+
+    if (res > a) {
+        res = 0;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline uint32_t ssubu32(CPURISCVState *env, int vxrm, uint32_t a,
+                               uint32_t b)
+{
+    uint32_t res = a - b;
+
+    if (res > a) {
+        res = 0;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline uint64_t ssubu64(CPURISCVState *env, int vxrm, uint64_t a,
+                               uint64_t b)
+{
+    uint64_t res = a - b;
+
+    if (res > a) {
+        res = 0;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+RVVCALL(OPIVV2_RM, vssubu_vv_b, OP_UUU_B, H1, H1, H1, ssubu8)
+RVVCALL(OPIVV2_RM, vssubu_vv_h, OP_UUU_H, H2, H2, H2, ssubu16)
+RVVCALL(OPIVV2_RM, vssubu_vv_w, OP_UUU_W, H4, H4, H4, ssubu32)
+RVVCALL(OPIVV2_RM, vssubu_vv_d, OP_UUU_D, H8, H8, H8, ssubu64)
+GEN_VEXT_VV_RM(vssubu_vv_b, 1)
+GEN_VEXT_VV_RM(vssubu_vv_h, 2)
+GEN_VEXT_VV_RM(vssubu_vv_w, 4)
+GEN_VEXT_VV_RM(vssubu_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vssubu_vx_b, OP_UUU_B, H1, H1, ssubu8)
+RVVCALL(OPIVX2_RM, vssubu_vx_h, OP_UUU_H, H2, H2, ssubu16)
+RVVCALL(OPIVX2_RM, vssubu_vx_w, OP_UUU_W, H4, H4, ssubu32)
+RVVCALL(OPIVX2_RM, vssubu_vx_d, OP_UUU_D, H8, H8, ssubu64)
+GEN_VEXT_VX_RM(vssubu_vx_b, 1)
+GEN_VEXT_VX_RM(vssubu_vx_h, 2)
+GEN_VEXT_VX_RM(vssubu_vx_w, 4)
+GEN_VEXT_VX_RM(vssubu_vx_d, 8)
+
+static inline int8_t ssub8(CPURISCVState *env, int vxrm, int8_t a, int8_t b)
+{
+    int8_t res = a - b;
+
+    if ((res ^ a) & (a ^ b) & INT8_MIN) {
+        res = a >= 0 ? INT8_MAX : INT8_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline int16_t ssub16(CPURISCVState *env, int vxrm, int16_t a,
+                             int16_t b)
+{
+    int16_t res = a - b;
+
+    if ((res ^ a) & (a ^ b) & INT16_MIN) {
+        res = a >= 0 ? INT16_MAX : INT16_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline int32_t ssub32(CPURISCVState *env, int vxrm, int32_t a,
+                             int32_t b)
+{
+    int32_t res = a - b;
+
+    if ((res ^ a) & (a ^ b) & INT32_MIN) {
+        res = a >= 0 ? INT32_MAX : INT32_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+static inline int64_t ssub64(CPURISCVState *env, int vxrm, int64_t a,
+                             int64_t b)
+{
+    int64_t res = a - b;
+
+    if ((res ^ a) & (a ^ b) & INT64_MIN) {
+        res = a >= 0 ? INT64_MAX : INT64_MIN;
+        env->vxsat = 1;
+    }
+    return res;
+}
+
+RVVCALL(OPIVV2_RM, vssub_vv_b, OP_SSS_B, H1, H1, H1, ssub8)
+RVVCALL(OPIVV2_RM, vssub_vv_h, OP_SSS_H, H2, H2, H2, ssub16)
+RVVCALL(OPIVV2_RM, vssub_vv_w, OP_SSS_W, H4, H4, H4, ssub32)
+RVVCALL(OPIVV2_RM, vssub_vv_d, OP_SSS_D, H8, H8, H8, ssub64)
+GEN_VEXT_VV_RM(vssub_vv_b, 1)
+GEN_VEXT_VV_RM(vssub_vv_h, 2)
+GEN_VEXT_VV_RM(vssub_vv_w, 4)
+GEN_VEXT_VV_RM(vssub_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vssub_vx_b, OP_SSS_B, H1, H1, ssub8)
+RVVCALL(OPIVX2_RM, vssub_vx_h, OP_SSS_H, H2, H2, ssub16)
+RVVCALL(OPIVX2_RM, vssub_vx_w, OP_SSS_W, H4, H4, ssub32)
+RVVCALL(OPIVX2_RM, vssub_vx_d, OP_SSS_D, H8, H8, ssub64)
+GEN_VEXT_VX_RM(vssub_vx_b, 1)
+GEN_VEXT_VX_RM(vssub_vx_h, 2)
+GEN_VEXT_VX_RM(vssub_vx_w, 4)
+GEN_VEXT_VX_RM(vssub_vx_d, 8)
+
+static inline uint8_t get_round(int vxrm, uint64_t v, uint8_t shift)
+{
+    uint8_t d;
+    uint8_t d1;
+    uint64_t d1_bits;
+    uint64_t d2_bits;
+
+    if (shift == 0 || shift > 64) {
+        return 0;
+    }
+
+    d = extract64(v, shift, 1);
+    d1 = extract64(v, shift - 1, 1);
+    d1_bits = extract64(v, 0, shift);
+    if (vxrm == 0) {
+        return d1;
+    } else if (vxrm == 1) {
+        if (shift > 1) {
+            d2_bits = extract64(v, 0, shift - 1);
+            return d1 & ((d2_bits != 0) | d);
+        } else {
+            return d1 & d;
+        }
+    } else if (vxrm == 3) {
+        return !d & (d1_bits != 0);
+    }
+    return 0;
+}
+
+static inline int32_t aadd32(CPURISCVState *env, int vxrm, int32_t a,
+                             int32_t b)
+{
+    int64_t res = (int64_t)a + b;
+    uint8_t round = get_round(vxrm, res, 1);
+
+    return (res >> 1) + round;
+}
+
+static inline int64_t aadd64(CPURISCVState *env, int vxrm, int64_t a,
+                             int64_t b)
+{
+    int64_t res = a + b;
+    uint8_t round = get_round(vxrm, res, 1);
+    int64_t over = (res ^ a) & (res ^ b) & INT64_MIN;
+
+    return ((res >> 1) ^ over) + round;
+}
+
+RVVCALL(OPIVV2_RM, vaadd_vv_b, OP_SSS_B, H1, H1, H1, aadd32)
+RVVCALL(OPIVV2_RM, vaadd_vv_h, OP_SSS_H, H2, H2, H2, aadd32)
+RVVCALL(OPIVV2_RM, vaadd_vv_w, OP_SSS_W, H4, H4, H4, aadd32)
+RVVCALL(OPIVV2_RM, vaadd_vv_d, OP_SSS_D, H8, H8, H8, aadd64)
+GEN_VEXT_VV_RM(vaadd_vv_b, 1)
+GEN_VEXT_VV_RM(vaadd_vv_h, 2)
+GEN_VEXT_VV_RM(vaadd_vv_w, 4)
+GEN_VEXT_VV_RM(vaadd_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vaadd_vx_b, OP_SSS_B, H1, H1, aadd32)
+RVVCALL(OPIVX2_RM, vaadd_vx_h, OP_SSS_H, H2, H2, aadd32)
+RVVCALL(OPIVX2_RM, vaadd_vx_w, OP_SSS_W, H4, H4, aadd32)
+RVVCALL(OPIVX2_RM, vaadd_vx_d, OP_SSS_D, H8, H8, aadd64)
+GEN_VEXT_VX_RM(vaadd_vx_b, 1)
+GEN_VEXT_VX_RM(vaadd_vx_h, 2)
+GEN_VEXT_VX_RM(vaadd_vx_w, 4)
+GEN_VEXT_VX_RM(vaadd_vx_d, 8)
+
+static inline uint32_t aaddu32(CPURISCVState *env, int vxrm, uint32_t a,
+                               uint32_t b)
+{
+    uint64_t res = (uint64_t)a + b;
+    uint8_t round = get_round(vxrm, res, 1);
+
+    return (res >> 1) + round;
+}
+
+static inline uint64_t aaddu64(CPURISCVState *env, int vxrm, uint64_t a,
+                               uint64_t b)
+{
+    uint64_t res = a + b;
+    uint8_t round = get_round(vxrm, res, 1);
+    uint64_t over = (uint64_t)(res < a) << 63;
+
+    return ((res >> 1) | over) + round;
+}
+
+RVVCALL(OPIVV2_RM, vaaddu_vv_b, OP_UUU_B, H1, H1, H1, aaddu32)
+RVVCALL(OPIVV2_RM, vaaddu_vv_h, OP_UUU_H, H2, H2, H2, aaddu32)
+RVVCALL(OPIVV2_RM, vaaddu_vv_w, OP_UUU_W, H4, H4, H4, aaddu32)
+RVVCALL(OPIVV2_RM, vaaddu_vv_d, OP_UUU_D, H8, H8, H8, aaddu64)
+GEN_VEXT_VV_RM(vaaddu_vv_b, 1)
+GEN_VEXT_VV_RM(vaaddu_vv_h, 2)
+GEN_VEXT_VV_RM(vaaddu_vv_w, 4)
+GEN_VEXT_VV_RM(vaaddu_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vaaddu_vx_b, OP_UUU_B, H1, H1, aaddu32)
+RVVCALL(OPIVX2_RM, vaaddu_vx_h, OP_UUU_H, H2, H2, aaddu32)
+RVVCALL(OPIVX2_RM, vaaddu_vx_w, OP_UUU_W, H4, H4, aaddu32)
+RVVCALL(OPIVX2_RM, vaaddu_vx_d, OP_UUU_D, H8, H8, aaddu64)
+GEN_VEXT_VX_RM(vaaddu_vx_b, 1)
+GEN_VEXT_VX_RM(vaaddu_vx_h, 2)
+GEN_VEXT_VX_RM(vaaddu_vx_w, 4)
+GEN_VEXT_VX_RM(vaaddu_vx_d, 8)
+
+static inline int32_t asub32(CPURISCVState *env, int vxrm, int32_t a,
+                             int32_t b)
+{
+    int64_t res = (int64_t)a - b;
+    uint8_t round = get_round(vxrm, res, 1);
+
+    return (res >> 1) + round;
+}
+
+static inline int64_t asub64(CPURISCVState *env, int vxrm, int64_t a,
+                             int64_t b)
+{
+    int64_t res = a - b;
+    uint8_t round = get_round(vxrm, res, 1);
+    int64_t over = (res ^ a) & (a ^ b) & INT64_MIN;
+
+    return ((res >> 1) ^ over) + round;
+}
+
+RVVCALL(OPIVV2_RM, vasub_vv_b, OP_SSS_B, H1, H1, H1, asub32)
+RVVCALL(OPIVV2_RM, vasub_vv_h, OP_SSS_H, H2, H2, H2, asub32)
+RVVCALL(OPIVV2_RM, vasub_vv_w, OP_SSS_W, H4, H4, H4, asub32)
+RVVCALL(OPIVV2_RM, vasub_vv_d, OP_SSS_D, H8, H8, H8, asub64)
+GEN_VEXT_VV_RM(vasub_vv_b, 1)
+GEN_VEXT_VV_RM(vasub_vv_h, 2)
+GEN_VEXT_VV_RM(vasub_vv_w, 4)
+GEN_VEXT_VV_RM(vasub_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vasub_vx_b, OP_SSS_B, H1, H1, asub32)
+RVVCALL(OPIVX2_RM, vasub_vx_h, OP_SSS_H, H2, H2, asub32)
+RVVCALL(OPIVX2_RM, vasub_vx_w, OP_SSS_W, H4, H4, asub32)
+RVVCALL(OPIVX2_RM, vasub_vx_d, OP_SSS_D, H8, H8, asub64)
+GEN_VEXT_VX_RM(vasub_vx_b, 1)
+GEN_VEXT_VX_RM(vasub_vx_h, 2)
+GEN_VEXT_VX_RM(vasub_vx_w, 4)
+GEN_VEXT_VX_RM(vasub_vx_d, 8)
+
+static inline uint32_t asubu32(CPURISCVState *env, int vxrm, uint32_t a,
+                               uint32_t b)
+{
+    int64_t res = (int64_t)a - b;
+    uint8_t round = get_round(vxrm, res, 1);
+
+    return (res >> 1) + round;
+}
+
+static inline uint64_t asubu64(CPURISCVState *env, int vxrm, uint64_t a,
+                               uint64_t b)
+{
+    uint64_t res = a - b;
+    uint8_t round = get_round(vxrm, res, 1);
+    uint64_t over = (uint64_t)(res > a) << 63;
+
+    return ((res >> 1) | over) + round;
+}
+
+RVVCALL(OPIVV2_RM, vasubu_vv_b, OP_UUU_B, H1, H1, H1, asubu32)
+RVVCALL(OPIVV2_RM, vasubu_vv_h, OP_UUU_H, H2, H2, H2, asubu32)
+RVVCALL(OPIVV2_RM, vasubu_vv_w, OP_UUU_W, H4, H4, H4, asubu32)
+RVVCALL(OPIVV2_RM, vasubu_vv_d, OP_UUU_D, H8, H8, H8, asubu64)
+GEN_VEXT_VV_RM(vasubu_vv_b, 1)
+GEN_VEXT_VV_RM(vasubu_vv_h, 2)
+GEN_VEXT_VV_RM(vasubu_vv_w, 4)
+GEN_VEXT_VV_RM(vasubu_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vasubu_vx_b, OP_UUU_B, H1, H1, asubu32)
+RVVCALL(OPIVX2_RM, vasubu_vx_h, OP_UUU_H, H2, H2, asubu32)
+RVVCALL(OPIVX2_RM, vasubu_vx_w, OP_UUU_W, H4, H4, asubu32)
+RVVCALL(OPIVX2_RM, vasubu_vx_d, OP_UUU_D, H8, H8, asubu64)
+GEN_VEXT_VX_RM(vasubu_vx_b, 1)
+GEN_VEXT_VX_RM(vasubu_vx_h, 2)
+GEN_VEXT_VX_RM(vasubu_vx_w, 4)
+GEN_VEXT_VX_RM(vasubu_vx_d, 8)
+
+static inline int8_t vsmul8(CPURISCVState *env, int vxrm, int8_t a, int8_t b)
+{
+    uint8_t round;
+    int16_t res;
+
+    res = (int16_t)a * (int16_t)b;
+    round = get_round(vxrm, res, 7);
+    res = (res >> 7) + round;
+
+    if (res > INT8_MAX) {
+        env->vxsat = 1;
+        return INT8_MAX;
+    } else if (res < INT8_MIN) {
+        env->vxsat = 1;
+        return INT8_MIN;
+    }
+    return res;
+}
+
+static int16_t vsmul16(CPURISCVState *env, int vxrm, int16_t a, int16_t b)
+{
+    uint8_t round;
+    int32_t res;
+
+    res = (int32_t)a * (int32_t)b;
+    round = get_round(vxrm, res, 15);
+    res = (res >> 15) + round;
+
+    if (res > INT16_MAX) {
+        env->vxsat = 1;
+        return INT16_MAX;
+    } else if (res < INT16_MIN) {
+        env->vxsat = 1;
+        return INT16_MIN;
+    }
+    return res;
+}
+
+static int32_t vsmul32(CPURISCVState *env, int vxrm, int32_t a, int32_t b)
+{
+    uint8_t round;
+    int64_t res;
+
+    res = (int64_t)a * (int64_t)b;
+    round = get_round(vxrm, res, 31);
+    res = (res >> 31) + round;
+
+    if (res > INT32_MAX) {
+        env->vxsat = 1;
+        return INT32_MAX;
+    } else if (res < INT32_MIN) {
+        env->vxsat = 1;
+        return INT32_MIN;
+    }
+    return res;
+}
+
+static int64_t vsmul64(CPURISCVState *env, int vxrm, int64_t a, int64_t b)
+{
+    uint8_t round;
+    uint64_t hi_64;
+    uint64_t lo_64;
+    int64_t res;
+
+    if (a == INT64_MIN && b == INT64_MIN) {
+        env->vxsat = 1;
+        return INT64_MAX;
+    }
+
+    muls64(&lo_64, &hi_64, a, b);
+    round = get_round(vxrm, lo_64, 63);
+    res = (hi_64 << 1) | (lo_64 >> 63);
+    if (round) {
+        if (res == INT64_MAX) {
+            env->vxsat = 1;
+        } else {
+            res += 1;
+        }
+    }
+    return res;
+}
+
+RVVCALL(OPIVV2_RM, vsmul_vv_b, OP_SSS_B, H1, H1, H1, vsmul8)
+RVVCALL(OPIVV2_RM, vsmul_vv_h, OP_SSS_H, H2, H2, H2, vsmul16)
+RVVCALL(OPIVV2_RM, vsmul_vv_w, OP_SSS_W, H4, H4, H4, vsmul32)
+RVVCALL(OPIVV2_RM, vsmul_vv_d, OP_SSS_D, H8, H8, H8, vsmul64)
+GEN_VEXT_VV_RM(vsmul_vv_b, 1)
+GEN_VEXT_VV_RM(vsmul_vv_h, 2)
+GEN_VEXT_VV_RM(vsmul_vv_w, 4)
+GEN_VEXT_VV_RM(vsmul_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vsmul_vx_b, OP_SSS_B, H1, H1, vsmul8)
+RVVCALL(OPIVX2_RM, vsmul_vx_h, OP_SSS_H, H2, H2, vsmul16)
+RVVCALL(OPIVX2_RM, vsmul_vx_w, OP_SSS_W, H4, H4, vsmul32)
+RVVCALL(OPIVX2_RM, vsmul_vx_d, OP_SSS_D, H8, H8, vsmul64)
+GEN_VEXT_VX_RM(vsmul_vx_b, 1)
+GEN_VEXT_VX_RM(vsmul_vx_h, 2)
+GEN_VEXT_VX_RM(vsmul_vx_w, 4)
+GEN_VEXT_VX_RM(vsmul_vx_d, 8)
+
+static inline uint8_t vssrl8(CPURISCVState *env, int vxrm, uint8_t a,
+                             uint8_t b)
+{
+    uint8_t shift = b & 0x7;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+static inline uint16_t vssrl16(CPURISCVState *env, int vxrm, uint16_t a,
+                               uint16_t b)
+{
+    uint8_t shift = b & 0xf;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+static inline uint32_t vssrl32(CPURISCVState *env, int vxrm, uint32_t a,
+                               uint32_t b)
+{
+    uint8_t shift = b & 0x1f;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+static inline uint64_t vssrl64(CPURISCVState *env, int vxrm, uint64_t a,
+                               uint64_t b)
+{
+    uint8_t shift = b & 0x3f;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+RVVCALL(OPIVV2_RM, vssrl_vv_b, OP_UUU_B, H1, H1, H1, vssrl8)
+RVVCALL(OPIVV2_RM, vssrl_vv_h, OP_UUU_H, H2, H2, H2, vssrl16)
+RVVCALL(OPIVV2_RM, vssrl_vv_w, OP_UUU_W, H4, H4, H4, vssrl32)
+RVVCALL(OPIVV2_RM, vssrl_vv_d, OP_UUU_D, H8, H8, H8, vssrl64)
+GEN_VEXT_VV_RM(vssrl_vv_b, 1)
+GEN_VEXT_VV_RM(vssrl_vv_h, 2)
+GEN_VEXT_VV_RM(vssrl_vv_w, 4)
+GEN_VEXT_VV_RM(vssrl_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vssrl_vx_b, OP_UUU_B, H1, H1, vssrl8)
+RVVCALL(OPIVX2_RM, vssrl_vx_h, OP_UUU_H, H2, H2, vssrl16)
+RVVCALL(OPIVX2_RM, vssrl_vx_w, OP_UUU_W, H4, H4, vssrl32)
+RVVCALL(OPIVX2_RM, vssrl_vx_d, OP_UUU_D, H8, H8, vssrl64)
+GEN_VEXT_VX_RM(vssrl_vx_b, 1)
+GEN_VEXT_VX_RM(vssrl_vx_h, 2)
+GEN_VEXT_VX_RM(vssrl_vx_w, 4)
+GEN_VEXT_VX_RM(vssrl_vx_d, 8)
+
+static inline int8_t vssra8(CPURISCVState *env, int vxrm, int8_t a,
+                            int8_t b)
+{
+    uint8_t shift = b & 0x7;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+static inline int16_t vssra16(CPURISCVState *env, int vxrm, int16_t a,
+                              int16_t b)
+{
+    uint8_t shift = b & 0xf;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+static inline int32_t vssra32(CPURISCVState *env, int vxrm, int32_t a,
+                              int32_t b)
+{
+    uint8_t shift = b & 0x1f;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+static inline int64_t vssra64(CPURISCVState *env, int vxrm, int64_t a,
+                              int64_t b)
+{
+    uint8_t shift = b & 0x3f;
+    uint8_t round = get_round(vxrm, a, shift);
+
+    return (a >> shift) + round;
+}
+
+RVVCALL(OPIVV2_RM, vssra_vv_b, OP_SSS_B, H1, H1, H1, vssra8)
+RVVCALL(OPIVV2_RM, vssra_vv_h, OP_SSS_H, H2, H2, H2, vssra16)
+RVVCALL(OPIVV2_RM, vssra_vv_w, OP_SSS_W, H4, H4, H4, vssra32)
+RVVCALL(OPIVV2_RM, vssra_vv_d, OP_SSS_D, H8, H8, H8, vssra64)
+GEN_VEXT_VV_RM(vssra_vv_b, 1)
+GEN_VEXT_VV_RM(vssra_vv_h, 2)
+GEN_VEXT_VV_RM(vssra_vv_w, 4)
+GEN_VEXT_VV_RM(vssra_vv_d, 8)
+
+RVVCALL(OPIVX2_RM, vssra_vx_b, OP_SSS_B, H1, H1, vssra8)
+RVVCALL(OPIVX2_RM, vssra_vx_h, OP_SSS_H, H2, H2, vssra16)
+RVVCALL(OPIVX2_RM, vssra_vx_w, OP_SSS_W, H4, H4, vssra32)
+RVVCALL(OPIVX2_RM, vssra_vx_d, OP_SSS_D, H8, H8, vssra64)
+GEN_VEXT_VX_RM(vssra_vx_b, 1)
+GEN_VEXT_VX_RM(vssra_vx_h, 2)
+GEN_VEXT_VX_RM(vssra_vx_w, 4)
+GEN_VEXT_VX_RM(vssra_vx_d, 8)
+
+static inline int8_t vnclip8(CPURISCVState *env, int vxrm, int16_t a,
+                             int8_t b)
+{
+    uint8_t shift = b & 0xf;
+    uint8_t round = get_round(vxrm, a, shift);
+    int16_t res = (a >> shift) + round;
+
+    if (res > INT8_MAX) {
+        env->vxsat = 1;
+        return INT8_MAX;
+    } else if (res < INT8_MIN) {
+        env->vxsat = 1;
+        return INT8_MIN;
+    }
+    return res;
+}
+
+static inline int16_t vnclip16(CPURISCVState *env, int vxrm, int32_t a,
+                               int16_t b)
+{
+    uint8_t shift = b & 0x1f;
+    uint8_t round = get_round(vxrm, a, shift);
+    int32_t res = (a >> shift) + round;
+
+    if (res > INT16_MAX) {
+        env->vxsat = 1;
+        return INT16_MAX;
+    } else if (res < INT16_MIN) {
+        env->vxsat = 1;
+        return INT16_MIN;
+    }
+    return res;
+}
+
+static inline int32_t vnclip32(CPURISCVState *env, int vxrm, int64_t a,
+                               int32_t b)
+{
+    uint8_t shift = b & 0x3f;
+    uint8_t round = get_round(vxrm, a, shift);
+    int64_t res = (a >> shift) + round;
+
+    if (res > INT32_MAX) {
+        env->vxsat = 1;
+        return INT32_MAX;
+    } else if (res < INT32_MIN) {
+        env->vxsat = 1;
+        return INT32_MIN;
+    }
+    return res;
+}
+
+RVVCALL(OPIVV2_RM, vnclip_wv_b, NOP_SSS_B, H1, H2, H1, vnclip8)
+RVVCALL(OPIVV2_RM, vnclip_wv_h, NOP_SSS_H, H2, H4, H2, vnclip16)
+RVVCALL(OPIVV2_RM, vnclip_wv_w, NOP_SSS_W, H4, H8, H4, vnclip32)
+GEN_VEXT_VV_RM(vnclip_wv_b, 1)
+GEN_VEXT_VV_RM(vnclip_wv_h, 2)
+GEN_VEXT_VV_RM(vnclip_wv_w, 4)
+
+RVVCALL(OPIVX2_RM, vnclip_wx_b, NOP_SSS_B, H1, H2, vnclip8)
+RVVCALL(OPIVX2_RM, vnclip_wx_h, NOP_SSS_H, H2, H4, vnclip16)
+RVVCALL(OPIVX2_RM, vnclip_wx_w, NOP_SSS_W, H4, H8, vnclip32)
+GEN_VEXT_VX_RM(vnclip_wx_b, 1)
+GEN_VEXT_VX_RM(vnclip_wx_h, 2)
+GEN_VEXT_VX_RM(vnclip_wx_w, 4)
+
+static inline uint8_t vnclipu8(CPURISCVState *env, int vxrm, uint16_t a,
+                               uint8_t b)
+{
+    uint8_t shift = b & 0xf;
+    uint8_t round = get_round(vxrm, a, shift);
+    uint16_t res = (a >> shift) + round;
+
+    if (res > UINT8_MAX) {
+        env->vxsat = 1;
+        return UINT8_MAX;
+    }
+    return res;
+}
+
+static inline uint16_t vnclipu16(CPURISCVState *env, int vxrm, uint32_t a,
+                                 uint16_t b)
+{
+    uint8_t shift = b & 0x1f;
+    uint8_t round = get_round(vxrm, a, shift);
+    uint32_t res = (a >> shift) + round;
+
+    if (res > UINT16_MAX) {
+        env->vxsat = 1;
+        return UINT16_MAX;
+    }
+    return res;
+}
+
+static inline uint32_t vnclipu32(CPURISCVState *env, int vxrm, uint64_t a,
+                                 uint32_t b)
+{
+    uint8_t shift = b & 0x3f;
+    uint8_t round = get_round(vxrm, a, shift);
+    uint64_t res = (a >> shift) + round;
+
+    if (res > UINT32_MAX) {
+        env->vxsat = 1;
+        return UINT32_MAX;
+    }
+    return res;
+}
+
+RVVCALL(OPIVV2_RM, vnclipu_wv_b, NOP_UUU_B, H1, H2, H1, vnclipu8)
+RVVCALL(OPIVV2_RM, vnclipu_wv_h, NOP_UUU_H, H2, H4, H2, vnclipu16)
+RVVCALL(OPIVV2_RM, vnclipu_wv_w, NOP_UUU_W, H4, H8, H4, vnclipu32)
+GEN_VEXT_VV_RM(vnclipu_wv_b, 1)
+GEN_VEXT_VV_RM(vnclipu_wv_h, 2)
+GEN_VEXT_VV_RM(vnclipu_wv_w, 4)
+
+RVVCALL(OPIVX2_RM, vnclipu_wx_b, NOP_UUU_B, H1, H2, vnclipu8)
+RVVCALL(OPIVX2_RM, vnclipu_wx_h, NOP_UUU_H, H2, H4, vnclipu16)
+RVVCALL(OPIVX2_RM, vnclipu_wx_w, NOP_UUU_W, H4, H8, vnclipu32)
+GEN_VEXT_VX_RM(vnclipu_wx_b, 1)
+GEN_VEXT_VX_RM(vnclipu_wx_h, 2)
+GEN_VEXT_VX_RM(vnclipu_wx_w, 4)
+
+#define GEN_VEXT_MASK_VV(NAME, OP)                        \
+void HELPER(NAME)(void *vd, void *v0, void *vs1,          \
+                  void *vs2, CPURISCVState *env,          \
+                  uint32_t desc)                          \
+{                                                         \
+    uint32_t vl = env->vl;                                \
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;    \
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);          \
+    uint32_t i;                                           \
+    int a;                                                \
+    int b;                                                \
+                                                          \
+    for (i = env->vstart; i < vl; i++) {                  \
+        a = vext_elem_mask(vs1, i);                       \
+        b = vext_elem_mask(vs2, i);                       \
+        vext_set_elem_mask(vd, i, OP(b, a));              \
+    }                                                     \
+    env->vstart = 0;                                      \
+    if (vta_all_1s) {                                     \
+        for (; i < total_elems; i++) {                    \
+            vext_set_elem_mask(vd, i, 1);                 \
+        }                                                 \
+    }                                                     \
+}
+
+#define DO_NAND(N, M) (!(N & M))
+#define DO_ANDNOT(N, M) (N & !M)
+#define DO_NOR(N, M) (!(N | M))
+#define DO_ORNOT(N, M) (N | !M)
+#define DO_XNOR(N, M) (!(N ^ M))
+
+GEN_VEXT_MASK_VV(vmand_mm, DO_AND)
+GEN_VEXT_MASK_VV(vmnand_mm, DO_NAND)
+GEN_VEXT_MASK_VV(vmandn_mm, DO_ANDNOT)
+GEN_VEXT_MASK_VV(vmxor_mm, DO_XOR)
+GEN_VEXT_MASK_VV(vmor_mm, DO_OR)
+GEN_VEXT_MASK_VV(vmnor_mm, DO_NOR)
+GEN_VEXT_MASK_VV(vmorn_mm, DO_ORNOT)
+GEN_VEXT_MASK_VV(vmxnor_mm, DO_XNOR)
+
+static void rvv_require_vstart_zero(CPURISCVState *env)
+{
+    if (env->vstart != 0) {
+        riscv_raise_exception(env, RISCV_EXCP_ILLEGAL_INST, GETPC());
+    }
+}
+
+#define GEN_VEXT_RED(NAME, TD, TS2, HD, HS2, OP)                  \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,      \
+                  CPURISCVState *env, uint32_t desc)             \
+{                                                                 \
+    uint32_t vm = vext_vm(desc);                                  \
+    uint32_t vl = env->vl;                                        \
+    uint32_t esz = sizeof(TD);                                    \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);  \
+    uint32_t vta = vext_vta(desc);                                \
+    uint32_t i;                                                   \
+    TD acc;                                                       \
+                                                                  \
+    rvv_require_vstart_zero(env);                                 \
+    if (vl == 0) {                                                \
+        return;                                                   \
+    }                                                             \
+    acc = *((TD *)vs1 + HD(0));                                   \
+    for (i = env->vstart; i < vl; i++) {                          \
+        TS2 s2 = *((TS2 *)vs2 + HS2(i));                          \
+                                                                  \
+        if (!vm && !vext_elem_mask(v0, i)) {                      \
+            continue;                                             \
+        }                                                         \
+        acc = OP(acc, (TD)s2);                                    \
+    }                                                             \
+    *((TD *)vd + HD(0)) = acc;                                    \
+    env->vstart = 0;                                              \
+    vext_set_elems_1s(vd, vta, esz, total_elems * esz);           \
+}
+
+GEN_VEXT_RED(vredsum_vs_b, int8_t, int8_t, H1, H1, DO_ADD)
+GEN_VEXT_RED(vredsum_vs_h, int16_t, int16_t, H2, H2, DO_ADD)
+GEN_VEXT_RED(vredsum_vs_w, int32_t, int32_t, H4, H4, DO_ADD)
+GEN_VEXT_RED(vredsum_vs_d, int64_t, int64_t, H8, H8, DO_ADD)
+
+GEN_VEXT_RED(vredmaxu_vs_b, uint8_t, uint8_t, H1, H1, DO_MAX)
+GEN_VEXT_RED(vredmaxu_vs_h, uint16_t, uint16_t, H2, H2, DO_MAX)
+GEN_VEXT_RED(vredmaxu_vs_w, uint32_t, uint32_t, H4, H4, DO_MAX)
+GEN_VEXT_RED(vredmaxu_vs_d, uint64_t, uint64_t, H8, H8, DO_MAX)
+
+GEN_VEXT_RED(vredmax_vs_b, int8_t, int8_t, H1, H1, DO_MAX)
+GEN_VEXT_RED(vredmax_vs_h, int16_t, int16_t, H2, H2, DO_MAX)
+GEN_VEXT_RED(vredmax_vs_w, int32_t, int32_t, H4, H4, DO_MAX)
+GEN_VEXT_RED(vredmax_vs_d, int64_t, int64_t, H8, H8, DO_MAX)
+
+GEN_VEXT_RED(vredminu_vs_b, uint8_t, uint8_t, H1, H1, DO_MIN)
+GEN_VEXT_RED(vredminu_vs_h, uint16_t, uint16_t, H2, H2, DO_MIN)
+GEN_VEXT_RED(vredminu_vs_w, uint32_t, uint32_t, H4, H4, DO_MIN)
+GEN_VEXT_RED(vredminu_vs_d, uint64_t, uint64_t, H8, H8, DO_MIN)
+
+GEN_VEXT_RED(vredmin_vs_b, int8_t, int8_t, H1, H1, DO_MIN)
+GEN_VEXT_RED(vredmin_vs_h, int16_t, int16_t, H2, H2, DO_MIN)
+GEN_VEXT_RED(vredmin_vs_w, int32_t, int32_t, H4, H4, DO_MIN)
+GEN_VEXT_RED(vredmin_vs_d, int64_t, int64_t, H8, H8, DO_MIN)
+
+GEN_VEXT_RED(vredand_vs_b, int8_t, int8_t, H1, H1, DO_AND)
+GEN_VEXT_RED(vredand_vs_h, int16_t, int16_t, H2, H2, DO_AND)
+GEN_VEXT_RED(vredand_vs_w, int32_t, int32_t, H4, H4, DO_AND)
+GEN_VEXT_RED(vredand_vs_d, int64_t, int64_t, H8, H8, DO_AND)
+
+GEN_VEXT_RED(vredor_vs_b, int8_t, int8_t, H1, H1, DO_OR)
+GEN_VEXT_RED(vredor_vs_h, int16_t, int16_t, H2, H2, DO_OR)
+GEN_VEXT_RED(vredor_vs_w, int32_t, int32_t, H4, H4, DO_OR)
+GEN_VEXT_RED(vredor_vs_d, int64_t, int64_t, H8, H8, DO_OR)
+
+GEN_VEXT_RED(vredxor_vs_b, int8_t, int8_t, H1, H1, DO_XOR)
+GEN_VEXT_RED(vredxor_vs_h, int16_t, int16_t, H2, H2, DO_XOR)
+GEN_VEXT_RED(vredxor_vs_w, int32_t, int32_t, H4, H4, DO_XOR)
+GEN_VEXT_RED(vredxor_vs_d, int64_t, int64_t, H8, H8, DO_XOR)
+
+GEN_VEXT_RED(vwredsum_vs_b, int16_t, int8_t, H2, H1, DO_ADD)
+GEN_VEXT_RED(vwredsum_vs_h, int32_t, int16_t, H4, H2, DO_ADD)
+GEN_VEXT_RED(vwredsum_vs_w, int64_t, int32_t, H8, H4, DO_ADD)
+
+GEN_VEXT_RED(vwredsumu_vs_b, uint16_t, uint8_t, H2, H1, DO_ADD)
+GEN_VEXT_RED(vwredsumu_vs_h, uint32_t, uint16_t, H4, H2, DO_ADD)
+GEN_VEXT_RED(vwredsumu_vs_w, uint64_t, uint32_t, H8, H4, DO_ADD)
+
+#define GEN_VEXT_FRED(NAME, TD, TS2, HD, HS2, OP)                 \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,      \
+                  CPURISCVState *env, uint32_t desc)             \
+{                                                                 \
+    uint32_t vm = vext_vm(desc);                                  \
+    uint32_t vl = env->vl;                                        \
+    uint32_t esz = sizeof(TD);                                    \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);  \
+    uint32_t vta = vext_vta(desc);                                \
+    uint32_t i;                                                   \
+    TD acc;                                                       \
+                                                                  \
+    rvv_require_vstart_zero(env);                                 \
+    if (vl == 0) {                                                \
+        return;                                                   \
+    }                                                             \
+    acc = *((TD *)vs1 + HD(0));                                   \
+    for (i = env->vstart; i < vl; i++) {                          \
+        TS2 s2 = *((TS2 *)vs2 + HS2(i));                          \
+                                                                  \
+        if (!vm && !vext_elem_mask(v0, i)) {                      \
+            continue;                                             \
+        }                                                         \
+        acc = OP(acc, (TD)s2, &env->fp_status);                   \
+    }                                                             \
+    *((TD *)vd + HD(0)) = acc;                                    \
+    env->vstart = 0;                                              \
+    vext_set_elems_1s(vd, vta, esz, total_elems * esz);           \
+}
+
+GEN_VEXT_FRED(vfredusum_vs_h, uint16_t, uint16_t, H2, H2, float16_add)
+GEN_VEXT_FRED(vfredusum_vs_w, uint32_t, uint32_t, H4, H4, float32_add)
+GEN_VEXT_FRED(vfredusum_vs_d, uint64_t, uint64_t, H8, H8, float64_add)
+
+GEN_VEXT_FRED(vfredosum_vs_h, uint16_t, uint16_t, H2, H2, float16_add)
+GEN_VEXT_FRED(vfredosum_vs_w, uint32_t, uint32_t, H4, H4, float32_add)
+GEN_VEXT_FRED(vfredosum_vs_d, uint64_t, uint64_t, H8, H8, float64_add)
+
+GEN_VEXT_FRED(vfredmin_vs_h, uint16_t, uint16_t, H2, H2,
+              float16_minimum_number)
+GEN_VEXT_FRED(vfredmin_vs_w, uint32_t, uint32_t, H4, H4,
+              float32_minimum_number)
+GEN_VEXT_FRED(vfredmin_vs_d, uint64_t, uint64_t, H8, H8,
+              float64_minimum_number)
+GEN_VEXT_FRED(vfredmax_vs_h, uint16_t, uint16_t, H2, H2,
+              float16_maximum_number)
+GEN_VEXT_FRED(vfredmax_vs_w, uint32_t, uint32_t, H4, H4,
+              float32_maximum_number)
+GEN_VEXT_FRED(vfredmax_vs_d, uint64_t, uint64_t, H8, H8,
+              float64_maximum_number)
+
+static uint32_t fwadd16(uint32_t a, uint16_t b, float_status *s)
+{
+    return float32_add(a, float16_to_float32(b, true, s), s);
+}
+
+static uint64_t fwadd32(uint64_t a, uint32_t b, float_status *s)
+{
+    return float64_add(a, float32_to_float64(b, s), s);
+}
+
+GEN_VEXT_FRED(vfwredusum_vs_h, uint32_t, uint16_t, H4, H2, fwadd16)
+GEN_VEXT_FRED(vfwredusum_vs_w, uint64_t, uint32_t, H8, H4, fwadd32)
+GEN_VEXT_FRED(vfwredosum_vs_h, uint32_t, uint16_t, H4, H2, fwadd16)
+GEN_VEXT_FRED(vfwredosum_vs_w, uint64_t, uint32_t, H8, H4, fwadd32)
+
+target_ulong HELPER(vcpop_m)(void *v0, void *vs2, CPURISCVState *env,
+                             uint32_t desc)
+{
+    target_ulong cnt = 0;
+    uint32_t vm = vext_vm(desc);
+    uint32_t vl = env->vl;
+    uint32_t i;
+
+    rvv_require_vstart_zero(env);
+    for (i = env->vstart; i < vl; i++) {
+        if ((vm || vext_elem_mask(v0, i)) && vext_elem_mask(vs2, i)) {
+            cnt++;
+        }
+    }
+    env->vstart = 0;
+    return cnt;
+}
+
+target_ulong HELPER(vfirst_m)(void *v0, void *vs2, CPURISCVState *env,
+                              uint32_t desc)
+{
+    uint32_t vm = vext_vm(desc);
+    uint32_t vl = env->vl;
+    uint32_t i;
+
+    rvv_require_vstart_zero(env);
+    for (i = env->vstart; i < vl; i++) {
+        if ((vm || vext_elem_mask(v0, i)) && vext_elem_mask(vs2, i)) {
+            env->vstart = 0;
+            return i;
+        }
+    }
+    env->vstart = 0;
+    return -1;
+}
+
+enum rvv_set_mask_type {
+    RVV_MASK_ONLY_FIRST = 1,
+    RVV_MASK_INCLUDE_FIRST,
+    RVV_MASK_BEFORE_FIRST,
+};
+
+static void vmsetm(void *vd, void *v0, void *vs2, CPURISCVState *env,
+                   uint32_t desc, enum rvv_set_mask_type type)
+{
+    uint32_t vm = vext_vm(desc);
+    uint32_t vl = env->vl;
+    uint32_t total_elems = env_archcpu(env)->cfg.vlen;
+    uint32_t vta_all_1s = vext_vta_all_1s(desc);
+    uint32_t vma = vext_vma(desc);
+    uint32_t i;
+    bool first_mask_bit = false;
+
+    rvv_require_vstart_zero(env);
+    if (vl == 0) {
+        return;
+    }
+    for (i = env->vstart; i < vl; i++) {
+        if (!vm && !vext_elem_mask(v0, i)) {
+            if (vma) {
+                vext_set_elem_mask(vd, i, 1);
+            }
+            continue;
+        }
+        if (first_mask_bit) {
+            vext_set_elem_mask(vd, i, 0);
+            continue;
+        }
+        if (vext_elem_mask(vs2, i)) {
+            first_mask_bit = true;
+            vext_set_elem_mask(vd, i, type == RVV_MASK_BEFORE_FIRST ? 0 : 1);
+        } else {
+            vext_set_elem_mask(vd, i, type == RVV_MASK_ONLY_FIRST ? 0 : 1);
+        }
+    }
+    env->vstart = 0;
+    if (vta_all_1s) {
+        for (; i < total_elems; i++) {
+            vext_set_elem_mask(vd, i, 1);
+        }
+    }
+}
+
+void HELPER(vmsbf_m)(void *vd, void *v0, void *vs2, CPURISCVState *env,
+                     uint32_t desc)
+{
+    vmsetm(vd, v0, vs2, env, desc, RVV_MASK_BEFORE_FIRST);
+}
+
+void HELPER(vmsif_m)(void *vd, void *v0, void *vs2, CPURISCVState *env,
+                     uint32_t desc)
+{
+    vmsetm(vd, v0, vs2, env, desc, RVV_MASK_INCLUDE_FIRST);
+}
+
+void HELPER(vmsof_m)(void *vd, void *v0, void *vs2, CPURISCVState *env,
+                     uint32_t desc)
+{
+    vmsetm(vd, v0, vs2, env, desc, RVV_MASK_ONLY_FIRST);
+}
+
+#define GEN_VEXT_VIOTA_M(NAME, ETYPE, H)                         \
+void HELPER(NAME)(void *vd, void *v0, void *vs2,                 \
+                  CPURISCVState *env, uint32_t desc)             \
+{                                                                \
+    uint32_t vm = vext_vm(desc);                                 \
+    uint32_t vl = env->vl;                                       \
+    uint32_t esz = sizeof(ETYPE);                                \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
+    uint32_t vta = vext_vta(desc);                               \
+    uint32_t vma = vext_vma(desc);                               \
+    uint32_t sum = 0;                                            \
+    uint32_t i;                                                  \
+                                                                 \
+    rvv_require_vstart_zero(env);                                \
+    if (vl == 0) {                                               \
+        return;                                                  \
+    }                                                            \
+    for (i = env->vstart; i < vl; i++) {                         \
+        if (!vm && !vext_elem_mask(v0, i)) {                     \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);  \
+            continue;                                            \
+        }                                                        \
+        *((ETYPE *)vd + H(i)) = sum;                             \
+        if (vext_elem_mask(vs2, i)) {                            \
+            sum++;                                               \
+        }                                                        \
+    }                                                            \
+    env->vstart = 0;                                             \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);     \
+}
+
+GEN_VEXT_VIOTA_M(viota_m_b, uint8_t, H1)
+GEN_VEXT_VIOTA_M(viota_m_h, uint16_t, H2)
+GEN_VEXT_VIOTA_M(viota_m_w, uint32_t, H4)
+GEN_VEXT_VIOTA_M(viota_m_d, uint64_t, H8)
+
+#define GEN_VEXT_VID_V(NAME, ETYPE, H)                            \
+void HELPER(NAME)(void *vd, void *v0, CPURISCVState *env,         \
+                  uint32_t desc)                                  \
+{                                                                 \
+    uint32_t vm = vext_vm(desc);                                  \
+    uint32_t vl = env->vl;                                        \
+    uint32_t esz = sizeof(ETYPE);                                 \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);  \
+    uint32_t vta = vext_vta(desc);                                \
+    uint32_t vma = vext_vma(desc);                                \
+    uint32_t i;                                                   \
+                                                                  \
+    for (i = env->vstart; i < vl; i++) {                          \
+        if (!vm && !vext_elem_mask(v0, i)) {                      \
+            vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);   \
+            continue;                                             \
+        }                                                         \
+        *((ETYPE *)vd + H(i)) = i;                                \
+    }                                                             \
+    env->vstart = 0;                                              \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);      \
+}
+
+GEN_VEXT_VID_V(vid_v_b, uint8_t, H1)
+GEN_VEXT_VID_V(vid_v_h, uint16_t, H2)
+GEN_VEXT_VID_V(vid_v_w, uint32_t, H4)
+GEN_VEXT_VID_V(vid_v_d, uint64_t, H8)
+
+#define GEN_VEXT_VMV_VV(NAME, ETYPE, H)                         \
+void HELPER(NAME)(void *vd, void *vs1, CPURISCVState *env,      \
+                  uint32_t desc)                                \
+{                                                               \
+    uint32_t i;                                                 \
+    uint32_t vl = env->vl;                                      \
+    uint32_t esz = sizeof(ETYPE);                               \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
+    uint32_t vta = vext_vta(desc);                              \
+                                                                \
+    for (i = env->vstart; i < vl; i++) {                        \
+        ETYPE s1 = *((ETYPE *)vs1 + H(i));                      \
+                                                                \
+        *((ETYPE *)vd + H(i)) = s1;                             \
+    }                                                           \
+    env->vstart = 0;                                            \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);    \
+}
+
+GEN_VEXT_VMV_VV(vmv_v_v_b, uint8_t, H1)
+GEN_VEXT_VMV_VV(vmv_v_v_h, uint16_t, H2)
+GEN_VEXT_VMV_VV(vmv_v_v_w, uint32_t, H4)
+GEN_VEXT_VMV_VV(vmv_v_v_d, uint64_t, H8)
+
+#define GEN_VEXT_VMV_VX(NAME, ETYPE, H)                         \
+void HELPER(NAME)(void *vd, uint64_t s1, CPURISCVState *env,    \
+                  uint32_t desc)                                \
+{                                                               \
+    uint32_t i;                                                 \
+    uint32_t vl = env->vl;                                      \
+    uint32_t esz = sizeof(ETYPE);                               \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz); \
+    uint32_t vta = vext_vta(desc);                              \
+                                                                \
+    for (i = env->vstart; i < vl; i++) {                        \
+        *((ETYPE *)vd + H(i)) = (ETYPE)s1;                      \
+    }                                                           \
+    env->vstart = 0;                                            \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);    \
+}
+
+GEN_VEXT_VMV_VX(vmv_v_x_b, uint8_t, H1)
+GEN_VEXT_VMV_VX(vmv_v_x_h, uint16_t, H2)
+GEN_VEXT_VMV_VX(vmv_v_x_w, uint32_t, H4)
+GEN_VEXT_VMV_VX(vmv_v_x_d, uint64_t, H8)
+
+#define GEN_VEXT_VMERGE_VV(NAME, ETYPE, H)                         \
+void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2,        \
+                  CPURISCVState *env, uint32_t desc)               \
+{                                                                  \
+    uint32_t i;                                                    \
+    uint32_t vl = env->vl;                                         \
+    uint32_t esz = sizeof(ETYPE);                                  \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);   \
+    uint32_t vta = vext_vta(desc);                                 \
+                                                                   \
+    for (i = env->vstart; i < vl; i++) {                           \
+        ETYPE *src = vext_elem_mask(v0, i) ?                       \
+                     (ETYPE *)vs1 : (ETYPE *)vs2;                  \
+                                                                   \
+        *((ETYPE *)vd + H(i)) = *(src + H(i));                     \
+    }                                                              \
+    env->vstart = 0;                                               \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);       \
+}
+
+GEN_VEXT_VMERGE_VV(vmerge_vvm_b, int8_t, H1)
+GEN_VEXT_VMERGE_VV(vmerge_vvm_h, int16_t, H2)
+GEN_VEXT_VMERGE_VV(vmerge_vvm_w, int32_t, H4)
+GEN_VEXT_VMERGE_VV(vmerge_vvm_d, int64_t, H8)
+
+#define GEN_VEXT_VMERGE_VX(NAME, ETYPE, H)                         \
+void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2,  \
+                  CPURISCVState *env, uint32_t desc)               \
+{                                                                  \
+    uint32_t i;                                                    \
+    uint32_t vl = env->vl;                                         \
+    uint32_t esz = sizeof(ETYPE);                                  \
+    uint32_t total_elems = vext_get_total_elems(env, desc, esz);   \
+    uint32_t vta = vext_vta(desc);                                 \
+                                                                   \
+    for (i = env->vstart; i < vl; i++) {                           \
+        ETYPE s2 = *((ETYPE *)vs2 + H(i));                         \
+        ETYPE data = vext_elem_mask(v0, i) ?                       \
+                     (ETYPE)(target_long)s1 : s2;                  \
+                                                                   \
+        *((ETYPE *)vd + H(i)) = data;                              \
+    }                                                              \
+    env->vstart = 0;                                               \
+    vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);       \
+}
+
+GEN_VEXT_VMERGE_VX(vmerge_vxm_b, int8_t, H1)
+GEN_VEXT_VMERGE_VX(vmerge_vxm_h, int16_t, H2)
+GEN_VEXT_VMERGE_VX(vmerge_vxm_w, int32_t, H4)
+GEN_VEXT_VMERGE_VX(vmerge_vxm_d, int64_t, H8)
diff --git a/qemu/target/s390x/cc_helper.c b/qemu/target/s390x/cc_helper.c
index 2208907dbd..e62a9111e9 100644
--- a/qemu/target/s390x/cc_helper.c
+++ b/qemu/target/s390x/cc_helper.c
@@ -390,6 +390,32 @@ static uint32_t cc_calc_vc(uint64_t low, uint64_t high)
     }
 }
 
+static uint32_t cc_calc_muls_32(int64_t res)
+{
+    const int64_t tmp = res >> 31;
+
+    if (!res) {
+        return 0;
+    } else if (tmp && tmp != -1) {
+        return 3;
+    } else if (res < 0) {
+        return 1;
+    }
+    return 2;
+}
+
+static uint32_t cc_calc_muls_64(int64_t res_high, uint64_t res_low)
+{
+    if (!res_high && !res_low) {
+        return 0;
+    } else if (res_high + (res_low >> 63) != 0) {
+        return 3;
+    } else if (res_high < 0) {
+        return 1;
+    }
+    return 2;
+}
+
 static uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
                                   uint64_t src, uint64_t dst, uint64_t vr)
 {
@@ -457,6 +483,9 @@ static uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
     case CC_OP_COMP_64:
         r =  cc_calc_comp_64(dst);
         break;
+    case CC_OP_MULS_64:
+        r = cc_calc_muls_64(src, dst);
+        break;
 
     case CC_OP_ADD_32:
         r =  cc_calc_add_32(src, dst, vr);
@@ -485,6 +514,9 @@ static uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
     case CC_OP_COMP_32:
         r =  cc_calc_comp_32(dst);
         break;
+    case CC_OP_MULS_32:
+        r = cc_calc_muls_32(dst);
+        break;
 
     case CC_OP_ICM:
         r =  cc_calc_icm(src, dst);
diff --git a/qemu/target/s390x/cpu.h b/qemu/target/s390x/cpu.h
index 426edb4de5..5507429566 100644
--- a/qemu/target/s390x/cpu.h
+++ b/qemu/target/s390x/cpu.h
@@ -52,7 +52,7 @@ struct CPUS390XState {
      * The floating point registers are part of the vector registers.
      * vregs[0][0] -> vregs[15][0] are 16 floating point registers
      */
-    uint64_t vregs[32][2] QEMU_ALIGNED(16);  /* vector registers */
+    QEMU_ALIGN(16, uint64_t vregs[32][2]);   /* vector registers */
     uint32_t aregs[16];    /* access registers */
     uint64_t gscb[4];      /* guarded storage control */
     uint64_t etoken;       /* etoken */
@@ -157,7 +157,7 @@ struct S390CPU {
     /*< public >*/
 
     CPUNegativeOffsetState neg;
-    CPUS390XState env;
+    QEMU_ALIGN(16, CPUS390XState env);
     S390CPUModel *model;
     /* needed for live migration */
     // void *irqstate;
diff --git a/qemu/target/s390x/cpu_models.c b/qemu/target/s390x/cpu_models.c
index d56635ccda..a6f5b883d0 100644
--- a/qemu/target/s390x/cpu_models.c
+++ b/qemu/target/s390x/cpu_models.c
@@ -434,6 +434,7 @@ static void s390_cpu_model_class_init(struct uc_struct *uc, CPUClass *oc, void *
 
     /* model that can change between QEMU versions */
     xcc->cpu_def = (const S390CPUDef *) data;
+    xcc->is_static = false;
     // xcc->is_migration_safe = true;
     // xcc->desc = xcc->cpu_def->desc;
 }
diff --git a/qemu/target/s390x/excp_helper.c b/qemu/target/s390x/excp_helper.c
index a91c407c9e..359696d2ff 100644
--- a/qemu/target/s390x/excp_helper.c
+++ b/qemu/target/s390x/excp_helper.c
@@ -82,6 +82,35 @@ void QEMU_NORETURN tcg_s390_vector_exception(CPUS390XState *env, uint32_t vxc,
     tcg_s390_program_interrupt(env, PGM_VECTOR_PROCESSING, ra);
 }
 
+static void QEMU_NORETURN monitor_event(CPUS390XState *env,
+                                        uint64_t monitor_code,
+                                        uint8_t monitor_class, uintptr_t ra)
+{
+#ifdef UNICORN_ARCH_POSTFIX
+    glue(stq_phys, UNICORN_ARCH_POSTFIX)(env->uc, env_cpu(env)->as,
+             env->psa + offsetof(LowCore, monitor_code), monitor_code);
+    glue(stw_phys, UNICORN_ARCH_POSTFIX)(env->uc, env_cpu(env)->as,
+             env->psa + offsetof(LowCore, mon_class_num), monitor_class);
+#else
+    stq_phys(env->uc, env_cpu(env)->as,
+             env->psa + offsetof(LowCore, monitor_code), monitor_code);
+    stw_phys(env->uc, env_cpu(env)->as,
+             env->psa + offsetof(LowCore, mon_class_num), monitor_class);
+#endif
+
+    tcg_s390_program_interrupt(env, PGM_MONITOR, ra);
+}
+
+void HELPER(monitor_call)(CPUS390XState *env, uint64_t monitor_code,
+                          uint32_t monitor_class)
+{
+    g_assert(monitor_class <= 0xf);
+
+    if (env->cregs[8] & (0x8000 >> monitor_class)) {
+        monitor_event(env, monitor_code, monitor_class, GETPC());
+    }
+}
+
 void HELPER(data_exception)(CPUS390XState *env, uint32_t dxc)
 {
     tcg_s390_data_exception(env, dxc, GETPC());
@@ -606,4 +635,4 @@ void helper_uc_s390x_exit(CPUS390XState *env)
     cs->exception_index = EXCP_HLT;
     cs->halted = 1;
     cpu_loop_exit(cs);
-}
\ No newline at end of file
+}
diff --git a/qemu/target/s390x/helper.c b/qemu/target/s390x/helper.c
index b23a8686a3..6192ae899b 100644
--- a/qemu/target/s390x/helper.c
+++ b/qemu/target/s390x/helper.c
@@ -331,6 +331,7 @@ const char *cc_name(enum cc_op cc_op)
         [CC_OP_SUBB_64]   = "CC_OP_SUBB_64",
         [CC_OP_ABS_64]    = "CC_OP_ABS_64",
         [CC_OP_NABS_64]   = "CC_OP_NABS_64",
+        [CC_OP_MULS_64]   = "CC_OP_MULS_64",
         [CC_OP_ADD_32]    = "CC_OP_ADD_32",
         [CC_OP_ADDU_32]   = "CC_OP_ADDU_32",
         [CC_OP_ADDC_32]   = "CC_OP_ADDC_32",
@@ -339,6 +340,7 @@ const char *cc_name(enum cc_op cc_op)
         [CC_OP_SUBB_32]   = "CC_OP_SUBB_32",
         [CC_OP_ABS_32]    = "CC_OP_ABS_32",
         [CC_OP_NABS_32]   = "CC_OP_NABS_32",
+        [CC_OP_MULS_32]   = "CC_OP_MULS_32",
         [CC_OP_COMP_32]   = "CC_OP_COMP_32",
         [CC_OP_COMP_64]   = "CC_OP_COMP_64",
         [CC_OP_TM_32]     = "CC_OP_TM_32",
diff --git a/qemu/target/s390x/helper.h b/qemu/target/s390x/helper.h
index abd8dd2a97..33dc9f3abb 100644
--- a/qemu/target/s390x/helper.h
+++ b/qemu/target/s390x/helper.h
@@ -8,6 +8,7 @@ DEF_HELPER_FLAGS_4(nc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
 DEF_HELPER_FLAGS_4(oc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
 DEF_HELPER_FLAGS_4(xc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
 DEF_HELPER_FLAGS_4(mvc, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_FLAGS_4(mvcrl, TCG_CALL_NO_WG, void, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(mvcin, TCG_CALL_NO_WG, void, env, i32, i64, i64)
 DEF_HELPER_FLAGS_4(clc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
 DEF_HELPER_3(mvcl, i32, env, i32, i32)
@@ -131,6 +132,7 @@ DEF_HELPER_FLAGS_3(probe_write_access, TCG_CALL_NO_WG, void, env, i64, i64)
 
 /* === Vector Support Instructions === */
 DEF_HELPER_FLAGS_4(vll, TCG_CALL_NO_WG, void, env, ptr, i64, i64)
+DEF_HELPER_FLAGS_4(gvec_vbperm, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vpk16, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vpk32, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vpk64, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
@@ -209,8 +211,11 @@ DEF_HELPER_FLAGS_4(gvec_verll16, TCG_CALL_NO_RWG, void, ptr, cptr, i64, i32)
 DEF_HELPER_FLAGS_4(gvec_verim8, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_verim16, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vsl, TCG_CALL_NO_RWG, void, ptr, cptr, i64, i32)
+DEF_HELPER_FLAGS_4(gvec_vsl_ve2, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vsra, TCG_CALL_NO_RWG, void, ptr, cptr, i64, i32)
+DEF_HELPER_FLAGS_4(gvec_vsra_ve2, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vsrl, TCG_CALL_NO_RWG, void, ptr, cptr, i64, i32)
+DEF_HELPER_FLAGS_4(gvec_vsrl_ve2, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vscbi8, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_FLAGS_4(gvec_vscbi16, TCG_CALL_NO_RWG, void, ptr, cptr, cptr, i32)
 DEF_HELPER_4(gvec_vtm, void, ptr, cptr, env, i32)
@@ -252,52 +257,89 @@ DEF_HELPER_6(gvec_vstrc_cc32, void, ptr, cptr, cptr, cptr, env, i32)
 DEF_HELPER_6(gvec_vstrc_cc_rt8, void, ptr, cptr, cptr, cptr, env, i32)
 DEF_HELPER_6(gvec_vstrc_cc_rt16, void, ptr, cptr, cptr, cptr, env, i32)
 DEF_HELPER_6(gvec_vstrc_cc_rt32, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_6(gvec_vstrs_8, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_6(gvec_vstrs_16, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_6(gvec_vstrs_32, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_6(gvec_vstrs_zs8, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_6(gvec_vstrs_zs16, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_6(gvec_vstrs_zs32, void, ptr, cptr, cptr, cptr, env, i32)
 
 /* === Vector Floating-Point Instructions */
+DEF_HELPER_FLAGS_5(gvec_vfa32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_5(gvec_vfa64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_5(gvec_vfa64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfa128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_4(gvec_wfc32, void, cptr, cptr, env, i32)
+DEF_HELPER_4(gvec_wfk32, void, cptr, cptr, env, i32)
 DEF_HELPER_4(gvec_wfc64, void, cptr, cptr, env, i32)
 DEF_HELPER_4(gvec_wfk64, void, cptr, cptr, env, i32)
+DEF_HELPER_4(gvec_wfc128, void, cptr, cptr, env, i32)
+DEF_HELPER_4(gvec_wfk128, void, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfce32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_5(gvec_vfce32_cc, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_5(gvec_vfce64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_5(gvec_vfce64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_5(gvec_vfce64_cc, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_5(gvec_vfce64s_cc, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfce128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_5(gvec_vfce128_cc, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfch32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_5(gvec_vfch32_cc, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_5(gvec_vfch64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_5(gvec_vfch64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_5(gvec_vfch64_cc, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_5(gvec_vfch64s_cc, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfch128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_5(gvec_vfch128_cc, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfche32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_5(gvec_vfche32_cc, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_5(gvec_vfche64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_5(gvec_vfche64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_5(gvec_vfche64_cc, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_5(gvec_vfche64s_cc, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfche128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_5(gvec_vfche128_cc, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vcdg32, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vcdlg32, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vcgd32, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vclgd32, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vcdg64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vcdg64s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vcdlg64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vcdlg64s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vcgd64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vcgd64s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vclgd64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vclgd64s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfd32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_5(gvec_vfd64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_5(gvec_vfd64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfd128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vfi32, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vfi64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vfi64s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vfi128, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vfll32, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vfll32s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vfll64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vflr64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vflr64s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vflr128, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfm32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_5(gvec_vfm64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_5(gvec_vfm64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfm128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfmax32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfmax64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfmax128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfmin32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfmin64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfmin128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfma32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_6(gvec_vfma64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_6(gvec_vfma64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfma128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfms32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_6(gvec_vfms64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_6(gvec_vfms64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfms128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfnma32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfnma64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfnma128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfnms32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfnms64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_6(gvec_vfnms128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vfsq32, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
 DEF_HELPER_FLAGS_4(gvec_vfsq64, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
-DEF_HELPER_FLAGS_4(gvec_vfsq64s, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_4(gvec_vfsq128, TCG_CALL_NO_WG, void, ptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfs32, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
 DEF_HELPER_FLAGS_5(gvec_vfs64, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
-DEF_HELPER_FLAGS_5(gvec_vfs64s, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_FLAGS_5(gvec_vfs128, TCG_CALL_NO_WG, void, ptr, cptr, cptr, env, i32)
+DEF_HELPER_4(gvec_vftci32, void, ptr, cptr, env, i32)
 DEF_HELPER_4(gvec_vftci64, void, ptr, cptr, env, i32)
-DEF_HELPER_4(gvec_vftci64s, void, ptr, cptr, env, i32)
+DEF_HELPER_4(gvec_vftci128, void, ptr, cptr, env, i32)
 
 #ifndef CONFIG_USER_ONLY
 DEF_HELPER_3(servc, i32, env, i64, i64)
@@ -316,6 +358,7 @@ DEF_HELPER_FLAGS_4(stctl, TCG_CALL_NO_WG, void, env, i32, i64, i32)
 DEF_HELPER_FLAGS_4(stctg, TCG_CALL_NO_WG, void, env, i32, i64, i32)
 DEF_HELPER_FLAGS_2(testblock, TCG_CALL_NO_WG, i32, env, i64)
 DEF_HELPER_FLAGS_3(tprot, TCG_CALL_NO_WG, i32, env, i64, i64)
+DEF_HELPER_3(monitor_call, void, env, i64, i32)
 DEF_HELPER_FLAGS_2(iske, TCG_CALL_NO_RWG_SE, i64, env, i64)
 DEF_HELPER_FLAGS_3(sske, TCG_CALL_NO_RWG, void, env, i64, i64)
 DEF_HELPER_FLAGS_2(rrbe, TCG_CALL_NO_RWG, i32, env, i64)
diff --git a/qemu/target/s390x/insn-data.def b/qemu/target/s390x/insn-data.def
index 1660c4d1f8..518e626a73 100644
--- a/qemu/target/s390x/insn-data.def
+++ b/qemu/target/s390x/insn-data.def
@@ -31,6 +31,7 @@
     C(0xb918, AGFR,    RRE,   Z,   r1, r2_32s, r1, 0, add, adds64)
     C(0xb9e8, AGRK,    RRF_a, DO,  r2, r3, r1, 0, add, adds64)
     C(0xe308, AG,      RXY_a, Z,   r1, m2_64, r1, 0, add, adds64)
+    C(0xe338, AGH,     RXY_a, MIE2,r1, m2_16s, r1, 0, add, adds64)
     C(0xe318, AGF,     RXY_a, Z,   r1, m2_32s, r1, 0, add, adds64)
     F(0xb30a, AEBR,    RRE,   Z,   e1, e2, new, e1, aeb, f32, IF_BFP)
     F(0xb31a, ADBR,    RRE,   Z,   f1, f2, new, f1, adb, f64, IF_BFP)
@@ -104,6 +105,9 @@
     D(0xa507, NILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, andi, 0, 0x1000)
     D(0x9400, NI,      SI,    Z,   la1, i2_8u, new, 0, ni, nz64, MO_UB)
     D(0xeb54, NIY,     SIY,   LD,  la1, i2_8u, new, 0, ni, nz64, MO_UB)
+/* AND WITH COMPLEMENT */
+    C(0xb9f5, NCRK,    RRF_a, MIE3, r2, r3, new, r1_32, andc, nz32)
+    C(0xb9e5, NCGRK,   RRF_a, MIE3, r2, r3, r1, 0, andc, nz64)
 
 /* BRANCH AND LINK */
     C(0x0500, BALR,    RR_a,  Z,   0, r2_nz, r1, 0, bal, 0)
@@ -117,6 +121,7 @@
 /* BRANCH ON CONDITION */
     C(0x0700, BCR,     RR_b,  Z,   0, r2_nz, 0, 0, bc, 0)
     C(0x4700, BC,      RX_b,  Z,   0, a2, 0, 0, bc, 0)
+    C(0xe347, BIC,     RXY_b, MIE2,0, m2_64w, 0, 0, bc, 0)
 /* BRANCH RELATIVE ON CONDITION */
     C(0xa704, BRC,     RI_c,  Z,   0, 0, 0, 0, bc, 0)
     C(0xc004, BRCL,    RIL_c, Z,   0, 0, 0, 0, bc, 0)
@@ -193,8 +198,8 @@
     C(0xe55c, CHSI,    SIL,   GIE, m1_32s, i2, 0, 0, 0, cmps64)
     C(0xe558, CGHSI,   SIL,   GIE, m1_64, i2, 0, 0, 0, cmps64)
 /* COMPARE HALFWORD RELATIVE LONG */
-    C(0xc605, CHRL,    RIL_b, GIE, r1_o, mri2_32s, 0, 0, 0, cmps32)
-    C(0xc604, CGHRL,   RIL_b, GIE, r1_o, mri2_64, 0, 0, 0, cmps64)
+    C(0xc605, CHRL,    RIL_b, GIE, r1_o, mri2_16s, 0, 0, 0, cmps32)
+    C(0xc604, CGHRL,   RIL_b, GIE, r1_o, mri2_16s, 0, 0, 0, cmps64)
 /* COMPARE HIGH */
     C(0xb9cd, CHHR,    RRE,   HW,  r1_sr32, r2_sr32, 0, 0, 0, cmps32)
     C(0xb9dd, CHLR,    RRE,   HW,  r1_sr32, r2_o, 0, 0, 0, cmps32)
@@ -284,8 +289,8 @@
     D(0xb961, CLGRT,   RRF_c, GIE, r1_o, r2_o, 0, 0, ct, 0, 1)
     D(0xeb23, CLT,     RSY_b, MIE, r1_32u, m2_32u, 0, 0, ct, 0, 1)
     D(0xeb2b, CLGT,    RSY_b, MIE, r1_o, m2_64, 0, 0, ct, 0, 1)
-    D(0xec73, CLFIT,   RIE_a, GIE, r1_32u, i2_32u, 0, 0, ct, 0, 1)
-    D(0xec71, CLGIT,   RIE_a, GIE, r1_o, i2_32u, 0, 0, ct, 0, 1)
+    D(0xec73, CLFIT,   RIE_a, GIE, r1_32u, i2_16u, 0, 0, ct, 0, 1)
+    D(0xec71, CLGIT,   RIE_a, GIE, r1_o, i2_16u, 0, 0, ct, 0, 1)
 
 /* CONVERT TO DECIMAL */
     C(0x4e00, CVD,     RX_a,  Z,   r1_o, a2, 0, 0, cvd, 0)
@@ -460,7 +465,7 @@
     C(0xe39f, LAT,     RXY_a, LAT, 0, m2_32u, r1, 0, lat, 0)
     C(0xe385, LGAT,    RXY_a, LAT, 0, a2, r1, 0, lgat, 0)
 /* LOAD AND ZERO RIGHTMOST BYTE */
-    C(0xe3eb, LZRF,    RXY_a, LZRB, 0, m2_32u, new, r1_32, lzrb, 0)
+    C(0xe33b, LZRF,    RXY_a, LZRB, 0, m2_32u, new, r1_32, lzrb, 0)
     C(0xe32a, LZRG,    RXY_a, LZRB, 0, m2_64, r1, 0, lzrb, 0)
 /* LOAD LOGICAL AND ZERO RIGHTMOST BYTE */
     C(0xe33a, LLZRGF,  RXY_a, LZRB, 0, m2_32u, r1, 0, lzrb, 0)
@@ -558,7 +563,7 @@
     C(0xec46, LOCGHI,  RIE_g, LOC2, r1, i2, r1, 0, loc, 0)
     C(0xec4e, LOCHHI,  RIE_g, LOC2, r1_sr32, i2, new, r1_32h, loc, 0)
 /* LOAD HIGH ON CONDITION */
-    C(0xb9e0, LOCFHR,  RRF_c, LOC2, r1_sr32, r2, new, r1_32h, loc, 0)
+    C(0xb9e0, LOCFHR,  RRF_c, LOC2, r1_sr32, r2_sr32, new, r1_32h, loc, 0)
     C(0xebe0, LOCFH,   RSY_b, LOC2, r1_sr32, m2_32u, new, r1_32h, loc, 0)
 /* LOAD PAIR DISJOINT */
     D(0xc804, LPD,     SSF,   ILA, 0, 0, new_P, r3_P32, lpd, 0, MO_TEUL)
@@ -617,8 +622,11 @@
     C(0x9a00, LAM,     RS_a,  Z,   0, a2, 0, 0, lam, 0)
     C(0xeb9a, LAMY,    RSY_a, LD,  0, a2, 0, 0, lam, 0)
 
+/* MONITOR CALL */
+    C(0xaf00, MC,      SI,    Z,   la1, 0, 0, 0, mc, 0)
 /* MOVE */
     C(0xd200, MVC,     SS_a,  Z,   la1, a2, 0, 0, mvc, 0)
+    C(0xe50a, MVCRL,   SSE,  MIE3, la1, a2, 0, 0, mvcrl, 0)
     C(0xe544, MVHHI,   SIL,   GIE, la1, i2, 0, m1_16, mov2, 0)
     C(0xe54c, MVHI,    SIL,   GIE, la1, i2, 0, m1_32, mov2, 0)
     C(0xe548, MVGHI,   SIL,   GIE, la1, i2, 0, m1_64, mov2, 0)
@@ -649,8 +657,10 @@
 
 /* MULTIPLY */
     C(0x1c00, MR,      RR_a,  Z,   r1p1_32s, r2_32s, new, r1_D32, mul, 0)
+    C(0xb9ec, MGRK,    RRF_a, MIE2,r3_o, r2_o, r1_P, 0, muls128, 0)
     C(0x5c00, M,       RX_a,  Z,   r1p1_32s, m2_32s, new, r1_D32, mul, 0)
     C(0xe35c, MFY,     RXY_a, GIE, r1p1_32s, m2_32s, new, r1_D32, mul, 0)
+    C(0xe384, MG,      RXY_a, MIE2,r1p1_o, m2_64, r1_P, 0, muls128, 0)
     F(0xb317, MEEBR,   RRE,   Z,   e1, e2, new, e1, meeb, 0, IF_BFP)
     F(0xb31c, MDBR,    RRE,   Z,   f1, f2, new, f1, mdb, 0, IF_BFP)
     F(0xb34c, MXBR,    RRE,   Z,   x2h, x2l, x1, x1, mxb, 0, IF_BFP)
@@ -663,6 +673,7 @@
 /* MULTIPLY HALFWORD */
     C(0x4c00, MH,      RX_a,  Z,   r1_o, m2_16s, new, r1_32, mul, 0)
     C(0xe37c, MHY,     RXY_a, GIE, r1_o, m2_16s, new, r1_32, mul, 0)
+    C(0xe33c, MGH,     RXY_a, MIE2,r1_o, m2_16s, r1, 0, mul, 0)
 /* MULTIPLY HALFWORD IMMEDIATE */
     C(0xa70c, MHI,     RI_a,  Z,   r1_o, i2, new, r1_32, mul, 0)
     C(0xa70d, MGHI,    RI_a,  Z,   r1_o, i2, r1, 0, mul, 0)
@@ -673,11 +684,15 @@
     C(0xe386, MLG,     RXY_a, Z,   r1p1, m2_64, r1_P, 0, mul128, 0)
 /* MULTIPLY SINGLE */
     C(0xb252, MSR,     RRE,   Z,   r1_o, r2_o, new, r1_32, mul, 0)
+    C(0xb9fd, MSRKC,   RRF_a, MIE2,r3_32s, r2_32s, new, r1_32, mul, muls32)
     C(0x7100, MS,      RX_a,  Z,   r1_o, m2_32s, new, r1_32, mul, 0)
     C(0xe351, MSY,     RXY_a, LD,  r1_o, m2_32s, new, r1_32, mul, 0)
+    C(0xe353, MSC,     RXY_a, MIE2,r1_32s, m2_32s, new, r1_32, mul, muls32)
     C(0xb90c, MSGR,    RRE,   Z,   r1_o, r2_o, r1, 0, mul, 0)
+    C(0xb9ed, MSGRKC,  RRF_a, MIE2,r3_o, r2_o, new_P, out2_r1, muls128, muls64)
     C(0xb91c, MSGFR,   RRE,   Z,   r1_o, r2_32s, r1, 0, mul, 0)
     C(0xe30c, MSG,     RXY_a, Z,   r1_o, m2_64, r1, 0, mul, 0)
+    C(0xe383, MSGC,    RXY_a, MIE2,r1_o, m2_64, new_P, out2_r1, muls128, muls64)
     C(0xe31c, MSGF,    RXY_a, Z,   r1_o, m2_32s, r1, 0, mul, 0)
 /* MULTIPLY SINGLE IMMEDIATE */
     C(0xc201, MSFI,    RIL_a, GIE, r1_o, i2, new, r1_32, mul, 0)
@@ -694,6 +709,16 @@
     F(0xed0f, MSEB,    RXF,   Z,   e1, m2_32u, new, e1, mseb, 0, IF_BFP)
     F(0xed1f, MSDB,    RXF,   Z,   f1, m2_64, new, f1, msdb, 0, IF_BFP)
 
+/* NAND */
+    C(0xb974, NNRK,    RRF_a, MIE3, r2, r3, new, r1_32, nand, nz32)
+    C(0xb964, NNGRK,   RRF_a, MIE3, r2, r3, r1, 0, nand, nz64)
+/* NOR */
+    C(0xb976, NORK,    RRF_a, MIE3, r2, r3, new, r1_32, nor, nz32)
+    C(0xb966, NOGRK,   RRF_a, MIE3, r2, r3, r1, 0, nor, nz64)
+/* NOT EXCLUSIVE OR */
+    C(0xb977, NXRK,    RRF_a, MIE3, r2, r3, new, r1_32, nxor, nz32)
+    C(0xb967, NXGRK,   RRF_a, MIE3, r2, r3, r1, 0, nxor, nz64)
+
 /* OR */
     C(0x1600, OR,      RR_a,  Z,   r1, r2, new, r1_32, or, nz32)
     C(0xb9f6, ORK,     RRF_a, DO,  r2, r3, new, r1_32, or, nz32)
@@ -712,6 +737,9 @@
     D(0xa50b, OILL,    RI_a,  Z,   r1_o, i2_16u, r1, 0, ori, 0, 0x1000)
     D(0x9600, OI,      SI,    Z,   la1, i2_8u, new, 0, oi, nz64, MO_UB)
     D(0xeb56, OIY,     SIY,   LD,  la1, i2_8u, new, 0, oi, nz64, MO_UB)
+/* OR WITH COMPLEMENT */
+    C(0xb975, OCRK,    RRF_a, MIE3, r2, r3, new, r1_32, orc, nz32)
+    C(0xb965, OCGRK,   RRF_a, MIE3, r2, r3, r1, 0, orc, nz64)
 
 /* PACK */
     /* Really format SS_b, but we pack both lengths into one argument
@@ -752,6 +780,12 @@
 /* SEARCH STRING UNICODE */
     C(0xb9be, SRSTU,   RRE,   ETF3, 0, 0, 0, 0, srstu, 0)
 
+/* SELECT */
+    C(0xb9f0, SELR,    RRF_a, MIE3, r3, r2, new, r1_32, loc, 0)
+    C(0xb9e3, SELGR,   RRF_a, MIE3, r3, r2, r1, 0, loc, 0)
+/* SELECT HIGH */
+    C(0xb9c0, SELFHR,  RRF_a, MIE3, r3_sr32, r2_sr32, new, r1_32h, loc, 0)
+
 /* SET ACCESS */
     C(0xb24e, SAR,     RRE,   Z,   0, r2_o, 0, 0, sar, 0)
 /* SET ADDRESSING MODE */
@@ -882,6 +916,7 @@
 /* SUBTRACT HALFWORD */
     C(0x4b00, SH,      RX_a,  Z,   r1, m2_16s, new, r1_32, sub, subs32)
     C(0xe37b, SHY,     RXY_a, LD,  r1, m2_16s, new, r1_32, sub, subs32)
+    C(0xe339, SGH,     RXY_a, MIE2,r1, m2_16s, r1, 0, sub, subs64)
 /* SUBTRACT HIGH */
     C(0xb9c9, SHHHR,   RRF_a, HW,  r2_sr32, r3_sr32, new, r1_32h, sub, subs32)
     C(0xb9d9, SHHLR,   RRF_a, HW,  r2_sr32, r3, new, r1_32h, sub, subs32)
@@ -968,12 +1003,15 @@
     D(0xb92d, KMCTR,   RRF_b, MSA4, 0, 0, 0, 0, msa, 0, S390_FEAT_TYPE_KMCTR)
     D(0xb92e, KM,      RRE,   MSA,  0, 0, 0, 0, msa, 0, S390_FEAT_TYPE_KM)
     D(0xb92f, KMC,     RRE,   MSA,  0, 0, 0, 0, msa, 0, S390_FEAT_TYPE_KMC)
+    D(0xb929, KMA,     RRF_b, MSA8, 0, 0, 0, 0, msa, 0, S390_FEAT_TYPE_KMA)
     D(0xb93c, PPNO,    RRE,   MSA5, 0, 0, 0, 0, msa, 0, S390_FEAT_TYPE_PPNO)
     D(0xb93e, KIMD,    RRE,   MSA,  0, 0, 0, 0, msa, 0, S390_FEAT_TYPE_KIMD)
     D(0xb93f, KLMD,    RRE,   MSA,  0, 0, 0, 0, msa, 0, S390_FEAT_TYPE_KLMD)
 
 /* === Vector Support Instructions === */
 
+/* VECTOR BIT PERMUTE */
+    E(0xe785, VBPERM,  VRR_c, VE,  0, 0, 0, 0, vbperm, 0, 0, IF_VEC)
 /* VECTOR GATHER ELEMENT */
     E(0xe713, VGEF,    VRV,   V,   la2, 0, 0, 0, vge, 0, ES_32, IF_VEC)
     E(0xe712, VGEG,    VRV,   V,   la2, 0, 0, 0, vge, 0, ES_64, IF_VEC)
@@ -986,6 +1024,16 @@
     F(0xe756, VLR,     VRR_a, V,   0, 0, 0, 0, vlr, 0, IF_VEC)
 /* VECTOR LOAD AND REPLICATE */
     F(0xe705, VLREP,   VRX,   V,   la2, 0, 0, 0, vlrep, 0, IF_VEC)
+/* VECTOR LOAD BYTE REVERSED ELEMENT */
+    E(0xe601, VLEBRH,  VRX,   VE2, la2, 0, 0, 0, vlebr, 0, ES_16, IF_VEC)
+    E(0xe603, VLEBRF,  VRX,   VE2, la2, 0, 0, 0, vlebr, 0, ES_32, IF_VEC)
+    E(0xe602, VLEBRG,  VRX,   VE2, la2, 0, 0, 0, vlebr, 0, ES_64, IF_VEC)
+/* VECTOR LOAD BYTE REVERSED ELEMENT AND REPLICATE */
+    F(0xe605, VLBRREP, VRX,   VE2, la2, 0, 0, 0, vlbrrep, 0, IF_VEC)
+/* VECTOR LOAD BYTE REVERSED ELEMENT AND ZERO */
+    F(0xe604, VLLEBRZ, VRX,   VE2, la2, 0, 0, 0, vllebrz, 0, IF_VEC)
+/* VECTOR LOAD BYTE REVERSED ELEMENTS */
+    F(0xe606, VLBR,    VRX,   VE2, la2, 0, 0, 0, vlbr, 0, IF_VEC)
 /* VECTOR LOAD ELEMENT */
     E(0xe700, VLEB,    VRX,   V,   la2, 0, 0, 0, vle, 0, ES_8, IF_VEC)
     E(0xe701, VLEH,    VRX,   V,   la2, 0, 0, 0, vle, 0, ES_16, IF_VEC)
@@ -996,6 +1044,8 @@
     E(0xe741, VLEIH,   VRI_a, V,   0, 0, 0, 0, vlei, 0, ES_16, IF_VEC)
     E(0xe743, VLEIF,   VRI_a, V,   0, 0, 0, 0, vlei, 0, ES_32, IF_VEC)
     E(0xe742, VLEIG,   VRI_a, V,   0, 0, 0, 0, vlei, 0, ES_64, IF_VEC)
+/* VECTOR LOAD ELEMENTS REVERSED */
+    F(0xe607, VLER,    VRX,   VE2, la2, 0, 0, 0, vler, 0, IF_VEC)
 /* VECTOR LOAD GR FROM VR ELEMENT */
     F(0xe721, VLGV,    VRS_c, V,   la2, 0, r1, 0, vlgv, 0, IF_VEC)
 /* VECTOR LOAD LOGICAL ELEMENT AND ZERO */
@@ -1036,11 +1086,19 @@
     F(0xe75f, VSEG,    VRR_a, V,   0, 0, 0, 0, vseg, 0, IF_VEC)
 /* VECTOR STORE */
     F(0xe70e, VST,     VRX,   V,   la2, 0, 0, 0, vst, 0, IF_VEC)
+/* VECTOR STORE BYTE REVERSED ELEMENT */
+    E(0xe609, VSTEBRH, VRX,   VE2, la2, 0, 0, 0, vstebr, 0, ES_16, IF_VEC)
+    E(0xe60b, VSTEBRF, VRX,   VE2, la2, 0, 0, 0, vstebr, 0, ES_32, IF_VEC)
+    E(0xe60a, VSTEBRG, VRX,   VE2, la2, 0, 0, 0, vstebr, 0, ES_64, IF_VEC)
+/* VECTOR STORE BYTE REVERSED ELEMENTS */
+    F(0xe60e, VSTBR,   VRX,   VE2, la2, 0, 0, 0, vstbr, 0, IF_VEC)
 /* VECTOR STORE ELEMENT */
     E(0xe708, VSTEB,   VRX,   V,   la2, 0, 0, 0, vste, 0, ES_8, IF_VEC)
     E(0xe709, VSTEH,   VRX,   V,   la2, 0, 0, 0, vste, 0, ES_16, IF_VEC)
     E(0xe70b, VSTEF,   VRX,   V,   la2, 0, 0, 0, vste, 0, ES_32, IF_VEC)
     E(0xe70a, VSTEG,   VRX,   V,   la2, 0, 0, 0, vste, 0, ES_64, IF_VEC)
+/* VECTOR STORE ELEMENTS REVERSED */
+    F(0xe60f, VSTER,   VRX,   VE2, la2, 0, 0, 0, vster, 0, IF_VEC)
 /* VECTOR STORE MULTIPLE */
     F(0xe73e, VSTM,    VRS_a, V,   la2, 0, 0, 0, vstm, 0, IF_VEC)
 /* VECTOR STORE WITH LENGTH */
@@ -1134,6 +1192,8 @@
     F(0xe7a7, VMO,     VRR_c, V,   0, 0, 0, 0, vm, 0, IF_VEC)
 /* VECTOR MULTIPLY LOGICAL ODD */
     F(0xe7a5, VMLO,    VRR_c, V,   0, 0, 0, 0, vm, 0, IF_VEC)
+/* VECTOR MULTIPLY SUM LOGICAL */
+    F(0xe7b8, VMSL,    VRR_d, VE,  0, 0, 0, 0, vmsl, 0, IF_VEC)
 /* VECTOR NAND */
     F(0xe76e, VNN,     VRR_c, VE,  0, 0, 0, 0, vnn, 0, IF_VEC)
 /* VECTOR NOR */
@@ -1161,19 +1221,23 @@
     F(0xe778, VESRLV,  VRR_c, V,   0, 0, 0, 0, vesv, 0, IF_VEC)
     F(0xe738, VESRL,   VRS_a, V,   la2, 0, 0, 0, ves, 0, IF_VEC)
 /* VECTOR SHIFT LEFT */
-    F(0xe774, VSL,     VRR_c, V,   0, 0, 0, 0, vsl, 0, IF_VEC)
+    E(0xe774, VSL,     VRR_c, V,   0, 0, 0, 0, vsl, 0, 0, IF_VEC)
 /* VECTOR SHIFT LEFT BY BYTE */
-    F(0xe775, VSLB,    VRR_c, V,   0, 0, 0, 0, vsl, 0, IF_VEC)
+    E(0xe775, VSLB,    VRR_c, V,   0, 0, 0, 0, vsl, 0, 1, IF_VEC)
+/* VECTOR SHIFT LEFT DOUBLE BY BIT */
+    E(0xe786, VSLD,    VRI_d, VE2, 0, 0, 0, 0, vsld, 0, 0, IF_VEC)
 /* VECTOR SHIFT LEFT DOUBLE BY BYTE */
-    F(0xe777, VSLDB,   VRI_d, V,   0, 0, 0, 0, vsldb, 0, IF_VEC)
+    E(0xe777, VSLDB,   VRI_d, V,   0, 0, 0, 0, vsld, 0, 1, IF_VEC)
 /* VECTOR SHIFT RIGHT ARITHMETIC */
-    F(0xe77e, VSRA,    VRR_c, V,   0, 0, 0, 0, vsra, 0, IF_VEC)
+    E(0xe77e, VSRA,    VRR_c, V,   0, 0, 0, 0, vsra, 0, 0, IF_VEC)
 /* VECTOR SHIFT RIGHT ARITHMETIC BY BYTE */
-    F(0xe77f, VSRAB,   VRR_c, V,   0, 0, 0, 0, vsra, 0, IF_VEC)
+    E(0xe77f, VSRAB,   VRR_c, V,   0, 0, 0, 0, vsra, 0, 1, IF_VEC)
+/* VECTOR SHIFT RIGHT DOUBLE BY BIT */
+    F(0xe787, VSRD,    VRI_d, VE2, 0, 0, 0, 0, vsrd, 0, IF_VEC)
 /* VECTOR SHIFT RIGHT LOGICAL */
-    F(0xe77c, VSRL,    VRR_c, V,   0, 0, 0, 0, vsrl, 0, IF_VEC)
+    E(0xe77c, VSRL,    VRR_c, V,   0, 0, 0, 0, vsrl, 0, 0, IF_VEC)
 /* VECTOR SHIFT RIGHT LOGICAL BY BYTE */
-    F(0xe77d, VSRLB,   VRR_c, V,   0, 0, 0, 0, vsrl, 0, IF_VEC)
+    E(0xe77d, VSRLB,   VRR_c, V,   0, 0, 0, 0, vsrl, 0, 1, IF_VEC)
 /* VECTOR SUBTRACT */
     F(0xe7f7, VS,      VRR_c, V,   0, 0, 0, 0, vs, 0, IF_VEC)
 /* VECTOR SUBTRACT COMPUTE BORROW INDICATION */
@@ -1203,6 +1267,8 @@
     F(0xe75c, VISTR,   VRR_a, V,   0, 0, 0, 0, vistr, 0, IF_VEC)
 /* VECTOR STRING RANGE COMPARE */
     F(0xe78a, VSTRC,   VRR_d, V,   0, 0, 0, 0, vstrc, 0, IF_VEC)
+/* VECTOR STRING SEARCH */
+    F(0xe78b, VSTRS,   VRR_d, VE2, 0, 0, 0, 0, vstrs, 0, IF_VEC)
 
 /* === Vector Floating-Point Instructions */
 
@@ -1234,12 +1300,20 @@
     F(0xe7c4, VFLL,    VRR_a, V,   0, 0, 0, 0, vfll, 0, IF_VEC)
 /* VECTOR LOAD ROUNDED */
     F(0xe7c5, VFLR,    VRR_a, V,   0, 0, 0, 0, vcdg, 0, IF_VEC)
+/* VECTOR FP MAXIMUM */
+    F(0xe7ef, VFMAX,   VRR_c, VE,  0, 0, 0, 0, vfmax, 0, IF_VEC)
+/* VECTOR FP MINIMUM */
+    F(0xe7ee, VFMIN,   VRR_c, VE,  0, 0, 0, 0, vfmax, 0, IF_VEC)
 /* VECTOR FP MULTIPLY */
     F(0xe7e7, VFM,     VRR_c, V,   0, 0, 0, 0, vfa, 0, IF_VEC)
 /* VECTOR FP MULTIPLY AND ADD */
     F(0xe78f, VFMA,    VRR_e, V,   0, 0, 0, 0, vfma, 0, IF_VEC)
 /* VECTOR FP MULTIPLY AND SUBTRACT */
     F(0xe78e, VFMS,    VRR_e, V,   0, 0, 0, 0, vfma, 0, IF_VEC)
+/* VECTOR FP NEGATIVE MULTIPLY AND ADD */
+    F(0xe79f, VFNMA,   VRR_e, VE,  0, 0, 0, 0, vfma, 0, IF_VEC)
+/* VECTOR FP NEGATIVE MULTIPLY AND SUBTRACT */
+    F(0xe79e, VFNMS,   VRR_e, VE,  0, 0, 0, 0, vfma, 0, IF_VEC)
 /* VECTOR FP PERFORM SIGN OPERATION */
     F(0xe7cc, VFPSO,   VRR_a, V,   0, 0, 0, 0, vfpso, 0, IF_VEC)
 /* VECTOR FP SQUARE ROOT */
diff --git a/qemu/target/s390x/internal.h b/qemu/target/s390x/internal.h
index 82cf8726be..4b9510c224 100644
--- a/qemu/target/s390x/internal.h
+++ b/qemu/target/s390x/internal.h
@@ -175,6 +175,7 @@ enum cc_op {
     CC_OP_SUBB_64,              /* overflow on unsigned sub-borrow (64bit) */
     CC_OP_ABS_64,               /* sign eval on abs (64bit) */
     CC_OP_NABS_64,              /* sign eval on nabs (64bit) */
+    CC_OP_MULS_64,              /* overflow on signed multiply (64bit) */
 
     CC_OP_ADD_32,               /* overflow on add (32bit) */
     CC_OP_ADDU_32,              /* overflow on unsigned add (32bit) */
@@ -184,6 +185,7 @@ enum cc_op {
     CC_OP_SUBB_32,              /* overflow on unsigned sub-borrow (32bit) */
     CC_OP_ABS_32,               /* sign eval on abs (64bit) */
     CC_OP_NABS_32,              /* sign eval on nabs (64bit) */
+    CC_OP_MULS_32,              /* overflow on signed multiply (32bit) */
 
     CC_OP_COMP_32,              /* complement */
     CC_OP_COMP_64,              /* complement */
@@ -269,6 +271,14 @@ uint8_t s390_softfloat_exc_to_ieee(unsigned int exc);
 int s390_swap_bfp_rounding_mode(CPUS390XState *env, int m3);
 void s390_restore_bfp_rounding_mode(CPUS390XState *env, int old_mode);
 int float_comp_to_cc(CPUS390XState *env, int float_compare);
+#define DCMASK_ZERO             0x0c00
+#define DCMASK_NORMAL           0x0300
+#define DCMASK_SUBNORMAL        0x00c0
+#define DCMASK_INFINITY         0x0030
+#define DCMASK_QUIET_NAN        0x000c
+#define DCMASK_SIGNALING_NAN    0x0003
+#define DCMASK_NAN              0x000f
+#define DCMASK_NEGATIVE         0x0555
 uint16_t float32_dcmask(CPUS390XState *env, float32 f1);
 uint16_t float64_dcmask(CPUS390XState *env, float64 f1);
 uint16_t float128_dcmask(CPUS390XState *env, float128 f1);
diff --git a/qemu/target/s390x/mem_helper.c b/qemu/target/s390x/mem_helper.c
index 88e07c5756..8ba5b9c2c4 100644
--- a/qemu/target/s390x/mem_helper.c
+++ b/qemu/target/s390x/mem_helper.c
@@ -463,6 +463,28 @@ void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
     do_helper_mvc(env, l, dest, src, GETPC());
 }
 
+/* move right to left */
+void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest,
+                   uint64_t src)
+{
+    const int mmu_idx = cpu_mmu_index(env, false);
+    uintptr_t ra = GETPC();
+    S390Access srca, desta;
+    int32_t i;
+
+    /* MVCRL always copies one more byte than specified - maximum is 256 */
+    l++;
+
+    srca = access_prepare(env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
+    desta = access_prepare(env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
+
+    for (i = l - 1; i >= 0; i--) {
+        uint8_t byte = access_get_byte(env, &srca, i, ra);
+
+        access_set_byte(env, &desta, i, byte, ra);
+    }
+}
+
 /* move inverse  */
 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
 {
@@ -2434,7 +2456,8 @@ void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr)
         insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32;
         break;
     case 6:
-        insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16;
+        insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32;
+        insn |= (uint64_t)cpu_lduw_code(env, addr + 4) << 16;
         break;
     default:
         g_assert_not_reached();
diff --git a/qemu/target/s390x/translate.c b/qemu/target/s390x/translate.c
index e41a3b73b0..7f08bf4bcb 100644
--- a/qemu/target/s390x/translate.c
+++ b/qemu/target/s390x/translate.c
@@ -389,7 +389,7 @@ static inline uint64_t ld_code2(CPUS390XState *env, uint64_t pc)
 
 static inline uint64_t ld_code4(CPUS390XState *env, uint64_t pc)
 {
-    return (uint64_t)(uint32_t)cpu_ldl_code(env, pc);
+    return (ld_code2(env, pc) << 16) | ld_code2(env, pc + 2);
 }
 
 static int get_mem_index(DisasContext *s)
@@ -650,6 +650,7 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_NZ_F64:
     case CC_OP_FLOGR:
     case CC_OP_LCBB:
+    case CC_OP_MULS_32:
         /* 1 argument */
         gen_helper_calc_cc(tcg_ctx, tcg_ctx->cc_op, tcg_ctx->cpu_env, local_cc_op, dummy, tcg_ctx->cc_dst, dummy);
         break;
@@ -663,6 +664,7 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_SLA:
     case CC_OP_NZ_F128:
     case CC_OP_VC:
+    case CC_OP_MULS_64:
         /* 2 arguments */
         gen_helper_calc_cc(tcg_ctx, tcg_ctx->cc_op, tcg_ctx->cpu_env, local_cc_op, tcg_ctx->cc_src, tcg_ctx->cc_dst, dummy);
         break;
@@ -1516,6 +1518,13 @@ static DisasJumpType op_and(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_andc(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_andc_i64(tcg_ctx, o->out, o->in1, o->in2);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_andi(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -1534,6 +1543,27 @@ static DisasJumpType op_andi(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_nand(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_nand_i64(tcg_ctx, o->out, o->in1, o->in2);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_nor(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_nor_i64(tcg_ctx, o->out, o->in1, o->in2);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_nxor(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_eqv_i64(tcg_ctx, o->out, o->in1, o->in2);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_ni(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -2783,6 +2813,12 @@ static DisasJumpType op_msa(DisasContext *s, DisasOps *o)
     TCGv_i32 t_r1, t_r2, t_r3, type;
 
     switch (s->insn->data) {
+    case S390_FEAT_TYPE_KMA:
+        if (r3 == r1 || r3 == r2) {
+            gen_program_exception(s, PGM_SPECIFICATION);
+            return DISAS_NORETURN;
+        }
+        /* FALL THROUGH */
     case S390_FEAT_TYPE_KMCTR:
         if (r3 & 1 || !r3) {
             gen_program_exception(s, PGM_SPECIFICATION);
@@ -2811,8 +2847,7 @@ static DisasJumpType op_msa(DisasContext *s, DisasOps *o)
     case S390_FEAT_TYPE_PCC:
         break;
     default:
-        // g_assert_not_reached();
-        break;
+        g_assert_not_reached();
     };
 
     t_r1 = tcg_const_i32(tcg_ctx, r1);
@@ -3090,7 +3125,11 @@ static DisasJumpType op_loc(DisasContext *s, DisasOps *o)
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     DisasCompare c;
 
-    disas_jcc(s, &c, get_field(s, m3));
+    if (have_field(s, m3)) {
+        disas_jcc(s, &c, get_field(s, m3));
+    } else {
+        disas_jcc(s, &c, get_field(s, m4));
+    }
 
     if (c.is_64) {
         tcg_gen_movcond_i64(tcg_ctx, c.cond, o->out, c.u.s64.a, c.u.s64.b,
@@ -3417,6 +3456,27 @@ static DisasJumpType op_lcbb(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_mc(DisasContext *s, DisasOps *o)
+{
+#if !defined(CONFIG_USER_ONLY)
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 i2;
+#endif
+    const uint8_t monitor_class = get_field(s, i2);
+
+    if (monitor_class & 0xf0) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+#if !defined(CONFIG_USER_ONLY)
+    i2 = tcg_const_i32(tcg_ctx, monitor_class);
+    gen_helper_monitor_call(tcg_ctx, tcg_ctx->cpu_env, o->addr1, i2);
+    tcg_temp_free_i32(tcg_ctx, i2);
+#endif
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_mov2(DisasContext *s, DisasOps *o)
 {
     o->out = o->in2;
@@ -3483,6 +3543,15 @@ static DisasJumpType op_mvc(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_mvcrl(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    gen_helper_mvcrl(tcg_ctx, tcg_ctx->cpu_env, tcg_ctx->regs[0],
+                     o->addr1, o->in2);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_mvcin(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -3647,6 +3716,13 @@ static DisasJumpType op_mul128(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_muls128(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_muls2_i64(tcg_ctx, o->out2, o->out, o->in1, o->in2);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_meeb(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -3811,6 +3887,13 @@ static DisasJumpType op_or(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_orc(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    tcg_gen_orc_i64(tcg_ctx, o->out, o->in1, o->in2);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_ori(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -5545,6 +5628,16 @@ static void cout_tm64(DisasContext *s, DisasOps *o)
     gen_op_update2_cc_i64(s, CC_OP_TM_64, o->in1, o->in2);
 }
 
+static void cout_muls32(DisasContext *s, DisasOps *o)
+{
+    gen_op_update1_cc_i64(s, CC_OP_MULS_32, o->out);
+}
+
+static void cout_muls64(DisasContext *s, DisasOps *o)
+{
+    gen_op_update2_cc_i64(s, CC_OP_MULS_64, o->out, o->out2);
+}
+
 /* ====================================================================== */
 /* The "PREParation" generators.  These initialize the DisasOps.OUT fields
    with the TCG register to which we will write.  Used in combination with
@@ -5606,6 +5699,13 @@ static void wout_r1(DisasContext *s, DisasOps *o)
 }
 #define SPEC_wout_r1 0
 
+static void wout_out2_r1(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    store_reg(tcg_ctx, get_field(s, r1), o->out2);
+}
+#define SPEC_wout_out2_r1 0
+
 static void wout_r1_8(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -5835,6 +5935,14 @@ static void in1_r1p1(DisasContext *s, DisasOps *o)
 }
 #define SPEC_in1_r1p1 SPEC_r1_even
 
+static void in1_r1p1_o(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    o->in1 = tcg_ctx->regs[get_field(s, r1) + 1];
+    o->g_in1 = true;
+}
+#define SPEC_in1_r1p1_o SPEC_r1_even
+
 static void in1_r1p1_32s(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -5882,6 +5990,14 @@ static void in1_r3(DisasContext *s, DisasOps *o)
 }
 #define SPEC_in1_r3 0
 
+static void in1_r3_sr32(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    o->in1 = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_shri_i64(tcg_ctx, o->in1, tcg_ctx->regs[get_field(s, r3)], 32);
+}
+#define SPEC_in1_r3_sr32 0
+
 static void in1_r3_o(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -6264,6 +6380,15 @@ static void in2_m2_64(DisasContext *s, DisasOps *o)
 }
 #define SPEC_in2_m2_64 0
 
+static void in2_m2_64w(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    in2_a2(s, o);
+    tcg_gen_qemu_ld64(tcg_ctx, o->in2, o->in2, get_mem_index(s));
+    gen_addi_and_wrap_i64(s, o->in2, o->in2, 0);
+}
+#define SPEC_in2_m2_64w 0
+
 static void in2_m2_64a(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -6280,6 +6405,14 @@ static void in2_mri2_16u(DisasContext *s, DisasOps *o)
 }
 #define SPEC_in2_mri2_16u 0
 
+static void in2_mri2_16s(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    in2_ri2(s, o);
+    tcg_gen_qemu_ld16s(tcg_ctx, o->in2, o->in2, get_mem_index(s));
+}
+#define SPEC_in2_mri2_16s 0
+
 static void in2_mri2_32s(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -6423,6 +6556,8 @@ enum DisasInsnEnum {
 #define FAC_HW          S390_FEAT_STFLE_45 /* high-word */
 #define FAC_IEEEE_SIM   S390_FEAT_FLOATING_POINT_SUPPPORT_ENH /* IEEE-exception-simulation */
 #define FAC_MIE         S390_FEAT_STFLE_49 /* misc-instruction-extensions */
+#define FAC_MIE2        S390_FEAT_MISC_INSTRUCTION_EXT /* miscellaneous-instruction-extensions facility 2 */
+#define FAC_MIE3        S390_FEAT_MISC_INSTRUCTION_EXT3 /* miscellaneous-instruction-extensions facility 3 */
 #define FAC_LAT         S390_FEAT_STFLE_49 /* load-and-trap */
 #define FAC_LOC         S390_FEAT_STFLE_45 /* load/store on condition 1 */
 #define FAC_LOC2        S390_FEAT_STFLE_53 /* load/store on condition 2 */
@@ -6443,11 +6578,13 @@ enum DisasInsnEnum {
 #define FAC_MSA3        S390_FEAT_MSA_EXT_3 /* msa-extension-3 facility */
 #define FAC_MSA4        S390_FEAT_MSA_EXT_4 /* msa-extension-4 facility */
 #define FAC_MSA5        S390_FEAT_MSA_EXT_5 /* msa-extension-5 facility */
+#define FAC_MSA8        S390_FEAT_MSA_EXT_8 /* msa-extension-8 facility */
 #define FAC_ECT         S390_FEAT_EXTRACT_CPU_TIME
 #define FAC_PCI         S390_FEAT_ZPCI /* z/PCI facility */
 #define FAC_AIS         S390_FEAT_ADAPTER_INT_SUPPRESSION
 #define FAC_V           S390_FEAT_VECTOR /* vector facility */
 #define FAC_VE          S390_FEAT_VECTOR_ENH /* vector enhancements facility 1 */
+#define FAC_VE2         S390_FEAT_VECTOR_ENH2 /* vector enhancements facility 2 */
 
 static const DisasInsn insn_info[] = {
 #include "insn-data.def"
diff --git a/qemu/target/s390x/translate_vx.inc.c b/qemu/target/s390x/translate_vx.inc.c
index 568b6a2acb..465715eb6d 100644
--- a/qemu/target/s390x/translate_vx.inc.c
+++ b/qemu/target/s390x/translate_vx.inc.c
@@ -251,6 +251,27 @@ static void get_vec_element_ptr_i64(TCGContext *tcg_ctx, TCGv_ptr ptr, uint8_t r
     tcg_gen_gvec_##fn(tcg_ctx, es, vec_full_reg_offset(v1), vec_full_reg_offset(v2), \
                       vec_full_reg_offset(v3), vec_full_reg_offset(v4), 16, 16)
 
+static void gen_hswap_i64(TCGContext *tcg_ctx, TCGv_i64 ret, TCGv_i64 arg)
+{
+    TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);
+    TCGv_i64 t1 = tcg_temp_new_i64(tcg_ctx);
+
+    tcg_gen_rotli_i64(tcg_ctx, t1, arg, 32);
+    tcg_gen_andi_i64(tcg_ctx, t0, t1, 0x0000ffff0000ffffull);
+    tcg_gen_shli_i64(tcg_ctx, t0, t0, 16);
+    tcg_gen_shri_i64(tcg_ctx, t1, t1, 16);
+    tcg_gen_andi_i64(tcg_ctx, t1, t1, 0x0000ffff0000ffffull);
+    tcg_gen_or_i64(tcg_ctx, ret, t0, t1);
+
+    tcg_temp_free_i64(tcg_ctx, t0);
+    tcg_temp_free_i64(tcg_ctx, t1);
+}
+
+static void gen_wswap_i64(TCGContext *tcg_ctx, TCGv_i64 ret, TCGv_i64 arg)
+{
+    tcg_gen_rotli_i64(tcg_ctx, ret, arg, 32);
+}
+
 /*
  * Helper to carry out a 128 bit vector computation using 2 i64 values per
  * vector.
@@ -354,6 +375,59 @@ static void gen_addi2_i64(TCGContext *tcg_ctx, TCGv_i64 dl, TCGv_i64 dh, TCGv_i6
     tcg_temp_free_i64(tcg_ctx, bh);
 }
 
+static DisasJumpType op_vbperm(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+
+    gen_gvec_3_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                   get_field(s, v3), 0, gen_helper_gvec_vbperm);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vmsl(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i64 l1, h1, l2, h2;
+
+    if (get_field(s, m5) != ES_64) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    l1 = tcg_temp_new_i64(tcg_ctx);
+    h1 = tcg_temp_new_i64(tcg_ctx);
+    l2 = tcg_temp_new_i64(tcg_ctx);
+    h2 = tcg_temp_new_i64(tcg_ctx);
+
+    read_vec_element_i64(tcg_ctx, l1, get_field(s, v2), 0, ES_64);
+    read_vec_element_i64(tcg_ctx, h1, get_field(s, v3), 0, ES_64);
+    tcg_gen_mulu2_i64(tcg_ctx, l1, h1, l1, h1);
+    if (extract32(get_field(s, m6), 3, 1)) {
+        tcg_gen_add2_i64(tcg_ctx, l1, h1, l1, h1, l1, h1);
+    }
+
+    read_vec_element_i64(tcg_ctx, l2, get_field(s, v2), 1, ES_64);
+    read_vec_element_i64(tcg_ctx, h2, get_field(s, v3), 1, ES_64);
+    tcg_gen_mulu2_i64(tcg_ctx, l2, h2, l2, h2);
+    if (extract32(get_field(s, m6), 2, 1)) {
+        tcg_gen_add2_i64(tcg_ctx, l2, h2, l2, h2, l2, h2);
+    }
+
+    tcg_gen_add2_i64(tcg_ctx, l1, h1, l1, h1, l2, h2);
+    read_vec_element_i64(tcg_ctx, h2, get_field(s, v4), 0, ES_64);
+    read_vec_element_i64(tcg_ctx, l2, get_field(s, v4), 1, ES_64);
+    tcg_gen_add2_i64(tcg_ctx, l1, h1, l1, h1, l2, h2);
+
+    write_vec_element_i64(tcg_ctx, h1, get_field(s, v1), 0, ES_64);
+    write_vec_element_i64(tcg_ctx, l1, get_field(s, v1), 1, ES_64);
+
+    tcg_temp_free_i64(tcg_ctx, l1);
+    tcg_temp_free_i64(tcg_ctx, h1);
+    tcg_temp_free_i64(tcg_ctx, l2);
+    tcg_temp_free_i64(tcg_ctx, h2);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_vge(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -482,6 +556,131 @@ static DisasJumpType op_vlrep(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_vlebr(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t es = s->insn->data;
+    const uint8_t enr = get_field(s, m3);
+    TCGv_i64 tmp;
+
+    if (!valid_vec_element(enr, es)) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_qemu_ld_i64(tcg_ctx, tmp, o->addr1, get_mem_index(s), MO_LE | es);
+    write_vec_element_i64(tcg_ctx, tmp, get_field(s, v1), enr, es);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vlbrrep(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t es = get_field(s, m3);
+    TCGv_i64 tmp;
+    int i;
+
+    if (es < ES_16 || es > ES_64) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_qemu_ld_i64(tcg_ctx, tmp, o->addr1, get_mem_index(s), MO_LE | es);
+    for (i = 0; i < NUM_VEC_ELEMENTS(es); i++) {
+        write_vec_element_i64(tcg_ctx, tmp, get_field(s, v1), i, es);
+    }
+    tcg_temp_free_i64(tcg_ctx, tmp);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vllebrz(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t m3 = get_field(s, m3);
+    TCGv_i64 tmp;
+    TCGv_i64 zero;
+    int es;
+    int lshift;
+
+    switch (m3) {
+    case ES_16:
+    case ES_32:
+    case ES_64:
+        es = m3;
+        lshift = 0;
+        break;
+    case 6:
+        es = ES_32;
+        lshift = 32;
+        break;
+    default:
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    zero = tcg_const_i64(tcg_ctx, 0);
+    tcg_gen_qemu_ld_i64(tcg_ctx, tmp, o->addr1, get_mem_index(s), MO_LE | es);
+    tcg_gen_shli_i64(tcg_ctx, tmp, tmp, lshift);
+    write_vec_element_i64(tcg_ctx, tmp, get_field(s, v1), 0, ES_64);
+    write_vec_element_i64(tcg_ctx, zero, get_field(s, v1), 1, ES_64);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+    tcg_temp_free_i64(tcg_ctx, zero);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vlbr(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t es = get_field(s, m3);
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+
+    if (es < ES_16 || es > ES_128) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+
+    if (es == ES_128) {
+        tcg_gen_qemu_ld_i64(tcg_ctx, t1, o->addr1, get_mem_index(s), MO_LEUQ);
+        gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
+        tcg_gen_qemu_ld_i64(tcg_ctx, t0, o->addr1, get_mem_index(s), MO_LEUQ);
+        goto write;
+    }
+
+    tcg_gen_qemu_ld_i64(tcg_ctx, t0, o->addr1, get_mem_index(s), MO_LEUQ);
+    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
+    tcg_gen_qemu_ld_i64(tcg_ctx, t1, o->addr1, get_mem_index(s), MO_LEUQ);
+
+    switch (es) {
+    case ES_16:
+        gen_hswap_i64(tcg_ctx, t0, t0);
+        gen_hswap_i64(tcg_ctx, t1, t1);
+        break;
+    case ES_32:
+        gen_wswap_i64(tcg_ctx, t0, t0);
+        gen_wswap_i64(tcg_ctx, t1, t1);
+        break;
+    case ES_64:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+write:
+    write_vec_element_i64(tcg_ctx, t0, get_field(s, v1), 0, ES_64);
+    write_vec_element_i64(tcg_ctx, t1, get_field(s, v1), 1, ES_64);
+    tcg_temp_free(tcg_ctx, t0);
+    tcg_temp_free(tcg_ctx, t1);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_vle(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -519,6 +718,46 @@ static DisasJumpType op_vlei(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_vler(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t es = get_field(s, m3);
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+
+    if (es < ES_16 || es > ES_64) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    tcg_gen_qemu_ld_i64(tcg_ctx, t1, o->addr1, get_mem_index(s), MO_TEUQ);
+    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
+    tcg_gen_qemu_ld_i64(tcg_ctx, t0, o->addr1, get_mem_index(s), MO_TEUQ);
+
+    switch (es) {
+    case ES_16:
+        gen_hswap_i64(tcg_ctx, t1, t1);
+        gen_hswap_i64(tcg_ctx, t0, t0);
+        break;
+    case ES_32:
+        gen_wswap_i64(tcg_ctx, t1, t1);
+        gen_wswap_i64(tcg_ctx, t0, t0);
+        break;
+    case ES_64:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    write_vec_element_i64(tcg_ctx, t0, get_field(s, v1), 0, ES_64);
+    write_vec_element_i64(tcg_ctx, t1, get_field(s, v1), 1, ES_64);
+    tcg_temp_free(tcg_ctx, t0);
+    tcg_temp_free(tcg_ctx, t1);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_vlgv(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -1004,6 +1243,78 @@ static DisasJumpType op_vst(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_vstebr(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t es = s->insn->data;
+    const uint8_t enr = get_field(s, m3);
+    TCGv_i64 tmp;
+
+    if (!valid_vec_element(enr, es)) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    read_vec_element_i64(tcg_ctx, tmp, get_field(s, v1), enr, es);
+    tcg_gen_qemu_st_i64(tcg_ctx, tmp, o->addr1, get_mem_index(s), MO_LE | es);
+    tcg_temp_free_i64(tcg_ctx, tmp);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vstbr(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t es = get_field(s, m3);
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 bytes;
+
+    if (es < ES_16 || es > ES_128) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    bytes = tcg_const_i64(tcg_ctx, 16);
+    gen_helper_probe_write_access(tcg_ctx, tcg_ctx->cpu_env, o->addr1, bytes);
+    tcg_temp_free_i64(tcg_ctx, bytes);
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+
+    if (es == ES_128) {
+        read_vec_element_i64(tcg_ctx, t1, get_field(s, v1), 0, ES_64);
+        read_vec_element_i64(tcg_ctx, t0, get_field(s, v1), 1, ES_64);
+        goto write;
+    }
+
+    read_vec_element_i64(tcg_ctx, t0, get_field(s, v1), 0, ES_64);
+    read_vec_element_i64(tcg_ctx, t1, get_field(s, v1), 1, ES_64);
+
+    switch (es) {
+    case ES_16:
+        gen_hswap_i64(tcg_ctx, t0, t0);
+        gen_hswap_i64(tcg_ctx, t1, t1);
+        break;
+    case ES_32:
+        gen_wswap_i64(tcg_ctx, t0, t0);
+        gen_wswap_i64(tcg_ctx, t1, t1);
+        break;
+    case ES_64:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+write:
+    tcg_gen_qemu_st_i64(tcg_ctx, t0, o->addr1, get_mem_index(s), MO_LEUQ);
+    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
+    tcg_gen_qemu_st_i64(tcg_ctx, t1, o->addr1, get_mem_index(s), MO_LEUQ);
+    tcg_temp_free(tcg_ctx, t0);
+    tcg_temp_free(tcg_ctx, t1);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_vste(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -1023,6 +1334,51 @@ static DisasJumpType op_vste(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
+static DisasJumpType op_vster(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t es = get_field(s, m3);
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 bytes;
+
+    if (es < ES_16 || es > ES_64) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    bytes = tcg_const_i64(tcg_ctx, 16);
+    gen_helper_probe_write_access(tcg_ctx, tcg_ctx->cpu_env, o->addr1, bytes);
+    tcg_temp_free_i64(tcg_ctx, bytes);
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    read_vec_element_i64(tcg_ctx, t1, get_field(s, v1), 0, ES_64);
+    read_vec_element_i64(tcg_ctx, t0, get_field(s, v1), 1, ES_64);
+
+    switch (es) {
+    case ES_16:
+        gen_hswap_i64(tcg_ctx, t1, t1);
+        gen_hswap_i64(tcg_ctx, t0, t0);
+        break;
+    case ES_32:
+        gen_wswap_i64(tcg_ctx, t1, t1);
+        gen_wswap_i64(tcg_ctx, t0, t0);
+        break;
+    case ES_64:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    tcg_gen_qemu_st_i64(tcg_ctx, t0, o->addr1, get_mem_index(s), MO_TEUQ);
+    gen_addi_and_wrap_i64(s, o->addr1, o->addr1, 8);
+    tcg_gen_qemu_st_i64(tcg_ctx, t1, o->addr1, get_mem_index(s), MO_TEUQ);
+    tcg_temp_free(tcg_ctx, t0);
+    tcg_temp_free(tcg_ctx, t1);
+    return DISAS_NEXT;
+}
+
 static DisasJumpType op_vstm(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
@@ -2101,30 +2457,43 @@ static DisasJumpType op_ves(DisasContext *s, DisasOps *o)
 static DisasJumpType op_vsl(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv_i64 shift = tcg_temp_new_i64(tcg_ctx);
+    const bool byte = s->insn->data;
 
-    read_vec_element_i64(tcg_ctx, shift, get_field(s, v3), 7, ES_8);
-    if (s->fields.op2 == 0x74) {
-        tcg_gen_andi_i64(tcg_ctx, shift, shift, 0x7);
+    if (!byte && s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH2)) {
+        gen_gvec_3_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                       get_field(s, v3), 0, gen_helper_gvec_vsl_ve2);
     } else {
-        tcg_gen_andi_i64(tcg_ctx, shift, shift, 0x78);
-    }
+        TCGv_i64 shift = tcg_temp_new_i64(tcg_ctx);
 
-    gen_gvec_2i_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
-                    shift, 0, gen_helper_gvec_vsl);
-    tcg_temp_free_i64(tcg_ctx, shift);
+        read_vec_element_i64(tcg_ctx, shift, get_field(s, v3), 7, ES_8);
+        tcg_gen_andi_i64(tcg_ctx, shift, shift, byte ? 0x78 : 7);
+        gen_gvec_2i_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                        shift, 0, gen_helper_gvec_vsl);
+        tcg_temp_free_i64(tcg_ctx, shift);
+    }
     return DISAS_NEXT;
 }
 
-static DisasJumpType op_vsldb(DisasContext *s, DisasOps *o)
+static DisasJumpType op_vsld(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    const uint8_t i4 = get_field(s, i4) & 0xf;
-    const int left_shift = (i4 & 7) * 8;
-    const int right_shift = 64 - left_shift;
-    TCGv_i64 t0 = tcg_temp_new_i64(tcg_ctx);
-    TCGv_i64 t1 = tcg_temp_new_i64(tcg_ctx);
-    TCGv_i64 t2 = tcg_temp_new_i64(tcg_ctx);
+    const bool byte = s->insn->data;
+    const uint8_t mask = byte ? 15 : 7;
+    const uint8_t mul = byte ? 8 : 1;
+    const uint8_t i4 = get_field(s, i4);
+    const int right_shift = 64 - (i4 & 7) * mul;
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 t2;
+
+    if (i4 & ~mask) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    t2 = tcg_temp_new_i64(tcg_ctx);
 
     if ((i4 & 8) == 0) {
         read_vec_element_i64(tcg_ctx, t0, get_field(s, v2), 0, ES_64);
@@ -2146,39 +2515,76 @@ static DisasJumpType op_vsldb(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
-static DisasJumpType op_vsra(DisasContext *s, DisasOps *o)
+static DisasJumpType op_vsrd(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv_i64 shift = tcg_temp_new_i64(tcg_ctx);
+    const uint8_t i4 = get_field(s, i4);
+    TCGv_i64 t0;
+    TCGv_i64 t1;
+    TCGv_i64 t2;
 
-    read_vec_element_i64(tcg_ctx, shift, get_field(s, v3), 7, ES_8);
-    if (s->fields.op2 == 0x7e) {
-        tcg_gen_andi_i64(tcg_ctx, shift, shift, 0x7);
-    } else {
-        tcg_gen_andi_i64(tcg_ctx, shift, shift, 0x78);
+    if (i4 & ~7) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
     }
 
-    gen_gvec_2i_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
-                    shift, 0, gen_helper_gvec_vsra);
-    tcg_temp_free_i64(tcg_ctx, shift);
-    return DISAS_NEXT;
-}
+    t0 = tcg_temp_new_i64(tcg_ctx);
+    t1 = tcg_temp_new_i64(tcg_ctx);
+    t2 = tcg_temp_new_i64(tcg_ctx);
 
-static DisasJumpType op_vsrl(DisasContext *s, DisasOps *o)
+    read_vec_element_i64(tcg_ctx, t0, get_field(s, v2), 1, ES_64);
+    read_vec_element_i64(tcg_ctx, t1, get_field(s, v3), 0, ES_64);
+    read_vec_element_i64(tcg_ctx, t2, get_field(s, v3), 1, ES_64);
+
+    tcg_gen_extract2_i64(tcg_ctx, t0, t1, t0, i4);
+    tcg_gen_extract2_i64(tcg_ctx, t1, t2, t1, i4);
+
+    write_vec_element_i64(tcg_ctx, t0, get_field(s, v1), 0, ES_64);
+    write_vec_element_i64(tcg_ctx, t1, get_field(s, v1), 1, ES_64);
+
+    tcg_temp_free(tcg_ctx, t0);
+    tcg_temp_free(tcg_ctx, t1);
+    tcg_temp_free(tcg_ctx, t2);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vsra(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    TCGv_i64 shift = tcg_temp_new_i64(tcg_ctx);
+    const bool byte = s->insn->data;
 
-    read_vec_element_i64(tcg_ctx, shift, get_field(s, v3), 7, ES_8);
-    if (s->fields.op2 == 0x7c) {
-        tcg_gen_andi_i64(tcg_ctx, shift, shift, 0x7);
+    if (!byte && s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH2)) {
+        gen_gvec_3_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                       get_field(s, v3), 0, gen_helper_gvec_vsra_ve2);
     } else {
-        tcg_gen_andi_i64(tcg_ctx, shift, shift, 0x78);
+        TCGv_i64 shift = tcg_temp_new_i64(tcg_ctx);
+
+        read_vec_element_i64(tcg_ctx, shift, get_field(s, v3), 7, ES_8);
+        tcg_gen_andi_i64(tcg_ctx, shift, shift, byte ? 0x78 : 7);
+        gen_gvec_2i_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                        shift, 0, gen_helper_gvec_vsra);
+        tcg_temp_free_i64(tcg_ctx, shift);
     }
+    return DISAS_NEXT;
+}
 
-    gen_gvec_2i_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
-                    shift, 0, gen_helper_gvec_vsrl);
-    tcg_temp_free_i64(tcg_ctx, shift);
+static DisasJumpType op_vsrl(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const bool byte = s->insn->data;
+
+    if (!byte && s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH2)) {
+        gen_gvec_3_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                       get_field(s, v3), 0, gen_helper_gvec_vsrl_ve2);
+    } else {
+        TCGv_i64 shift = tcg_temp_new_i64(tcg_ctx);
+
+        read_vec_element_i64(tcg_ctx, shift, get_field(s, v3), 7, ES_8);
+        tcg_gen_andi_i64(tcg_ctx, shift, shift, byte ? 0x78 : 7);
+        gen_gvec_2i_ool(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                        shift, 0, gen_helper_gvec_vsrl);
+        tcg_temp_free_i64(tcg_ctx, shift);
+    }
     return DISAS_NEXT;
 }
 
@@ -2594,37 +3000,125 @@ static DisasJumpType op_vstrc(DisasContext *s, DisasOps *o)
     return DISAS_NEXT;
 }
 
-static DisasJumpType op_vfa(DisasContext *s, DisasOps *o)
+static DisasJumpType op_vstrs(DisasContext *s, DisasOps *o)
 {
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
-    const uint8_t fpf = get_field(s, m4);
-    const uint8_t m5 = get_field(s, m5);
-    const bool se = extract32(m5, 3, 1);
-    gen_helper_gvec_3_ptr *fn;
+    static gen_helper_gvec_4_ptr * const fns[3][2] = {
+        { gen_helper_gvec_vstrs_8, gen_helper_gvec_vstrs_zs8 },
+        { gen_helper_gvec_vstrs_16, gen_helper_gvec_vstrs_zs16 },
+        { gen_helper_gvec_vstrs_32, gen_helper_gvec_vstrs_zs32 },
+    };
+    const uint8_t es = get_field(s, m5);
+    const uint8_t m6 = get_field(s, m6);
+    const bool zs = extract32(m6, 1, 1);
 
-    if (fpf != FPF_LONG || extract32(m5, 0, 3)) {
+    if (es > ES_32 || (m6 & ~2)) {
         gen_program_exception(s, PGM_SPECIFICATION);
         return DISAS_NORETURN;
     }
 
+    gen_gvec_4_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                   get_field(s, v3), get_field(s, v4),
+                   tcg_ctx->cpu_env, 0, fns[es][zs]);
+    set_cc_static(s);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vfa(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t fpf = get_field(s, m4);
+    const uint8_t m5 = get_field(s, m5);
+    gen_helper_gvec_3_ptr *fn = NULL;
+
     switch (s->fields.op2) {
     case 0xe3:
-        fn = se ? gen_helper_gvec_vfa64s : gen_helper_gvec_vfa64;
+        switch (fpf) {
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfa32;
+            }
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfa64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfa128;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xe5:
-        fn = se ? gen_helper_gvec_vfd64s : gen_helper_gvec_vfd64;
+        switch (fpf) {
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfd32;
+            }
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfd64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfd128;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xe7:
-        fn = se ? gen_helper_gvec_vfm64s : gen_helper_gvec_vfm64;
+        switch (fpf) {
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfm32;
+            }
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfm64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfm128;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xe2:
-        fn = se ? gen_helper_gvec_vfs64s : gen_helper_gvec_vfs64;
+        switch (fpf) {
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfs32;
+            }
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfs64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfs128;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     default:
         g_assert_not_reached();
     }
+
+    if (!fn || extract32(m5, 0, 3)) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
     gen_gvec_3_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
-                   get_field(s, v3), tcg_ctx->cpu_env, 0, fn);
+                   get_field(s, v3), tcg_ctx->cpu_env, m5, fn);
     return DISAS_NEXT;
 }
 
@@ -2633,19 +3127,42 @@ static DisasJumpType op_wfc(DisasContext *s, DisasOps *o)
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     const uint8_t fpf = get_field(s, m3);
     const uint8_t m4 = get_field(s, m4);
+    gen_helper_gvec_2_ptr *fn = NULL;
+
+    switch (fpf) {
+    case FPF_SHORT:
+        if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+            fn = gen_helper_gvec_wfk32;
+            if (s->fields.op2 == 0xcb) {
+                fn = gen_helper_gvec_wfc32;
+            }
+        }
+        break;
+    case FPF_LONG:
+        fn = gen_helper_gvec_wfk64;
+        if (s->fields.op2 == 0xcb) {
+            fn = gen_helper_gvec_wfc64;
+        }
+        break;
+    case FPF_EXT:
+        if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+            fn = gen_helper_gvec_wfk128;
+            if (s->fields.op2 == 0xcb) {
+                fn = gen_helper_gvec_wfc128;
+            }
+        }
+        break;
+    default:
+        break;
+    }
 
-    if (fpf != FPF_LONG || m4) {
+    if (!fn || m4) {
         gen_program_exception(s, PGM_SPECIFICATION);
         return DISAS_NORETURN;
     }
 
-    if (s->fields.op2 == 0xcb) {
-        gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
-                       tcg_ctx->cpu_env, 0, gen_helper_gvec_wfc64);
-    } else {
-        gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
-                       tcg_ctx->cpu_env, 0, gen_helper_gvec_wfk64);
-    }
+    gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                   tcg_ctx->cpu_env, 0, fn);
     set_cc_static(s);
     return DISAS_NEXT;
 }
@@ -2656,46 +3173,68 @@ static DisasJumpType op_vfc(DisasContext *s, DisasOps *o)
     const uint8_t fpf = get_field(s, m4);
     const uint8_t m5 = get_field(s, m5);
     const uint8_t m6 = get_field(s, m6);
-    const bool se = extract32(m5, 3, 1);
     const bool cs = extract32(m6, 0, 1);
-    gen_helper_gvec_3_ptr *fn;
-
-    if (fpf != FPF_LONG || extract32(m5, 0, 3) || extract32(m6, 1, 3)) {
-        gen_program_exception(s, PGM_SPECIFICATION);
-        return DISAS_NORETURN;
-    }
+    const bool sq = extract32(m5, 2, 1);
+    gen_helper_gvec_3_ptr *fn = NULL;
 
-    if (cs) {
-        switch (s->fields.op2) {
-        case 0xe8:
-            fn = se ? gen_helper_gvec_vfce64s_cc : gen_helper_gvec_vfce64_cc;
+    switch (s->fields.op2) {
+    case 0xe8:
+        switch (fpf) {
+        case FPF_SHORT:
+            fn = cs ? gen_helper_gvec_vfce32_cc : gen_helper_gvec_vfce32;
             break;
-        case 0xeb:
-            fn = se ? gen_helper_gvec_vfch64s_cc : gen_helper_gvec_vfch64_cc;
+        case FPF_LONG:
+            fn = cs ? gen_helper_gvec_vfce64_cc : gen_helper_gvec_vfce64;
             break;
-        case 0xea:
-            fn = se ? gen_helper_gvec_vfche64s_cc : gen_helper_gvec_vfche64_cc;
+        case FPF_EXT:
+            fn = cs ? gen_helper_gvec_vfce128_cc : gen_helper_gvec_vfce128;
             break;
         default:
-            g_assert_not_reached();
+            break;
         }
-    } else {
-        switch (s->fields.op2) {
-        case 0xe8:
-            fn = se ? gen_helper_gvec_vfce64s : gen_helper_gvec_vfce64;
+        break;
+    case 0xeb:
+        switch (fpf) {
+        case FPF_SHORT:
+            fn = cs ? gen_helper_gvec_vfch32_cc : gen_helper_gvec_vfch32;
             break;
-        case 0xeb:
-            fn = se ? gen_helper_gvec_vfch64s : gen_helper_gvec_vfch64;
+        case FPF_LONG:
+            fn = cs ? gen_helper_gvec_vfch64_cc : gen_helper_gvec_vfch64;
             break;
-        case 0xea:
-            fn = se ? gen_helper_gvec_vfche64s : gen_helper_gvec_vfche64;
+        case FPF_EXT:
+            fn = cs ? gen_helper_gvec_vfch128_cc : gen_helper_gvec_vfch128;
             break;
         default:
-            g_assert_not_reached();
+            break;
+        }
+        break;
+    case 0xea:
+        switch (fpf) {
+        case FPF_SHORT:
+            fn = cs ? gen_helper_gvec_vfche32_cc : gen_helper_gvec_vfche32;
+            break;
+        case FPF_LONG:
+            fn = cs ? gen_helper_gvec_vfche64_cc : gen_helper_gvec_vfche64;
+            break;
+        case FPF_EXT:
+            fn = cs ? gen_helper_gvec_vfche128_cc : gen_helper_gvec_vfche128;
+            break;
+        default:
+            break;
         }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if (!fn || extract32(m5, 0, 2) || extract32(m6, 1, 3) ||
+        (!s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH) &&
+         (fpf != FPF_LONG || sq))) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
     }
     gen_gvec_3_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
-                   get_field(s, v3), tcg_ctx->cpu_env, 0, fn);
+                   get_field(s, v3), tcg_ctx->cpu_env, m5, fn);
     if (cs) {
         set_cc_static(s);
     }
@@ -2708,36 +3247,107 @@ static DisasJumpType op_vcdg(DisasContext *s, DisasOps *o)
     const uint8_t fpf = get_field(s, m3);
     const uint8_t m4 = get_field(s, m4);
     const uint8_t erm = get_field(s, m5);
-    const bool se = extract32(m4, 3, 1);
-    gen_helper_gvec_2_ptr *fn;
-
-    if (fpf != FPF_LONG || extract32(m4, 0, 2) || erm > 7 || erm == 2) {
-        gen_program_exception(s, PGM_SPECIFICATION);
-        return DISAS_NORETURN;
-    }
+    gen_helper_gvec_2_ptr *fn = NULL;
 
     switch (s->fields.op2) {
     case 0xc3:
-        fn = se ? gen_helper_gvec_vcdg64s : gen_helper_gvec_vcdg64;
+        switch (fpf) {
+        case FPF_LONG:
+            fn = gen_helper_gvec_vcdg64;
+            break;
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH2)) {
+                fn = gen_helper_gvec_vcdg32;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xc1:
-        fn = se ? gen_helper_gvec_vcdlg64s : gen_helper_gvec_vcdlg64;
+        switch (fpf) {
+        case FPF_LONG:
+            fn = gen_helper_gvec_vcdlg64;
+            break;
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH2)) {
+                fn = gen_helper_gvec_vcdlg32;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xc2:
-        fn = se ? gen_helper_gvec_vcgd64s : gen_helper_gvec_vcgd64;
+        switch (fpf) {
+        case FPF_LONG:
+            fn = gen_helper_gvec_vcgd64;
+            break;
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH2)) {
+                fn = gen_helper_gvec_vcgd32;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xc0:
-        fn = se ? gen_helper_gvec_vclgd64s : gen_helper_gvec_vclgd64;
+        switch (fpf) {
+        case FPF_LONG:
+            fn = gen_helper_gvec_vclgd64;
+            break;
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH2)) {
+                fn = gen_helper_gvec_vclgd32;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xc7:
-        fn = se ? gen_helper_gvec_vfi64s : gen_helper_gvec_vfi64;
+        switch (fpf) {
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfi32;
+            }
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfi64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfi128;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     case 0xc5:
-        fn = se ? gen_helper_gvec_vflr64s : gen_helper_gvec_vflr64;
+        switch (fpf) {
+        case FPF_LONG:
+            fn = gen_helper_gvec_vflr64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vflr128;
+            }
+            break;
+        default:
+            break;
+        }
         break;
     default:
         g_assert_not_reached();
     }
+
+    if (!fn || extract32(m4, 0, 2) || erm > 7 || erm == 2) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
     gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2), tcg_ctx->cpu_env,
                    deposit32(m4, 4, 4, erm), fn);
     return DISAS_NEXT;
@@ -2748,18 +3358,74 @@ static DisasJumpType op_vfll(DisasContext *s, DisasOps *o)
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     const uint8_t fpf = get_field(s, m3);
     const uint8_t m4 = get_field(s, m4);
-    gen_helper_gvec_2_ptr *fn = gen_helper_gvec_vfll32;
+    gen_helper_gvec_2_ptr *fn = NULL;
 
-    if (fpf != FPF_SHORT || extract32(m4, 0, 3)) {
+    switch (fpf) {
+    case FPF_SHORT:
+        fn = gen_helper_gvec_vfll32;
+        break;
+    case FPF_LONG:
+        if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+            fn = gen_helper_gvec_vfll64;
+        }
+        break;
+    default:
+        break;
+    }
+
+    if (!fn || extract32(m4, 0, 3)) {
         gen_program_exception(s, PGM_SPECIFICATION);
         return DISAS_NORETURN;
     }
 
-    if (extract32(m4, 3, 1)) {
-        fn = gen_helper_gvec_vfll32s;
-    }
     gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2), tcg_ctx->cpu_env,
-                   0, fn);
+                   m4, fn);
+    return DISAS_NEXT;
+}
+
+static DisasJumpType op_vfmax(DisasContext *s, DisasOps *o)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    const uint8_t fpf = get_field(s, m4);
+    const uint8_t m6 = get_field(s, m6);
+    const uint8_t m5 = get_field(s, m5);
+    gen_helper_gvec_3_ptr *fn;
+
+    if (m6 == 5 || m6 == 6 || m6 == 7 || m6 >= 13 || (m5 & 7)) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    switch (fpf) {
+    case FPF_SHORT:
+        if (s->fields.op2 == 0xef) {
+            fn = gen_helper_gvec_vfmax32;
+        } else {
+            fn = gen_helper_gvec_vfmin32;
+        }
+        break;
+    case FPF_LONG:
+        if (s->fields.op2 == 0xef) {
+            fn = gen_helper_gvec_vfmax64;
+        } else {
+            fn = gen_helper_gvec_vfmin64;
+        }
+        break;
+    case FPF_EXT:
+        if (s->fields.op2 == 0xef) {
+            fn = gen_helper_gvec_vfmax128;
+        } else {
+            fn = gen_helper_gvec_vfmin128;
+        }
+        break;
+    default:
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    gen_gvec_3_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
+                   get_field(s, v3), tcg_ctx->cpu_env,
+                   deposit32(m5, 4, 4, m6), fn);
     return DISAS_NEXT;
 }
 
@@ -2768,22 +3434,89 @@ static DisasJumpType op_vfma(DisasContext *s, DisasOps *o)
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     const uint8_t m5 = get_field(s, m5);
     const uint8_t fpf = get_field(s, m6);
-    const bool se = extract32(m5, 3, 1);
-    gen_helper_gvec_4_ptr *fn;
+    gen_helper_gvec_4_ptr *fn = NULL;
 
-    if (fpf != FPF_LONG || extract32(m5, 0, 3)) {
+    switch (s->fields.op2) {
+    case 0x8f:
+        switch (fpf) {
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfma32;
+            }
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfma64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfma128;
+            }
+            break;
+        default:
+            break;
+        }
+        break;
+    case 0x8e:
+        switch (fpf) {
+        case FPF_SHORT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfms32;
+            }
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfms64;
+            break;
+        case FPF_EXT:
+            if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+                fn = gen_helper_gvec_vfms128;
+            }
+            break;
+        default:
+            break;
+        }
+        break;
+    case 0x9f:
+        switch (fpf) {
+        case FPF_SHORT:
+            fn = gen_helper_gvec_vfnma32;
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfnma64;
+            break;
+        case FPF_EXT:
+            fn = gen_helper_gvec_vfnma128;
+            break;
+        default:
+            break;
+        }
+        break;
+    case 0x9e:
+        switch (fpf) {
+        case FPF_SHORT:
+            fn = gen_helper_gvec_vfnms32;
+            break;
+        case FPF_LONG:
+            fn = gen_helper_gvec_vfnms64;
+            break;
+        case FPF_EXT:
+            fn = gen_helper_gvec_vfnms128;
+            break;
+        default:
+            break;
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if (!fn || extract32(m5, 0, 3)) {
         gen_program_exception(s, PGM_SPECIFICATION);
         return DISAS_NORETURN;
     }
 
-    if (s->fields.op2 == 0x8f) {
-        fn = se ? gen_helper_gvec_vfma64s : gen_helper_gvec_vfma64;
-    } else {
-        fn = se ? gen_helper_gvec_vfms64s : gen_helper_gvec_vfms64;
-    }
     gen_gvec_4_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2),
                    get_field(s, v3), get_field(s, v4), tcg_ctx->cpu_env,
-                   0, fn);
+                   m5, fn);
     return DISAS_NEXT;
 }
 
@@ -2795,48 +3528,76 @@ static DisasJumpType op_vfpso(DisasContext *s, DisasOps *o)
     const uint8_t fpf = get_field(s, m3);
     const uint8_t m4 = get_field(s, m4);
     const uint8_t m5 = get_field(s, m5);
+    const bool se = extract32(m4, 3, 1);
     TCGv_i64 tmp;
 
-    if (fpf != FPF_LONG || extract32(m4, 0, 3) || m5 > 2) {
+    if ((fpf != FPF_LONG && !s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) ||
+        extract32(m4, 0, 3) || m5 > 2) {
         gen_program_exception(s, PGM_SPECIFICATION);
         return DISAS_NORETURN;
     }
 
-    if (extract32(m4, 3, 1)) {
-        tmp = tcg_temp_new_i64(tcg_ctx);
-        read_vec_element_i64(tcg_ctx, tmp, v2, 0, ES_64);
-        switch (m5) {
-        case 0:
-            /* sign bit is inverted (complement) */
-            tcg_gen_xori_i64(tcg_ctx, tmp, tmp, 1ull << 63);
-            break;
-        case 1:
-            /* sign bit is set to one (negative) */
-            tcg_gen_ori_i64(tcg_ctx, tmp, tmp, 1ull << 63);
-            break;
-        case 2:
-            /* sign bit is set to zero (positive) */
-            tcg_gen_andi_i64(tcg_ctx, tmp, tmp, (1ull << 63) - 1);
-            break;
+    switch (fpf) {
+    case FPF_SHORT:
+        if (!se) {
+            switch (m5) {
+            case 0:
+                gen_gvec_fn_2i(tcg_ctx, xori, ES_32, v1, v2, 1ull << 31);
+                break;
+            case 1:
+                gen_gvec_fn_2i(tcg_ctx, ori, ES_32, v1, v2, 1ull << 31);
+                break;
+            case 2:
+                gen_gvec_fn_2i(tcg_ctx, andi, ES_32, v1, v2, (1ull << 31) - 1);
+                break;
+            }
+            return DISAS_NEXT;
         }
-        write_vec_element_i64(tcg_ctx, tmp, v1, 0, ES_64);
-        tcg_temp_free_i64(tcg_ctx, tmp);
-    } else {
-        switch (m5) {
-        case 0:
-            /* sign bit is inverted (complement) */
-            gen_gvec_fn_2i(tcg_ctx, xori, ES_64, v1, v2, 1ull << 63);
-            break;
-        case 1:
-            /* sign bit is set to one (negative) */
-            gen_gvec_fn_2i(tcg_ctx, ori, ES_64, v1, v2, 1ull << 63);
-            break;
-        case 2:
-            /* sign bit is set to zero (positive) */
-            gen_gvec_fn_2i(tcg_ctx, andi, ES_64, v1, v2, (1ull << 63) - 1);
-            break;
+        break;
+    case FPF_LONG:
+        if (!se) {
+            switch (m5) {
+            case 0:
+                gen_gvec_fn_2i(tcg_ctx, xori, ES_64, v1, v2, 1ull << 63);
+                break;
+            case 1:
+                gen_gvec_fn_2i(tcg_ctx, ori, ES_64, v1, v2, 1ull << 63);
+                break;
+            case 2:
+                gen_gvec_fn_2i(tcg_ctx, andi, ES_64, v1, v2, (1ull << 63) - 1);
+                break;
+            }
+            return DISAS_NEXT;
         }
+        break;
+    case FPF_EXT:
+        break;
+    default:
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return DISAS_NORETURN;
+    }
+
+    tmp = tcg_temp_new_i64(tcg_ctx);
+    read_vec_element_i64(tcg_ctx, tmp, v2, 0, ES_64);
+    switch (m5) {
+    case 0:
+        tcg_gen_xori_i64(tcg_ctx, tmp, tmp, 1ull << 63);
+        break;
+    case 1:
+        tcg_gen_ori_i64(tcg_ctx, tmp, tmp, 1ull << 63);
+        break;
+    case 2:
+        tcg_gen_andi_i64(tcg_ctx, tmp, tmp, (1ull << 63) - 1);
+        break;
     }
+    write_vec_element_i64(tcg_ctx, tmp, v1, 0, ES_64);
+
+    if (fpf == FPF_EXT) {
+        read_vec_element_i64(tcg_ctx, tmp, v2, 1, ES_64);
+        write_vec_element_i64(tcg_ctx, tmp, v1, 1, ES_64);
+    }
+
+    tcg_temp_free_i64(tcg_ctx, tmp);
     return DISAS_NEXT;
 }
 
@@ -2845,18 +3606,33 @@ static DisasJumpType op_vfsq(DisasContext *s, DisasOps *o)
     TCGContext *tcg_ctx = s->uc->tcg_ctx;
     const uint8_t fpf = get_field(s, m3);
     const uint8_t m4 = get_field(s, m4);
-    gen_helper_gvec_2_ptr *fn = gen_helper_gvec_vfsq64;
+    gen_helper_gvec_2_ptr *fn = NULL;
+
+    switch (fpf) {
+    case FPF_SHORT:
+        if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+            fn = gen_helper_gvec_vfsq32;
+        }
+        break;
+    case FPF_LONG:
+        fn = gen_helper_gvec_vfsq64;
+        break;
+    case FPF_EXT:
+        if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+            fn = gen_helper_gvec_vfsq128;
+        }
+        break;
+    default:
+        break;
+    }
 
-    if (fpf != FPF_LONG || extract32(m4, 0, 3)) {
+    if (!fn || extract32(m4, 0, 3)) {
         gen_program_exception(s, PGM_SPECIFICATION);
         return DISAS_NORETURN;
     }
 
-    if (extract32(m4, 3, 1)) {
-        fn = gen_helper_gvec_vfsq64s;
-    }
     gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2), tcg_ctx->cpu_env,
-                   0, fn);
+                   m4, fn);
     return DISAS_NEXT;
 }
 
@@ -2866,17 +3642,33 @@ static DisasJumpType op_vftci(DisasContext *s, DisasOps *o)
     const uint16_t i3 = get_field(s, i3);
     const uint8_t fpf = get_field(s, m4);
     const uint8_t m5 = get_field(s, m5);
-    gen_helper_gvec_2_ptr *fn = gen_helper_gvec_vftci64;
+    gen_helper_gvec_2_ptr *fn = NULL;
 
-    if (fpf != FPF_LONG || extract32(m5, 0, 3)) {
+    switch (fpf) {
+    case FPF_SHORT:
+        if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+            fn = gen_helper_gvec_vftci32;
+        }
+        break;
+    case FPF_LONG:
+        fn = gen_helper_gvec_vftci64;
+        break;
+    case FPF_EXT:
+        if (s390_has_feat(s->uc, S390_FEAT_VECTOR_ENH)) {
+            fn = gen_helper_gvec_vftci128;
+        }
+        break;
+    default:
+        break;
+    }
+
+    if (!fn || extract32(m5, 0, 3)) {
         gen_program_exception(s, PGM_SPECIFICATION);
         return DISAS_NORETURN;
     }
 
-    if (extract32(m5, 3, 1)) {
-        fn = gen_helper_gvec_vftci64s;
-    }
-    gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2), tcg_ctx->cpu_env, i3, fn);
+    gen_gvec_2_ptr(tcg_ctx, get_field(s, v1), get_field(s, v2), tcg_ctx->cpu_env,
+                   deposit32(m5, 4, 12, i3), fn);
     set_cc_static(s);
     return DISAS_NEXT;
 }
diff --git a/qemu/target/s390x/vec_fpu_helper.c b/qemu/target/s390x/vec_fpu_helper.c
index e87ef56f04..006279d930 100644
--- a/qemu/target/s390x/vec_fpu_helper.c
+++ b/qemu/target/s390x/vec_fpu_helper.c
@@ -78,6 +78,62 @@ static void handle_ieee_exc(CPUS390XState *env, uint8_t vxc, uint8_t vec_exc,
     }
 }
 
+static float32 s390_vec_read_float32(const S390Vector *v, uint8_t enr)
+{
+    return make_float32(s390_vec_read_element32(v, enr));
+}
+
+static float64 s390_vec_read_float64(const S390Vector *v, uint8_t enr)
+{
+    return make_float64(s390_vec_read_element64(v, enr));
+}
+
+static float128 s390_vec_read_float128(const S390Vector *v)
+{
+    return make_float128(s390_vec_read_element64(v, 0),
+                         s390_vec_read_element64(v, 1));
+}
+
+static void s390_vec_write_float32(S390Vector *v, uint8_t enr, float32 data)
+{
+    s390_vec_write_element32(v, enr, data);
+}
+
+static void s390_vec_write_float64(S390Vector *v, uint8_t enr, float64 data)
+{
+    s390_vec_write_element64(v, enr, data);
+}
+
+static void s390_vec_write_float128(S390Vector *v, float128 data)
+{
+    s390_vec_write_element64(v, 0, data.high);
+    s390_vec_write_element64(v, 1, data.low);
+}
+
+typedef float32 (*vop32_2_fn)(float32 a, float_status *s);
+static void vop32_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
+                    bool s, bool XxC, uint8_t erm, vop32_2_fn fn,
+                    uintptr_t retaddr)
+{
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    int i, old_mode;
+
+    old_mode = s390_swap_bfp_rounding_mode(env, erm);
+    for (i = 0; i < 4; i++) {
+        const float32 a = s390_vec_read_float32(v2, i);
+
+        s390_vec_write_float32(&tmp, i, fn(a, &env->fpu_status));
+        vxc = check_ieee_exc(env, i, XxC, &vec_exc);
+        if (s || vxc) {
+            break;
+        }
+    }
+    s390_restore_bfp_rounding_mode(env, old_mode);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
+}
+
 typedef uint64_t (*vop64_2_fn)(uint64_t a, float_status *s);
 static void vop64_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
                     bool s, bool XxC, uint8_t erm, vop64_2_fn fn,
@@ -102,6 +158,47 @@ static void vop64_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
     *v1 = tmp;
 }
 
+typedef float128 (*vop128_2_fn)(float128 a, float_status *s);
+static void vop128_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
+                     bool s, bool XxC, uint8_t erm, vop128_2_fn fn,
+                     uintptr_t retaddr)
+{
+    const float128 a = s390_vec_read_float128(v2);
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    int old_mode;
+
+    old_mode = s390_swap_bfp_rounding_mode(env, erm);
+    s390_vec_write_float128(&tmp, fn(a, &env->fpu_status));
+    vxc = check_ieee_exc(env, 0, XxC, &vec_exc);
+    s390_restore_bfp_rounding_mode(env, old_mode);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
+}
+
+typedef float32 (*vop32_3_fn)(float32 a, float32 b, float_status *s);
+static void vop32_3(S390Vector *v1, const S390Vector *v2,
+                    const S390Vector *v3, CPUS390XState *env, bool s,
+                    vop32_3_fn fn, uintptr_t retaddr)
+{
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        const float32 a = s390_vec_read_float32(v2, i);
+        const float32 b = s390_vec_read_float32(v3, i);
+
+        s390_vec_write_float32(&tmp, i, fn(a, b, &env->fpu_status));
+        vxc = check_ieee_exc(env, i, false, &vec_exc);
+        if (s || vxc) {
+            break;
+        }
+    }
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
+}
+
 typedef uint64_t (*vop64_3_fn)(uint64_t a, uint64_t b, float_status *s);
 static void vop64_3(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                     CPUS390XState *env, bool s, vop64_3_fn fn,
@@ -125,21 +222,78 @@ static void vop64_3(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
     *v1 = tmp;
 }
 
+typedef float128 (*vop128_3_fn)(float128 a, float128 b, float_status *s);
+static void vop128_3(S390Vector *v1, const S390Vector *v2,
+                     const S390Vector *v3, CPUS390XState *env, bool s,
+                     vop128_3_fn fn, uintptr_t retaddr)
+{
+    const float128 a = s390_vec_read_float128(v2);
+    const float128 b = s390_vec_read_float128(v3);
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+
+    s390_vec_write_float128(&tmp, fn(a, b, &env->fpu_status));
+    vxc = check_ieee_exc(env, 0, false, &vec_exc);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
+}
+
+static float32 vfa32(float32 a, float32 b, float_status *s)
+{
+    return float32_add(a, b, s);
+}
+
 static uint64_t vfa64(uint64_t a, uint64_t b, float_status *s)
 {
     return float64_add(a, b, s);
 }
 
+static float128 vfa128(float128 a, float128 b, float_status *s)
+{
+    return float128_add(a, b, s);
+}
+
+void HELPER(gvec_vfa32)(void *v1, const void *v2, const void *v3,
+                        CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop32_3(v1, v2, v3, env, se, vfa32, GETPC());
+}
+
 void HELPER(gvec_vfa64)(void *v1, const void *v2, const void *v3,
                         CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, false, vfa64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop64_3(v1, v2, v3, env, se, vfa64, GETPC());
 }
 
-void HELPER(gvec_vfa64s)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfa128)(void *v1, const void *v2, const void *v3,
                          CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, true, vfa64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop128_3(v1, v2, v3, env, se, vfa128, GETPC());
+}
+
+static int wfc32(const S390Vector *v1, const S390Vector *v2,
+                 CPUS390XState *env, bool signal, uintptr_t retaddr)
+{
+    const float32 a = s390_vec_read_float32(v1, 0);
+    const float32 b = s390_vec_read_float32(v2, 0);
+    uint8_t vxc, vec_exc = 0;
+    int cmp;
+
+    if (signal) {
+        cmp = float32_compare(a, b, &env->fpu_status);
+    } else {
+        cmp = float32_compare_quiet(a, b, &env->fpu_status);
+    }
+    vxc = check_ieee_exc(env, 0, false, &vec_exc);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+
+    return float_comp_to_cc(env, cmp);
 }
 
 static int wfc64(const S390Vector *v1, const S390Vector *v2,
@@ -162,6 +316,37 @@ static int wfc64(const S390Vector *v1, const S390Vector *v2,
     return float_comp_to_cc(env, cmp);
 }
 
+static int wfc128(const S390Vector *v1, const S390Vector *v2,
+                  CPUS390XState *env, bool signal, uintptr_t retaddr)
+{
+    const float128 a = s390_vec_read_float128(v1);
+    const float128 b = s390_vec_read_float128(v2);
+    uint8_t vxc, vec_exc = 0;
+    int cmp;
+
+    if (signal) {
+        cmp = float128_compare(a, b, &env->fpu_status);
+    } else {
+        cmp = float128_compare_quiet(a, b, &env->fpu_status);
+    }
+    vxc = check_ieee_exc(env, 0, false, &vec_exc);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+
+    return float_comp_to_cc(env, cmp);
+}
+
+void HELPER(gvec_wfc32)(const void *v1, const void *v2, CPUS390XState *env,
+                        uint32_t desc)
+{
+    env->cc_op = wfc32(v1, v2, env, false, GETPC());
+}
+
+void HELPER(gvec_wfk32)(const void *v1, const void *v2, CPUS390XState *env,
+                        uint32_t desc)
+{
+    env->cc_op = wfc32(v1, v2, env, true, GETPC());
+}
+
 void HELPER(gvec_wfc64)(const void *v1, const void *v2, CPUS390XState *env,
                         uint32_t desc)
 {
@@ -174,7 +359,50 @@ void HELPER(gvec_wfk64)(const void *v1, const void *v2, CPUS390XState *env,
     env->cc_op = wfc64(v1, v2, env, true, GETPC());
 }
 
-typedef int (*vfc64_fn)(float64 a, float64 b, float_status *status);
+void HELPER(gvec_wfc128)(const void *v1, const void *v2, CPUS390XState *env,
+                         uint32_t desc)
+{
+    env->cc_op = wfc128(v1, v2, env, false, GETPC());
+}
+
+void HELPER(gvec_wfk128)(const void *v1, const void *v2, CPUS390XState *env,
+                         uint32_t desc)
+{
+    env->cc_op = wfc128(v1, v2, env, true, GETPC());
+}
+
+typedef bool (*vfc32_fn)(float32 a, float32 b, float_status *status);
+static int vfc32(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
+                 CPUS390XState *env, bool s, vfc32_fn fn, uintptr_t retaddr)
+{
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    int match = 0;
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        const float32 a = s390_vec_read_float32(v2, i);
+        const float32 b = s390_vec_read_float32(v3, i);
+
+        if (fn(b, a, &env->fpu_status)) {
+            match++;
+            s390_vec_write_element32(&tmp, i, -1u);
+        }
+        vxc = check_ieee_exc(env, i, false, &vec_exc);
+        if (s || vxc) {
+            break;
+        }
+    }
+
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
+    if (match) {
+        return s || match == 4 ? 0 : 1;
+    }
+    return 3;
+}
+
+typedef bool (*vfc64_fn)(float64 a, float64 b, float_status *status);
 static int vfc64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
                  CPUS390XState *env, bool s, vfc64_fn fn, uintptr_t retaddr)
 {
@@ -206,168 +434,338 @@ static int vfc64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
     return 3;
 }
 
-void HELPER(gvec_vfce64)(void *v1, const void *v2, const void *v3,
-                         CPUS390XState *env, uint32_t desc)
+typedef bool (*vfc128_fn)(float128 a, float128 b, float_status *status);
+static int vfc128(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
+                  CPUS390XState *env, bool s, vfc128_fn fn, uintptr_t retaddr)
 {
-    vfc64(v1, v2, v3, env, false, float64_eq_quiet, GETPC());
+    const float128 a = s390_vec_read_float128(v2);
+    const float128 b = s390_vec_read_float128(v3);
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    bool match = false;
+
+    if (fn(b, a, &env->fpu_status)) {
+        match = true;
+        s390_vec_write_element64(&tmp, 0, -1ull);
+        s390_vec_write_element64(&tmp, 1, -1ull);
+    }
+    vxc = check_ieee_exc(env, 0, false, &vec_exc);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
+    return match ? 0 : 3;
 }
 
-void HELPER(gvec_vfce64s)(void *v1, const void *v2, const void *v3,
-                          CPUS390XState *env, uint32_t desc)
+void HELPER(gvec_vfce32)(void *v1, const void *v2, const void *v3,
+                         CPUS390XState *env, uint32_t desc)
 {
-    vfc64(v1, v2, v3, env, true, float64_eq_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc32_fn fn = sq ? float32_eq : float32_eq_quiet;
+
+    vfc32(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfce64_cc)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfce32_cc)(void *v1, const void *v2, const void *v3,
                             CPUS390XState *env, uint32_t desc)
 {
-    env->cc_op = vfc64(v1, v2, v3, env, false, float64_eq_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc32_fn fn = sq ? float32_eq : float32_eq_quiet;
+
+    env->cc_op = vfc32(v1, v2, v3, env, se, fn, GETPC());
+}
+
+void HELPER(gvec_vfce64)(void *v1, const void *v2, const void *v3,
+                         CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc64_fn fn = sq ? float64_eq : float64_eq_quiet;
+
+    vfc64(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfce64s_cc)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfce64_cc)(void *v1, const void *v2, const void *v3,
                             CPUS390XState *env, uint32_t desc)
 {
-    env->cc_op = vfc64(v1, v2, v3, env, true, float64_eq_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc64_fn fn = sq ? float64_eq : float64_eq_quiet;
+
+    env->cc_op = vfc64(v1, v2, v3, env, se, fn, GETPC());
 }
 
 void HELPER(gvec_vfch64)(void *v1, const void *v2, const void *v3,
                          CPUS390XState *env, uint32_t desc)
 {
-    vfc64(v1, v2, v3, env, false, float64_lt_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc64_fn fn = sq ? float64_lt : float64_lt_quiet;
+
+    vfc64(v1, v2, v3, env, se, fn, GETPC());
+}
+
+void HELPER(gvec_vfch64_cc)(void *v1, const void *v2, const void *v3,
+                            CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc64_fn fn = sq ? float64_lt : float64_lt_quiet;
+
+    env->cc_op = vfc64(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfch64s)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfche64)(void *v1, const void *v2, const void *v3,
                           CPUS390XState *env, uint32_t desc)
 {
-    vfc64(v1, v2, v3, env, true, float64_lt_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc64_fn fn = sq ? float64_le : float64_le_quiet;
+
+    vfc64(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfch64_cc)(void *v1, const void *v2, const void *v3,
-                            CPUS390XState *env, uint32_t desc)
+void HELPER(gvec_vfche64_cc)(void *v1, const void *v2, const void *v3,
+                             CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc64_fn fn = sq ? float64_le : float64_le_quiet;
+
+    env->cc_op = vfc64(v1, v2, v3, env, se, fn, GETPC());
+}
+
+void HELPER(gvec_vfce128)(void *v1, const void *v2, const void *v3,
+                          CPUS390XState *env, uint32_t desc)
 {
-    env->cc_op = vfc64(v1, v2, v3, env, false, float64_lt_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc128_fn fn = sq ? float128_eq : float128_eq_quiet;
+
+    vfc128(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfch64s_cc)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfce128_cc)(void *v1, const void *v2, const void *v3,
                              CPUS390XState *env, uint32_t desc)
 {
-    env->cc_op = vfc64(v1, v2, v3, env, true, float64_lt_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc128_fn fn = sq ? float128_eq : float128_eq_quiet;
+
+    env->cc_op = vfc128(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfche64)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfch32)(void *v1, const void *v2, const void *v3,
+                         CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc32_fn fn = sq ? float32_lt : float32_lt_quiet;
+
+    vfc32(v1, v2, v3, env, se, fn, GETPC());
+}
+
+void HELPER(gvec_vfch32_cc)(void *v1, const void *v2, const void *v3,
+                            CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc32_fn fn = sq ? float32_lt : float32_lt_quiet;
+
+    env->cc_op = vfc32(v1, v2, v3, env, se, fn, GETPC());
+}
+
+void HELPER(gvec_vfch128)(void *v1, const void *v2, const void *v3,
                           CPUS390XState *env, uint32_t desc)
 {
-    vfc64(v1, v2, v3, env, false, float64_le_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc128_fn fn = sq ? float128_lt : float128_lt_quiet;
+
+    vfc128(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfche64s)(void *v1, const void *v2, const void *v3,
-                           CPUS390XState *env, uint32_t desc)
+void HELPER(gvec_vfch128_cc)(void *v1, const void *v2, const void *v3,
+                             CPUS390XState *env, uint32_t desc)
 {
-    vfc64(v1, v2, v3, env, true, float64_le_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc128_fn fn = sq ? float128_lt : float128_lt_quiet;
+
+    env->cc_op = vfc128(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfche64_cc)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfche32)(void *v1, const void *v2, const void *v3,
+                          CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc32_fn fn = sq ? float32_le : float32_le_quiet;
+
+    vfc32(v1, v2, v3, env, se, fn, GETPC());
+}
+
+void HELPER(gvec_vfche32_cc)(void *v1, const void *v2, const void *v3,
                              CPUS390XState *env, uint32_t desc)
 {
-    env->cc_op = vfc64(v1, v2, v3, env, false, float64_le_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc32_fn fn = sq ? float32_le : float32_le_quiet;
+
+    env->cc_op = vfc32(v1, v2, v3, env, se, fn, GETPC());
+}
+
+void HELPER(gvec_vfche128)(void *v1, const void *v2, const void *v3,
+                           CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc128_fn fn = sq ? float128_le : float128_le_quiet;
+
+    vfc128(v1, v2, v3, env, se, fn, GETPC());
 }
 
-void HELPER(gvec_vfche64s_cc)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfche128_cc)(void *v1, const void *v2, const void *v3,
                               CPUS390XState *env, uint32_t desc)
 {
-    env->cc_op = vfc64(v1, v2, v3, env, true, float64_le_quiet, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool sq = extract32(simd_data(desc), 2, 1);
+    vfc128_fn fn = sq ? float128_le : float128_le_quiet;
+
+    env->cc_op = vfc128(v1, v2, v3, env, se, fn, GETPC());
 }
 
-static uint64_t vcdg64(uint64_t a, float_status *s)
+static float32 vcdg32(float32 a, float_status *s)
 {
-    return int64_to_float64(a, s);
+    return int32_to_float32(a, s);
 }
 
-void HELPER(gvec_vcdg64)(void *v1, const void *v2, CPUS390XState *env,
+static float32 vcdlg32(float32 a, float_status *s)
+{
+    return uint32_to_float32(a, s);
+}
+
+static float32 vcgd32(float32 a, float_status *s)
+{
+    const float32 tmp = float32_to_int32(a, s);
+
+    return float32_is_any_nan(a) ? INT32_MIN : tmp;
+}
+
+static float32 vclgd32(float32 a, float_status *s)
+{
+    const float32 tmp = float32_to_uint32(a, s);
+
+    return float32_is_any_nan(a) ? 0 : tmp;
+}
+
+void HELPER(gvec_vcdg32)(void *v1, const void *v2, CPUS390XState *env,
                          uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, false, XxC, erm, vcdg64, GETPC());
+    vop32_2(v1, v2, env, se, XxC, erm, vcdg32, GETPC());
 }
 
-void HELPER(gvec_vcdg64s)(void *v1, const void *v2, CPUS390XState *env,
+void HELPER(gvec_vcdlg32)(void *v1, const void *v2, CPUS390XState *env,
                           uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, true, XxC, erm, vcdg64, GETPC());
+    vop32_2(v1, v2, env, se, XxC, erm, vcdlg32, GETPC());
 }
 
-static uint64_t vcdlg64(uint64_t a, float_status *s)
+void HELPER(gvec_vcgd32)(void *v1, const void *v2, CPUS390XState *env,
+                         uint32_t desc)
 {
-    return uint64_to_float64(a, s);
+    const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool XxC = extract32(simd_data(desc), 2, 1);
+
+    vop32_2(v1, v2, env, se, XxC, erm, vcgd32, GETPC());
 }
 
-void HELPER(gvec_vcdlg64)(void *v1, const void *v2, CPUS390XState *env,
+void HELPER(gvec_vclgd32)(void *v1, const void *v2, CPUS390XState *env,
                           uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, false, XxC, erm, vcdlg64, GETPC());
+    vop32_2(v1, v2, env, se, XxC, erm, vclgd32, GETPC());
 }
 
-void HELPER(gvec_vcdlg64s)(void *v1, const void *v2, CPUS390XState *env,
-                           uint32_t desc)
+static uint64_t vcdg64(uint64_t a, float_status *s)
+{
+    return int64_to_float64(a, s);
+}
+
+void HELPER(gvec_vcdg64)(void *v1, const void *v2, CPUS390XState *env,
+                         uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, true, XxC, erm, vcdlg64, GETPC());
+    vop64_2(v1, v2, env, se, XxC, erm, vcdg64, GETPC());
 }
 
-static uint64_t vcgd64(uint64_t a, float_status *s)
+static uint64_t vcdlg64(uint64_t a, float_status *s)
 {
-    return float64_to_int64(a, s);
+    return uint64_to_float64(a, s);
 }
 
-void HELPER(gvec_vcgd64)(void *v1, const void *v2, CPUS390XState *env,
-                         uint32_t desc)
+void HELPER(gvec_vcdlg64)(void *v1, const void *v2, CPUS390XState *env,
+                          uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, false, XxC, erm, vcgd64, GETPC());
+    vop64_2(v1, v2, env, se, XxC, erm, vcdlg64, GETPC());
 }
 
-void HELPER(gvec_vcgd64s)(void *v1, const void *v2, CPUS390XState *env,
-                          uint32_t desc)
+static uint64_t vcgd64(uint64_t a, float_status *s)
+{
+    const uint64_t tmp = float64_to_int64(a, s);
+
+    return float64_is_any_nan(a) ? INT64_MIN : tmp;
+}
+
+void HELPER(gvec_vcgd64)(void *v1, const void *v2, CPUS390XState *env,
+                         uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, true, XxC, erm, vcgd64, GETPC());
+    vop64_2(v1, v2, env, se, XxC, erm, vcgd64, GETPC());
 }
 
 static uint64_t vclgd64(uint64_t a, float_status *s)
 {
-    return float64_to_uint64(a, s);
+    const uint64_t tmp = float64_to_uint64(a, s);
+
+    return float64_is_any_nan(a) ? 0 : tmp;
 }
 
 void HELPER(gvec_vclgd64)(void *v1, const void *v2, CPUS390XState *env,
                           uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, false, XxC, erm, vclgd64, GETPC());
+    vop64_2(v1, v2, env, se, XxC, erm, vclgd64, GETPC());
 }
 
-void HELPER(gvec_vclgd64s)(void *v1, const void *v2, CPUS390XState *env,
-                           uint32_t desc)
+static float32 vfd32(float32 a, float32 b, float_status *s)
 {
-    const uint8_t erm = extract32(simd_data(desc), 4, 4);
-    const bool XxC = extract32(simd_data(desc), 2, 1);
-
-    vop64_2(v1, v2, env, true, XxC, erm, vclgd64, GETPC());
+    return float32_div(a, b, s);
 }
 
 static uint64_t vfd64(uint64_t a, uint64_t b, float_status *s)
@@ -375,16 +773,38 @@ static uint64_t vfd64(uint64_t a, uint64_t b, float_status *s)
     return float64_div(a, b, s);
 }
 
+static float128 vfd128(float128 a, float128 b, float_status *s)
+{
+    return float128_div(a, b, s);
+}
+
+void HELPER(gvec_vfd32)(void *v1, const void *v2, const void *v3,
+                        CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop32_3(v1, v2, v3, env, se, vfd32, GETPC());
+}
+
 void HELPER(gvec_vfd64)(void *v1, const void *v2, const void *v3,
                         CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, false, vfd64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop64_3(v1, v2, v3, env, se, vfd64, GETPC());
 }
 
-void HELPER(gvec_vfd64s)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfd128)(void *v1, const void *v2, const void *v3,
                          CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, true, vfd64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop128_3(v1, v2, v3, env, se, vfd128, GETPC());
+}
+
+static float32 vfi32(float32 a, float_status *s)
+{
+    return float32_round_to_int(a, s);
 }
 
 static uint64_t vfi64(uint64_t a, float_status *s)
@@ -392,22 +812,39 @@ static uint64_t vfi64(uint64_t a, float_status *s)
     return float64_round_to_int(a, s);
 }
 
+static float128 vfi128(float128 a, float_status *s)
+{
+    return float128_round_to_int(a, s);
+}
+
+void HELPER(gvec_vfi32)(void *v1, const void *v2, CPUS390XState *env,
+                        uint32_t desc)
+{
+    const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const bool XxC = extract32(simd_data(desc), 2, 1);
+
+    vop32_2(v1, v2, env, se, XxC, erm, vfi32, GETPC());
+}
+
 void HELPER(gvec_vfi64)(void *v1, const void *v2, CPUS390XState *env,
                         uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, false, XxC, erm, vfi64, GETPC());
+    vop64_2(v1, v2, env, se, XxC, erm, vfi64, GETPC());
 }
 
-void HELPER(gvec_vfi64s)(void *v1, const void *v2, CPUS390XState *env,
+void HELPER(gvec_vfi128)(void *v1, const void *v2, CPUS390XState *env,
                          uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vop64_2(v1, v2, env, true, XxC, erm, vfi64, GETPC());
+    vop128_2(v1, v2, env, se, XxC, erm, vfi128, GETPC());
 }
 
 static void vfll32(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
@@ -436,13 +873,21 @@ static void vfll32(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
 void HELPER(gvec_vfll32)(void *v1, const void *v2, CPUS390XState *env,
                          uint32_t desc)
 {
-    vfll32(v1, v2, env, false, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vfll32(v1, v2, env, se, GETPC());
 }
 
-void HELPER(gvec_vfll32s)(void *v1, const void *v2, CPUS390XState *env,
-                          uint32_t desc)
+void HELPER(gvec_vfll64)(void *v1, const void *v2, CPUS390XState *env,
+                         uint32_t desc)
 {
-    vfll32(v1, v2, env, true, GETPC());
+    const float128 ret = float64_to_float128(s390_vec_read_float64(v2, 0),
+                                             &env->fpu_status);
+    uint8_t vxc, vec_exc = 0;
+
+    vxc = check_ieee_exc(env, 0, false, &vec_exc);
+    handle_ieee_exc(env, vxc, vec_exc, GETPC());
+    s390_vec_write_float128(v1, ret);
 }
 
 static void vflr64(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
@@ -474,18 +919,32 @@ void HELPER(gvec_vflr64)(void *v1, const void *v2, CPUS390XState *env,
                          uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
+    const bool se = extract32(simd_data(desc), 3, 1);
     const bool XxC = extract32(simd_data(desc), 2, 1);
 
-    vflr64(v1, v2, env, false, XxC, erm, GETPC());
+    vflr64(v1, v2, env, se, XxC, erm, GETPC());
 }
 
-void HELPER(gvec_vflr64s)(void *v1, const void *v2, CPUS390XState *env,
+void HELPER(gvec_vflr128)(void *v1, const void *v2, CPUS390XState *env,
                           uint32_t desc)
 {
     const uint8_t erm = extract32(simd_data(desc), 4, 4);
     const bool XxC = extract32(simd_data(desc), 2, 1);
+    uint8_t vxc, vec_exc = 0;
+    int old_mode;
+    float64 ret;
+
+    old_mode = s390_swap_bfp_rounding_mode(env, erm);
+    ret = float128_to_float64(s390_vec_read_float128(v2), &env->fpu_status);
+    vxc = check_ieee_exc(env, 0, XxC, &vec_exc);
+    s390_restore_bfp_rounding_mode(env, old_mode);
+    handle_ieee_exc(env, vxc, vec_exc, GETPC());
+    s390_vec_write_float64(v1, 0, ret);
+}
 
-    vflr64(v1, v2, env, true, XxC, erm, GETPC());
+static float32 vfm32(float32 a, float32 b, float_status *s)
+{
+    return float32_mul(a, b, s);
 }
 
 static uint64_t vfm64(uint64_t a, uint64_t b, float_status *s)
@@ -493,16 +952,57 @@ static uint64_t vfm64(uint64_t a, uint64_t b, float_status *s)
     return float64_mul(a, b, s);
 }
 
+static float128 vfm128(float128 a, float128 b, float_status *s)
+{
+    return float128_mul(a, b, s);
+}
+
+void HELPER(gvec_vfm32)(void *v1, const void *v2, const void *v3,
+                        CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop32_3(v1, v2, v3, env, se, vfm32, GETPC());
+}
+
 void HELPER(gvec_vfm64)(void *v1, const void *v2, const void *v3,
                         CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, false, vfm64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop64_3(v1, v2, v3, env, se, vfm64, GETPC());
 }
 
-void HELPER(gvec_vfm64s)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfm128)(void *v1, const void *v2, const void *v3,
                          CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, true, vfm64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop128_3(v1, v2, v3, env, se, vfm128, GETPC());
+}
+
+static void vfma32(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
+                   const S390Vector *v4, CPUS390XState *env, bool s, int flags,
+                   uintptr_t retaddr)
+{
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        const float32 a = s390_vec_read_float32(v2, i);
+        const float32 b = s390_vec_read_float32(v3, i);
+        const float32 c = s390_vec_read_float32(v4, i);
+        float32 ret = float32_muladd(a, b, c, flags, &env->fpu_status);
+
+        s390_vec_write_float32(&tmp, i, ret);
+        vxc = check_ieee_exc(env, i, false, &vec_exc);
+        if (s || vxc) {
+            break;
+        }
+    }
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
 }
 
 static void vfma64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
@@ -514,12 +1014,13 @@ static void vfma64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
     int i;
 
     for (i = 0; i < 2; i++) {
-        const uint64_t a = s390_vec_read_element64(v2, i);
-        const uint64_t b = s390_vec_read_element64(v3, i);
-        const uint64_t c = s390_vec_read_element64(v4, i);
-        uint64_t ret = float64_muladd(a, b, c, flags, &env->fpu_status);
+        const float64 a = s390_vec_read_float64(v2, i);
+        const float64 b = s390_vec_read_float64(v3, i);
+        const float64 c = s390_vec_read_float64(v4, i);
+        const float64 ret = float64_muladd(a, b, c, flags,
+                                           &env->fpu_status);
 
-        s390_vec_write_element64(&tmp, i, ret);
+        s390_vec_write_float64(&tmp, i, ret);
         vxc = check_ieee_exc(env, i, false, &vec_exc);
         if (s || vxc) {
             break;
@@ -529,28 +1030,373 @@ static void vfma64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
     *v1 = tmp;
 }
 
-void HELPER(gvec_vfma64)(void *v1, const void *v2, const void *v3,
-                         const void *v4, CPUS390XState *env, uint32_t desc)
+static void vfma128(S390Vector *v1, const S390Vector *v2, const S390Vector *v3,
+                    const S390Vector *v4, CPUS390XState *env, bool s,
+                    int flags, uintptr_t retaddr)
 {
-    vfma64(v1, v2, v3, v4, env, false, 0, GETPC());
+    const float128 a = s390_vec_read_float128(v2);
+    const float128 b = s390_vec_read_float128(v3);
+    const float128 c = s390_vec_read_float128(v4);
+    uint8_t vxc, vec_exc = 0;
+    float128 ret;
+
+    ret = float128_muladd(a, b, c, flags, &env->fpu_status);
+    vxc = check_ieee_exc(env, 0, false, &vec_exc);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    s390_vec_write_float128(v1, ret);
+}
+
+#define DEF_GVEC_VFMA_B(NAME, FLAGS, BITS)                               \
+void HELPER(gvec_##NAME##BITS)(void *v1, const void *v2, const void *v3, \
+                               const void *v4, CPUS390XState *env,       \
+                               uint32_t desc)                            \
+{                                                                        \
+    const bool se = extract32(simd_data(desc), 3, 1);                    \
+                                                                         \
+    vfma##BITS(v1, v2, v3, v4, env, se, FLAGS, GETPC());                 \
+}
+
+#define DEF_GVEC_VFMA(NAME, FLAGS)                                       \
+    DEF_GVEC_VFMA_B(NAME, FLAGS, 32)                                     \
+    DEF_GVEC_VFMA_B(NAME, FLAGS, 64)                                     \
+    DEF_GVEC_VFMA_B(NAME, FLAGS, 128)
+
+DEF_GVEC_VFMA(vfma, 0)
+DEF_GVEC_VFMA(vfms, float_muladd_negate_c)
+DEF_GVEC_VFMA(vfnma, float_muladd_negate_result)
+DEF_GVEC_VFMA(vfnms, float_muladd_negate_c | float_muladd_negate_result)
+
+typedef enum S390MinMaxType {
+    S390_MINMAX_TYPE_IEEE = 0,
+    S390_MINMAX_TYPE_JAVA,
+    S390_MINMAX_TYPE_C_MACRO,
+    S390_MINMAX_TYPE_CPP,
+    S390_MINMAX_TYPE_F,
+} S390MinMaxType;
+
+typedef enum S390MinMaxRes {
+    S390_MINMAX_RES_MINMAX = 0,
+    S390_MINMAX_RES_A,
+    S390_MINMAX_RES_B,
+    S390_MINMAX_RES_SILENCE_A,
+    S390_MINMAX_RES_SILENCE_B,
+} S390MinMaxRes;
+
+static S390MinMaxRes vfmin_res(uint16_t dcmask_a, uint16_t dcmask_b,
+                               S390MinMaxType type, float_status *s)
+{
+    const bool neg_a = dcmask_a & DCMASK_NEGATIVE;
+    const bool nan_a = dcmask_a & DCMASK_NAN;
+    const bool nan_b = dcmask_b & DCMASK_NAN;
+
+    g_assert(type > S390_MINMAX_TYPE_IEEE && type <= S390_MINMAX_TYPE_F);
+
+    if (unlikely((dcmask_a | dcmask_b) & DCMASK_NAN)) {
+        const bool sig_a = dcmask_a & DCMASK_SIGNALING_NAN;
+        const bool sig_b = dcmask_b & DCMASK_SIGNALING_NAN;
+
+        if ((dcmask_a | dcmask_b) & DCMASK_SIGNALING_NAN) {
+            s->float_exception_flags |= float_flag_invalid;
+        }
+        switch (type) {
+        case S390_MINMAX_TYPE_JAVA:
+            if (sig_a) {
+                return S390_MINMAX_RES_SILENCE_A;
+            } else if (sig_b) {
+                return S390_MINMAX_RES_SILENCE_B;
+            }
+            return nan_a ? S390_MINMAX_RES_A : S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_F:
+            return nan_b ? S390_MINMAX_RES_A : S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_C_MACRO:
+            s->float_exception_flags |= float_flag_invalid;
+            return S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_CPP:
+            s->float_exception_flags |= float_flag_invalid;
+            return S390_MINMAX_RES_A;
+        default:
+            g_assert_not_reached();
+        }
+    } else if (unlikely((dcmask_a & DCMASK_ZERO) && (dcmask_b & DCMASK_ZERO))) {
+        switch (type) {
+        case S390_MINMAX_TYPE_JAVA:
+            return neg_a ? S390_MINMAX_RES_A : S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_C_MACRO:
+            return S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_F:
+            return !neg_a ? S390_MINMAX_RES_B : S390_MINMAX_RES_A;
+        case S390_MINMAX_TYPE_CPP:
+            return S390_MINMAX_RES_A;
+        default:
+            g_assert_not_reached();
+        }
+    }
+    return S390_MINMAX_RES_MINMAX;
+}
+
+static S390MinMaxRes vfmax_res(uint16_t dcmask_a, uint16_t dcmask_b,
+                               S390MinMaxType type, float_status *s)
+{
+    g_assert(type > S390_MINMAX_TYPE_IEEE && type <= S390_MINMAX_TYPE_F);
+
+    if (unlikely((dcmask_a | dcmask_b) & DCMASK_NAN)) {
+        const bool sig_a = dcmask_a & DCMASK_SIGNALING_NAN;
+        const bool sig_b = dcmask_b & DCMASK_SIGNALING_NAN;
+        const bool nan_a = dcmask_a & DCMASK_NAN;
+        const bool nan_b = dcmask_b & DCMASK_NAN;
+
+        if ((dcmask_a | dcmask_b) & DCMASK_SIGNALING_NAN) {
+            s->float_exception_flags |= float_flag_invalid;
+        }
+        switch (type) {
+        case S390_MINMAX_TYPE_JAVA:
+            if (sig_a) {
+                return S390_MINMAX_RES_SILENCE_A;
+            } else if (sig_b) {
+                return S390_MINMAX_RES_SILENCE_B;
+            }
+            return nan_a ? S390_MINMAX_RES_A : S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_F:
+            return nan_b ? S390_MINMAX_RES_A : S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_C_MACRO:
+            s->float_exception_flags |= float_flag_invalid;
+            return S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_CPP:
+            s->float_exception_flags |= float_flag_invalid;
+            return S390_MINMAX_RES_A;
+        default:
+            g_assert_not_reached();
+        }
+    } else if (unlikely((dcmask_a & DCMASK_ZERO) && (dcmask_b & DCMASK_ZERO))) {
+        const bool neg_a = dcmask_a & DCMASK_NEGATIVE;
+
+        switch (type) {
+        case S390_MINMAX_TYPE_JAVA:
+        case S390_MINMAX_TYPE_F:
+            return neg_a ? S390_MINMAX_RES_B : S390_MINMAX_RES_A;
+        case S390_MINMAX_TYPE_C_MACRO:
+            return S390_MINMAX_RES_B;
+        case S390_MINMAX_TYPE_CPP:
+            return S390_MINMAX_RES_A;
+        default:
+            g_assert_not_reached();
+        }
+    }
+    return S390_MINMAX_RES_MINMAX;
 }
 
-void HELPER(gvec_vfma64s)(void *v1, const void *v2, const void *v3,
-                         const void *v4, CPUS390XState *env, uint32_t desc)
+static S390MinMaxRes vfminmax_res(uint16_t dcmask_a, uint16_t dcmask_b,
+                                  S390MinMaxType type, bool is_min,
+                                  float_status *s)
 {
-    vfma64(v1, v2, v3, v4, env, true, 0, GETPC());
+    return is_min ? vfmin_res(dcmask_a, dcmask_b, type, s) :
+                    vfmax_res(dcmask_a, dcmask_b, type, s);
 }
 
-void HELPER(gvec_vfms64)(void *v1, const void *v2, const void *v3,
-                         const void *v4, CPUS390XState *env, uint32_t desc)
+static void vfminmax32(S390Vector *v1, const S390Vector *v2,
+                       const S390Vector *v3, CPUS390XState *env,
+                       S390MinMaxType type, bool is_min, bool is_abs, bool se,
+                       uintptr_t retaddr)
 {
-    vfma64(v1, v2, v3, v4, env, false, float_muladd_negate_c, GETPC());
+    float_status *s = &env->fpu_status;
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        float32 a = s390_vec_read_float32(v2, i);
+        float32 b = s390_vec_read_float32(v3, i);
+        float32 result;
+
+        if (type != S390_MINMAX_TYPE_IEEE) {
+            S390MinMaxRes res;
+
+            if (is_abs) {
+                a = float32_abs(a);
+                b = float32_abs(b);
+            }
+
+            res = vfminmax_res(float32_dcmask(env, a), float32_dcmask(env, b),
+                               type, is_min, s);
+            switch (res) {
+            case S390_MINMAX_RES_MINMAX:
+                result = is_min ? float32_min(a, b, s) : float32_max(a, b, s);
+                break;
+            case S390_MINMAX_RES_A:
+                result = a;
+                break;
+            case S390_MINMAX_RES_B:
+                result = b;
+                break;
+            case S390_MINMAX_RES_SILENCE_A:
+                result = float32_silence_nan(a, s);
+                break;
+            case S390_MINMAX_RES_SILENCE_B:
+                result = float32_silence_nan(b, s);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+        } else if (!is_abs) {
+            result = is_min ? float32_minnum(a, b, &env->fpu_status) :
+                              float32_maxnum(a, b, &env->fpu_status);
+        } else {
+            result = is_min ? float32_minnummag(a, b, &env->fpu_status) :
+                              float32_maxnummag(a, b, &env->fpu_status);
+        }
+
+        s390_vec_write_float32(&tmp, i, result);
+        vxc = check_ieee_exc(env, i, false, &vec_exc);
+        if (se || vxc) {
+            break;
+        }
+    }
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
 }
 
-void HELPER(gvec_vfms64s)(void *v1, const void *v2, const void *v3,
-                         const void *v4, CPUS390XState *env, uint32_t desc)
+static void vfminmax64(S390Vector *v1, const S390Vector *v2,
+                       const S390Vector *v3, CPUS390XState *env,
+                       S390MinMaxType type, bool is_min, bool is_abs, bool se,
+                       uintptr_t retaddr)
 {
-    vfma64(v1, v2, v3, v4, env, true, float_muladd_negate_c, GETPC());
+    float_status *s = &env->fpu_status;
+    uint8_t vxc, vec_exc = 0;
+    S390Vector tmp = { 0 };
+    int i;
+
+    for (i = 0; i < 2; i++) {
+        float64 a = s390_vec_read_float64(v2, i);
+        float64 b = s390_vec_read_float64(v3, i);
+        float64 result;
+
+        if (type != S390_MINMAX_TYPE_IEEE) {
+            S390MinMaxRes res;
+
+            if (is_abs) {
+                a = float64_abs(a);
+                b = float64_abs(b);
+            }
+
+            res = vfminmax_res(float64_dcmask(env, a), float64_dcmask(env, b),
+                               type, is_min, s);
+            switch (res) {
+            case S390_MINMAX_RES_MINMAX:
+                result = is_min ? float64_min(a, b, s) : float64_max(a, b, s);
+                break;
+            case S390_MINMAX_RES_A:
+                result = a;
+                break;
+            case S390_MINMAX_RES_B:
+                result = b;
+                break;
+            case S390_MINMAX_RES_SILENCE_A:
+                result = float64_silence_nan(a, s);
+                break;
+            case S390_MINMAX_RES_SILENCE_B:
+                result = float64_silence_nan(b, s);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+        } else if (!is_abs) {
+            result = is_min ? float64_minnum(a, b, &env->fpu_status) :
+                              float64_maxnum(a, b, &env->fpu_status);
+        } else {
+            result = is_min ? float64_minnummag(a, b, &env->fpu_status) :
+                              float64_maxnummag(a, b, &env->fpu_status);
+        }
+
+        s390_vec_write_float64(&tmp, i, result);
+        vxc = check_ieee_exc(env, i, false, &vec_exc);
+        if (se || vxc) {
+            break;
+        }
+    }
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    *v1 = tmp;
+}
+
+static void vfminmax128(S390Vector *v1, const S390Vector *v2,
+                        const S390Vector *v3, CPUS390XState *env,
+                        S390MinMaxType type, bool is_min, bool is_abs, bool se,
+                        uintptr_t retaddr)
+{
+    float128 a = s390_vec_read_float128(v2);
+    float128 b = s390_vec_read_float128(v3);
+    float_status *s = &env->fpu_status;
+    uint8_t vxc, vec_exc = 0;
+    float128 result;
+
+    if (type != S390_MINMAX_TYPE_IEEE) {
+        S390MinMaxRes res;
+
+        if (is_abs) {
+            a = float128_abs(a);
+            b = float128_abs(b);
+        }
+
+        res = vfminmax_res(float128_dcmask(env, a), float128_dcmask(env, b),
+                           type, is_min, s);
+        switch (res) {
+        case S390_MINMAX_RES_MINMAX:
+            result = is_min ? float128_min(a, b, s) : float128_max(a, b, s);
+            break;
+        case S390_MINMAX_RES_A:
+            result = a;
+            break;
+        case S390_MINMAX_RES_B:
+            result = b;
+            break;
+        case S390_MINMAX_RES_SILENCE_A:
+            result = float128_silence_nan(a, s);
+            break;
+        case S390_MINMAX_RES_SILENCE_B:
+            result = float128_silence_nan(b, s);
+            break;
+        default:
+            g_assert_not_reached();
+        }
+    } else if (!is_abs) {
+        result = is_min ? float128_minnum(a, b, &env->fpu_status) :
+                          float128_maxnum(a, b, &env->fpu_status);
+    } else {
+        result = is_min ? float128_minnummag(a, b, &env->fpu_status) :
+                          float128_maxnummag(a, b, &env->fpu_status);
+    }
+
+    vxc = check_ieee_exc(env, 0, false, &vec_exc);
+    handle_ieee_exc(env, vxc, vec_exc, retaddr);
+    s390_vec_write_float128(v1, result);
+}
+
+#define DEF_GVEC_VFMINMAX_B(NAME, IS_MIN, BITS)                          \
+void HELPER(gvec_##NAME##BITS)(void *v1, const void *v2, const void *v3, \
+                               CPUS390XState *env, uint32_t desc)        \
+{                                                                        \
+    const bool se = extract32(simd_data(desc), 3, 1);                    \
+    uint8_t type = extract32(simd_data(desc), 4, 4);                     \
+    bool is_abs = false;                                                 \
+                                                                         \
+    if (type >= 8) {                                                     \
+        is_abs = true;                                                   \
+        type -= 8;                                                       \
+    }                                                                    \
+                                                                         \
+    vfminmax##BITS(v1, v2, v3, env, type, IS_MIN, is_abs, se, GETPC());  \
+}
+
+#define DEF_GVEC_VFMINMAX(NAME, IS_MIN)                                  \
+    DEF_GVEC_VFMINMAX_B(NAME, IS_MIN, 32)                                \
+    DEF_GVEC_VFMINMAX_B(NAME, IS_MIN, 64)                                \
+    DEF_GVEC_VFMINMAX_B(NAME, IS_MIN, 128)
+
+DEF_GVEC_VFMINMAX(vfmax, false)
+DEF_GVEC_VFMINMAX(vfmin, true)
+
+static float32 vfsq32(float32 a, float_status *s)
+{
+    return float32_sqrt(a, s);
 }
 
 static uint64_t vfsq64(uint64_t a, float_status *s)
@@ -558,16 +1404,38 @@ static uint64_t vfsq64(uint64_t a, float_status *s)
     return float64_sqrt(a, s);
 }
 
+static float128 vfsq128(float128 a, float_status *s)
+{
+    return float128_sqrt(a, s);
+}
+
+void HELPER(gvec_vfsq32)(void *v1, const void *v2, CPUS390XState *env,
+                         uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop32_2(v1, v2, env, se, false, 0, vfsq32, GETPC());
+}
+
 void HELPER(gvec_vfsq64)(void *v1, const void *v2, CPUS390XState *env,
                          uint32_t desc)
 {
-    vop64_2(v1, v2, env, false, false, 0, vfsq64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop64_2(v1, v2, env, se, false, 0, vfsq64, GETPC());
 }
 
-void HELPER(gvec_vfsq64s)(void *v1, const void *v2, CPUS390XState *env,
+void HELPER(gvec_vfsq128)(void *v1, const void *v2, CPUS390XState *env,
                           uint32_t desc)
 {
-    vop64_2(v1, v2, env, true, false, 0, vfsq64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop128_2(v1, v2, env, se, false, 0, vfsq128, GETPC());
+}
+
+static float32 vfs32(float32 a, float32 b, float_status *s)
+{
+    return float32_sub(a, b, s);
 }
 
 static uint64_t vfs64(uint64_t a, uint64_t b, float_status *s)
@@ -575,16 +1443,64 @@ static uint64_t vfs64(uint64_t a, uint64_t b, float_status *s)
     return float64_sub(a, b, s);
 }
 
+static float128 vfs128(float128 a, float128 b, float_status *s)
+{
+    return float128_sub(a, b, s);
+}
+
+void HELPER(gvec_vfs32)(void *v1, const void *v2, const void *v3,
+                        CPUS390XState *env, uint32_t desc)
+{
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop32_3(v1, v2, v3, env, se, vfs32, GETPC());
+}
+
 void HELPER(gvec_vfs64)(void *v1, const void *v2, const void *v3,
                         CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, false, vfs64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop64_3(v1, v2, v3, env, se, vfs64, GETPC());
 }
 
-void HELPER(gvec_vfs64s)(void *v1, const void *v2, const void *v3,
+void HELPER(gvec_vfs128)(void *v1, const void *v2, const void *v3,
                          CPUS390XState *env, uint32_t desc)
 {
-    vop64_3(v1, v2, v3, env, true, vfs64, GETPC());
+    const bool se = extract32(simd_data(desc), 3, 1);
+
+    vop128_3(v1, v2, v3, env, se, vfs128, GETPC());
+}
+
+void HELPER(gvec_vftci32)(void *v1, const void *v2, CPUS390XState *env,
+                          uint32_t desc)
+{
+    const uint16_t i3 = extract32(simd_data(desc), 4, 12);
+    const bool se = extract32(simd_data(desc), 3, 1);
+    int match = 0;
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        float32 a = s390_vec_read_float32(v2, i);
+
+        if (float32_dcmask(env, a) & i3) {
+            match++;
+            s390_vec_write_element32(v1, i, -1u);
+        } else {
+            s390_vec_write_element32(v1, i, 0);
+        }
+        if (se) {
+            break;
+        }
+    }
+
+    if (match == 4 || (se && match)) {
+        env->cc_op = 0;
+    } else if (match) {
+        env->cc_op = 1;
+    } else {
+        env->cc_op = 3;
+    }
 }
 
 static int vftci64(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
@@ -615,11 +1531,25 @@ static int vftci64(S390Vector *v1, const S390Vector *v2, CPUS390XState *env,
 void HELPER(gvec_vftci64)(void *v1, const void *v2, CPUS390XState *env,
                           uint32_t desc)
 {
-    env->cc_op = vftci64(v1, v2, env, false, simd_data(desc));
+    const bool se = extract32(simd_data(desc), 3, 1);
+    const uint16_t i3 = extract32(simd_data(desc), 4, 12);
+
+    env->cc_op = vftci64(v1, v2, env, se, i3);
 }
 
-void HELPER(gvec_vftci64s)(void *v1, const void *v2, CPUS390XState *env,
+void HELPER(gvec_vftci128)(void *v1, const void *v2, CPUS390XState *env,
                            uint32_t desc)
 {
-    env->cc_op = vftci64(v1, v2, env, true, simd_data(desc));
+    const float128 a = s390_vec_read_float128(v2);
+    const uint16_t i3 = extract32(simd_data(desc), 4, 12);
+
+    if (float128_dcmask(env, a) & i3) {
+        env->cc_op = 0;
+        s390_vec_write_element64(v1, 0, -1ull);
+        s390_vec_write_element64(v1, 1, -1ull);
+    } else {
+        env->cc_op = 3;
+        s390_vec_write_element64(v1, 0, 0);
+        s390_vec_write_element64(v1, 1, 0);
+    }
 }
diff --git a/qemu/target/s390x/vec_helper.c b/qemu/target/s390x/vec_helper.c
index fb774da37d..d9a2c0f268 100644
--- a/qemu/target/s390x/vec_helper.c
+++ b/qemu/target/s390x/vec_helper.c
@@ -19,6 +19,28 @@
 #include "exec/cpu_ldst.h"
 #include "exec/exec-all.h"
 
+void HELPER(gvec_vbperm)(void *v1, const void *v2, const void *v3,
+                         uint32_t desc)
+{
+    S390Vector tmp = { 0 };
+    uint16_t result = 0;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        const uint8_t bit_nr = s390_vec_read_element8(v3, i);
+        uint16_t bit;
+
+        if (bit_nr >= 128) {
+            continue;
+        }
+        bit = (s390_vec_read_element8(v2, bit_nr / 8)
+               >> (7 - (bit_nr % 8))) & 1;
+        result |= (bit << (15 - i));
+    }
+    s390_vec_write_element16(&tmp, 3, result);
+    *(S390Vector *)v1 = tmp;
+}
+
 void HELPER(vll)(CPUS390XState *env, void *v1, uint64_t addr, uint64_t bytes)
 {
     if (likely(bytes >= 16)) {
diff --git a/qemu/target/s390x/vec_int_helper.c b/qemu/target/s390x/vec_int_helper.c
index b81441395c..078af898f7 100644
--- a/qemu/target/s390x/vec_int_helper.c
+++ b/qemu/target/s390x/vec_int_helper.c
@@ -571,18 +571,77 @@ void HELPER(gvec_vsl)(void *v1, const void *v2, uint64_t count,
     s390_vec_shl(v1, v2, count);
 }
 
+void HELPER(gvec_vsl_ve2)(void *v1, const void *v2, const void *v3,
+                          uint32_t desc)
+{
+    S390Vector tmp;
+    uint32_t sh;
+    uint32_t e0;
+    uint32_t e1 = 0;
+    int i;
+
+    for (i = 15; i >= 0; --i, e1 = e0) {
+        e0 = s390_vec_read_element8(v2, i);
+        sh = s390_vec_read_element8(v3, i) & 7;
+        s390_vec_write_element8(&tmp, i, rol32(e0 | (e1 << 24), sh));
+    }
+
+    *(S390Vector *)v1 = tmp;
+}
+
 void HELPER(gvec_vsra)(void *v1, const void *v2, uint64_t count,
                        uint32_t desc)
 {
     s390_vec_sar(v1, v2, count);
 }
 
+void HELPER(gvec_vsra_ve2)(void *v1, const void *v2, const void *v3,
+                           uint32_t desc)
+{
+    S390Vector tmp;
+    uint32_t sh;
+    uint32_t e0;
+    uint32_t e1 = 0;
+    int i = 0;
+
+    e0 = (int32_t)(int8_t)s390_vec_read_element8(v2, i);
+    sh = s390_vec_read_element8(v3, i) & 7;
+    s390_vec_write_element8(&tmp, i, e0 >> sh);
+
+    e1 = e0;
+    for (i = 1; i < 16; ++i, e1 = e0) {
+        e0 = s390_vec_read_element8(v2, i);
+        sh = s390_vec_read_element8(v3, i) & 7;
+        s390_vec_write_element8(&tmp, i, (e0 | (e1 << 8)) >> sh);
+    }
+
+    *(S390Vector *)v1 = tmp;
+}
+
 void HELPER(gvec_vsrl)(void *v1, const void *v2, uint64_t count,
                        uint32_t desc)
 {
     s390_vec_shr(v1, v2, count);
 }
 
+void HELPER(gvec_vsrl_ve2)(void *v1, const void *v2, const void *v3,
+                           uint32_t desc)
+{
+    S390Vector tmp;
+    uint32_t sh;
+    uint32_t e0;
+    uint32_t e1 = 0;
+    int i;
+
+    for (i = 0; i < 16; ++i, e1 = e0) {
+        e0 = s390_vec_read_element8(v2, i);
+        sh = s390_vec_read_element8(v3, i) & 7;
+        s390_vec_write_element8(&tmp, i, (e0 | (e1 << 8)) >> sh);
+    }
+
+    *(S390Vector *)v1 = tmp;
+}
+
 #define DEF_VSCBI(BITS)                                                        \
 void HELPER(gvec_vscbi##BITS)(void *v1, const void *v2, const void *v3,        \
                               uint32_t desc)                                   \
diff --git a/qemu/target/s390x/vec_string_helper.c b/qemu/target/s390x/vec_string_helper.c
index c75152feb5..90d2c54a9b 100644
--- a/qemu/target/s390x/vec_string_helper.c
+++ b/qemu/target/s390x/vec_string_helper.c
@@ -471,3 +471,86 @@ void HELPER(gvec_vstrc_cc_rt##BITS)(void *v1, const void *v2, const void *v3,  \
 DEF_VSTRC_CC_RT_HELPER(8)
 DEF_VSTRC_CC_RT_HELPER(16)
 DEF_VSTRC_CC_RT_HELPER(32)
+
+static int vstrs(void *v1, const void *v2, const void *v3, const void *v4,
+                 uint8_t es, bool zs)
+{
+    int substr_elen, i, j, k, cc;
+    int nelem = 16 >> es;
+    int str_leftmost_0;
+
+    substr_elen = s390_vec_read_element8(v4, 7) >> es;
+
+    if (zs) {
+        substr_elen = MIN(substr_elen, nelem);
+        for (i = 0; i < substr_elen; i++) {
+            if (s390_vec_read_element(v3, i, es) == 0) {
+                substr_elen = i;
+                break;
+            }
+        }
+    }
+
+    if (substr_elen == 0) {
+        cc = 2;
+        k = 0;
+        goto done;
+    }
+
+    str_leftmost_0 = nelem;
+    if (zs) {
+        for (k = 0; k < nelem; k++) {
+            if (s390_vec_read_element(v2, k, es) == 0) {
+                str_leftmost_0 = k;
+                break;
+            }
+        }
+    }
+
+    cc = str_leftmost_0 == nelem ? 0 : 1;
+    for (k = 0; k < nelem; k++) {
+        i = MIN(nelem, k + substr_elen);
+        for (j = k; j < i; j++) {
+            uint32_t e2 = s390_vec_read_element(v2, j, es);
+            uint32_t e3 = s390_vec_read_element(v3, j - k, es);
+
+            if (e2 != e3) {
+                break;
+            }
+        }
+        if (j == i) {
+            if (k > str_leftmost_0) {
+                cc = 1;
+                k = nelem;
+            } else if (i - k == substr_elen) {
+                cc = 2;
+            } else {
+                cc = 3;
+            }
+            break;
+        }
+    }
+
+done:
+    s390_vec_write_element64(v1, 0, k << es);
+    s390_vec_write_element64(v1, 1, 0);
+    return cc;
+}
+
+#define DEF_VSTRS_HELPER(BITS)                                                \
+void HELPER(gvec_vstrs_##BITS)(void *v1, const void *v2, const void *v3,      \
+                               const void *v4, CPUS390XState *env,            \
+                               uint32_t desc)                                 \
+{                                                                              \
+    env->cc_op = vstrs(v1, v2, v3, v4, MO_##BITS, false);                     \
+}                                                                              \
+void HELPER(gvec_vstrs_zs##BITS)(void *v1, const void *v2, const void *v3,    \
+                                 const void *v4, CPUS390XState *env,          \
+                                 uint32_t desc)                               \
+{                                                                              \
+    env->cc_op = vstrs(v1, v2, v3, v4, MO_##BITS, true);                      \
+}
+
+DEF_VSTRS_HELPER(8)
+DEF_VSTRS_HELPER(16)
+DEF_VSTRS_HELPER(32)
diff --git a/qemu/target/sparc/fop_helper.c b/qemu/target/sparc/fop_helper.c
index 9eb9b75718..e6dd3fc313 100644
--- a/qemu/target/sparc/fop_helper.c
+++ b/qemu/target/sparc/fop_helper.c
@@ -264,7 +264,7 @@ void helper_fsqrtq(CPUSPARCState *env)
 #define GEN_FCMP(name, size, reg1, reg2, FS, E)                         \
     target_ulong glue(helper_, name) (CPUSPARCState *env)               \
     {                                                                   \
-        int ret;                                                        \
+        FloatRelation ret;                                              \
         target_ulong fsr;                                               \
         if (E) {                                                        \
             ret = glue(size, _compare)(reg1, reg2, &env->fp_status);    \
@@ -295,7 +295,7 @@ void helper_fsqrtq(CPUSPARCState *env)
 #define GEN_FCMP_T(name, size, FS, E)                                   \
     target_ulong glue(helper_, name)(CPUSPARCState *env, size src1, size src2)\
     {                                                                   \
-        int ret;                                                        \
+        FloatRelation ret;                                              \
         target_ulong fsr;                                               \
         if (E) {                                                        \
             ret = glue(size, _compare)(src1, src2, &env->fp_status);    \
diff --git a/qemu/target/sparc/unicorn.c b/qemu/target/sparc/unicorn.c
index a2607282e7..21d9777513 100644
--- a/qemu/target/sparc/unicorn.c
+++ b/qemu/target/sparc/unicorn.c
@@ -29,6 +29,60 @@ static uint64_t sparc_get_pc(struct uc_struct *uc)
     return ((CPUSPARCState *)uc->cpu->env_ptr)->pc;
 }
 
+static uint32_t sparc_get_fpr_f(CPUSPARCState *env, unsigned int reg)
+{
+    CPU_DoubleU *fpr = &env->fpr[reg / 2];
+
+    return (reg & 1) ? fpr->l.lower : fpr->l.upper;
+}
+
+static void sparc_set_fpr_f(CPUSPARCState *env, unsigned int reg,
+                            uint32_t value)
+{
+    CPU_DoubleU *fpr = &env->fpr[reg / 2];
+
+    if (reg & 1) {
+        fpr->l.lower = value;
+    } else {
+        fpr->l.upper = value;
+    }
+}
+
+static uint32_t sparc_get_fcc(CPUSPARCState *env, unsigned int offset)
+{
+    return ((env->fsr >> (FSR_FCC0_SHIFT + offset)) & 1) |
+           (((env->fsr >> (FSR_FCC1_SHIFT + offset)) & 1) << 1);
+}
+
+static void sparc_set_fcc(CPUSPARCState *env, unsigned int offset,
+                          uint32_t value)
+{
+    target_ulong mask = (FSR_FCC0 | FSR_FCC1) << offset;
+
+    env->fsr &= ~mask;
+    if (value & 1) {
+        env->fsr |= FSR_FCC0 << offset;
+    }
+    if (value & 2) {
+        env->fsr |= FSR_FCC1 << offset;
+    }
+}
+
+static uint32_t sparc_get_icc(CPUSPARCState *env)
+{
+    if (env->cc_op != CC_OP_FLAGS && env->cc_op != CC_OP_DYNAMIC) {
+        cpu_get_psr(env);
+    }
+    return (env->psr & PSR_ICC) >> PSR_CARRY_SHIFT;
+}
+
+static void sparc_set_icc(CPUSPARCState *env, uint32_t value)
+{
+    env->psr = (env->psr & ~PSR_ICC) |
+               ((value & 0xf) << PSR_CARRY_SHIFT);
+    env->cc_op = CC_OP_FLAGS;
+}
+
 static void sparc_release(void *ctx)
 {
     int i;
@@ -71,6 +125,10 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
     if (regid >= UC_SPARC_REG_G0 && regid <= UC_SPARC_REG_G7) {
         CHECK_REG_TYPE(uint32_t);
         *(uint32_t *)value = env->gregs[regid - UC_SPARC_REG_G0];
+    } else if (regid >= UC_SPARC_REG_F0 && regid <= UC_SPARC_REG_F31) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = sparc_get_fpr_f(env,
+                                             regid - UC_SPARC_REG_F0);
     } else if (regid >= UC_SPARC_REG_O0 && regid <= UC_SPARC_REG_O7) {
         CHECK_REG_TYPE(uint32_t);
         *(uint32_t *)value = env->regwptr[regid - UC_SPARC_REG_O0];
@@ -82,10 +140,19 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
         *(uint32_t *)value = env->regwptr[16 + regid - UC_SPARC_REG_I0];
     } else if (regid == UC_SPARC_REG_PSR) {
         CHECK_REG_TYPE(uint32_t);
-        if (env->cc_op != CC_OP_FLAGS) {
+        if (env->cc_op != CC_OP_FLAGS && env->cc_op != CC_OP_DYNAMIC) {
             cpu_get_psr(env);
         }
         *(uint32_t *)value = env->psr;
+    } else if (regid == UC_SPARC_REG_FCC0) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = sparc_get_fcc(env, 0);
+    } else if (regid == UC_SPARC_REG_ICC) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = sparc_get_icc(env);
+    } else if (regid == UC_SPARC_REG_Y) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = env->y;
     } else {
         switch (regid) {
         default:
@@ -111,6 +178,9 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
     if (regid >= UC_SPARC_REG_G0 && regid <= UC_SPARC_REG_G7) {
         CHECK_REG_TYPE(uint32_t);
         env->gregs[regid - UC_SPARC_REG_G0] = *(uint32_t *)value;
+    } else if (regid >= UC_SPARC_REG_F0 && regid <= UC_SPARC_REG_F31) {
+        CHECK_REG_TYPE(uint32_t);
+        sparc_set_fpr_f(env, regid - UC_SPARC_REG_F0, *(uint32_t *)value);
     } else if (regid >= UC_SPARC_REG_O0 && regid <= UC_SPARC_REG_O7) {
         CHECK_REG_TYPE(uint32_t);
         env->regwptr[regid - UC_SPARC_REG_O0] = *(uint32_t *)value;
@@ -124,6 +194,15 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
         CHECK_REG_TYPE(uint32_t);
         env->psr = *(uint32_t *)value;
         cpu_put_psr(env, env->psr);
+    } else if (regid == UC_SPARC_REG_FCC0) {
+        CHECK_REG_TYPE(uint32_t);
+        sparc_set_fcc(env, 0, *(uint32_t *)value);
+    } else if (regid == UC_SPARC_REG_ICC) {
+        CHECK_REG_TYPE(uint32_t);
+        sparc_set_icc(env, *(uint32_t *)value);
+    } else if (regid == UC_SPARC_REG_Y) {
+        CHECK_REG_TYPE(uint32_t);
+        env->y = *(uint32_t *)value;
     } else {
         switch (regid) {
         default:
diff --git a/qemu/target/sparc/unicorn64.c b/qemu/target/sparc/unicorn64.c
index 11c46c8b0c..cfb609d1b1 100644
--- a/qemu/target/sparc/unicorn64.c
+++ b/qemu/target/sparc/unicorn64.c
@@ -12,11 +12,11 @@ const int SPARC64_REGS_STORAGE_SIZE = offsetof(CPUSPARCState, irq_manager);
 
 static bool sparc_stop_interrupt(struct uc_struct *uc, int intno)
 {
-    switch(intno) {
-        default:
-            return false;
-        case TT_ILL_INSN:
-            return true;
+    switch (intno) {
+    default:
+        return false;
+    case TT_ILL_INSN:
+        return true;
     }
 }
 
@@ -31,6 +31,74 @@ static uint64_t sparc_get_pc(struct uc_struct *uc)
     return ((CPUSPARCState *)uc->cpu->env_ptr)->pc;
 }
 
+static uint32_t sparc_get_fpr_f(CPUSPARCState *env, unsigned int reg)
+{
+    CPU_DoubleU *fpr = &env->fpr[reg / 2];
+
+    return (reg & 1) ? fpr->l.lower : fpr->l.upper;
+}
+
+static void sparc_set_fpr_f(CPUSPARCState *env, unsigned int reg,
+                            uint32_t value)
+{
+    CPU_DoubleU *fpr = &env->fpr[reg / 2];
+
+    if (reg & 1) {
+        fpr->l.lower = value;
+    } else {
+        fpr->l.upper = value;
+    }
+}
+
+static uint32_t sparc64_fpr_number(unsigned int regid)
+{
+    return 32 + (regid - UC_SPARC_REG_F32) * 2;
+}
+
+static uint32_t sparc_get_fcc(CPUSPARCState *env, unsigned int offset)
+{
+    return ((env->fsr >> (FSR_FCC0_SHIFT + offset)) & 1) |
+           (((env->fsr >> (FSR_FCC1_SHIFT + offset)) & 1) << 1);
+}
+
+static void sparc_set_fcc(CPUSPARCState *env, unsigned int offset,
+                          uint32_t value)
+{
+    target_ulong mask = (FSR_FCC0 | FSR_FCC1) << offset;
+
+    env->fsr &= ~mask;
+    if (value & 1) {
+        env->fsr |= FSR_FCC0 << offset;
+    }
+    if (value & 2) {
+        env->fsr |= FSR_FCC1 << offset;
+    }
+}
+
+static unsigned int sparc_fcc_offset(unsigned int regid)
+{
+    static const unsigned int fcc_offsets[] = { 0, 22, 24, 26 };
+
+    return fcc_offsets[regid - UC_SPARC_REG_FCC0];
+}
+
+static target_ulong sparc_get_ccr(CPUSPARCState *env)
+{
+    if (env->cc_op != CC_OP_FLAGS && env->cc_op != CC_OP_DYNAMIC) {
+        return cpu_get_ccr(env);
+    }
+    return ((env->xcc >> PSR_CARRY_SHIFT) << 4) |
+           ((env->psr & PSR_ICC) >> PSR_CARRY_SHIFT);
+}
+
+static void sparc_set_ccr(CPUSPARCState *env, target_ulong value)
+{
+    env->xcc = ((value >> 4) & 0xf) << PSR_CARRY_SHIFT;
+    env->psr = (env->psr & ~PSR_ICC) |
+               ((value & 0xf) << PSR_CARRY_SHIFT);
+    env->cc_op = CC_OP_FLAGS;
+}
+
 static void sparc_release(void *ctx)
 {
     release_common(ctx);
@@ -85,7 +153,7 @@ static void reg_reset(struct uc_struct *uc)
 
 DEFAULT_VISIBILITY
 uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
-                       size_t *size)
+                size_t *size)
 {
     CPUSPARCState *env = _env;
     uc_err ret = UC_ERR_ARG;
@@ -93,6 +161,15 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
     if (regid >= UC_SPARC_REG_G0 && regid <= UC_SPARC_REG_G7) {
         CHECK_REG_TYPE(uint64_t);
         *(uint64_t *)value = env->gregs[regid - UC_SPARC_REG_G0];
+    } else if (regid >= UC_SPARC_REG_F0 && regid <= UC_SPARC_REG_F31) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = sparc_get_fpr_f(env,
+                                             regid - UC_SPARC_REG_F0);
+    } else if (regid >= UC_SPARC_REG_F32 && regid <= UC_SPARC_REG_F62) {
+        uint32_t reg = sparc64_fpr_number(regid);
+
+        CHECK_REG_TYPE(uint64_t);
+        *(uint64_t *)value = env->fpr[reg / 2].ll;
     } else if (regid >= UC_SPARC_REG_O0 && regid <= UC_SPARC_REG_O7) {
         CHECK_REG_TYPE(uint64_t);
         *(uint64_t *)value = env->regwptr[regid - UC_SPARC_REG_O0];
@@ -102,8 +179,20 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
     } else if (regid >= UC_SPARC_REG_I0 && regid <= UC_SPARC_REG_I7) {
         CHECK_REG_TYPE(uint64_t);
         *(uint64_t *)value = env->regwptr[16 + regid - UC_SPARC_REG_I0];
+    } else if (regid >= UC_SPARC_REG_FCC0 && regid <= UC_SPARC_REG_FCC3) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = sparc_get_fcc(env, sparc_fcc_offset(regid));
+    } else if (regid == UC_SPARC_REG_ICC) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = sparc_get_ccr(env) & 0xf;
+    } else if (regid == UC_SPARC_REG_Y) {
+        CHECK_REG_TYPE(uint64_t);
+        *(uint64_t *)value = env->y;
+    } else if (regid == UC_SPARC_REG_XCC) {
+        CHECK_REG_TYPE(uint32_t);
+        *(uint32_t *)value = (sparc_get_ccr(env) >> 4) & 0xf;
     } else {
-        switch(regid) {
+        switch (regid) {
         default:
             break;
         case UC_SPARC_REG_PC:
@@ -113,12 +202,13 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
         }
     }
 
+    CHECK_RET_DEPRECATE(ret, regid);
     return ret;
 }
 
 DEFAULT_VISIBILITY
 uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
-                        size_t *size, int *setpc)
+                 size_t *size, int *setpc)
 {
     CPUSPARCState *env = _env;
     uc_err ret = UC_ERR_ARG;
@@ -126,6 +216,14 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
     if (regid >= UC_SPARC_REG_G0 && regid <= UC_SPARC_REG_G7) {
         CHECK_REG_TYPE(uint64_t);
         env->gregs[regid - UC_SPARC_REG_G0] = *(uint64_t *)value;
+    } else if (regid >= UC_SPARC_REG_F0 && regid <= UC_SPARC_REG_F31) {
+        CHECK_REG_TYPE(uint32_t);
+        sparc_set_fpr_f(env, regid - UC_SPARC_REG_F0, *(uint32_t *)value);
+    } else if (regid >= UC_SPARC_REG_F32 && regid <= UC_SPARC_REG_F62) {
+        uint32_t reg = sparc64_fpr_number(regid);
+
+        CHECK_REG_TYPE(uint64_t);
+        env->fpr[reg / 2].ll = *(uint64_t *)value;
     } else if (regid >= UC_SPARC_REG_O0 && regid <= UC_SPARC_REG_O7) {
         CHECK_REG_TYPE(uint64_t);
         env->regwptr[regid - UC_SPARC_REG_O0] = *(uint64_t *)value;
@@ -135,8 +233,28 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
     } else if (regid >= UC_SPARC_REG_I0 && regid <= UC_SPARC_REG_I7) {
         CHECK_REG_TYPE(uint64_t);
         env->regwptr[16 + regid - UC_SPARC_REG_I0] = *(uint64_t *)value;
+    } else if (regid >= UC_SPARC_REG_FCC0 && regid <= UC_SPARC_REG_FCC3) {
+        CHECK_REG_TYPE(uint32_t);
+        sparc_set_fcc(env, sparc_fcc_offset(regid), *(uint32_t *)value);
+    } else if (regid == UC_SPARC_REG_ICC) {
+        target_ulong ccr;
+
+        CHECK_REG_TYPE(uint32_t);
+        ccr = sparc_get_ccr(env) & ~0xf;
+        ccr |= *(uint32_t *)value & 0xf;
+        sparc_set_ccr(env, ccr);
+    } else if (regid == UC_SPARC_REG_Y) {
+        CHECK_REG_TYPE(uint64_t);
+        env->y = *(uint64_t *)value;
+    } else if (regid == UC_SPARC_REG_XCC) {
+        target_ulong ccr;
+
+        CHECK_REG_TYPE(uint32_t);
+        ccr = sparc_get_ccr(env) & ~0xf0;
+        ccr |= (*(uint32_t *)value & 0xf) << 4;
+        sparc_set_ccr(env, ccr);
     } else {
-        switch(regid) {
+        switch (regid) {
         default:
             break;
         case UC_SPARC_REG_PC:
@@ -148,6 +266,7 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
         }
     }
 
+    CHECK_RET_DEPRECATE(ret, regid);
     return ret;
 }
 
diff --git a/qemu/target/sparc/vis_helper.c b/qemu/target/sparc/vis_helper.c
index 8a9b763d0b..b51e308dfc 100644
--- a/qemu/target/sparc/vis_helper.c
+++ b/qemu/target/sparc/vis_helper.c
@@ -42,7 +42,7 @@ target_ulong helper_array8(target_ulong pixel_addr, target_ulong cubesize)
         GET_FIELD_SP(pixel_addr, 11, 12);
 }
 
-#ifdef HOST_WORDS_BIGENDIAN
+#if HOST_BIG_ENDIAN
 #define VIS_B64(n) b[7 - (n)]
 #define VIS_W64(n) w[3 - (n)]
 #define VIS_SW64(n) sw[3 - (n)]
@@ -470,7 +470,7 @@ uint64_t helper_bshuffle(uint64_t gsr, uint64_t src1, uint64_t src2)
     uint32_t i, mask, host;
 
     /* Set up S such that we can index across all of the bytes.  */
-#ifdef HOST_WORDS_BIGENDIAN
+#if HOST_BIG_ENDIAN
     s.ll[0] = src1;
     s.ll[1] = src2;
     host = 0;
diff --git a/qemu/target/tricore/unicorn.c b/qemu/target/tricore/unicorn.c
index ba27a26481..7c539ea999 100644
--- a/qemu/target/tricore/unicorn.c
+++ b/qemu/target/tricore/unicorn.c
@@ -27,6 +27,18 @@ static uint64_t tricore_get_pc(struct uc_struct *uc)
     return ((CPUTriCoreState *)uc->cpu->env_ptr)->PC;
 }
 
+#define TRICORE_READ_REG_FIELD(reg, field)                                    \
+    case reg:                                                                 \
+        CHECK_REG_TYPE(uint32_t);                                             \
+        *(uint32_t *)value = env->field;                                      \
+        break
+
+#define TRICORE_WRITE_REG_FIELD(reg, field)                                   \
+    case reg:                                                                 \
+        CHECK_REG_TYPE(uint32_t);                                             \
+        env->field = *(uint32_t *)value;                                      \
+        break
+
 static void reg_reset(struct uc_struct *uc)
 {
     CPUTriCoreState *env;
@@ -77,7 +89,7 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
             break;
         case UC_TRICORE_REG_PSW:
             CHECK_REG_TYPE(uint32_t);
-            *(uint32_t *)value = env->PSW;
+            *(uint32_t *)value = psw_read(env);
             break;
         case UC_TRICORE_REG_PSW_USB_C:
             CHECK_REG_TYPE(uint32_t);
@@ -135,6 +147,60 @@ uc_err reg_read(void *_env, int mode, unsigned int regid, void *value,
             CHECK_REG_TYPE(uint32_t);
             *(uint32_t *)value = env->COMPAT;
             break;
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR0_U, DPR0_0U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR1_U, DPR0_1U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR2_U, DPR0_2U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR3_U, DPR0_3U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR0_L, DPR0_0L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR1_L, DPR0_1L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR2_L, DPR0_2L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPR3_L, DPR0_3L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR0_U, CPR0_0U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR1_U, CPR0_1U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR2_U, CPR0_2U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR3_U, CPR0_3U);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR0_L, CPR0_0L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR1_L, CPR0_1L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR2_L, CPR0_2L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPR3_L, CPR0_3L);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPM0, DPM0);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPM1, DPM1);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPM2, DPM2);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DPM3, DPM3);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPM0, CPM0);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPM1, CPM1);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPM2, CPM2);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CPM3, CPM3);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_MMU_CON, MMU_CON);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_MMU_ASI, MMU_ASI);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_MMU_TVA, MMU_TVA);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_MMU_TPA, MMU_TPA);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_MMU_TPX, MMU_TPX);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_MMU_TFA, MMU_TFA);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_BMACON, BMACON);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_SMACON, SMACON);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DIEAR, DIEAR);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DIETR, DIETR);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CCDIER, CCDIER);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_MIECON, MIECON);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_PIEAR, PIEAR);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_PIETR, PIETR);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CCPIER, CCPIER);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DBGSR, DBGSR);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_EXEVT, EXEVT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CREVT, CREVT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_SWEVT, SWEVT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_TR0EVT, TR0EVT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_TR1EVT, TR1EVT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DMS, DMS);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DCX, DCX);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_DBGTCR, DBGTCR);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CCTRL, CCTRL);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_CCNT, CCNT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_ICNT, ICNT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_M1CNT, M1CNT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_M2CNT, M2CNT);
+        TRICORE_READ_REG_FIELD(UC_TRICORE_REG_M3CNT, M3CNT);
         }
     }
 
@@ -181,7 +247,7 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             break;
         case UC_TRICORE_REG_PSW:
             CHECK_REG_TYPE(uint32_t);
-            env->PSW = *(uint32_t *)value;
+            psw_write(env, *(uint32_t *)value);
             break;
         case UC_TRICORE_REG_PSW_USB_C:
             CHECK_REG_TYPE(uint32_t);
@@ -239,6 +305,60 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
             CHECK_REG_TYPE(uint32_t);
             env->COMPAT = *(uint32_t *)value;
             break;
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR0_U, DPR0_0U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR1_U, DPR0_1U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR2_U, DPR0_2U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR3_U, DPR0_3U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR0_L, DPR0_0L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR1_L, DPR0_1L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR2_L, DPR0_2L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPR3_L, DPR0_3L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR0_U, CPR0_0U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR1_U, CPR0_1U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR2_U, CPR0_2U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR3_U, CPR0_3U);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR0_L, CPR0_0L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR1_L, CPR0_1L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR2_L, CPR0_2L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPR3_L, CPR0_3L);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPM0, DPM0);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPM1, DPM1);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPM2, DPM2);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DPM3, DPM3);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPM0, CPM0);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPM1, CPM1);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPM2, CPM2);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CPM3, CPM3);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_MMU_CON, MMU_CON);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_MMU_ASI, MMU_ASI);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_MMU_TVA, MMU_TVA);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_MMU_TPA, MMU_TPA);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_MMU_TPX, MMU_TPX);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_MMU_TFA, MMU_TFA);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_BMACON, BMACON);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_SMACON, SMACON);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DIEAR, DIEAR);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DIETR, DIETR);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CCDIER, CCDIER);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_MIECON, MIECON);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_PIEAR, PIEAR);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_PIETR, PIETR);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CCPIER, CCPIER);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DBGSR, DBGSR);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_EXEVT, EXEVT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CREVT, CREVT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_SWEVT, SWEVT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_TR0EVT, TR0EVT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_TR1EVT, TR1EVT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DMS, DMS);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DCX, DCX);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_DBGTCR, DBGTCR);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CCTRL, CCTRL);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_CCNT, CCNT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_ICNT, ICNT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_M1CNT, M1CNT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_M2CNT, M2CNT);
+        TRICORE_WRITE_REG_FIELD(UC_TRICORE_REG_M3CNT, M3CNT);
         }
     }
 
@@ -246,6 +366,9 @@ uc_err reg_write(void *_env, int mode, unsigned int regid, const void *value,
     return ret;
 }
 
+#undef TRICORE_READ_REG_FIELD
+#undef TRICORE_WRITE_REG_FIELD
+
 static int tricore_cpus_init(struct uc_struct *uc, const char *cpu_model)
 {
     TriCoreCPU *cpu;
diff --git a/qemu/tcg/README b/qemu/tcg/README
index bfa2e4ed24..bc15cc3b32 100644
--- a/qemu/tcg/README
+++ b/qemu/tcg/README
@@ -254,6 +254,12 @@ t0 = t1 ? clz(t1) : t2
 
 t0 = t1 ? ctz(t1) : t2
 
+* ctpop_i32/i64 t0, t1
+
+t0 = number of bits set in t1
+With "ctpop" short for "count population", matching
+the function name used in include/qemu/host-utils.h.
+
 ********* Shifts/Rotates
 
 * shl_i32/i64 t0, t1, t2
@@ -295,19 +301,25 @@ ext32u_i64 t0, t1
 
 8, 16 or 32 bit sign/zero extension (both operands must have the same type)
 
-* bswap16_i32/i64 t0, t1
+* bswap16_i32/i64 t0, t1, flags
 
-16 bit byte swap on a 32/64 bit value. It assumes that the two/six high order
-bytes are set to zero.
+16 bit byte swap on the low bits of a 32/64 bit input.
+If flags & TCG_BSWAP_IZ, then t1 is known to be zero-extended from bit 15.
+If flags & TCG_BSWAP_OZ, then t0 will be zero-extended from bit 15.
+If flags & TCG_BSWAP_OS, then t0 will be sign-extended from bit 15.
+If neither TCG_BSWAP_OZ nor TCG_BSWAP_OS are set, then the bits of
+t0 above bit 15 may contain any value.
 
-* bswap32_i32/i64 t0, t1
+* bswap32_i64 t0, t1, flags
 
-32 bit byte swap on a 32/64 bit value. With a 64 bit value, it assumes that
-the four high order bytes are set to zero.
+32 bit byte swap on a 64-bit value.  The flags are the same as for bswap16,
+except they apply from bit 31 instead of bit 15.
 
-* bswap64_i64 t0, t1
+* bswap32_i32 t0, t1, flags
+* bswap64_i64 t0, t1, flags
 
-64 bit byte swap
+32/64 bit byte swap.  The flags are ignored, but still present
+for consistency with the other bswap opcodes.
 
 * discard_i32/i64 t0
 
@@ -461,7 +473,7 @@ when MTTCG is enabled.
 The guest translators should generate this opcode for all guest instructions
 which have ordering side effects.
 
-Please see docs/devel/atomics.txt for more information on memory barriers.
+Please see docs/devel/atomics.rst for more information on memory barriers.
 
 ********* 64-bit guest on 32-bit host support
 
@@ -502,6 +514,7 @@ goto_ptr opcode, emitting this op is equivalent to emitting exit_tb(0).
 
 * qemu_ld_i32/i64 t0, t1, flags, memidx
 * qemu_st_i32/i64 t0, t1, flags, memidx
+* qemu_st8_i32 t0, t1, flags, memidx
 
 Load data at the guest address t1 into t0, or store data in t0 at guest
 address t1.  The _i32/_i64 size applies to the size of the input/output
@@ -518,6 +531,10 @@ of the memory access.
 For a 32-bit host, qemu_ld/st_i64 is guaranteed to only be used with a
 64-bit memory access specified in flags.
 
+For i386, qemu_st8_i32 is exactly like qemu_st_i32, except the size of
+the memory operation is known to be 8-bit.  This allows the backend to
+provide a different set of register constraints.
+
 ********* Host vector operations
 
 All of the vector ops have two parameters, TCGOP_VECL & TCGOP_VECE.
@@ -605,10 +622,11 @@ E.g. VECL=1 -> 64 << 1 -> v128, and VECE=2 -> 1 << 2 -> i32.
 
 * shri_vec   v0, v1, i2
 * sari_vec   v0, v1, i2
+* rotli_vec  v0, v1, i2
 * shrs_vec   v0, v1, s2
 * sars_vec   v0, v1, s2
 
-  Similarly for logical and arithmetic right shift.
+  Similarly for logical and arithmetic right shift, and left rotate.
 
 * shlv_vec   v0, v1, v2
 
@@ -620,8 +638,10 @@ E.g. VECL=1 -> 64 << 1 -> v128, and VECE=2 -> 1 << 2 -> i32.
 
 * shrv_vec   v0, v1, v2
 * sarv_vec   v0, v1, v2
+* rotlv_vec  v0, v1, v2
+* rotrv_vec  v0, v1, v2
 
-  Similarly for logical and arithmetic right shift.
+  Similarly for logical and arithmetic right shift, and rotates.
 
 * cmp_vec  v0, v1, v2, cond
 
@@ -649,7 +669,7 @@ function tcg_gen_xxx(args).
 
 4) Backend
 
-tcg-target.h contains the target specific definitions. tcg-target.inc.c
+tcg-target.h contains the target specific definitions. tcg-target.c.inc
 contains the target specific code; it is #included by tcg/tcg.c, rather
 than being a standalone C file.
 
diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index 50c9e595bb..44eceb3a6c 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -1550,7 +1550,7 @@ static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d,
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     TCGMemOpIdx oi, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
     [MO_LEUL] = helper_le_ldul_mmu,
@@ -1564,7 +1564,7 @@ static void * const qemu_ld_helpers[16] = {
  *                                     uintxx_t val, TCGMemOpIdx oi,
  *                                     uintptr_t ra)
  */
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/aarch64/tcg-target.opc.h b/qemu/tcg/aarch64/tcg-target.opc.h
index 26bfd9c460..bce30accd9 100644
--- a/qemu/tcg/aarch64/tcg-target.opc.h
+++ b/qemu/tcg/aarch64/tcg-target.opc.h
@@ -12,3 +12,4 @@
  */
 
 DEF(aa64_sshl_vec, 1, 2, 0, IMPLVEC)
+DEF(aa64_sli_vec, 1, 2, 1, IMPLVEC)
diff --git a/qemu/tcg/arm/tcg-target.h b/qemu/tcg/arm/tcg-target.h
index 63d5b253b5..486269eff1 100644
--- a/qemu/tcg/arm/tcg-target.h
+++ b/qemu/tcg/arm/tcg-target.h
@@ -26,40 +26,14 @@
 #ifndef ARM_TCG_TARGET_H
 #define ARM_TCG_TARGET_H
 
-/* The __ARM_ARCH define is provided by gcc 4.8.  Construct it otherwise.  */
-#ifndef __ARM_ARCH
-# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
-     || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \
-     || defined(__ARM_ARCH_7EM__)
-#  define __ARM_ARCH 7
-# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \
-       || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) \
-       || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6T2__)
-#  define __ARM_ARCH 6
-# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5E__) \
-       || defined(__ARM_ARCH_5T__) || defined(__ARM_ARCH_5TE__) \
-       || defined(__ARM_ARCH_5TEJ__)
-#  define __ARM_ARCH 5
-# else
-#  define __ARM_ARCH 4
-# endif
-#endif
-
 extern int arm_arch;
 
-#if defined(__ARM_ARCH_5T__) \
-    || defined(__ARM_ARCH_5TE__) || defined(__ARM_ARCH_5TEJ__)
-# define use_armv5t_instructions 1
-#else
-# define use_armv5t_instructions use_armv6_instructions
-#endif
-
-#define use_armv6_instructions  (__ARM_ARCH >= 6 || arm_arch >= 6)
 #define use_armv7_instructions  (__ARM_ARCH >= 7 || arm_arch >= 7)
 
 #undef TCG_TARGET_STACK_GROWSUP
 #define TCG_TARGET_INSN_UNIT_SIZE 4
 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 16
+#define MAX_CODE_GEN_BUFFER_SIZE  UINT32_MAX
 
 typedef enum {
     TCG_REG_R0 = 0,
@@ -78,17 +52,40 @@ typedef enum {
     TCG_REG_R13,
     TCG_REG_R14,
     TCG_REG_PC,
+
+    TCG_REG_Q0,
+    TCG_REG_Q1,
+    TCG_REG_Q2,
+    TCG_REG_Q3,
+    TCG_REG_Q4,
+    TCG_REG_Q5,
+    TCG_REG_Q6,
+    TCG_REG_Q7,
+    TCG_REG_Q8,
+    TCG_REG_Q9,
+    TCG_REG_Q10,
+    TCG_REG_Q11,
+    TCG_REG_Q12,
+    TCG_REG_Q13,
+    TCG_REG_Q14,
+    TCG_REG_Q15,
+
+    TCG_AREG0 = TCG_REG_R6,
+    TCG_REG_CALL_STACK = TCG_REG_R13,
 } TCGReg;
 
-#define TCG_TARGET_NB_REGS 16
+#define TCG_TARGET_NB_REGS 32
 
 #ifndef __ARM_ARCH_EXT_IDIV__
 extern bool use_idiv_instructions;  // Unicorn: Don't have the same name with macro
 #endif
-
+#ifdef __ARM_NEON__
+#define use_neon_instructions  1
+#else
+extern bool use_neon_instructions;
+#endif
 
 /* used for function call generation */
-#define TCG_REG_CALL_STACK		TCG_REG_R13
 #define TCG_TARGET_STACK_ALIGN		8
 #define TCG_TARGET_CALL_ALIGN_ARGS	1
 #define TCG_TARGET_CALL_STACK_OFFSET	0
@@ -108,7 +105,7 @@ extern bool use_idiv_instructions;  // Unicorn: Don't have the same name with ma
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
-#define TCG_TARGET_HAS_clz_i32          use_armv5t_instructions
+#define TCG_TARGET_HAS_clz_i32          1
 #define TCG_TARGET_HAS_ctz_i32          use_armv7_instructions
 #define TCG_TARGET_HAS_ctpop_i32        0
 #define TCG_TARGET_HAS_deposit_i32      use_armv7_instructions
@@ -126,27 +123,40 @@ extern bool use_idiv_instructions;  // Unicorn: Don't have the same name with ma
 #define TCG_TARGET_HAS_div_i32          use_idiv_instructions
 #endif
 #define TCG_TARGET_HAS_rem_i32          0
-#define TCG_TARGET_HAS_goto_ptr         1
 #define TCG_TARGET_HAS_direct_jump      0
-
-enum {
-    TCG_AREG0 = TCG_REG_R6,
-};
+#define TCG_TARGET_HAS_qemu_st8_i32     0
+
+#define TCG_TARGET_HAS_v64              use_neon_instructions
+#define TCG_TARGET_HAS_v128             use_neon_instructions
+#define TCG_TARGET_HAS_v256             0
+
+#define TCG_TARGET_HAS_andc_vec         1
+#define TCG_TARGET_HAS_orc_vec          1
+#define TCG_TARGET_HAS_nand_vec         0
+#define TCG_TARGET_HAS_nor_vec          0
+#define TCG_TARGET_HAS_eqv_vec          0
+#define TCG_TARGET_HAS_not_vec          1
+#define TCG_TARGET_HAS_neg_vec          1
+#define TCG_TARGET_HAS_abs_vec          1
+#define TCG_TARGET_HAS_roti_vec         0
+#define TCG_TARGET_HAS_rots_vec         0
+#define TCG_TARGET_HAS_rotv_vec         0
+#define TCG_TARGET_HAS_shi_vec          1
+#define TCG_TARGET_HAS_shs_vec          0
+#define TCG_TARGET_HAS_shv_vec          0
+#define TCG_TARGET_HAS_mul_vec          1
+#define TCG_TARGET_HAS_sat_vec          1
+#define TCG_TARGET_HAS_minmax_vec       1
+#define TCG_TARGET_HAS_bitsel_vec       1
+#define TCG_TARGET_HAS_cmpsel_vec       0
 
 #define TCG_TARGET_DEFAULT_MO (0)
-#define TCG_TARGET_HAS_MEMORY_BSWAP     1
-
-static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
-{
-    __builtin___clear_cache((char *) start, (char *) stop);
-}
+#define TCG_TARGET_HAS_MEMORY_BSWAP     0
 
 /* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
-#ifdef CONFIG_SOFTMMU
 #define TCG_TARGET_NEED_LDST_LABELS
-#endif
 #define TCG_TARGET_NEED_POOL_LABELS
 
 #endif
diff --git a/qemu/tcg/arm/tcg-target.inc.c b/qemu/tcg/arm/tcg-target.inc.c
index 467d063690..22f4e47220 100644
--- a/qemu/tcg/arm/tcg-target.inc.c
+++ b/qemu/tcg/arm/tcg-target.inc.c
@@ -1136,7 +1136,7 @@ static TCGCond tcg_out_cmp2(TCGContext *s, const TCGArg *args,
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
 
@@ -1156,7 +1156,7 @@ static void * const qemu_ld_helpers[16] = {
 /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
  *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/i386/tcg-target.inc.c b/qemu/tcg/i386/tcg-target.inc.c
index 15cc1c05d9..b4fd036d4c 100644
--- a/qemu/tcg/i386/tcg-target.inc.c
+++ b/qemu/tcg/i386/tcg-target.inc.c
@@ -1656,7 +1656,7 @@ static void tcg_out_nopn(TCGContext *s, int n)
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
     [MO_LEUL] = helper_le_ldul_mmu,
@@ -1669,7 +1669,7 @@ static void * const qemu_ld_helpers[16] = {
 /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
  *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/loongarch64/tcg-target.inc.c b/qemu/tcg/loongarch64/tcg-target.inc.c
index cdcd11800a..0f2aa5304f 100644
--- a/qemu/tcg/loongarch64/tcg-target.inc.c
+++ b/qemu/tcg/loongarch64/tcg-target.inc.c
@@ -893,7 +893,7 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
 /*
  * Load/store helpers for SoftMMU, and qemu_ld/st implementations
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
@@ -910,7 +910,7 @@ static void * const qemu_ld_helpers[16] = {
 #endif
 };
 
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/mips/tcg-target.h b/qemu/tcg/mips/tcg-target.h
index c6b091d849..7669213175 100644
--- a/qemu/tcg/mips/tcg-target.h
+++ b/qemu/tcg/mips/tcg-target.h
@@ -39,6 +39,8 @@
 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 16
 #define TCG_TARGET_NB_REGS 32
 
+#define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
+
 typedef enum {
     TCG_REG_ZERO = 0,
     TCG_REG_AT,
@@ -130,8 +132,7 @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_muluh_i32        1
 #define TCG_TARGET_HAS_mulsh_i32        1
 #define TCG_TARGET_HAS_bswap32_i32      1
-#define TCG_TARGET_HAS_goto_ptr         1
-#define TCG_TARGET_HAS_direct_jump      1
+#define TCG_TARGET_HAS_direct_jump      0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_add2_i32         0
@@ -169,6 +170,7 @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_clz_i32          use_mips32r2_instructions
 #define TCG_TARGET_HAS_ctz_i32          0
 #define TCG_TARGET_HAS_ctpop_i32        0
+#define TCG_TARGET_HAS_qemu_st8_i32     0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_movcond_i64      use_movnz_instructions
@@ -198,24 +200,13 @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_ext16u_i64       0 /* andi rt, rs, 0xffff */
 #endif
 
-#ifdef __OpenBSD__
-#include <machine/sysarch.h>
-#else
-#include <sys/cachectl.h>
-#endif
-
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
-static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
-{
-    cacheflush ((void *)start, stop-start, ICACHE);
-}
-
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+/* not defined -- call should be eliminated at compile time */
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t)
+    QEMU_ERROR("code path is reachable");
 
-#ifdef CONFIG_SOFTMMU
 #define TCG_TARGET_NEED_LDST_LABELS
-#endif
 
 #endif
diff --git a/qemu/tcg/mips/tcg-target.inc.c b/qemu/tcg/mips/tcg-target.inc.c
index ed5a9356c3..7eb598685b 100644
--- a/qemu/tcg/mips/tcg-target.inc.c
+++ b/qemu/tcg/mips/tcg-target.inc.c
@@ -1109,7 +1109,7 @@ static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
 #if defined(CONFIG_SOFTMMU)
 #include "../tcg-ldst.inc.c"
 
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
@@ -1126,7 +1126,7 @@ static void * const qemu_ld_helpers[16] = {
 #endif
 };
 
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/ppc/tcg-target.h b/qemu/tcg/ppc/tcg-target.h
index 4fa21f0e71..b5cd225cfa 100644
--- a/qemu/tcg/ppc/tcg-target.h
+++ b/qemu/tcg/ppc/tcg-target.h
@@ -27,8 +27,10 @@
 
 #ifdef _ARCH_PPC64
 # define TCG_TARGET_REG_BITS  64
+# define MAX_CODE_GEN_BUFFER_SIZE  (2 * GiB)
 #else
 # define TCG_TARGET_REG_BITS  32
+# define MAX_CODE_GEN_BUFFER_SIZE  (32 * MiB)
 #endif
 
 #define TCG_TARGET_NB_REGS 64
@@ -63,6 +65,7 @@ typedef enum {
     tcg_isa_2_06,
     tcg_isa_2_07,
     tcg_isa_3_00,
+    tcg_isa_3_10,
 } TCGPowerISA;
 
 extern TCGPowerISA have_isa;
@@ -72,6 +75,7 @@ extern bool have_vsx;
 #define have_isa_2_06  (have_isa >= tcg_isa_2_06)
 #define have_isa_2_07  (have_isa >= tcg_isa_2_07)
 #define have_isa_3_00  (have_isa >= tcg_isa_3_00)
+#define have_isa_3_10  (have_isa >= tcg_isa_3_10)
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_ext8u_i32        0 /* andi */
@@ -79,7 +83,7 @@ extern bool have_vsx;
 
 /* optional instructions */
 #define TCG_TARGET_HAS_div_i32          1
-#define TCG_TARGET_HAS_rem_i32          0
+#define TCG_TARGET_HAS_rem_i32          have_isa_3_00
 #define TCG_TARGET_HAS_rot_i32          1
 #define TCG_TARGET_HAS_ext8s_i32        1
 #define TCG_TARGET_HAS_ext16s_i32       1
@@ -104,8 +108,8 @@ extern bool have_vsx;
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        1
 #define TCG_TARGET_HAS_mulsh_i32        1
-#define TCG_TARGET_HAS_goto_ptr         1
 #define TCG_TARGET_HAS_direct_jump      1
+#define TCG_TARGET_HAS_qemu_st8_i32     0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_add2_i32         0
@@ -113,7 +117,7 @@ extern bool have_vsx;
 #define TCG_TARGET_HAS_extrl_i64_i32    0
 #define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div_i64          1
-#define TCG_TARGET_HAS_rem_i64          0
+#define TCG_TARGET_HAS_rem_i64          have_isa_3_00
 #define TCG_TARGET_HAS_rot_i64          1
 #define TCG_TARGET_HAS_ext8s_i64        1
 #define TCG_TARGET_HAS_ext16s_i64       1
@@ -158,28 +162,30 @@ extern bool have_vsx;
 
 #define TCG_TARGET_HAS_andc_vec         1
 #define TCG_TARGET_HAS_orc_vec          have_isa_2_07
+#define TCG_TARGET_HAS_nand_vec         have_isa_2_07
+#define TCG_TARGET_HAS_nor_vec          1
+#define TCG_TARGET_HAS_eqv_vec          have_isa_2_07
 #define TCG_TARGET_HAS_not_vec          1
 #define TCG_TARGET_HAS_neg_vec          have_isa_3_00
 #define TCG_TARGET_HAS_abs_vec          0
+#define TCG_TARGET_HAS_roti_vec         0
+#define TCG_TARGET_HAS_rots_vec         0
+#define TCG_TARGET_HAS_rotv_vec         1
 #define TCG_TARGET_HAS_shi_vec          0
 #define TCG_TARGET_HAS_shs_vec          0
 #define TCG_TARGET_HAS_shv_vec          1
-#define TCG_TARGET_HAS_cmp_vec          1
 #define TCG_TARGET_HAS_mul_vec          1
 #define TCG_TARGET_HAS_sat_vec          1
 #define TCG_TARGET_HAS_minmax_vec       1
 #define TCG_TARGET_HAS_bitsel_vec       have_vsx
 #define TCG_TARGET_HAS_cmpsel_vec       0
 
-void flush_icache_range(uintptr_t start, uintptr_t stop);
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
-#ifdef CONFIG_SOFTMMU
 #define TCG_TARGET_NEED_LDST_LABELS
-#endif
 #define TCG_TARGET_NEED_POOL_LABELS
 
 #endif
diff --git a/qemu/tcg/ppc/tcg-target.inc.c b/qemu/tcg/ppc/tcg-target.inc.c
index 7f9dbb3b07..2f5c1c6f32 100644
--- a/qemu/tcg/ppc/tcg-target.inc.c
+++ b/qemu/tcg/ppc/tcg-target.inc.c
@@ -1850,7 +1850,7 @@ static const uint32_t qemu_exts_opc[4] = {
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
  *                                 int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
     [MO_LEUL] = helper_le_ldul_mmu,
@@ -1863,7 +1863,7 @@ static void * const qemu_ld_helpers[16] = {
 /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
  *                                 uintxx_t val, int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/ppc/tcg-target.opc.h b/qemu/tcg/ppc/tcg-target.opc.h
index 1373f77e82..db514403c3 100644
--- a/qemu/tcg/ppc/tcg-target.opc.h
+++ b/qemu/tcg/ppc/tcg-target.opc.h
@@ -30,4 +30,3 @@ DEF(ppc_msum_vec, 1, 3, 0, IMPLVEC)
 DEF(ppc_muleu_vec, 1, 2, 0, IMPLVEC)
 DEF(ppc_mulou_vec, 1, 2, 0, IMPLVEC)
 DEF(ppc_pkum_vec, 1, 2, 0, IMPLVEC)
-DEF(ppc_rotl_vec, 1, 2, 0, IMPLVEC)
diff --git a/qemu/tcg/riscv/tcg-target.h b/qemu/tcg/riscv/tcg-target.h
index 032439d806..11c9b3e4f4 100644
--- a/qemu/tcg/riscv/tcg-target.h
+++ b/qemu/tcg/riscv/tcg-target.h
@@ -34,6 +34,7 @@
 #define TCG_TARGET_INSN_UNIT_SIZE 4
 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 20
 #define TCG_TARGET_NB_REGS 32
+#define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
 
 typedef enum {
     TCG_REG_ZERO,
@@ -84,7 +85,6 @@ typedef enum {
 #define TCG_TARGET_CALL_STACK_OFFSET    0
 
 /* optional instructions */
-#define TCG_TARGET_HAS_goto_ptr         1
 #define TCG_TARGET_HAS_movcond_i32      0
 #define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_rem_i32          1
@@ -119,6 +119,7 @@ typedef enum {
 #define TCG_TARGET_HAS_direct_jump      0
 #define TCG_TARGET_HAS_brcond2          1
 #define TCG_TARGET_HAS_setcond2         1
+#define TCG_TARGET_HAS_qemu_st8_i32     0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_movcond_i64      0
@@ -159,19 +160,12 @@ typedef enum {
 #define TCG_TARGET_HAS_mulsh_i64        1
 #endif
 
-static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
-{
-    __builtin___clear_cache((char *)start, (char *)stop);
-}
-
 /* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
 #define TCG_TARGET_DEFAULT_MO (0)
 
-#ifdef CONFIG_SOFTMMU
 #define TCG_TARGET_NEED_LDST_LABELS
-#endif
 #define TCG_TARGET_NEED_POOL_LABELS
 
 #define TCG_TARGET_HAS_MEMORY_BSWAP 0
diff --git a/qemu/tcg/riscv/tcg-target.inc.c b/qemu/tcg/riscv/tcg-target.inc.c
index bff876c336..d1d7613349 100644
--- a/qemu/tcg/riscv/tcg-target.inc.c
+++ b/qemu/tcg/riscv/tcg-target.inc.c
@@ -926,7 +926,7 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     TCGMemOpIdx oi, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
@@ -949,7 +949,7 @@ static void * const qemu_ld_helpers[16] = {
  *                                     uintxx_t val, TCGMemOpIdx oi,
  *                                     uintptr_t ra)
  */
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/s390/tcg-target.inc.c b/qemu/tcg/s390/tcg-target.inc.c
index b82e5e84e7..85c010f892 100644
--- a/qemu/tcg/s390/tcg-target.inc.c
+++ b/qemu/tcg/s390/tcg-target.inc.c
@@ -337,7 +337,7 @@ static const uint8_t tcg_cond_to_ltr_cond[] = {
 };
 
 #ifdef CONFIG_SOFTMMU
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
@@ -352,7 +352,7 @@ static void * const qemu_ld_helpers[16] = {
     [MO_BEQ]  = helper_be_ldq_mmu,
 };
 
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/sparc/tcg-target.inc.c b/qemu/tcg/sparc/tcg-target.inc.c
index d4bc69d3b5..fb13db72f0 100644
--- a/qemu/tcg/sparc/tcg-target.inc.c
+++ b/qemu/tcg/sparc/tcg-target.inc.c
@@ -901,7 +901,7 @@ static void emit_extend(TCGContext *s, TCGReg r, int op)
 
 static void build_trampolines(TCGContext *s)
 {
-    static void * const qemu_ld_helpers[16] = {
+    static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
         [MO_UB]   = helper_ret_ldub_mmu,
         [MO_SB]   = helper_ret_ldsb_mmu,
         [MO_LEUW] = helper_le_lduw_mmu,
@@ -913,7 +913,7 @@ static void build_trampolines(TCGContext *s)
         [MO_BEUL] = helper_be_ldul_mmu,
         [MO_BEQ]  = helper_be_ldq_mmu,
     };
-    static void * const qemu_st_helpers[16] = {
+    static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
         [MO_UB]   = helper_ret_stb_mmu,
         [MO_LEUW] = helper_le_stw_mmu,
         [MO_LEUL] = helper_le_stl_mmu,
diff --git a/qemu/tcg/tcg-op-gvec.c b/qemu/tcg/tcg-op-gvec.c
index cab429c44a..ca507905ad 100644
--- a/qemu/tcg/tcg-op-gvec.c
+++ b/qemu/tcg/tcg-op-gvec.c
@@ -36,11 +36,21 @@
    of the operand offsets so that we can check them all at once.  */
 static void check_size_align(uint32_t oprsz, uint32_t maxsz, uint32_t ofs)
 {
-    uint32_t opr_align = oprsz >= 16 ? 15 : 7;
-    uint32_t max_align = maxsz >= 16 || oprsz >= 16 ? 15 : 7;
-    tcg_debug_assert(oprsz > 0);
-    tcg_debug_assert(oprsz <= maxsz);
-    tcg_debug_assert((oprsz & opr_align) == 0);
+    uint32_t max_align;
+
+    switch (oprsz) {
+    case 8:
+    case 16:
+    case 32:
+        tcg_debug_assert(oprsz <= maxsz);
+        break;
+    default:
+        tcg_debug_assert(oprsz == maxsz);
+        break;
+    }
+    tcg_debug_assert(maxsz <= (8 << SIMD_MAXSZ_BITS));
+
+    max_align = maxsz >= 16 ? 15 : 7;
     tcg_debug_assert((maxsz & max_align) == 0);
     tcg_debug_assert((ofs & max_align) == 0);
 }
@@ -76,12 +86,17 @@ uint32_t simd_desc(uint32_t oprsz, uint32_t maxsz, int32_t data)
 {
     uint32_t desc = 0;
 
-    assert(oprsz % 8 == 0 && oprsz <= (8 << SIMD_OPRSZ_BITS));
-    assert(maxsz % 8 == 0 && maxsz <= (8 << SIMD_MAXSZ_BITS));
-    assert(data == sextract32(data, 0, SIMD_DATA_BITS));
+    check_size_align(oprsz, maxsz, 0);
+
+    tcg_debug_assert(data == sextract32(data, 0, SIMD_DATA_BITS) ||
+                     data == extract32(data, 0, SIMD_DATA_BITS));
 
     oprsz = (oprsz / 8) - 1;
     maxsz = (maxsz / 8) - 1;
+    if (oprsz == maxsz) {
+        oprsz = 2;
+    }
+
     desc = deposit32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS, oprsz);
     desc = deposit32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS, maxsz);
     desc = deposit32(desc, SIMD_DATA_SHIFT, SIMD_DATA_BITS, data);
@@ -1571,6 +1586,13 @@ void tcg_gen_gvec_dup8i(TCGContext *tcg_ctx, uint32_t dofs, uint32_t oprsz,
     do_dup(tcg_ctx, MO_8, dofs, oprsz, maxsz, NULL, NULL, x);
 }
 
+void tcg_gen_gvec_dup_imm(TCGContext *tcg_ctx, unsigned vece, uint32_t dofs,
+                          uint32_t oprsz, uint32_t maxsz, uint64_t x)
+{
+    check_size_align(oprsz, maxsz, dofs);
+    do_dup(tcg_ctx, vece, dofs, oprsz, maxsz, NULL, NULL, x);
+}
+
 void tcg_gen_gvec_not(TCGContext *tcg_ctx, unsigned vece, uint32_t dofs, uint32_t aofs,
                       uint32_t oprsz, uint32_t maxsz)
 {
diff --git a/qemu/tcg/tcg-op-vec.c b/qemu/tcg/tcg-op-vec.c
index 99343962ac..a2e863f4ed 100644
--- a/qemu/tcg/tcg-op-vec.c
+++ b/qemu/tcg/tcg-op-vec.c
@@ -250,6 +250,16 @@ TCGv_vec tcg_const_ones_vec_matching(TCGContext *tcg_ctx, TCGv_vec m)
     return tcg_const_ones_vec(tcg_ctx, t->base_type);
 }
 
+TCGv_vec tcg_constant_vec_matching(TCGContext *tcg_ctx, TCGv_vec match,
+                                   unsigned vece, int64_t val)
+{
+    TCGTemp *t = tcgv_vec_temp(tcg_ctx, match);
+    TCGv_vec ret = tcg_temp_new_vec(tcg_ctx, t->base_type);
+
+    do_dupi_vec(tcg_ctx, ret, MO_REG, dup_const(vece, val));
+    return ret;
+}
+
 void tcg_gen_dup64i_vec(TCGContext *tcg_ctx, TCGv_vec r, uint64_t a)
 {
     if (TCG_TARGET_REG_BITS == 32 && a == deposit64(a, 32, 32, a)) {
diff --git a/qemu/tcg/tcg-op.c b/qemu/tcg/tcg-op.c
index d0223ea294..d76153d587 100644
--- a/qemu/tcg/tcg-op.c
+++ b/qemu/tcg/tcg-op.c
@@ -3121,7 +3121,7 @@ typedef void (*gen_atomic_op_i64)(TCGContext *tcg_ctx, TCGv_i64, TCGv_env, TCGv,
 # define WITH_ATOMIC64(X)
 #endif
 
-static void * const table_cmpxchg[16] = {
+static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_8] = gen_helper_atomic_cmpxchgb,
     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
@@ -3331,7 +3331,7 @@ static void do_atomic_op_i64(TCGContext *tcg_ctx, TCGv_i64 ret, TCGv addr, TCGv_
 }
 
 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
-static void * const table_##NAME[16] = {                                \
+static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = {          \
     [MO_8] = gen_helper_atomic_##NAME##b,                               \
     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
diff --git a/qemu/tcg/tcg.c b/qemu/tcg/tcg.c
index bc62051f19..0b542afe3f 100644
--- a/qemu/tcg/tcg.c
+++ b/qemu/tcg/tcg.c
@@ -2147,6 +2147,19 @@ TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
     return new_op;
 }
 
+void tcg_remove_ops_after(TCGContext *tcg_ctx, TCGOp *op)
+{
+    TCGOp *last;
+
+    for (;;) {
+        last = tcg_last_op(tcg_ctx);
+        if (last == op) {
+            return;
+        }
+        tcg_op_remove(tcg_ctx, last);
+    }
+}
+
 /* Reachable analysis : remove unreachable code.  */
 static void reachable_code_pass(TCGContext *s)
 {
diff --git a/qemu/tricore.h b/qemu/tricore.h
index 937eb5d257..43c5806a1c 100644
--- a/qemu/tricore.h
+++ b/qemu/tricore.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_tricore
 #define float32_squash_input_denormal float32_squash_input_denormal_tricore
 #define float64_squash_input_denormal float64_squash_input_denormal_tricore
+#define bfloat16_add bfloat16_add_tricore
+#define bfloat16_compare bfloat16_compare_tricore
+#define bfloat16_compare_quiet bfloat16_compare_quiet_tricore
+#define bfloat16_default_nan bfloat16_default_nan_tricore
+#define bfloat16_div bfloat16_div_tricore
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_tricore
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_tricore
+#define bfloat16_max bfloat16_max_tricore
+#define bfloat16_maximum_number bfloat16_maximum_number_tricore
+#define bfloat16_maxnum bfloat16_maxnum_tricore
+#define bfloat16_maxnummag bfloat16_maxnummag_tricore
+#define bfloat16_min bfloat16_min_tricore
+#define bfloat16_minimum_number bfloat16_minimum_number_tricore
+#define bfloat16_minnum bfloat16_minnum_tricore
+#define bfloat16_minnummag bfloat16_minnummag_tricore
+#define bfloat16_mul bfloat16_mul_tricore
+#define bfloat16_muladd bfloat16_muladd_tricore
+#define bfloat16_round_to_int bfloat16_round_to_int_tricore
+#define bfloat16_scalbn bfloat16_scalbn_tricore
+#define bfloat16_silence_nan bfloat16_silence_nan_tricore
+#define bfloat16_sqrt bfloat16_sqrt_tricore
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_tricore
+#define bfloat16_sub bfloat16_sub_tricore
+#define bfloat16_to_float32 bfloat16_to_float32_tricore
+#define bfloat16_to_float64 bfloat16_to_float64_tricore
+#define bfloat16_to_int16 bfloat16_to_int16_tricore
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_tricore
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_tricore
+#define bfloat16_to_int32 bfloat16_to_int32_tricore
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_tricore
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_tricore
+#define bfloat16_to_int64 bfloat16_to_int64_tricore
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_tricore
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_tricore
+#define bfloat16_to_uint16 bfloat16_to_uint16_tricore
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_tricore
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_tricore
+#define bfloat16_to_uint32 bfloat16_to_uint32_tricore
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_tricore
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_tricore
+#define bfloat16_to_uint64 bfloat16_to_uint64_tricore
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_tricore
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_tricore
+#define float128_maximum_number float128_maximum_number_tricore
+#define float128_max float128_max_tricore
+#define float128_maxnum float128_maxnum_tricore
+#define float128_maxnummag float128_maxnummag_tricore
+#define float128_min float128_min_tricore
+#define float128_minimum_number float128_minimum_number_tricore
+#define float128_minnum float128_minnum_tricore
+#define float128_minnummag float128_minnummag_tricore
+#define float128_muladd float128_muladd_tricore
+#define float128_to_int128 float128_to_int128_tricore
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_tricore
+#define float128_to_uint128 float128_to_uint128_tricore
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_tricore
+#define float16_maximum_number float16_maximum_number_tricore
+#define float16_minimum_number float16_minimum_number_tricore
+#define float16_to_int8 float16_to_int8_tricore
+#define float16_to_int8_scalbn float16_to_int8_scalbn_tricore
+#define float16_to_uint8 float16_to_uint8_tricore
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_tricore
+#define float32_maximum_number float32_maximum_number_tricore
+#define float32_minimum_number float32_minimum_number_tricore
+#define float32_to_bfloat16 float32_to_bfloat16_tricore
+#define float64_maximum_number float64_maximum_number_tricore
+#define float64_minimum_number float64_minimum_number_tricore
+#define float64_to_bfloat16 float64_to_bfloat16_tricore
+#define float64r32_add float64r32_add_tricore
+#define float64r32_div float64r32_div_tricore
+#define float64r32_mul float64r32_mul_tricore
+#define float64r32_muladd float64r32_muladd_tricore
+#define float64r32_sqrt float64r32_sqrt_tricore
+#define float64r32_sub float64r32_sub_tricore
+#define floatx80_mod floatx80_mod_tricore
+#define floatx80_modrem floatx80_modrem_tricore
+#define int128_to_float128 int128_to_float128_tricore
+#define int16_to_bfloat16 int16_to_bfloat16_tricore
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_tricore
+#define int32_to_bfloat16 int32_to_bfloat16_tricore
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_tricore
+#define int64_to_bfloat16 int64_to_bfloat16_tricore
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_tricore
+#define int8_to_float16 int8_to_float16_tricore
+#define uint128_to_float128 uint128_to_float128_tricore
+#define uint16_to_bfloat16 uint16_to_bfloat16_tricore
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_tricore
+#define uint32_to_bfloat16 uint32_to_bfloat16_tricore
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_tricore
+#define uint64_to_bfloat16 uint64_to_bfloat16_tricore
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_tricore
+#define uint8_to_float16 uint8_to_float16_tricore
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_tricore
 #define roundAndPackFloatx80 roundAndPackFloatx80_tricore
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_tricore
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_tricore
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_tricore
 #define helper_exit_atomic helper_exit_atomic_tricore
+#define helper_memset helper_memset_tricore
+#define helper_emu_stop helper_emu_stop_tricore
+#define tcg_remove_ops_after tcg_remove_ops_after_tricore
+#define tcg_constant_vec_matching tcg_constant_vec_matching_tricore
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_tricore
 #define helper_gvec_add8 helper_gvec_add8_tricore
 #define helper_gvec_add16 helper_gvec_add16_tricore
 #define helper_gvec_add32 helper_gvec_add32_tricore
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_tricore
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_tricore
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_tricore
+#define gen_helper_mve_vctp gen_helper_mve_vctp_tricore
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_tricore
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_tricore
+#define gen_helper_mve_vdup gen_helper_mve_vdup_tricore
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_tricore
+#define gen_helper_mve_vandi gen_helper_mve_vandi_tricore
+#define gen_helper_mve_vorri gen_helper_mve_vorri_tricore
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_tricore
+#define gen_helper_mve_viduph gen_helper_mve_viduph_tricore
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_tricore
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_tricore
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_tricore
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_tricore
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_tricore
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_tricore
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_tricore
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_tricore
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_tricore
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_tricore
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_tricore
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_tricore
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_tricore
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_tricore
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_tricore
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_tricore
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_tricore
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_tricore
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_tricore
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_tricore
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_tricore
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_tricore
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_tricore
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_tricore
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_tricore
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_tricore
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_tricore
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_tricore
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_tricore
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_tricore
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_tricore
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_tricore
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_tricore
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_tricore
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_tricore
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_tricore
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_tricore
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_tricore
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_tricore
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_tricore
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_tricore
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_tricore
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_tricore
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_tricore
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_tricore
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_tricore
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_tricore
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_tricore
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_tricore
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_tricore
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_tricore
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_tricore
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_tricore
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_tricore
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_tricore
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_tricore
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_tricore
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_tricore
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_tricore
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_tricore
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_tricore
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_tricore
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_tricore
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_tricore
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_tricore
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_tricore
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_tricore
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_tricore
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_tricore
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_tricore
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_tricore
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_tricore
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_tricore
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_tricore
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_tricore
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_tricore
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_tricore
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_tricore
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_tricore
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_tricore
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_tricore
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_tricore
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_tricore
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_tricore
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_tricore
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_tricore
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_tricore
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_tricore
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_tricore
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_tricore
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_tricore
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_tricore
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_tricore
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_tricore
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_tricore
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_tricore
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_tricore
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_tricore
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_tricore
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_tricore
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_tricore
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_tricore
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_tricore
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_tricore
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_tricore
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_tricore
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_tricore
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_tricore
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_tricore
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_tricore
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_tricore
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_tricore
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_tricore
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_tricore
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_tricore
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_tricore
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_tricore
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_tricore
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_tricore
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_tricore
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_tricore
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_tricore
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_tricore
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_tricore
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_tricore
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_tricore
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_tricore
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_tricore
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_tricore
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_tricore
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_tricore
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_tricore
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_tricore
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_tricore
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_tricore
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_tricore
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_tricore
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_tricore
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_tricore
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_tricore
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_tricore
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_tricore
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_tricore
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_tricore
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_tricore
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_tricore
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_tricore
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_tricore
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_tricore
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_tricore
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_tricore
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_tricore
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_tricore
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_tricore
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_tricore
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_tricore
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_tricore
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_tricore
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_tricore
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_tricore
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_tricore
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_tricore
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_tricore
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_tricore
+#define gen_helper_mve_vand gen_helper_mve_vand_tricore
+#define gen_helper_mve_vbic gen_helper_mve_vbic_tricore
+#define gen_helper_mve_vorr gen_helper_mve_vorr_tricore
+#define gen_helper_mve_vorn gen_helper_mve_vorn_tricore
+#define gen_helper_mve_veor gen_helper_mve_veor_tricore
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_tricore
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_tricore
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_tricore
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_tricore
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_tricore
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_tricore
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_tricore
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_tricore
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_tricore
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_tricore
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_tricore
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_tricore
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_tricore
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_tricore
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_tricore
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_tricore
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_tricore
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_tricore
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_tricore
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_tricore
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_tricore
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_tricore
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_tricore
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_tricore
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_tricore
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_tricore
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_tricore
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_tricore
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_tricore
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_tricore
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_tricore
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_tricore
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_tricore
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_tricore
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_tricore
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_tricore
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_tricore
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_tricore
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_tricore
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_tricore
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_tricore
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_tricore
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_tricore
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_tricore
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_tricore
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_tricore
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_tricore
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_tricore
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_tricore
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_tricore
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_tricore
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_tricore
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_tricore
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_tricore
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_tricore
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_tricore
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_tricore
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_tricore
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_tricore
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_tricore
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_tricore
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_tricore
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_tricore
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_tricore
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_tricore
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_tricore
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_tricore
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_tricore
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_tricore
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_tricore
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_tricore
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_tricore
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_tricore
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_tricore
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_tricore
+#define gen_helper_mve_vminub gen_helper_mve_vminub_tricore
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_tricore
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_tricore
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_tricore
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_tricore
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_tricore
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_tricore
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_tricore
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_tricore
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_tricore
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_tricore
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_tricore
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_tricore
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_tricore
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_tricore
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_tricore
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_tricore
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_tricore
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_tricore
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_tricore
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_tricore
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_tricore
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_tricore
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_tricore
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_tricore
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_tricore
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_tricore
+#define gen_helper_mve_vadc gen_helper_mve_vadc_tricore
+#define gen_helper_mve_vadci gen_helper_mve_vadci_tricore
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_tricore
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_tricore
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_tricore
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_tricore
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_tricore
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_tricore
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_tricore
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_tricore
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_tricore
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_tricore
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_tricore
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_tricore
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_tricore
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_tricore
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_tricore
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_tricore
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_tricore
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_tricore
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_tricore
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_tricore
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_tricore
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_tricore
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_tricore
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_tricore
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_tricore
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_tricore
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_tricore
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_tricore
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_tricore
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_tricore
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_tricore
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_tricore
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_tricore
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_tricore
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_tricore
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_tricore
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_tricore
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_tricore
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_tricore
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_tricore
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_tricore
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_tricore
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_tricore
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_tricore
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_tricore
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_tricore
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_tricore
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_tricore
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_tricore
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_tricore
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_tricore
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_tricore
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_tricore
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_tricore
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_tricore
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_tricore
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_tricore
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_tricore
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_tricore
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_tricore
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_tricore
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_tricore
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_tricore
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_tricore
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_tricore
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_tricore
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_tricore
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_tricore
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_tricore
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_tricore
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_tricore
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_tricore
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_tricore
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_tricore
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_tricore
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_tricore
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_tricore
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_tricore
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_tricore
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_tricore
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_tricore
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_tricore
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_tricore
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_tricore
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_tricore
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_tricore
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_tricore
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_tricore
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_tricore
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_tricore
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_tricore
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_tricore
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_tricore
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_tricore
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_tricore
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_tricore
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_tricore
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_tricore
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_tricore
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_tricore
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_tricore
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_tricore
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_tricore
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_tricore
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_tricore
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_tricore
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_tricore
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_tricore
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_tricore
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_tricore
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_tricore
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_tricore
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_tricore
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_tricore
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_tricore
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_tricore
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_tricore
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_tricore
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_tricore
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_tricore
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_tricore
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_tricore
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_tricore
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_tricore
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_tricore
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_tricore
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_tricore
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_tricore
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_tricore
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_tricore
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_tricore
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_tricore
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_tricore
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_tricore
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_tricore
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_tricore
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_tricore
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_tricore
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_tricore
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_tricore
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_tricore
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_tricore
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_tricore
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_tricore
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_tricore
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_tricore
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_tricore
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_tricore
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_tricore
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_tricore
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_tricore
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_tricore
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_tricore
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_tricore
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_tricore
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_tricore
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_tricore
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_tricore
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_tricore
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_tricore
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_tricore
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_tricore
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_tricore
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_tricore
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_tricore
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_tricore
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_tricore
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_tricore
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_tricore
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_tricore
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_tricore
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_tricore
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_tricore
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_tricore
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_tricore
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_tricore
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_tricore
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_tricore
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_tricore
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_tricore
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_tricore
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_tricore
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_tricore
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_tricore
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_tricore
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_tricore
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_tricore
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_tricore
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_tricore
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_tricore
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_tricore
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_tricore
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_tricore
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_tricore
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_tricore
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_tricore
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_tricore
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_tricore
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_tricore
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_tricore
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_tricore
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_tricore
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_tricore
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_tricore
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_tricore
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_tricore
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_tricore
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_tricore
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_tricore
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_tricore
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_tricore
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_tricore
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_tricore
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_tricore
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_tricore
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_tricore
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_tricore
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_tricore
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_tricore
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_tricore
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_tricore
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_tricore
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_tricore
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_tricore
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_tricore
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_tricore
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_tricore
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_tricore
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_tricore
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_tricore
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_tricore
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_tricore
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_tricore
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_tricore
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_tricore
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_tricore
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_tricore
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_tricore
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_tricore
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_tricore
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_tricore
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_tricore
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_tricore
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_tricore
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_tricore
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_tricore
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_tricore
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_tricore
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_tricore
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_tricore
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_tricore
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_tricore
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_tricore
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_tricore
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_tricore
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_tricore
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_tricore
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_tricore
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_tricore
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_tricore
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_tricore
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_tricore
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_tricore
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_tricore
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_tricore
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_tricore
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_tricore
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_tricore
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_tricore
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_tricore
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_tricore
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_tricore
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_tricore
+#define gen_helper_mve_ushll gen_helper_mve_ushll_tricore
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_tricore
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_tricore
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_tricore
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_tricore
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_tricore
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_tricore
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_tricore
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_tricore
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_tricore
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_tricore
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_tricore
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_tricore
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_tricore
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_tricore
+#define gen_helper_mve_vslib gen_helper_mve_vslib_tricore
+#define gen_helper_mve_vslih gen_helper_mve_vslih_tricore
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_tricore
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_tricore
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_tricore
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_tricore
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_tricore
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_tricore
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_tricore
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_tricore
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_tricore
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_tricore
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_tricore
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_tricore
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_tricore
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_tricore
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_tricore
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_tricore
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_tricore
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_tricore
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_tricore
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_tricore
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_tricore
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_tricore
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_tricore
+#define gen_helper_mve_vminab gen_helper_mve_vminab_tricore
+#define gen_helper_mve_vminah gen_helper_mve_vminah_tricore
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_tricore
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_tricore
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_tricore
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_tricore
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_tricore
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_tricore
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_tricore
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_tricore
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_tricore
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_tricore
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_tricore
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_tricore
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_tricore
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_tricore
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_tricore
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_tricore
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_tricore
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_tricore
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_tricore
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_tricore
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_tricore
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_tricore
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_tricore
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_tricore
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_tricore
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_tricore
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_tricore
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_tricore
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_tricore
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_tricore
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_tricore
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_tricore
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_tricore
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_tricore
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_tricore
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_tricore
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_tricore
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_tricore
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_tricore
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_tricore
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_tricore
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_tricore
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_tricore
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_tricore
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_tricore
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_tricore
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_tricore
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_tricore
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_tricore
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_tricore
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_tricore
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_tricore
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_tricore
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_tricore
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_tricore
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_tricore
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_tricore
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_tricore
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_tricore
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_tricore
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_tricore
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_tricore
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_tricore
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_tricore
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_tricore
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_tricore
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_tricore
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_tricore
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_tricore
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_tricore
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_tricore
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_tricore
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_tricore
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_tricore
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_tricore
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_tricore
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_tricore
 #define gen_helper_cpsr_read gen_helper_cpsr_read_tricore
 #define gen_helper_cpsr_write gen_helper_cpsr_write_tricore
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_tricore
diff --git a/qemu/util/guest-random.c b/qemu/util/guest-random.c
index 328da3f84e..46ebbdefe0 100644
--- a/qemu/util/guest-random.c
+++ b/qemu/util/guest-random.c
@@ -44,7 +44,7 @@ static int glib_random_bytes(void *buf, size_t len)
     }
     if (i < len) {
         x = g_rand_int(rand);
-        __builtin_memcpy(buf + i, &x, i - len);
+        __builtin_memcpy(buf + i, &x, len - i);
     }
 #else
     uint32_t x;
@@ -56,15 +56,16 @@ static int glib_random_bytes(void *buf, size_t len)
         } else {
             srand(time(NULL));
         }
+        initialized = true;
+    }
 
-        for (i = 0; i + 4 <= len; i += 4) {
-            x = rand();
-            memcpy(((uint8_t*)buf) + i, &x, 4);
-        }
-        if (i < len) {
-            x = rand();
-            memcpy(((uint8_t*)buf) + i, &x, i - len);
-        }
+    for (i = 0; i + 4 <= len; i += 4) {
+        x = rand();
+        memcpy(((uint8_t *)buf) + i, &x, 4);
+    }
+    if (i < len) {
+        x = rand();
+        memcpy(((uint8_t *)buf) + i, &x, len - i);
     }
 #endif
     return 0;
@@ -101,4 +102,3 @@ void qemu_guest_random_seed_thread_part2(uint64_t seed)
     }
 #endif
 }
-
diff --git a/qemu/util/host-utils.c b/qemu/util/host-utils.c
index 7b9322071d..fb91bcba82 100644
--- a/qemu/util/host-utils.c
+++ b/qemu/util/host-utils.c
@@ -34,7 +34,7 @@ static inline void mul64(uint64_t *plow, uint64_t *phigh,
     typedef union {
         uint64_t ll;
         struct {
-#ifdef HOST_WORDS_BIGENDIAN
+#if HOST_BIG_ENDIAN
             uint32_t high, low;
 #else
             uint32_t low, high;
@@ -86,78 +86,119 @@ void muls64 (uint64_t *plow, uint64_t *phigh, int64_t a, int64_t b)
     *phigh = rh;
 }
 
-/* Unsigned 128x64 division.  Returns 1 if overflow (divide by zero or */
-/* quotient exceeds 64 bits).  Otherwise returns quotient via plow and */
-/* remainder via phigh. */
-int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor)
+/*
+ * Unsigned 128-by-64 division.
+ * Returns the remainder.
+ * Returns quotient via plow and phigh.
+ * Also returns the remainder via the function return value.
+ */
+uint64_t divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor)
 {
     uint64_t dhi = *phigh;
     uint64_t dlo = *plow;
-    unsigned i;
-    uint64_t carry = 0;
+    uint64_t rem, dhighest;
+    int sh;
 
-    if (divisor == 0) {
-        return 1;
-    } else if (dhi == 0) {
+    if (divisor == 0 || dhi == 0) {
         *plow  = dlo / divisor;
-        *phigh = dlo % divisor;
-        return 0;
-    } else if (dhi > divisor) {
-        return 1;
+        *phigh = 0;
+        return dlo % divisor;
     } else {
+        sh = clz64(divisor);
 
-        for (i = 0; i < 64; i++) {
-            carry = dhi >> 63;
-            dhi = (dhi << 1) | (dlo >> 63);
-            if (carry || (dhi >= divisor)) {
-                dhi -= divisor;
-                carry = 1;
+        if (dhi < divisor) {
+            if (sh != 0) {
+                /* normalize the divisor, shifting the dividend accordingly */
+                divisor <<= sh;
+                dhi = (dhi << sh) | (dlo >> (64 - sh));
+                dlo <<= sh;
+            }
+
+            *phigh = 0;
+            *plow = udiv_qrnnd(&rem, dhi, dlo, divisor);
+        } else {
+            if (sh != 0) {
+                /* normalize the divisor, shifting the dividend accordingly */
+                divisor <<= sh;
+                dhighest = dhi >> (64 - sh);
+                dhi = (dhi << sh) | (dlo >> (64 - sh));
+                dlo <<= sh;
+
+                *phigh = udiv_qrnnd(&dhi, dhighest, dhi, divisor);
             } else {
-                carry = 0;
+                /**
+                 * dhi >= divisor
+                 * Since the MSB of divisor is set (sh == 0),
+                 * (dhi - divisor) < divisor
+                 *
+                 * Thus, the high part of the quotient is 1, and we can
+                 * calculate the low part with a single call to udiv_qrnnd
+                 * after subtracting divisor from dhi
+                 */
+                dhi -= divisor;
+                *phigh = 1;
             }
-            dlo = (dlo << 1) | carry;
+
+            *plow = udiv_qrnnd(&rem, dhi, dlo, divisor);
         }
 
-        *plow = dlo;
-        *phigh = dhi;
-        return 0;
+        /*
+         * since the dividend/divisor might have been normalized,
+         * the remainder might also have to be shifted back
+         */
+        return rem >> sh;
     }
 }
 
-int divs128(int64_t *plow, int64_t *phigh, int64_t divisor)
+/*
+ * Signed 128-by-64 division.
+ * Returns quotient via plow and phigh.
+ * Also returns the remainder via the function return value.
+ */
+int64_t divs128(uint64_t *plow, int64_t *phigh, int64_t divisor)
 {
-    int sgn_dvdnd = *phigh < 0;
-    int sgn_divsr = divisor < 0;
-    int overflow = 0;
-
-    if (sgn_dvdnd) {
-        *plow = ~(*plow);
-        *phigh = ~(*phigh);
-        if (*plow == (int64_t)-1) {
-            *plow = 0;
-            (*phigh)++;
-         } else {
-            (*plow)++;
-         }
-    }
+    bool neg_quotient = false, neg_remainder = false;
+    uint64_t unsig_hi = *phigh, unsig_lo = *plow;
+    uint64_t rem;
 
-    if (sgn_divsr) {
-        divisor = 0 - divisor;
+    if (*phigh < 0) {
+        neg_quotient = !neg_quotient;
+        neg_remainder = !neg_remainder;
+
+        if (unsig_lo == 0) {
+            unsig_hi = -unsig_hi;
+        } else {
+            unsig_hi = ~unsig_hi;
+            unsig_lo = -unsig_lo;
+        }
     }
 
-    overflow = divu128((uint64_t *)plow, (uint64_t *)phigh, (uint64_t)divisor);
+    if (divisor < 0) {
+        neg_quotient = !neg_quotient;
 
-    if (sgn_dvdnd  ^ sgn_divsr) {
-        *plow = 0 - *plow;
+        divisor = -divisor;
     }
 
-    if (!overflow) {
-        if ((*plow < 0) ^ (sgn_dvdnd ^ sgn_divsr)) {
-            overflow = 1;
+    rem = divu128(&unsig_lo, &unsig_hi, (uint64_t)divisor);
+
+    if (neg_quotient) {
+        if (unsig_lo == 0) {
+            *phigh = -unsig_hi;
+            *plow = 0;
+        } else {
+            *phigh = ~unsig_hi;
+            *plow = -unsig_lo;
         }
+    } else {
+        *phigh = unsig_hi;
+        *plow = unsig_lo;
     }
 
-    return overflow;
+    if (neg_remainder) {
+        return -rem;
+    } else {
+        return rem;
+    }
 }
 #endif
 
@@ -225,3 +266,183 @@ void ulshift(uint64_t *plow, uint64_t *phigh, int32_t shift, bool *overflow)
         *plow = *plow << shift;
     }
 }
+
+/*
+ * Unsigned 256-by-128 division.
+ * Returns the remainder via r.
+ * Returns lower 128 bit of quotient.
+ * Needs a normalized divisor (most significant bit set to 1).
+ *
+ * Adapted from include/qemu/host-utils.h udiv_qrnnd,
+ * from the GNU Multi Precision Library - longlong.h __udiv_qrnnd
+ * (https://gmplib.org/repo/gmp/file/tip/longlong.h)
+ *
+ * Licensed under the GPLv2/LGPLv3
+ */
+static Int128 udiv256_qrnnd(Int128 *r, Int128 n1, Int128 n0, Int128 d)
+{
+    Int128 d0, d1, q0, q1, r1, r0, m;
+    uint64_t mp0, mp1;
+
+    d0 = int128_make64(int128_getlo(d));
+    d1 = int128_make64(int128_gethi(d));
+
+    r1 = int128_remu(n1, d1);
+    q1 = int128_divu(n1, d1);
+    mp0 = int128_getlo(q1);
+    mp1 = int128_gethi(q1);
+    mulu128(&mp0, &mp1, int128_getlo(d0));
+    m = int128_make128(mp0, mp1);
+    r1 = int128_make128(int128_gethi(n0), int128_getlo(r1));
+    if (int128_ult(r1, m)) {
+        q1 = int128_sub(q1, int128_one());
+        r1 = int128_add(r1, d);
+        if (int128_uge(r1, d)) {
+            if (int128_ult(r1, m)) {
+                q1 = int128_sub(q1, int128_one());
+                r1 = int128_add(r1, d);
+            }
+        }
+    }
+    r1 = int128_sub(r1, m);
+
+    r0 = int128_remu(r1, d1);
+    q0 = int128_divu(r1, d1);
+    mp0 = int128_getlo(q0);
+    mp1 = int128_gethi(q0);
+    mulu128(&mp0, &mp1, int128_getlo(d0));
+    m = int128_make128(mp0, mp1);
+    r0 = int128_make128(int128_getlo(n0), int128_getlo(r0));
+    if (int128_ult(r0, m)) {
+        q0 = int128_sub(q0, int128_one());
+        r0 = int128_add(r0, d);
+        if (int128_uge(r0, d)) {
+            if (int128_ult(r0, m)) {
+                q0 = int128_sub(q0, int128_one());
+                r0 = int128_add(r0, d);
+            }
+        }
+    }
+    r0 = int128_sub(r0, m);
+
+    *r = r0;
+    return int128_or(int128_lshift(q1, 64), q0);
+}
+
+/*
+ * Unsigned 256-by-128 division.
+ * Returns the remainder.
+ * Returns quotient via plow and phigh.
+ * Also returns the remainder via the function return value.
+ */
+Int128 divu256(Int128 *plow, Int128 *phigh, Int128 divisor)
+{
+    Int128 dhi = *phigh;
+    Int128 dlo = *plow;
+    Int128 rem, dhighest;
+    int sh;
+
+    if (!int128_nz(divisor) || !int128_nz(dhi)) {
+        *plow  = int128_divu(dlo, divisor);
+        *phigh = int128_zero();
+        return int128_remu(dlo, divisor);
+    } else {
+        sh = clz128(divisor);
+
+        if (int128_ult(dhi, divisor)) {
+            if (sh != 0) {
+                /* normalize the divisor, shifting the dividend accordingly */
+                divisor = int128_lshift(divisor, sh);
+                dhi = int128_or(int128_lshift(dhi, sh),
+                                int128_urshift(dlo, (128 - sh)));
+                dlo = int128_lshift(dlo, sh);
+            }
+
+            *phigh = int128_zero();
+            *plow = udiv256_qrnnd(&rem, dhi, dlo, divisor);
+        } else {
+            if (sh != 0) {
+                /* normalize the divisor, shifting the dividend accordingly */
+                divisor = int128_lshift(divisor, sh);
+                dhighest = int128_rshift(dhi, (128 - sh));
+                dhi = int128_or(int128_lshift(dhi, sh),
+                                int128_urshift(dlo, (128 - sh)));
+                dlo = int128_lshift(dlo, sh);
+
+                *phigh = udiv256_qrnnd(&dhi, dhighest, dhi, divisor);
+            } else {
+                /*
+                 * dhi >= divisor
+                 * Since the MSB of divisor is set (sh == 0),
+                 * (dhi - divisor) < divisor
+                 *
+                 * Thus, the high part of the quotient is 1, and we can
+                 * calculate the low part with a single call to udiv_qrnnd
+                 * after subtracting divisor from dhi
+                 */
+                dhi = int128_sub(dhi, divisor);
+                *phigh = int128_one();
+            }
+
+            *plow = udiv256_qrnnd(&rem, dhi, dlo, divisor);
+        }
+
+        /*
+         * since the dividend/divisor might have been normalized,
+         * the remainder might also have to be shifted back
+         */
+        rem = int128_urshift(rem, sh);
+        return rem;
+    }
+}
+
+/*
+ * Signed 256-by-128 division.
+ * Returns quotient via plow and phigh.
+ * Also returns the remainder via the function return value.
+ */
+Int128 divs256(Int128 *plow, Int128 *phigh, Int128 divisor)
+{
+    bool neg_quotient = false, neg_remainder = false;
+    Int128 unsig_hi = *phigh, unsig_lo = *plow;
+    Int128 rem;
+
+    if (!int128_nonneg(*phigh)) {
+        neg_quotient = !neg_quotient;
+        neg_remainder = !neg_remainder;
+
+        if (!int128_nz(unsig_lo)) {
+            unsig_hi = int128_neg(unsig_hi);
+        } else {
+            unsig_hi = int128_not(unsig_hi);
+            unsig_lo = int128_neg(unsig_lo);
+        }
+    }
+
+    if (!int128_nonneg(divisor)) {
+        neg_quotient = !neg_quotient;
+
+        divisor = int128_neg(divisor);
+    }
+
+    rem = divu256(&unsig_lo, &unsig_hi, divisor);
+
+    if (neg_quotient) {
+        if (!int128_nz(unsig_lo)) {
+            *phigh = int128_neg(unsig_hi);
+            *plow = int128_zero();
+        } else {
+            *phigh = int128_not(unsig_hi);
+            *plow = int128_neg(unsig_lo);
+        }
+    } else {
+        *phigh = unsig_hi;
+        *plow = unsig_lo;
+    }
+
+    if (neg_remainder) {
+        return int128_neg(rem);
+    } else {
+        return rem;
+    }
+}
diff --git a/qemu/util/int128.c b/qemu/util/int128.c
new file mode 100644
index 0000000000..ed8f25fef1
--- /dev/null
+++ b/qemu/util/int128.c
@@ -0,0 +1,147 @@
+/*
+ * 128-bit division and remainder for compilers not supporting __int128
+ *
+ * Copyright (c) 2021 Frédéric Pétrot <frederic.petrot@univ-grenoble-alpes.fr>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/host-utils.h"
+#include "qemu/int128.h"
+
+#ifndef CONFIG_INT128
+
+/*
+ * Division and remainder algorithms for 128-bit due to Stefan Kanthak,
+ * https://skanthak.homepage.t-online.de/integer.html#udivmodti4
+ * Preconditions:
+ *     - function should never be called with v equals to 0, it has to
+ *       be dealt with beforehand
+ *     - quotien pointer must be valid
+ */
+static Int128 divrem128(Int128 u, Int128 v, Int128 *q)
+{
+    Int128 qq;
+    uint64_t hi, lo, tmp;
+    int s = clz64(v.hi);
+
+    if (s == 64) {
+        /* we have uu÷0v => let's use divu128 */
+        hi = u.hi;
+        lo = u.lo;
+        tmp = divu128(&lo, &hi, v.lo);
+        *q = int128_make128(lo, hi);
+        return int128_make128(tmp, 0);
+    } else {
+        hi = int128_gethi(int128_lshift(v, s));
+
+        if (hi > u.hi) {
+            lo = u.lo;
+            tmp = u.hi;
+            divu128(&lo, &tmp, hi);
+            lo = int128_gethi(int128_lshift(int128_make128(lo, 0), s));
+        } else { /* prevent overflow */
+            lo = u.lo;
+            tmp = u.hi - hi;
+            divu128(&lo, &tmp, hi);
+            lo = int128_gethi(int128_lshift(int128_make128(lo, 1), s));
+        }
+
+        qq = int128_make64(lo);
+
+        tmp = lo * v.hi;
+        mulu64(&lo, &hi, lo, v.lo);
+        hi += tmp;
+
+        if (hi < tmp     /* quotient * divisor >= 2**128 > dividend */
+            || hi > u.hi /* quotient * divisor > dividend */
+            || (hi == u.hi && lo > u.lo)) {
+            qq.lo -= 1;
+            mulu64(&lo, &hi, qq.lo, v.lo);
+            hi += qq.lo * v.hi;
+        }
+
+        *q = qq;
+        u.hi -= hi + (u.lo < lo);
+        u.lo -= lo;
+        return u;
+    }
+}
+
+Int128 int128_divu(Int128 a, Int128 b)
+{
+    Int128 q;
+    divrem128(a, b, &q);
+    return q;
+}
+
+Int128 int128_remu(Int128 a, Int128 b)
+{
+    Int128 q;
+    return divrem128(a, b, &q);
+}
+
+Int128 int128_divs(Int128 a, Int128 b)
+{
+    Int128 q;
+    bool sgna = !int128_nonneg(a);
+    bool sgnb = !int128_nonneg(b);
+
+    if (sgna) {
+        a = int128_neg(a);
+    }
+
+    if (sgnb) {
+        b = int128_neg(b);
+    }
+
+    divrem128(a, b, &q);
+
+    if (sgna != sgnb) {
+        q = int128_neg(q);
+    }
+
+    return q;
+}
+
+Int128 int128_rems(Int128 a, Int128 b)
+{
+    Int128 q, r;
+    bool sgna = !int128_nonneg(a);
+    bool sgnb = !int128_nonneg(b);
+
+    if (sgna) {
+        a = int128_neg(a);
+    }
+
+    if (sgnb) {
+        b = int128_neg(b);
+    }
+
+    r = divrem128(a, b, &q);
+
+    if (sgna) {
+        r = int128_neg(r);
+    }
+
+    return r;
+}
+
+#endif
diff --git a/qemu/util/qemu-timer-common.c b/qemu/util/qemu-timer-common.c
index 2dcebf4d2f..9429f8ed42 100644
--- a/qemu/util/qemu-timer-common.c
+++ b/qemu/util/qemu-timer-common.c
@@ -27,6 +27,8 @@
 /***********************************************************/
 /* real time host monotonic timer */
 
+int64_t clock_start;
+
 #ifdef _WIN32
 
 int64_t clock_freq;
@@ -40,6 +42,7 @@ void init_get_clock(void)
         exit(1);
     }
     clock_freq = freq.QuadPart;
+    clock_start = get_clock();
 }
 
 #else
@@ -54,5 +57,6 @@ void init_get_clock(void)
     if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
         use_rt_clock = 1;
     }
+    clock_start = get_clock();
 }
 #endif
diff --git a/qemu/x86_64.h b/qemu/x86_64.h
index 637b88524a..a64dc44c84 100644
--- a/qemu/x86_64.h
+++ b/qemu/x86_64.h
@@ -329,6 +329,98 @@
 #define float16_squash_input_denormal float16_squash_input_denormal_x86_64
 #define float32_squash_input_denormal float32_squash_input_denormal_x86_64
 #define float64_squash_input_denormal float64_squash_input_denormal_x86_64
+#define bfloat16_add bfloat16_add_x86_64
+#define bfloat16_compare bfloat16_compare_x86_64
+#define bfloat16_compare_quiet bfloat16_compare_quiet_x86_64
+#define bfloat16_default_nan bfloat16_default_nan_x86_64
+#define bfloat16_div bfloat16_div_x86_64
+#define bfloat16_is_quiet_nan bfloat16_is_quiet_nan_x86_64
+#define bfloat16_is_signaling_nan bfloat16_is_signaling_nan_x86_64
+#define bfloat16_max bfloat16_max_x86_64
+#define bfloat16_maximum_number bfloat16_maximum_number_x86_64
+#define bfloat16_maxnum bfloat16_maxnum_x86_64
+#define bfloat16_maxnummag bfloat16_maxnummag_x86_64
+#define bfloat16_min bfloat16_min_x86_64
+#define bfloat16_minimum_number bfloat16_minimum_number_x86_64
+#define bfloat16_minnum bfloat16_minnum_x86_64
+#define bfloat16_minnummag bfloat16_minnummag_x86_64
+#define bfloat16_mul bfloat16_mul_x86_64
+#define bfloat16_muladd bfloat16_muladd_x86_64
+#define bfloat16_round_to_int bfloat16_round_to_int_x86_64
+#define bfloat16_scalbn bfloat16_scalbn_x86_64
+#define bfloat16_silence_nan bfloat16_silence_nan_x86_64
+#define bfloat16_sqrt bfloat16_sqrt_x86_64
+#define bfloat16_squash_input_denormal bfloat16_squash_input_denormal_x86_64
+#define bfloat16_sub bfloat16_sub_x86_64
+#define bfloat16_to_float32 bfloat16_to_float32_x86_64
+#define bfloat16_to_float64 bfloat16_to_float64_x86_64
+#define bfloat16_to_int16 bfloat16_to_int16_x86_64
+#define bfloat16_to_int16_round_to_zero bfloat16_to_int16_round_to_zero_x86_64
+#define bfloat16_to_int16_scalbn bfloat16_to_int16_scalbn_x86_64
+#define bfloat16_to_int32 bfloat16_to_int32_x86_64
+#define bfloat16_to_int32_round_to_zero bfloat16_to_int32_round_to_zero_x86_64
+#define bfloat16_to_int32_scalbn bfloat16_to_int32_scalbn_x86_64
+#define bfloat16_to_int64 bfloat16_to_int64_x86_64
+#define bfloat16_to_int64_round_to_zero bfloat16_to_int64_round_to_zero_x86_64
+#define bfloat16_to_int64_scalbn bfloat16_to_int64_scalbn_x86_64
+#define bfloat16_to_uint16 bfloat16_to_uint16_x86_64
+#define bfloat16_to_uint16_round_to_zero bfloat16_to_uint16_round_to_zero_x86_64
+#define bfloat16_to_uint16_scalbn bfloat16_to_uint16_scalbn_x86_64
+#define bfloat16_to_uint32 bfloat16_to_uint32_x86_64
+#define bfloat16_to_uint32_round_to_zero bfloat16_to_uint32_round_to_zero_x86_64
+#define bfloat16_to_uint32_scalbn bfloat16_to_uint32_scalbn_x86_64
+#define bfloat16_to_uint64 bfloat16_to_uint64_x86_64
+#define bfloat16_to_uint64_round_to_zero bfloat16_to_uint64_round_to_zero_x86_64
+#define bfloat16_to_uint64_scalbn bfloat16_to_uint64_scalbn_x86_64
+#define float128_maximum_number float128_maximum_number_x86_64
+#define float128_max float128_max_x86_64
+#define float128_maxnum float128_maxnum_x86_64
+#define float128_maxnummag float128_maxnummag_x86_64
+#define float128_min float128_min_x86_64
+#define float128_minimum_number float128_minimum_number_x86_64
+#define float128_minnum float128_minnum_x86_64
+#define float128_minnummag float128_minnummag_x86_64
+#define float128_muladd float128_muladd_x86_64
+#define float128_to_int128 float128_to_int128_x86_64
+#define float128_to_int128_round_to_zero float128_to_int128_round_to_zero_x86_64
+#define float128_to_uint128 float128_to_uint128_x86_64
+#define float128_to_uint128_round_to_zero float128_to_uint128_round_to_zero_x86_64
+#define float16_maximum_number float16_maximum_number_x86_64
+#define float16_minimum_number float16_minimum_number_x86_64
+#define float16_to_int8 float16_to_int8_x86_64
+#define float16_to_int8_scalbn float16_to_int8_scalbn_x86_64
+#define float16_to_uint8 float16_to_uint8_x86_64
+#define float16_to_uint8_scalbn float16_to_uint8_scalbn_x86_64
+#define float32_maximum_number float32_maximum_number_x86_64
+#define float32_minimum_number float32_minimum_number_x86_64
+#define float32_to_bfloat16 float32_to_bfloat16_x86_64
+#define float64_maximum_number float64_maximum_number_x86_64
+#define float64_minimum_number float64_minimum_number_x86_64
+#define float64_to_bfloat16 float64_to_bfloat16_x86_64
+#define float64r32_add float64r32_add_x86_64
+#define float64r32_div float64r32_div_x86_64
+#define float64r32_mul float64r32_mul_x86_64
+#define float64r32_muladd float64r32_muladd_x86_64
+#define float64r32_sqrt float64r32_sqrt_x86_64
+#define float64r32_sub float64r32_sub_x86_64
+#define floatx80_mod floatx80_mod_x86_64
+#define floatx80_modrem floatx80_modrem_x86_64
+#define int128_to_float128 int128_to_float128_x86_64
+#define int16_to_bfloat16 int16_to_bfloat16_x86_64
+#define int16_to_bfloat16_scalbn int16_to_bfloat16_scalbn_x86_64
+#define int32_to_bfloat16 int32_to_bfloat16_x86_64
+#define int32_to_bfloat16_scalbn int32_to_bfloat16_scalbn_x86_64
+#define int64_to_bfloat16 int64_to_bfloat16_x86_64
+#define int64_to_bfloat16_scalbn int64_to_bfloat16_scalbn_x86_64
+#define int8_to_float16 int8_to_float16_x86_64
+#define uint128_to_float128 uint128_to_float128_x86_64
+#define uint16_to_bfloat16 uint16_to_bfloat16_x86_64
+#define uint16_to_bfloat16_scalbn uint16_to_bfloat16_scalbn_x86_64
+#define uint32_to_bfloat16 uint32_to_bfloat16_x86_64
+#define uint32_to_bfloat16_scalbn uint32_to_bfloat16_scalbn_x86_64
+#define uint64_to_bfloat16 uint64_to_bfloat16_x86_64
+#define uint64_to_bfloat16_scalbn uint64_to_bfloat16_scalbn_x86_64
+#define uint8_to_float16 uint8_to_float16_x86_64
 #define normalizeFloatx80Subnormal normalizeFloatx80Subnormal_x86_64
 #define roundAndPackFloatx80 roundAndPackFloatx80_x86_64
 #define normalizeRoundAndPackFloatx80 normalizeRoundAndPackFloatx80_x86_64
@@ -1126,6 +1218,11 @@
 #define helper_ctpop_i64 helper_ctpop_i64_x86_64
 #define helper_lookup_tb_ptr helper_lookup_tb_ptr_x86_64
 #define helper_exit_atomic helper_exit_atomic_x86_64
+#define helper_memset helper_memset_x86_64
+#define helper_emu_stop helper_emu_stop_x86_64
+#define tcg_remove_ops_after tcg_remove_ops_after_x86_64
+#define tcg_constant_vec_matching tcg_constant_vec_matching_x86_64
+#define tcg_gen_gvec_dup_imm tcg_gen_gvec_dup_imm_x86_64
 #define helper_gvec_add8 helper_gvec_add8_x86_64
 #define helper_gvec_add16 helper_gvec_add16_x86_64
 #define helper_gvec_add32 helper_gvec_add32_x86_64
@@ -1289,6 +1386,680 @@
 #define gen_helper_raise_interrupt gen_helper_raise_interrupt_x86_64
 #define gen_helper_vfp_get_fpscr gen_helper_vfp_get_fpscr_x86_64
 #define gen_helper_vfp_set_fpscr gen_helper_vfp_set_fpscr_x86_64
+#define gen_helper_mve_vctp gen_helper_mve_vctp_x86_64
+#define gen_helper_mve_vpnot gen_helper_mve_vpnot_x86_64
+#define gen_helper_mve_vpsel gen_helper_mve_vpsel_x86_64
+#define gen_helper_mve_vdup gen_helper_mve_vdup_x86_64
+#define gen_helper_mve_vmovi gen_helper_mve_vmovi_x86_64
+#define gen_helper_mve_vandi gen_helper_mve_vandi_x86_64
+#define gen_helper_mve_vorri gen_helper_mve_vorri_x86_64
+#define gen_helper_mve_vidupb gen_helper_mve_vidupb_x86_64
+#define gen_helper_mve_viduph gen_helper_mve_viduph_x86_64
+#define gen_helper_mve_vidupw gen_helper_mve_vidupw_x86_64
+#define gen_helper_mve_viwdupb gen_helper_mve_viwdupb_x86_64
+#define gen_helper_mve_viwduph gen_helper_mve_viwduph_x86_64
+#define gen_helper_mve_viwdupw gen_helper_mve_viwdupw_x86_64
+#define gen_helper_mve_vdwdupb gen_helper_mve_vdwdupb_x86_64
+#define gen_helper_mve_vdwduph gen_helper_mve_vdwduph_x86_64
+#define gen_helper_mve_vdwdupw gen_helper_mve_vdwdupw_x86_64
+#define gen_helper_mve_vcmpeqb gen_helper_mve_vcmpeqb_x86_64
+#define gen_helper_mve_vcmpeqh gen_helper_mve_vcmpeqh_x86_64
+#define gen_helper_mve_vcmpeqw gen_helper_mve_vcmpeqw_x86_64
+#define gen_helper_mve_vcmpeq_scalarb gen_helper_mve_vcmpeq_scalarb_x86_64
+#define gen_helper_mve_vcmpeq_scalarh gen_helper_mve_vcmpeq_scalarh_x86_64
+#define gen_helper_mve_vcmpeq_scalarw gen_helper_mve_vcmpeq_scalarw_x86_64
+#define gen_helper_mve_vcmpneb gen_helper_mve_vcmpneb_x86_64
+#define gen_helper_mve_vcmpneh gen_helper_mve_vcmpneh_x86_64
+#define gen_helper_mve_vcmpnew gen_helper_mve_vcmpnew_x86_64
+#define gen_helper_mve_vcmpne_scalarb gen_helper_mve_vcmpne_scalarb_x86_64
+#define gen_helper_mve_vcmpne_scalarh gen_helper_mve_vcmpne_scalarh_x86_64
+#define gen_helper_mve_vcmpne_scalarw gen_helper_mve_vcmpne_scalarw_x86_64
+#define gen_helper_mve_vcmpcsb gen_helper_mve_vcmpcsb_x86_64
+#define gen_helper_mve_vcmpcsh gen_helper_mve_vcmpcsh_x86_64
+#define gen_helper_mve_vcmpcsw gen_helper_mve_vcmpcsw_x86_64
+#define gen_helper_mve_vcmpcs_scalarb gen_helper_mve_vcmpcs_scalarb_x86_64
+#define gen_helper_mve_vcmpcs_scalarh gen_helper_mve_vcmpcs_scalarh_x86_64
+#define gen_helper_mve_vcmpcs_scalarw gen_helper_mve_vcmpcs_scalarw_x86_64
+#define gen_helper_mve_vcmphib gen_helper_mve_vcmphib_x86_64
+#define gen_helper_mve_vcmphih gen_helper_mve_vcmphih_x86_64
+#define gen_helper_mve_vcmphiw gen_helper_mve_vcmphiw_x86_64
+#define gen_helper_mve_vcmphi_scalarb gen_helper_mve_vcmphi_scalarb_x86_64
+#define gen_helper_mve_vcmphi_scalarh gen_helper_mve_vcmphi_scalarh_x86_64
+#define gen_helper_mve_vcmphi_scalarw gen_helper_mve_vcmphi_scalarw_x86_64
+#define gen_helper_mve_vcmpgeb gen_helper_mve_vcmpgeb_x86_64
+#define gen_helper_mve_vcmpgeh gen_helper_mve_vcmpgeh_x86_64
+#define gen_helper_mve_vcmpgew gen_helper_mve_vcmpgew_x86_64
+#define gen_helper_mve_vcmpge_scalarb gen_helper_mve_vcmpge_scalarb_x86_64
+#define gen_helper_mve_vcmpge_scalarh gen_helper_mve_vcmpge_scalarh_x86_64
+#define gen_helper_mve_vcmpge_scalarw gen_helper_mve_vcmpge_scalarw_x86_64
+#define gen_helper_mve_vcmpltb gen_helper_mve_vcmpltb_x86_64
+#define gen_helper_mve_vcmplth gen_helper_mve_vcmplth_x86_64
+#define gen_helper_mve_vcmpltw gen_helper_mve_vcmpltw_x86_64
+#define gen_helper_mve_vcmplt_scalarb gen_helper_mve_vcmplt_scalarb_x86_64
+#define gen_helper_mve_vcmplt_scalarh gen_helper_mve_vcmplt_scalarh_x86_64
+#define gen_helper_mve_vcmplt_scalarw gen_helper_mve_vcmplt_scalarw_x86_64
+#define gen_helper_mve_vcmpgtb gen_helper_mve_vcmpgtb_x86_64
+#define gen_helper_mve_vcmpgth gen_helper_mve_vcmpgth_x86_64
+#define gen_helper_mve_vcmpgtw gen_helper_mve_vcmpgtw_x86_64
+#define gen_helper_mve_vcmpgt_scalarb gen_helper_mve_vcmpgt_scalarb_x86_64
+#define gen_helper_mve_vcmpgt_scalarh gen_helper_mve_vcmpgt_scalarh_x86_64
+#define gen_helper_mve_vcmpgt_scalarw gen_helper_mve_vcmpgt_scalarw_x86_64
+#define gen_helper_mve_vcmpleb gen_helper_mve_vcmpleb_x86_64
+#define gen_helper_mve_vcmpleh gen_helper_mve_vcmpleh_x86_64
+#define gen_helper_mve_vcmplew gen_helper_mve_vcmplew_x86_64
+#define gen_helper_mve_vcmple_scalarb gen_helper_mve_vcmple_scalarb_x86_64
+#define gen_helper_mve_vcmple_scalarh gen_helper_mve_vcmple_scalarh_x86_64
+#define gen_helper_mve_vcmple_scalarw gen_helper_mve_vcmple_scalarw_x86_64
+#define gen_helper_mve_vfcmpeqh gen_helper_mve_vfcmpeqh_x86_64
+#define gen_helper_mve_vfcmpeqs gen_helper_mve_vfcmpeqs_x86_64
+#define gen_helper_mve_vfcmpneh gen_helper_mve_vfcmpneh_x86_64
+#define gen_helper_mve_vfcmpnes gen_helper_mve_vfcmpnes_x86_64
+#define gen_helper_mve_vfcmpgeh gen_helper_mve_vfcmpgeh_x86_64
+#define gen_helper_mve_vfcmpges gen_helper_mve_vfcmpges_x86_64
+#define gen_helper_mve_vfcmplth gen_helper_mve_vfcmplth_x86_64
+#define gen_helper_mve_vfcmplts gen_helper_mve_vfcmplts_x86_64
+#define gen_helper_mve_vfcmpgth gen_helper_mve_vfcmpgth_x86_64
+#define gen_helper_mve_vfcmpgts gen_helper_mve_vfcmpgts_x86_64
+#define gen_helper_mve_vfcmpleh gen_helper_mve_vfcmpleh_x86_64
+#define gen_helper_mve_vfcmples gen_helper_mve_vfcmples_x86_64
+#define gen_helper_mve_vfcmpeq_scalarh gen_helper_mve_vfcmpeq_scalarh_x86_64
+#define gen_helper_mve_vfcmpeq_scalars gen_helper_mve_vfcmpeq_scalars_x86_64
+#define gen_helper_mve_vfcmpne_scalarh gen_helper_mve_vfcmpne_scalarh_x86_64
+#define gen_helper_mve_vfcmpne_scalars gen_helper_mve_vfcmpne_scalars_x86_64
+#define gen_helper_mve_vfcmpge_scalarh gen_helper_mve_vfcmpge_scalarh_x86_64
+#define gen_helper_mve_vfcmpge_scalars gen_helper_mve_vfcmpge_scalars_x86_64
+#define gen_helper_mve_vfcmplt_scalarh gen_helper_mve_vfcmplt_scalarh_x86_64
+#define gen_helper_mve_vfcmplt_scalars gen_helper_mve_vfcmplt_scalars_x86_64
+#define gen_helper_mve_vfcmpgt_scalarh gen_helper_mve_vfcmpgt_scalarh_x86_64
+#define gen_helper_mve_vfcmpgt_scalars gen_helper_mve_vfcmpgt_scalars_x86_64
+#define gen_helper_mve_vfcmple_scalarh gen_helper_mve_vfcmple_scalarh_x86_64
+#define gen_helper_mve_vfcmple_scalars gen_helper_mve_vfcmple_scalars_x86_64
+#define gen_helper_mve_vfabsh gen_helper_mve_vfabsh_x86_64
+#define gen_helper_mve_vfabss gen_helper_mve_vfabss_x86_64
+#define gen_helper_mve_vfnegh gen_helper_mve_vfnegh_x86_64
+#define gen_helper_mve_vfnegs gen_helper_mve_vfnegs_x86_64
+#define gen_helper_mve_vldrb gen_helper_mve_vldrb_x86_64
+#define gen_helper_mve_vldrh gen_helper_mve_vldrh_x86_64
+#define gen_helper_mve_vldrw gen_helper_mve_vldrw_x86_64
+#define gen_helper_mve_vldrb_sh gen_helper_mve_vldrb_sh_x86_64
+#define gen_helper_mve_vldrb_uh gen_helper_mve_vldrb_uh_x86_64
+#define gen_helper_mve_vldrb_sw gen_helper_mve_vldrb_sw_x86_64
+#define gen_helper_mve_vldrb_uw gen_helper_mve_vldrb_uw_x86_64
+#define gen_helper_mve_vldrh_sw gen_helper_mve_vldrh_sw_x86_64
+#define gen_helper_mve_vldrh_uw gen_helper_mve_vldrh_uw_x86_64
+#define gen_helper_mve_vstrb gen_helper_mve_vstrb_x86_64
+#define gen_helper_mve_vstrh gen_helper_mve_vstrh_x86_64
+#define gen_helper_mve_vstrw gen_helper_mve_vstrw_x86_64
+#define gen_helper_mve_vstrb_h gen_helper_mve_vstrb_h_x86_64
+#define gen_helper_mve_vstrb_w gen_helper_mve_vstrb_w_x86_64
+#define gen_helper_mve_vstrh_w gen_helper_mve_vstrh_w_x86_64
+#define gen_helper_mve_vldrb_sg_sh gen_helper_mve_vldrb_sg_sh_x86_64
+#define gen_helper_mve_vldrb_sg_sw gen_helper_mve_vldrb_sg_sw_x86_64
+#define gen_helper_mve_vldrh_sg_sw gen_helper_mve_vldrh_sg_sw_x86_64
+#define gen_helper_mve_vldrb_sg_ub gen_helper_mve_vldrb_sg_ub_x86_64
+#define gen_helper_mve_vldrb_sg_uh gen_helper_mve_vldrb_sg_uh_x86_64
+#define gen_helper_mve_vldrb_sg_uw gen_helper_mve_vldrb_sg_uw_x86_64
+#define gen_helper_mve_vldrh_sg_uh gen_helper_mve_vldrh_sg_uh_x86_64
+#define gen_helper_mve_vldrh_sg_uw gen_helper_mve_vldrh_sg_uw_x86_64
+#define gen_helper_mve_vldrw_sg_uw gen_helper_mve_vldrw_sg_uw_x86_64
+#define gen_helper_mve_vldrd_sg_ud gen_helper_mve_vldrd_sg_ud_x86_64
+#define gen_helper_mve_vldrh_sg_os_sw gen_helper_mve_vldrh_sg_os_sw_x86_64
+#define gen_helper_mve_vldrh_sg_os_uh gen_helper_mve_vldrh_sg_os_uh_x86_64
+#define gen_helper_mve_vldrh_sg_os_uw gen_helper_mve_vldrh_sg_os_uw_x86_64
+#define gen_helper_mve_vldrw_sg_os_uw gen_helper_mve_vldrw_sg_os_uw_x86_64
+#define gen_helper_mve_vldrd_sg_os_ud gen_helper_mve_vldrd_sg_os_ud_x86_64
+#define gen_helper_mve_vstrb_sg_ub gen_helper_mve_vstrb_sg_ub_x86_64
+#define gen_helper_mve_vstrb_sg_uh gen_helper_mve_vstrb_sg_uh_x86_64
+#define gen_helper_mve_vstrb_sg_uw gen_helper_mve_vstrb_sg_uw_x86_64
+#define gen_helper_mve_vstrh_sg_uh gen_helper_mve_vstrh_sg_uh_x86_64
+#define gen_helper_mve_vstrh_sg_uw gen_helper_mve_vstrh_sg_uw_x86_64
+#define gen_helper_mve_vstrw_sg_uw gen_helper_mve_vstrw_sg_uw_x86_64
+#define gen_helper_mve_vstrd_sg_ud gen_helper_mve_vstrd_sg_ud_x86_64
+#define gen_helper_mve_vstrh_sg_os_uh gen_helper_mve_vstrh_sg_os_uh_x86_64
+#define gen_helper_mve_vstrh_sg_os_uw gen_helper_mve_vstrh_sg_os_uw_x86_64
+#define gen_helper_mve_vstrw_sg_os_uw gen_helper_mve_vstrw_sg_os_uw_x86_64
+#define gen_helper_mve_vstrd_sg_os_ud gen_helper_mve_vstrd_sg_os_ud_x86_64
+#define gen_helper_mve_vldrw_sg_wb_uw gen_helper_mve_vldrw_sg_wb_uw_x86_64
+#define gen_helper_mve_vldrd_sg_wb_ud gen_helper_mve_vldrd_sg_wb_ud_x86_64
+#define gen_helper_mve_vstrw_sg_wb_uw gen_helper_mve_vstrw_sg_wb_uw_x86_64
+#define gen_helper_mve_vstrd_sg_wb_ud gen_helper_mve_vstrd_sg_wb_ud_x86_64
+#define gen_helper_mve_vld20b gen_helper_mve_vld20b_x86_64
+#define gen_helper_mve_vld20h gen_helper_mve_vld20h_x86_64
+#define gen_helper_mve_vld20w gen_helper_mve_vld20w_x86_64
+#define gen_helper_mve_vld21b gen_helper_mve_vld21b_x86_64
+#define gen_helper_mve_vld21h gen_helper_mve_vld21h_x86_64
+#define gen_helper_mve_vld21w gen_helper_mve_vld21w_x86_64
+#define gen_helper_mve_vld40b gen_helper_mve_vld40b_x86_64
+#define gen_helper_mve_vld40h gen_helper_mve_vld40h_x86_64
+#define gen_helper_mve_vld40w gen_helper_mve_vld40w_x86_64
+#define gen_helper_mve_vld41b gen_helper_mve_vld41b_x86_64
+#define gen_helper_mve_vld41h gen_helper_mve_vld41h_x86_64
+#define gen_helper_mve_vld41w gen_helper_mve_vld41w_x86_64
+#define gen_helper_mve_vld42b gen_helper_mve_vld42b_x86_64
+#define gen_helper_mve_vld42h gen_helper_mve_vld42h_x86_64
+#define gen_helper_mve_vld42w gen_helper_mve_vld42w_x86_64
+#define gen_helper_mve_vld43b gen_helper_mve_vld43b_x86_64
+#define gen_helper_mve_vld43h gen_helper_mve_vld43h_x86_64
+#define gen_helper_mve_vld43w gen_helper_mve_vld43w_x86_64
+#define gen_helper_mve_vst20b gen_helper_mve_vst20b_x86_64
+#define gen_helper_mve_vst20h gen_helper_mve_vst20h_x86_64
+#define gen_helper_mve_vst20w gen_helper_mve_vst20w_x86_64
+#define gen_helper_mve_vst21b gen_helper_mve_vst21b_x86_64
+#define gen_helper_mve_vst21h gen_helper_mve_vst21h_x86_64
+#define gen_helper_mve_vst21w gen_helper_mve_vst21w_x86_64
+#define gen_helper_mve_vst40b gen_helper_mve_vst40b_x86_64
+#define gen_helper_mve_vst40h gen_helper_mve_vst40h_x86_64
+#define gen_helper_mve_vst40w gen_helper_mve_vst40w_x86_64
+#define gen_helper_mve_vst41b gen_helper_mve_vst41b_x86_64
+#define gen_helper_mve_vst41h gen_helper_mve_vst41h_x86_64
+#define gen_helper_mve_vst41w gen_helper_mve_vst41w_x86_64
+#define gen_helper_mve_vst42b gen_helper_mve_vst42b_x86_64
+#define gen_helper_mve_vst42h gen_helper_mve_vst42h_x86_64
+#define gen_helper_mve_vst42w gen_helper_mve_vst42w_x86_64
+#define gen_helper_mve_vst43b gen_helper_mve_vst43b_x86_64
+#define gen_helper_mve_vst43h gen_helper_mve_vst43h_x86_64
+#define gen_helper_mve_vst43w gen_helper_mve_vst43w_x86_64
+#define gen_helper_mve_vand gen_helper_mve_vand_x86_64
+#define gen_helper_mve_vbic gen_helper_mve_vbic_x86_64
+#define gen_helper_mve_vorr gen_helper_mve_vorr_x86_64
+#define gen_helper_mve_vorn gen_helper_mve_vorn_x86_64
+#define gen_helper_mve_veor gen_helper_mve_veor_x86_64
+#define gen_helper_mve_vaddb gen_helper_mve_vaddb_x86_64
+#define gen_helper_mve_vaddh gen_helper_mve_vaddh_x86_64
+#define gen_helper_mve_vaddw gen_helper_mve_vaddw_x86_64
+#define gen_helper_mve_vadd_scalarb gen_helper_mve_vadd_scalarb_x86_64
+#define gen_helper_mve_vadd_scalarh gen_helper_mve_vadd_scalarh_x86_64
+#define gen_helper_mve_vadd_scalarw gen_helper_mve_vadd_scalarw_x86_64
+#define gen_helper_mve_vsubb gen_helper_mve_vsubb_x86_64
+#define gen_helper_mve_vsubh gen_helper_mve_vsubh_x86_64
+#define gen_helper_mve_vsubw gen_helper_mve_vsubw_x86_64
+#define gen_helper_mve_vsub_scalarb gen_helper_mve_vsub_scalarb_x86_64
+#define gen_helper_mve_vsub_scalarh gen_helper_mve_vsub_scalarh_x86_64
+#define gen_helper_mve_vsub_scalarw gen_helper_mve_vsub_scalarw_x86_64
+#define gen_helper_mve_vmulb gen_helper_mve_vmulb_x86_64
+#define gen_helper_mve_vmulh gen_helper_mve_vmulh_x86_64
+#define gen_helper_mve_vmulw gen_helper_mve_vmulw_x86_64
+#define gen_helper_mve_vmul_scalarb gen_helper_mve_vmul_scalarb_x86_64
+#define gen_helper_mve_vmul_scalarh gen_helper_mve_vmul_scalarh_x86_64
+#define gen_helper_mve_vmul_scalarw gen_helper_mve_vmul_scalarw_x86_64
+#define gen_helper_mve_vmulhsb gen_helper_mve_vmulhsb_x86_64
+#define gen_helper_mve_vmulhsh gen_helper_mve_vmulhsh_x86_64
+#define gen_helper_mve_vmulhsw gen_helper_mve_vmulhsw_x86_64
+#define gen_helper_mve_vmulhub gen_helper_mve_vmulhub_x86_64
+#define gen_helper_mve_vmulhuh gen_helper_mve_vmulhuh_x86_64
+#define gen_helper_mve_vmulhuw gen_helper_mve_vmulhuw_x86_64
+#define gen_helper_mve_vrmulhsb gen_helper_mve_vrmulhsb_x86_64
+#define gen_helper_mve_vrmulhsh gen_helper_mve_vrmulhsh_x86_64
+#define gen_helper_mve_vrmulhsw gen_helper_mve_vrmulhsw_x86_64
+#define gen_helper_mve_vrmulhub gen_helper_mve_vrmulhub_x86_64
+#define gen_helper_mve_vrmulhuh gen_helper_mve_vrmulhuh_x86_64
+#define gen_helper_mve_vrmulhuw gen_helper_mve_vrmulhuw_x86_64
+#define gen_helper_mve_vmullbsb gen_helper_mve_vmullbsb_x86_64
+#define gen_helper_mve_vmullbsh gen_helper_mve_vmullbsh_x86_64
+#define gen_helper_mve_vmullbsw gen_helper_mve_vmullbsw_x86_64
+#define gen_helper_mve_vmullbub gen_helper_mve_vmullbub_x86_64
+#define gen_helper_mve_vmullbuh gen_helper_mve_vmullbuh_x86_64
+#define gen_helper_mve_vmullbuw gen_helper_mve_vmullbuw_x86_64
+#define gen_helper_mve_vmulltsb gen_helper_mve_vmulltsb_x86_64
+#define gen_helper_mve_vmulltsh gen_helper_mve_vmulltsh_x86_64
+#define gen_helper_mve_vmulltsw gen_helper_mve_vmulltsw_x86_64
+#define gen_helper_mve_vmulltub gen_helper_mve_vmulltub_x86_64
+#define gen_helper_mve_vmulltuh gen_helper_mve_vmulltuh_x86_64
+#define gen_helper_mve_vmulltuw gen_helper_mve_vmulltuw_x86_64
+#define gen_helper_mve_vmullpbh gen_helper_mve_vmullpbh_x86_64
+#define gen_helper_mve_vmullpth gen_helper_mve_vmullpth_x86_64
+#define gen_helper_mve_vmullpbw gen_helper_mve_vmullpbw_x86_64
+#define gen_helper_mve_vmullptw gen_helper_mve_vmullptw_x86_64
+#define gen_helper_mve_vqdmullbh gen_helper_mve_vqdmullbh_x86_64
+#define gen_helper_mve_vqdmullbw gen_helper_mve_vqdmullbw_x86_64
+#define gen_helper_mve_vqdmullth gen_helper_mve_vqdmullth_x86_64
+#define gen_helper_mve_vqdmulltw gen_helper_mve_vqdmulltw_x86_64
+#define gen_helper_mve_vqdmullb_scalarh gen_helper_mve_vqdmullb_scalarh_x86_64
+#define gen_helper_mve_vqdmullb_scalarw gen_helper_mve_vqdmullb_scalarw_x86_64
+#define gen_helper_mve_vqdmullt_scalarh gen_helper_mve_vqdmullt_scalarh_x86_64
+#define gen_helper_mve_vqdmullt_scalarw gen_helper_mve_vqdmullt_scalarw_x86_64
+#define gen_helper_mve_vcadd90b gen_helper_mve_vcadd90b_x86_64
+#define gen_helper_mve_vcadd90h gen_helper_mve_vcadd90h_x86_64
+#define gen_helper_mve_vcadd90w gen_helper_mve_vcadd90w_x86_64
+#define gen_helper_mve_vcadd270b gen_helper_mve_vcadd270b_x86_64
+#define gen_helper_mve_vcadd270h gen_helper_mve_vcadd270h_x86_64
+#define gen_helper_mve_vcadd270w gen_helper_mve_vcadd270w_x86_64
+#define gen_helper_mve_vhcadd90b gen_helper_mve_vhcadd90b_x86_64
+#define gen_helper_mve_vhcadd90h gen_helper_mve_vhcadd90h_x86_64
+#define gen_helper_mve_vhcadd90w gen_helper_mve_vhcadd90w_x86_64
+#define gen_helper_mve_vhcadd270b gen_helper_mve_vhcadd270b_x86_64
+#define gen_helper_mve_vhcadd270h gen_helper_mve_vhcadd270h_x86_64
+#define gen_helper_mve_vhcadd270w gen_helper_mve_vhcadd270w_x86_64
+#define gen_helper_mve_vmaxsb gen_helper_mve_vmaxsb_x86_64
+#define gen_helper_mve_vmaxsh gen_helper_mve_vmaxsh_x86_64
+#define gen_helper_mve_vmaxsw gen_helper_mve_vmaxsw_x86_64
+#define gen_helper_mve_vmaxub gen_helper_mve_vmaxub_x86_64
+#define gen_helper_mve_vmaxuh gen_helper_mve_vmaxuh_x86_64
+#define gen_helper_mve_vmaxuw gen_helper_mve_vmaxuw_x86_64
+#define gen_helper_mve_vminsb gen_helper_mve_vminsb_x86_64
+#define gen_helper_mve_vminsh gen_helper_mve_vminsh_x86_64
+#define gen_helper_mve_vminsw gen_helper_mve_vminsw_x86_64
+#define gen_helper_mve_vminub gen_helper_mve_vminub_x86_64
+#define gen_helper_mve_vminuh gen_helper_mve_vminuh_x86_64
+#define gen_helper_mve_vminuw gen_helper_mve_vminuw_x86_64
+#define gen_helper_mve_vabdsb gen_helper_mve_vabdsb_x86_64
+#define gen_helper_mve_vabdsh gen_helper_mve_vabdsh_x86_64
+#define gen_helper_mve_vabdsw gen_helper_mve_vabdsw_x86_64
+#define gen_helper_mve_vabdub gen_helper_mve_vabdub_x86_64
+#define gen_helper_mve_vabduh gen_helper_mve_vabduh_x86_64
+#define gen_helper_mve_vabduw gen_helper_mve_vabduw_x86_64
+#define gen_helper_mve_vhaddsb gen_helper_mve_vhaddsb_x86_64
+#define gen_helper_mve_vhaddsh gen_helper_mve_vhaddsh_x86_64
+#define gen_helper_mve_vhaddsw gen_helper_mve_vhaddsw_x86_64
+#define gen_helper_mve_vhaddub gen_helper_mve_vhaddub_x86_64
+#define gen_helper_mve_vhadduh gen_helper_mve_vhadduh_x86_64
+#define gen_helper_mve_vhadduw gen_helper_mve_vhadduw_x86_64
+#define gen_helper_mve_vhadds_scalarb gen_helper_mve_vhadds_scalarb_x86_64
+#define gen_helper_mve_vhadds_scalarh gen_helper_mve_vhadds_scalarh_x86_64
+#define gen_helper_mve_vhadds_scalarw gen_helper_mve_vhadds_scalarw_x86_64
+#define gen_helper_mve_vhaddu_scalarb gen_helper_mve_vhaddu_scalarb_x86_64
+#define gen_helper_mve_vhaddu_scalarh gen_helper_mve_vhaddu_scalarh_x86_64
+#define gen_helper_mve_vhaddu_scalarw gen_helper_mve_vhaddu_scalarw_x86_64
+#define gen_helper_mve_vrhaddsb gen_helper_mve_vrhaddsb_x86_64
+#define gen_helper_mve_vrhaddsh gen_helper_mve_vrhaddsh_x86_64
+#define gen_helper_mve_vrhaddsw gen_helper_mve_vrhaddsw_x86_64
+#define gen_helper_mve_vrhaddub gen_helper_mve_vrhaddub_x86_64
+#define gen_helper_mve_vrhadduh gen_helper_mve_vrhadduh_x86_64
+#define gen_helper_mve_vrhadduw gen_helper_mve_vrhadduw_x86_64
+#define gen_helper_mve_vadc gen_helper_mve_vadc_x86_64
+#define gen_helper_mve_vadci gen_helper_mve_vadci_x86_64
+#define gen_helper_mve_vsbc gen_helper_mve_vsbc_x86_64
+#define gen_helper_mve_vsbci gen_helper_mve_vsbci_x86_64
+#define gen_helper_mve_vhsubsb gen_helper_mve_vhsubsb_x86_64
+#define gen_helper_mve_vhsubsh gen_helper_mve_vhsubsh_x86_64
+#define gen_helper_mve_vhsubsw gen_helper_mve_vhsubsw_x86_64
+#define gen_helper_mve_vhsubub gen_helper_mve_vhsubub_x86_64
+#define gen_helper_mve_vhsubuh gen_helper_mve_vhsubuh_x86_64
+#define gen_helper_mve_vhsubuw gen_helper_mve_vhsubuw_x86_64
+#define gen_helper_mve_vhsubs_scalarb gen_helper_mve_vhsubs_scalarb_x86_64
+#define gen_helper_mve_vhsubs_scalarh gen_helper_mve_vhsubs_scalarh_x86_64
+#define gen_helper_mve_vhsubs_scalarw gen_helper_mve_vhsubs_scalarw_x86_64
+#define gen_helper_mve_vhsubu_scalarb gen_helper_mve_vhsubu_scalarb_x86_64
+#define gen_helper_mve_vhsubu_scalarh gen_helper_mve_vhsubu_scalarh_x86_64
+#define gen_helper_mve_vhsubu_scalarw gen_helper_mve_vhsubu_scalarw_x86_64
+#define gen_helper_mve_vqaddsb gen_helper_mve_vqaddsb_x86_64
+#define gen_helper_mve_vqaddsh gen_helper_mve_vqaddsh_x86_64
+#define gen_helper_mve_vqaddsw gen_helper_mve_vqaddsw_x86_64
+#define gen_helper_mve_vqaddub gen_helper_mve_vqaddub_x86_64
+#define gen_helper_mve_vqadduh gen_helper_mve_vqadduh_x86_64
+#define gen_helper_mve_vqadduw gen_helper_mve_vqadduw_x86_64
+#define gen_helper_mve_vqsubsb gen_helper_mve_vqsubsb_x86_64
+#define gen_helper_mve_vqsubsh gen_helper_mve_vqsubsh_x86_64
+#define gen_helper_mve_vqsubsw gen_helper_mve_vqsubsw_x86_64
+#define gen_helper_mve_vqsubub gen_helper_mve_vqsubub_x86_64
+#define gen_helper_mve_vqsubuh gen_helper_mve_vqsubuh_x86_64
+#define gen_helper_mve_vqsubuw gen_helper_mve_vqsubuw_x86_64
+#define gen_helper_mve_vqdmulhb gen_helper_mve_vqdmulhb_x86_64
+#define gen_helper_mve_vqdmulhh gen_helper_mve_vqdmulhh_x86_64
+#define gen_helper_mve_vqdmulhw gen_helper_mve_vqdmulhw_x86_64
+#define gen_helper_mve_vqrdmulhb gen_helper_mve_vqrdmulhb_x86_64
+#define gen_helper_mve_vqrdmulhh gen_helper_mve_vqrdmulhh_x86_64
+#define gen_helper_mve_vqrdmulhw gen_helper_mve_vqrdmulhw_x86_64
+#define gen_helper_mve_vqadds_scalarb gen_helper_mve_vqadds_scalarb_x86_64
+#define gen_helper_mve_vqadds_scalarh gen_helper_mve_vqadds_scalarh_x86_64
+#define gen_helper_mve_vqadds_scalarw gen_helper_mve_vqadds_scalarw_x86_64
+#define gen_helper_mve_vqaddu_scalarb gen_helper_mve_vqaddu_scalarb_x86_64
+#define gen_helper_mve_vqaddu_scalarh gen_helper_mve_vqaddu_scalarh_x86_64
+#define gen_helper_mve_vqaddu_scalarw gen_helper_mve_vqaddu_scalarw_x86_64
+#define gen_helper_mve_vqsubs_scalarb gen_helper_mve_vqsubs_scalarb_x86_64
+#define gen_helper_mve_vqsubs_scalarh gen_helper_mve_vqsubs_scalarh_x86_64
+#define gen_helper_mve_vqsubs_scalarw gen_helper_mve_vqsubs_scalarw_x86_64
+#define gen_helper_mve_vqsubu_scalarb gen_helper_mve_vqsubu_scalarb_x86_64
+#define gen_helper_mve_vqsubu_scalarh gen_helper_mve_vqsubu_scalarh_x86_64
+#define gen_helper_mve_vqsubu_scalarw gen_helper_mve_vqsubu_scalarw_x86_64
+#define gen_helper_mve_vqdmulh_scalarb gen_helper_mve_vqdmulh_scalarb_x86_64
+#define gen_helper_mve_vqdmulh_scalarh gen_helper_mve_vqdmulh_scalarh_x86_64
+#define gen_helper_mve_vqdmulh_scalarw gen_helper_mve_vqdmulh_scalarw_x86_64
+#define gen_helper_mve_vqrdmulh_scalarb gen_helper_mve_vqrdmulh_scalarb_x86_64
+#define gen_helper_mve_vqrdmulh_scalarh gen_helper_mve_vqrdmulh_scalarh_x86_64
+#define gen_helper_mve_vqrdmulh_scalarw gen_helper_mve_vqrdmulh_scalarw_x86_64
+#define gen_helper_mve_vmlab gen_helper_mve_vmlab_x86_64
+#define gen_helper_mve_vmlah gen_helper_mve_vmlah_x86_64
+#define gen_helper_mve_vmlaw gen_helper_mve_vmlaw_x86_64
+#define gen_helper_mve_vmlasb gen_helper_mve_vmlasb_x86_64
+#define gen_helper_mve_vmlash gen_helper_mve_vmlash_x86_64
+#define gen_helper_mve_vmlasw gen_helper_mve_vmlasw_x86_64
+#define gen_helper_mve_vqdmlahb gen_helper_mve_vqdmlahb_x86_64
+#define gen_helper_mve_vqdmlahh gen_helper_mve_vqdmlahh_x86_64
+#define gen_helper_mve_vqdmlahw gen_helper_mve_vqdmlahw_x86_64
+#define gen_helper_mve_vqrdmlahb gen_helper_mve_vqrdmlahb_x86_64
+#define gen_helper_mve_vqrdmlahh gen_helper_mve_vqrdmlahh_x86_64
+#define gen_helper_mve_vqrdmlahw gen_helper_mve_vqrdmlahw_x86_64
+#define gen_helper_mve_vqdmlashb gen_helper_mve_vqdmlashb_x86_64
+#define gen_helper_mve_vqdmlashh gen_helper_mve_vqdmlashh_x86_64
+#define gen_helper_mve_vqdmlashw gen_helper_mve_vqdmlashw_x86_64
+#define gen_helper_mve_vqrdmlashb gen_helper_mve_vqrdmlashb_x86_64
+#define gen_helper_mve_vqrdmlashh gen_helper_mve_vqrdmlashh_x86_64
+#define gen_helper_mve_vqrdmlashw gen_helper_mve_vqrdmlashw_x86_64
+#define gen_helper_mve_vfaddh gen_helper_mve_vfaddh_x86_64
+#define gen_helper_mve_vfadds gen_helper_mve_vfadds_x86_64
+#define gen_helper_mve_vfsubh gen_helper_mve_vfsubh_x86_64
+#define gen_helper_mve_vfsubs gen_helper_mve_vfsubs_x86_64
+#define gen_helper_mve_vfmulh gen_helper_mve_vfmulh_x86_64
+#define gen_helper_mve_vfmuls gen_helper_mve_vfmuls_x86_64
+#define gen_helper_mve_vfabdh gen_helper_mve_vfabdh_x86_64
+#define gen_helper_mve_vfabds gen_helper_mve_vfabds_x86_64
+#define gen_helper_mve_vmaxnmh gen_helper_mve_vmaxnmh_x86_64
+#define gen_helper_mve_vmaxnms gen_helper_mve_vmaxnms_x86_64
+#define gen_helper_mve_vminnmh gen_helper_mve_vminnmh_x86_64
+#define gen_helper_mve_vminnms gen_helper_mve_vminnms_x86_64
+#define gen_helper_mve_vmaxnmah gen_helper_mve_vmaxnmah_x86_64
+#define gen_helper_mve_vmaxnmas gen_helper_mve_vmaxnmas_x86_64
+#define gen_helper_mve_vminnmah gen_helper_mve_vminnmah_x86_64
+#define gen_helper_mve_vminnmas gen_helper_mve_vminnmas_x86_64
+#define gen_helper_mve_vfcadd90h gen_helper_mve_vfcadd90h_x86_64
+#define gen_helper_mve_vfcadd90s gen_helper_mve_vfcadd90s_x86_64
+#define gen_helper_mve_vfcadd270h gen_helper_mve_vfcadd270h_x86_64
+#define gen_helper_mve_vfcadd270s gen_helper_mve_vfcadd270s_x86_64
+#define gen_helper_mve_vcmul0h gen_helper_mve_vcmul0h_x86_64
+#define gen_helper_mve_vcmul0s gen_helper_mve_vcmul0s_x86_64
+#define gen_helper_mve_vcmul90h gen_helper_mve_vcmul90h_x86_64
+#define gen_helper_mve_vcmul90s gen_helper_mve_vcmul90s_x86_64
+#define gen_helper_mve_vcmul180h gen_helper_mve_vcmul180h_x86_64
+#define gen_helper_mve_vcmul180s gen_helper_mve_vcmul180s_x86_64
+#define gen_helper_mve_vcmul270h gen_helper_mve_vcmul270h_x86_64
+#define gen_helper_mve_vcmul270s gen_helper_mve_vcmul270s_x86_64
+#define gen_helper_mve_vcmla0h gen_helper_mve_vcmla0h_x86_64
+#define gen_helper_mve_vcmla0s gen_helper_mve_vcmla0s_x86_64
+#define gen_helper_mve_vcmla90h gen_helper_mve_vcmla90h_x86_64
+#define gen_helper_mve_vcmla90s gen_helper_mve_vcmla90s_x86_64
+#define gen_helper_mve_vcmla180h gen_helper_mve_vcmla180h_x86_64
+#define gen_helper_mve_vcmla180s gen_helper_mve_vcmla180s_x86_64
+#define gen_helper_mve_vcmla270h gen_helper_mve_vcmla270h_x86_64
+#define gen_helper_mve_vcmla270s gen_helper_mve_vcmla270s_x86_64
+#define gen_helper_mve_vfmah gen_helper_mve_vfmah_x86_64
+#define gen_helper_mve_vfmas gen_helper_mve_vfmas_x86_64
+#define gen_helper_mve_vfmsh gen_helper_mve_vfmsh_x86_64
+#define gen_helper_mve_vfmss gen_helper_mve_vfmss_x86_64
+#define gen_helper_mve_vfadd_scalarh gen_helper_mve_vfadd_scalarh_x86_64
+#define gen_helper_mve_vfadd_scalars gen_helper_mve_vfadd_scalars_x86_64
+#define gen_helper_mve_vfsub_scalarh gen_helper_mve_vfsub_scalarh_x86_64
+#define gen_helper_mve_vfsub_scalars gen_helper_mve_vfsub_scalars_x86_64
+#define gen_helper_mve_vfmul_scalarh gen_helper_mve_vfmul_scalarh_x86_64
+#define gen_helper_mve_vfmul_scalars gen_helper_mve_vfmul_scalars_x86_64
+#define gen_helper_mve_vfma_scalarh gen_helper_mve_vfma_scalarh_x86_64
+#define gen_helper_mve_vfma_scalars gen_helper_mve_vfma_scalars_x86_64
+#define gen_helper_mve_vfmas_scalarh gen_helper_mve_vfmas_scalarh_x86_64
+#define gen_helper_mve_vfmas_scalars gen_helper_mve_vfmas_scalars_x86_64
+#define gen_helper_mve_vcvt_sh gen_helper_mve_vcvt_sh_x86_64
+#define gen_helper_mve_vcvt_uh gen_helper_mve_vcvt_uh_x86_64
+#define gen_helper_mve_vcvt_hs gen_helper_mve_vcvt_hs_x86_64
+#define gen_helper_mve_vcvt_hu gen_helper_mve_vcvt_hu_x86_64
+#define gen_helper_mve_vcvt_sf gen_helper_mve_vcvt_sf_x86_64
+#define gen_helper_mve_vcvt_uf gen_helper_mve_vcvt_uf_x86_64
+#define gen_helper_mve_vcvt_fs gen_helper_mve_vcvt_fs_x86_64
+#define gen_helper_mve_vcvt_fu gen_helper_mve_vcvt_fu_x86_64
+#define gen_helper_mve_vcvtb_sh gen_helper_mve_vcvtb_sh_x86_64
+#define gen_helper_mve_vcvtt_sh gen_helper_mve_vcvtt_sh_x86_64
+#define gen_helper_mve_vcvtb_hs gen_helper_mve_vcvtb_hs_x86_64
+#define gen_helper_mve_vcvtt_hs gen_helper_mve_vcvtt_hs_x86_64
+#define gen_helper_mve_vcvt_rm_sh gen_helper_mve_vcvt_rm_sh_x86_64
+#define gen_helper_mve_vcvt_rm_uh gen_helper_mve_vcvt_rm_uh_x86_64
+#define gen_helper_mve_vcvt_rm_ss gen_helper_mve_vcvt_rm_ss_x86_64
+#define gen_helper_mve_vcvt_rm_us gen_helper_mve_vcvt_rm_us_x86_64
+#define gen_helper_mve_vrint_rm_h gen_helper_mve_vrint_rm_h_x86_64
+#define gen_helper_mve_vrint_rm_s gen_helper_mve_vrint_rm_s_x86_64
+#define gen_helper_mve_vrintx_h gen_helper_mve_vrintx_h_x86_64
+#define gen_helper_mve_vrintx_s gen_helper_mve_vrintx_s_x86_64
+#define gen_helper_mve_vshlsb gen_helper_mve_vshlsb_x86_64
+#define gen_helper_mve_vshlsh gen_helper_mve_vshlsh_x86_64
+#define gen_helper_mve_vshlsw gen_helper_mve_vshlsw_x86_64
+#define gen_helper_mve_vshlub gen_helper_mve_vshlub_x86_64
+#define gen_helper_mve_vshluh gen_helper_mve_vshluh_x86_64
+#define gen_helper_mve_vshluw gen_helper_mve_vshluw_x86_64
+#define gen_helper_mve_vrshlsb gen_helper_mve_vrshlsb_x86_64
+#define gen_helper_mve_vrshlsh gen_helper_mve_vrshlsh_x86_64
+#define gen_helper_mve_vrshlsw gen_helper_mve_vrshlsw_x86_64
+#define gen_helper_mve_vrshlub gen_helper_mve_vrshlub_x86_64
+#define gen_helper_mve_vrshluh gen_helper_mve_vrshluh_x86_64
+#define gen_helper_mve_vrshluw gen_helper_mve_vrshluw_x86_64
+#define gen_helper_mve_vqshlsb gen_helper_mve_vqshlsb_x86_64
+#define gen_helper_mve_vqshlsh gen_helper_mve_vqshlsh_x86_64
+#define gen_helper_mve_vqshlsw gen_helper_mve_vqshlsw_x86_64
+#define gen_helper_mve_vqshlub gen_helper_mve_vqshlub_x86_64
+#define gen_helper_mve_vqshluh gen_helper_mve_vqshluh_x86_64
+#define gen_helper_mve_vqshluw gen_helper_mve_vqshluw_x86_64
+#define gen_helper_mve_vqrshlsb gen_helper_mve_vqrshlsb_x86_64
+#define gen_helper_mve_vqrshlsh gen_helper_mve_vqrshlsh_x86_64
+#define gen_helper_mve_vqrshlsw gen_helper_mve_vqrshlsw_x86_64
+#define gen_helper_mve_vqrshlub gen_helper_mve_vqrshlub_x86_64
+#define gen_helper_mve_vqrshluh gen_helper_mve_vqrshluh_x86_64
+#define gen_helper_mve_vqrshluw gen_helper_mve_vqrshluw_x86_64
+#define gen_helper_mve_vqdmladhb gen_helper_mve_vqdmladhb_x86_64
+#define gen_helper_mve_vqdmladhh gen_helper_mve_vqdmladhh_x86_64
+#define gen_helper_mve_vqdmladhw gen_helper_mve_vqdmladhw_x86_64
+#define gen_helper_mve_vqdmladhxb gen_helper_mve_vqdmladhxb_x86_64
+#define gen_helper_mve_vqdmladhxh gen_helper_mve_vqdmladhxh_x86_64
+#define gen_helper_mve_vqdmladhxw gen_helper_mve_vqdmladhxw_x86_64
+#define gen_helper_mve_vqrdmladhb gen_helper_mve_vqrdmladhb_x86_64
+#define gen_helper_mve_vqrdmladhh gen_helper_mve_vqrdmladhh_x86_64
+#define gen_helper_mve_vqrdmladhw gen_helper_mve_vqrdmladhw_x86_64
+#define gen_helper_mve_vqrdmladhxb gen_helper_mve_vqrdmladhxb_x86_64
+#define gen_helper_mve_vqrdmladhxh gen_helper_mve_vqrdmladhxh_x86_64
+#define gen_helper_mve_vqrdmladhxw gen_helper_mve_vqrdmladhxw_x86_64
+#define gen_helper_mve_vqdmlsdhb gen_helper_mve_vqdmlsdhb_x86_64
+#define gen_helper_mve_vqdmlsdhh gen_helper_mve_vqdmlsdhh_x86_64
+#define gen_helper_mve_vqdmlsdhw gen_helper_mve_vqdmlsdhw_x86_64
+#define gen_helper_mve_vqdmlsdhxb gen_helper_mve_vqdmlsdhxb_x86_64
+#define gen_helper_mve_vqdmlsdhxh gen_helper_mve_vqdmlsdhxh_x86_64
+#define gen_helper_mve_vqdmlsdhxw gen_helper_mve_vqdmlsdhxw_x86_64
+#define gen_helper_mve_vqrdmlsdhb gen_helper_mve_vqrdmlsdhb_x86_64
+#define gen_helper_mve_vqrdmlsdhh gen_helper_mve_vqrdmlsdhh_x86_64
+#define gen_helper_mve_vqrdmlsdhw gen_helper_mve_vqrdmlsdhw_x86_64
+#define gen_helper_mve_vqrdmlsdhxb gen_helper_mve_vqrdmlsdhxb_x86_64
+#define gen_helper_mve_vqrdmlsdhxh gen_helper_mve_vqrdmlsdhxh_x86_64
+#define gen_helper_mve_vqrdmlsdhxw gen_helper_mve_vqrdmlsdhxw_x86_64
+#define gen_helper_mve_vbrsrb gen_helper_mve_vbrsrb_x86_64
+#define gen_helper_mve_vbrsrh gen_helper_mve_vbrsrh_x86_64
+#define gen_helper_mve_vbrsrw gen_helper_mve_vbrsrw_x86_64
+#define gen_helper_mve_vshli_sb gen_helper_mve_vshli_sb_x86_64
+#define gen_helper_mve_vshli_sh gen_helper_mve_vshli_sh_x86_64
+#define gen_helper_mve_vshli_sw gen_helper_mve_vshli_sw_x86_64
+#define gen_helper_mve_vshli_ub gen_helper_mve_vshli_ub_x86_64
+#define gen_helper_mve_vshli_uh gen_helper_mve_vshli_uh_x86_64
+#define gen_helper_mve_vshli_uw gen_helper_mve_vshli_uw_x86_64
+#define gen_helper_mve_vrshli_sb gen_helper_mve_vrshli_sb_x86_64
+#define gen_helper_mve_vrshli_sh gen_helper_mve_vrshli_sh_x86_64
+#define gen_helper_mve_vrshli_sw gen_helper_mve_vrshli_sw_x86_64
+#define gen_helper_mve_vrshli_ub gen_helper_mve_vrshli_ub_x86_64
+#define gen_helper_mve_vrshli_uh gen_helper_mve_vrshli_uh_x86_64
+#define gen_helper_mve_vrshli_uw gen_helper_mve_vrshli_uw_x86_64
+#define gen_helper_mve_vqshli_sb gen_helper_mve_vqshli_sb_x86_64
+#define gen_helper_mve_vqshli_sh gen_helper_mve_vqshli_sh_x86_64
+#define gen_helper_mve_vqshli_sw gen_helper_mve_vqshli_sw_x86_64
+#define gen_helper_mve_vqshli_ub gen_helper_mve_vqshli_ub_x86_64
+#define gen_helper_mve_vqshli_uh gen_helper_mve_vqshli_uh_x86_64
+#define gen_helper_mve_vqshli_uw gen_helper_mve_vqshli_uw_x86_64
+#define gen_helper_mve_vqrshli_sb gen_helper_mve_vqrshli_sb_x86_64
+#define gen_helper_mve_vqrshli_sh gen_helper_mve_vqrshli_sh_x86_64
+#define gen_helper_mve_vqrshli_sw gen_helper_mve_vqrshli_sw_x86_64
+#define gen_helper_mve_vqrshli_ub gen_helper_mve_vqrshli_ub_x86_64
+#define gen_helper_mve_vqrshli_uh gen_helper_mve_vqrshli_uh_x86_64
+#define gen_helper_mve_vqrshli_uw gen_helper_mve_vqrshli_uw_x86_64
+#define gen_helper_mve_vqshlui_sb gen_helper_mve_vqshlui_sb_x86_64
+#define gen_helper_mve_vqshlui_sh gen_helper_mve_vqshlui_sh_x86_64
+#define gen_helper_mve_vqshlui_sw gen_helper_mve_vqshlui_sw_x86_64
+#define gen_helper_mve_vshllbsb gen_helper_mve_vshllbsb_x86_64
+#define gen_helper_mve_vshllbsh gen_helper_mve_vshllbsh_x86_64
+#define gen_helper_mve_vshllbub gen_helper_mve_vshllbub_x86_64
+#define gen_helper_mve_vshllbuh gen_helper_mve_vshllbuh_x86_64
+#define gen_helper_mve_vshlltsb gen_helper_mve_vshlltsb_x86_64
+#define gen_helper_mve_vshlltsh gen_helper_mve_vshlltsh_x86_64
+#define gen_helper_mve_vshlltub gen_helper_mve_vshlltub_x86_64
+#define gen_helper_mve_vshlltuh gen_helper_mve_vshlltuh_x86_64
+#define gen_helper_mve_vshrnbb gen_helper_mve_vshrnbb_x86_64
+#define gen_helper_mve_vshrnbh gen_helper_mve_vshrnbh_x86_64
+#define gen_helper_mve_vshrntb gen_helper_mve_vshrntb_x86_64
+#define gen_helper_mve_vshrnth gen_helper_mve_vshrnth_x86_64
+#define gen_helper_mve_vrshrnbb gen_helper_mve_vrshrnbb_x86_64
+#define gen_helper_mve_vrshrnbh gen_helper_mve_vrshrnbh_x86_64
+#define gen_helper_mve_vrshrntb gen_helper_mve_vrshrntb_x86_64
+#define gen_helper_mve_vrshrnth gen_helper_mve_vrshrnth_x86_64
+#define gen_helper_mve_vqshrnb_sb gen_helper_mve_vqshrnb_sb_x86_64
+#define gen_helper_mve_vqshrnb_sh gen_helper_mve_vqshrnb_sh_x86_64
+#define gen_helper_mve_vqshrnt_sb gen_helper_mve_vqshrnt_sb_x86_64
+#define gen_helper_mve_vqshrnt_sh gen_helper_mve_vqshrnt_sh_x86_64
+#define gen_helper_mve_vqshrnb_ub gen_helper_mve_vqshrnb_ub_x86_64
+#define gen_helper_mve_vqshrnb_uh gen_helper_mve_vqshrnb_uh_x86_64
+#define gen_helper_mve_vqshrnt_ub gen_helper_mve_vqshrnt_ub_x86_64
+#define gen_helper_mve_vqshrnt_uh gen_helper_mve_vqshrnt_uh_x86_64
+#define gen_helper_mve_vqshrunbb gen_helper_mve_vqshrunbb_x86_64
+#define gen_helper_mve_vqshrunbh gen_helper_mve_vqshrunbh_x86_64
+#define gen_helper_mve_vqshruntb gen_helper_mve_vqshruntb_x86_64
+#define gen_helper_mve_vqshrunth gen_helper_mve_vqshrunth_x86_64
+#define gen_helper_mve_vqrshrnb_sb gen_helper_mve_vqrshrnb_sb_x86_64
+#define gen_helper_mve_vqrshrnb_sh gen_helper_mve_vqrshrnb_sh_x86_64
+#define gen_helper_mve_vqrshrnt_sb gen_helper_mve_vqrshrnt_sb_x86_64
+#define gen_helper_mve_vqrshrnt_sh gen_helper_mve_vqrshrnt_sh_x86_64
+#define gen_helper_mve_vqrshrnb_ub gen_helper_mve_vqrshrnb_ub_x86_64
+#define gen_helper_mve_vqrshrnb_uh gen_helper_mve_vqrshrnb_uh_x86_64
+#define gen_helper_mve_vqrshrnt_ub gen_helper_mve_vqrshrnt_ub_x86_64
+#define gen_helper_mve_vqrshrnt_uh gen_helper_mve_vqrshrnt_uh_x86_64
+#define gen_helper_mve_vqrshrunbb gen_helper_mve_vqrshrunbb_x86_64
+#define gen_helper_mve_vqrshrunbh gen_helper_mve_vqrshrunbh_x86_64
+#define gen_helper_mve_vqrshruntb gen_helper_mve_vqrshruntb_x86_64
+#define gen_helper_mve_vqrshrunth gen_helper_mve_vqrshrunth_x86_64
+#define gen_helper_mve_vmovnbb gen_helper_mve_vmovnbb_x86_64
+#define gen_helper_mve_vmovnbh gen_helper_mve_vmovnbh_x86_64
+#define gen_helper_mve_vmovntb gen_helper_mve_vmovntb_x86_64
+#define gen_helper_mve_vmovnth gen_helper_mve_vmovnth_x86_64
+#define gen_helper_mve_vqmovnbsb gen_helper_mve_vqmovnbsb_x86_64
+#define gen_helper_mve_vqmovnbsh gen_helper_mve_vqmovnbsh_x86_64
+#define gen_helper_mve_vqmovntsb gen_helper_mve_vqmovntsb_x86_64
+#define gen_helper_mve_vqmovntsh gen_helper_mve_vqmovntsh_x86_64
+#define gen_helper_mve_vqmovnbub gen_helper_mve_vqmovnbub_x86_64
+#define gen_helper_mve_vqmovnbuh gen_helper_mve_vqmovnbuh_x86_64
+#define gen_helper_mve_vqmovntub gen_helper_mve_vqmovntub_x86_64
+#define gen_helper_mve_vqmovntuh gen_helper_mve_vqmovntuh_x86_64
+#define gen_helper_mve_vqmovunbb gen_helper_mve_vqmovunbb_x86_64
+#define gen_helper_mve_vqmovunbh gen_helper_mve_vqmovunbh_x86_64
+#define gen_helper_mve_vqmovuntb gen_helper_mve_vqmovuntb_x86_64
+#define gen_helper_mve_vqmovunth gen_helper_mve_vqmovunth_x86_64
+#define gen_helper_mve_sshrl gen_helper_mve_sshrl_x86_64
+#define gen_helper_mve_ushll gen_helper_mve_ushll_x86_64
+#define gen_helper_mve_sqshll gen_helper_mve_sqshll_x86_64
+#define gen_helper_mve_uqshll gen_helper_mve_uqshll_x86_64
+#define gen_helper_mve_sqrshrl gen_helper_mve_sqrshrl_x86_64
+#define gen_helper_mve_uqrshll gen_helper_mve_uqrshll_x86_64
+#define gen_helper_mve_sqrshrl48 gen_helper_mve_sqrshrl48_x86_64
+#define gen_helper_mve_uqrshll48 gen_helper_mve_uqrshll48_x86_64
+#define gen_helper_mve_uqshl gen_helper_mve_uqshl_x86_64
+#define gen_helper_mve_sqshl gen_helper_mve_sqshl_x86_64
+#define gen_helper_mve_uqrshl gen_helper_mve_uqrshl_x86_64
+#define gen_helper_mve_sqrshr gen_helper_mve_sqrshr_x86_64
+#define gen_helper_mve_vshlc gen_helper_mve_vshlc_x86_64
+#define gen_helper_mve_vsrib gen_helper_mve_vsrib_x86_64
+#define gen_helper_mve_vsrih gen_helper_mve_vsrih_x86_64
+#define gen_helper_mve_vsriw gen_helper_mve_vsriw_x86_64
+#define gen_helper_mve_vslib gen_helper_mve_vslib_x86_64
+#define gen_helper_mve_vslih gen_helper_mve_vslih_x86_64
+#define gen_helper_mve_vsliw gen_helper_mve_vsliw_x86_64
+#define gen_helper_mve_vclsb gen_helper_mve_vclsb_x86_64
+#define gen_helper_mve_vclsh gen_helper_mve_vclsh_x86_64
+#define gen_helper_mve_vclsw gen_helper_mve_vclsw_x86_64
+#define gen_helper_mve_vclzb gen_helper_mve_vclzb_x86_64
+#define gen_helper_mve_vclzh gen_helper_mve_vclzh_x86_64
+#define gen_helper_mve_vclzw gen_helper_mve_vclzw_x86_64
+#define gen_helper_mve_vrev16b gen_helper_mve_vrev16b_x86_64
+#define gen_helper_mve_vrev32b gen_helper_mve_vrev32b_x86_64
+#define gen_helper_mve_vrev32h gen_helper_mve_vrev32h_x86_64
+#define gen_helper_mve_vrev64b gen_helper_mve_vrev64b_x86_64
+#define gen_helper_mve_vrev64h gen_helper_mve_vrev64h_x86_64
+#define gen_helper_mve_vrev64w gen_helper_mve_vrev64w_x86_64
+#define gen_helper_mve_vmvn gen_helper_mve_vmvn_x86_64
+#define gen_helper_mve_vabsb gen_helper_mve_vabsb_x86_64
+#define gen_helper_mve_vabsh gen_helper_mve_vabsh_x86_64
+#define gen_helper_mve_vabsw gen_helper_mve_vabsw_x86_64
+#define gen_helper_mve_vnegb gen_helper_mve_vnegb_x86_64
+#define gen_helper_mve_vnegh gen_helper_mve_vnegh_x86_64
+#define gen_helper_mve_vnegw gen_helper_mve_vnegw_x86_64
+#define gen_helper_mve_vmaxab gen_helper_mve_vmaxab_x86_64
+#define gen_helper_mve_vmaxah gen_helper_mve_vmaxah_x86_64
+#define gen_helper_mve_vmaxaw gen_helper_mve_vmaxaw_x86_64
+#define gen_helper_mve_vminab gen_helper_mve_vminab_x86_64
+#define gen_helper_mve_vminah gen_helper_mve_vminah_x86_64
+#define gen_helper_mve_vminaw gen_helper_mve_vminaw_x86_64
+#define gen_helper_mve_vqabsb gen_helper_mve_vqabsb_x86_64
+#define gen_helper_mve_vqabsh gen_helper_mve_vqabsh_x86_64
+#define gen_helper_mve_vqabsw gen_helper_mve_vqabsw_x86_64
+#define gen_helper_mve_vqnegb gen_helper_mve_vqnegb_x86_64
+#define gen_helper_mve_vqnegh gen_helper_mve_vqnegh_x86_64
+#define gen_helper_mve_vqnegw gen_helper_mve_vqnegw_x86_64
+#define gen_helper_mve_vmlaldavsh gen_helper_mve_vmlaldavsh_x86_64
+#define gen_helper_mve_vmlaldavsw gen_helper_mve_vmlaldavsw_x86_64
+#define gen_helper_mve_vmlaldavxsh gen_helper_mve_vmlaldavxsh_x86_64
+#define gen_helper_mve_vmlaldavxsw gen_helper_mve_vmlaldavxsw_x86_64
+#define gen_helper_mve_vmlaldavuh gen_helper_mve_vmlaldavuh_x86_64
+#define gen_helper_mve_vmlaldavuw gen_helper_mve_vmlaldavuw_x86_64
+#define gen_helper_mve_vmlsldavsh gen_helper_mve_vmlsldavsh_x86_64
+#define gen_helper_mve_vmlsldavsw gen_helper_mve_vmlsldavsw_x86_64
+#define gen_helper_mve_vmlsldavxsh gen_helper_mve_vmlsldavxsh_x86_64
+#define gen_helper_mve_vmlsldavxsw gen_helper_mve_vmlsldavxsw_x86_64
+#define gen_helper_mve_vrmlaldavhsw gen_helper_mve_vrmlaldavhsw_x86_64
+#define gen_helper_mve_vrmlaldavhxsw gen_helper_mve_vrmlaldavhxsw_x86_64
+#define gen_helper_mve_vrmlaldavhuw gen_helper_mve_vrmlaldavhuw_x86_64
+#define gen_helper_mve_vrmlsldavhsw gen_helper_mve_vrmlsldavhsw_x86_64
+#define gen_helper_mve_vrmlsldavhxsw gen_helper_mve_vrmlsldavhxsw_x86_64
+#define gen_helper_mve_vmladavsb gen_helper_mve_vmladavsb_x86_64
+#define gen_helper_mve_vmladavsh gen_helper_mve_vmladavsh_x86_64
+#define gen_helper_mve_vmladavsw gen_helper_mve_vmladavsw_x86_64
+#define gen_helper_mve_vmladavub gen_helper_mve_vmladavub_x86_64
+#define gen_helper_mve_vmladavuh gen_helper_mve_vmladavuh_x86_64
+#define gen_helper_mve_vmladavuw gen_helper_mve_vmladavuw_x86_64
+#define gen_helper_mve_vmlsdavb gen_helper_mve_vmlsdavb_x86_64
+#define gen_helper_mve_vmlsdavh gen_helper_mve_vmlsdavh_x86_64
+#define gen_helper_mve_vmlsdavw gen_helper_mve_vmlsdavw_x86_64
+#define gen_helper_mve_vmladavsxb gen_helper_mve_vmladavsxb_x86_64
+#define gen_helper_mve_vmladavsxh gen_helper_mve_vmladavsxh_x86_64
+#define gen_helper_mve_vmladavsxw gen_helper_mve_vmladavsxw_x86_64
+#define gen_helper_mve_vmlsdavxb gen_helper_mve_vmlsdavxb_x86_64
+#define gen_helper_mve_vmlsdavxh gen_helper_mve_vmlsdavxh_x86_64
+#define gen_helper_mve_vmlsdavxw gen_helper_mve_vmlsdavxw_x86_64
+#define gen_helper_mve_vaddvsb gen_helper_mve_vaddvsb_x86_64
+#define gen_helper_mve_vaddvsh gen_helper_mve_vaddvsh_x86_64
+#define gen_helper_mve_vaddvsw gen_helper_mve_vaddvsw_x86_64
+#define gen_helper_mve_vaddvub gen_helper_mve_vaddvub_x86_64
+#define gen_helper_mve_vaddvuh gen_helper_mve_vaddvuh_x86_64
+#define gen_helper_mve_vaddvuw gen_helper_mve_vaddvuw_x86_64
+#define gen_helper_mve_vmaxvsb gen_helper_mve_vmaxvsb_x86_64
+#define gen_helper_mve_vmaxvsh gen_helper_mve_vmaxvsh_x86_64
+#define gen_helper_mve_vmaxvsw gen_helper_mve_vmaxvsw_x86_64
+#define gen_helper_mve_vmaxvub gen_helper_mve_vmaxvub_x86_64
+#define gen_helper_mve_vmaxvuh gen_helper_mve_vmaxvuh_x86_64
+#define gen_helper_mve_vmaxvuw gen_helper_mve_vmaxvuw_x86_64
+#define gen_helper_mve_vmaxavb gen_helper_mve_vmaxavb_x86_64
+#define gen_helper_mve_vmaxavh gen_helper_mve_vmaxavh_x86_64
+#define gen_helper_mve_vmaxavw gen_helper_mve_vmaxavw_x86_64
+#define gen_helper_mve_vminvsb gen_helper_mve_vminvsb_x86_64
+#define gen_helper_mve_vminvsh gen_helper_mve_vminvsh_x86_64
+#define gen_helper_mve_vminvsw gen_helper_mve_vminvsw_x86_64
+#define gen_helper_mve_vminvub gen_helper_mve_vminvub_x86_64
+#define gen_helper_mve_vminvuh gen_helper_mve_vminvuh_x86_64
+#define gen_helper_mve_vminvuw gen_helper_mve_vminvuw_x86_64
+#define gen_helper_mve_vminavb gen_helper_mve_vminavb_x86_64
+#define gen_helper_mve_vminavh gen_helper_mve_vminavh_x86_64
+#define gen_helper_mve_vminavw gen_helper_mve_vminavw_x86_64
+#define gen_helper_mve_vmaxnmvh gen_helper_mve_vmaxnmvh_x86_64
+#define gen_helper_mve_vmaxnmvs gen_helper_mve_vmaxnmvs_x86_64
+#define gen_helper_mve_vminnmvh gen_helper_mve_vminnmvh_x86_64
+#define gen_helper_mve_vminnmvs gen_helper_mve_vminnmvs_x86_64
+#define gen_helper_mve_vmaxnmavh gen_helper_mve_vmaxnmavh_x86_64
+#define gen_helper_mve_vmaxnmavs gen_helper_mve_vmaxnmavs_x86_64
+#define gen_helper_mve_vminnmavh gen_helper_mve_vminnmavh_x86_64
+#define gen_helper_mve_vminnmavs gen_helper_mve_vminnmavs_x86_64
+#define gen_helper_mve_vaddlv_s gen_helper_mve_vaddlv_s_x86_64
+#define gen_helper_mve_vaddlv_u gen_helper_mve_vaddlv_u_x86_64
+#define gen_helper_mve_vabavsb gen_helper_mve_vabavsb_x86_64
+#define gen_helper_mve_vabavsh gen_helper_mve_vabavsh_x86_64
+#define gen_helper_mve_vabavsw gen_helper_mve_vabavsw_x86_64
+#define gen_helper_mve_vabavub gen_helper_mve_vabavub_x86_64
+#define gen_helper_mve_vabavuh gen_helper_mve_vabavuh_x86_64
+#define gen_helper_mve_vabavuw gen_helper_mve_vabavuw_x86_64
 #define gen_helper_cpsr_read gen_helper_cpsr_read_x86_64
 #define gen_helper_cpsr_write gen_helper_cpsr_write_x86_64
 #define tlb_reset_dirty_by_vaddr tlb_reset_dirty_by_vaddr_x86_64
@@ -1572,119 +2343,69 @@
 #define helper_maskmov_xmm helper_maskmov_xmm_x86_64
 #define helper_movl_mm_T0_xmm helper_movl_mm_T0_xmm_x86_64
 #define helper_movq_mm_T0_xmm helper_movq_mm_T0_xmm_x86_64
-#define helper_shufps helper_shufps_x86_64
-#define helper_shufpd helper_shufpd_x86_64
 #define helper_pshufd_xmm helper_pshufd_xmm_x86_64
 #define helper_pshuflw_xmm helper_pshuflw_xmm_x86_64
 #define helper_pshufhw_xmm helper_pshufhw_xmm_x86_64
-#define helper_addps helper_addps_x86_64
 #define helper_addss helper_addss_x86_64
-#define helper_addpd helper_addpd_x86_64
 #define helper_addsd helper_addsd_x86_64
-#define helper_subps helper_subps_x86_64
 #define helper_subss helper_subss_x86_64
-#define helper_subpd helper_subpd_x86_64
 #define helper_subsd helper_subsd_x86_64
-#define helper_mulps helper_mulps_x86_64
 #define helper_mulss helper_mulss_x86_64
-#define helper_mulpd helper_mulpd_x86_64
 #define helper_mulsd helper_mulsd_x86_64
-#define helper_divps helper_divps_x86_64
 #define helper_divss helper_divss_x86_64
-#define helper_divpd helper_divpd_x86_64
 #define helper_divsd helper_divsd_x86_64
-#define helper_minps helper_minps_x86_64
 #define helper_minss helper_minss_x86_64
-#define helper_minpd helper_minpd_x86_64
 #define helper_minsd helper_minsd_x86_64
-#define helper_maxps helper_maxps_x86_64
 #define helper_maxss helper_maxss_x86_64
-#define helper_maxpd helper_maxpd_x86_64
 #define helper_maxsd helper_maxsd_x86_64
-#define helper_sqrtps helper_sqrtps_x86_64
 #define helper_sqrtss helper_sqrtss_x86_64
-#define helper_sqrtpd helper_sqrtpd_x86_64
 #define helper_sqrtsd helper_sqrtsd_x86_64
-#define helper_cvtps2pd helper_cvtps2pd_x86_64
-#define helper_cvtpd2ps helper_cvtpd2ps_x86_64
 #define helper_cvtss2sd helper_cvtss2sd_x86_64
 #define helper_cvtsd2ss helper_cvtsd2ss_x86_64
-#define helper_cvtdq2ps helper_cvtdq2ps_x86_64
-#define helper_cvtdq2pd helper_cvtdq2pd_x86_64
 #define helper_cvtpi2ps helper_cvtpi2ps_x86_64
 #define helper_cvtpi2pd helper_cvtpi2pd_x86_64
 #define helper_cvtsi2ss helper_cvtsi2ss_x86_64
 #define helper_cvtsi2sd helper_cvtsi2sd_x86_64
 #define helper_cvtsq2ss helper_cvtsq2ss_x86_64
 #define helper_cvtsq2sd helper_cvtsq2sd_x86_64
-#define helper_cvtps2dq helper_cvtps2dq_x86_64
-#define helper_cvtpd2dq helper_cvtpd2dq_x86_64
 #define helper_cvtps2pi helper_cvtps2pi_x86_64
 #define helper_cvtpd2pi helper_cvtpd2pi_x86_64
 #define helper_cvtss2si helper_cvtss2si_x86_64
 #define helper_cvtsd2si helper_cvtsd2si_x86_64
 #define helper_cvtss2sq helper_cvtss2sq_x86_64
 #define helper_cvtsd2sq helper_cvtsd2sq_x86_64
-#define helper_cvttps2dq helper_cvttps2dq_x86_64
-#define helper_cvttpd2dq helper_cvttpd2dq_x86_64
 #define helper_cvttps2pi helper_cvttps2pi_x86_64
 #define helper_cvttpd2pi helper_cvttpd2pi_x86_64
 #define helper_cvttss2si helper_cvttss2si_x86_64
 #define helper_cvttsd2si helper_cvttsd2si_x86_64
 #define helper_cvttss2sq helper_cvttss2sq_x86_64
 #define helper_cvttsd2sq helper_cvttsd2sq_x86_64
-#define helper_rsqrtps helper_rsqrtps_x86_64
 #define helper_rsqrtss helper_rsqrtss_x86_64
-#define helper_rcpps helper_rcpps_x86_64
 #define helper_rcpss helper_rcpss_x86_64
 #define helper_extrq_r helper_extrq_r_x86_64
 #define helper_extrq_i helper_extrq_i_x86_64
 #define helper_insertq_r helper_insertq_r_x86_64
 #define helper_insertq_i helper_insertq_i_x86_64
-#define helper_haddps helper_haddps_x86_64
-#define helper_haddpd helper_haddpd_x86_64
-#define helper_hsubps helper_hsubps_x86_64
-#define helper_hsubpd helper_hsubpd_x86_64
-#define helper_addsubps helper_addsubps_x86_64
-#define helper_addsubpd helper_addsubpd_x86_64
-#define helper_cmpeqps helper_cmpeqps_x86_64
 #define helper_cmpeqss helper_cmpeqss_x86_64
-#define helper_cmpeqpd helper_cmpeqpd_x86_64
 #define helper_cmpeqsd helper_cmpeqsd_x86_64
-#define helper_cmpltps helper_cmpltps_x86_64
 #define helper_cmpltss helper_cmpltss_x86_64
-#define helper_cmpltpd helper_cmpltpd_x86_64
 #define helper_cmpltsd helper_cmpltsd_x86_64
-#define helper_cmpleps helper_cmpleps_x86_64
 #define helper_cmpless helper_cmpless_x86_64
-#define helper_cmplepd helper_cmplepd_x86_64
 #define helper_cmplesd helper_cmplesd_x86_64
-#define helper_cmpunordps helper_cmpunordps_x86_64
 #define helper_cmpunordss helper_cmpunordss_x86_64
-#define helper_cmpunordpd helper_cmpunordpd_x86_64
 #define helper_cmpunordsd helper_cmpunordsd_x86_64
-#define helper_cmpneqps helper_cmpneqps_x86_64
 #define helper_cmpneqss helper_cmpneqss_x86_64
-#define helper_cmpneqpd helper_cmpneqpd_x86_64
 #define helper_cmpneqsd helper_cmpneqsd_x86_64
-#define helper_cmpnltps helper_cmpnltps_x86_64
 #define helper_cmpnltss helper_cmpnltss_x86_64
-#define helper_cmpnltpd helper_cmpnltpd_x86_64
 #define helper_cmpnltsd helper_cmpnltsd_x86_64
-#define helper_cmpnleps helper_cmpnleps_x86_64
 #define helper_cmpnless helper_cmpnless_x86_64
-#define helper_cmpnlepd helper_cmpnlepd_x86_64
 #define helper_cmpnlesd helper_cmpnlesd_x86_64
-#define helper_cmpordps helper_cmpordps_x86_64
 #define helper_cmpordss helper_cmpordss_x86_64
-#define helper_cmpordpd helper_cmpordpd_x86_64
 #define helper_cmpordsd helper_cmpordsd_x86_64
 #define helper_ucomiss helper_ucomiss_x86_64
 #define helper_comiss helper_comiss_x86_64
 #define helper_ucomisd helper_ucomisd_x86_64
 #define helper_comisd helper_comisd_x86_64
-#define helper_movmskps helper_movmskps_x86_64
-#define helper_movmskpd helper_movmskpd_x86_64
 #define helper_pmovmskb_xmm helper_pmovmskb_xmm_x86_64
 #define helper_packsswb_xmm helper_packsswb_xmm_x86_64
 #define helper_packuswb_xmm helper_packuswb_xmm_x86_64
@@ -1895,4 +2616,104 @@
 #define x86_cpu_xrstor_all_areas x86_cpu_xrstor_all_areas_x86_64
 #define cpu_get_fp80 cpu_get_fp80_x86_64
 #define cpu_set_fp80 cpu_set_fp80_x86_64
+#define helper_addpd_xmm helper_addpd_xmm_x86_64
+#define helper_addpd_ymm helper_addpd_ymm_x86_64
+#define helper_addps_xmm helper_addps_xmm_x86_64
+#define helper_addps_ymm helper_addps_ymm_x86_64
+#define helper_addsubpd_xmm helper_addsubpd_xmm_x86_64
+#define helper_addsubpd_ymm helper_addsubpd_ymm_x86_64
+#define helper_addsubps_xmm helper_addsubps_xmm_x86_64
+#define helper_addsubps_ymm helper_addsubps_ymm_x86_64
+#define helper_cmpeqpd_xmm helper_cmpeqpd_xmm_x86_64
+#define helper_cmpeqpd_ymm helper_cmpeqpd_ymm_x86_64
+#define helper_cmpeqps_xmm helper_cmpeqps_xmm_x86_64
+#define helper_cmpeqps_ymm helper_cmpeqps_ymm_x86_64
+#define helper_cmplepd_xmm helper_cmplepd_xmm_x86_64
+#define helper_cmplepd_ymm helper_cmplepd_ymm_x86_64
+#define helper_cmpleps_xmm helper_cmpleps_xmm_x86_64
+#define helper_cmpleps_ymm helper_cmpleps_ymm_x86_64
+#define helper_cmpltpd_xmm helper_cmpltpd_xmm_x86_64
+#define helper_cmpltpd_ymm helper_cmpltpd_ymm_x86_64
+#define helper_cmpltps_xmm helper_cmpltps_xmm_x86_64
+#define helper_cmpltps_ymm helper_cmpltps_ymm_x86_64
+#define helper_cmpneqpd_xmm helper_cmpneqpd_xmm_x86_64
+#define helper_cmpneqpd_ymm helper_cmpneqpd_ymm_x86_64
+#define helper_cmpneqps_xmm helper_cmpneqps_xmm_x86_64
+#define helper_cmpneqps_ymm helper_cmpneqps_ymm_x86_64
+#define helper_cmpnlepd_xmm helper_cmpnlepd_xmm_x86_64
+#define helper_cmpnlepd_ymm helper_cmpnlepd_ymm_x86_64
+#define helper_cmpnleps_xmm helper_cmpnleps_xmm_x86_64
+#define helper_cmpnleps_ymm helper_cmpnleps_ymm_x86_64
+#define helper_cmpnltpd_xmm helper_cmpnltpd_xmm_x86_64
+#define helper_cmpnltpd_ymm helper_cmpnltpd_ymm_x86_64
+#define helper_cmpnltps_xmm helper_cmpnltps_xmm_x86_64
+#define helper_cmpnltps_ymm helper_cmpnltps_ymm_x86_64
+#define helper_cmpordpd_xmm helper_cmpordpd_xmm_x86_64
+#define helper_cmpordpd_ymm helper_cmpordpd_ymm_x86_64
+#define helper_cmpordps_xmm helper_cmpordps_xmm_x86_64
+#define helper_cmpordps_ymm helper_cmpordps_ymm_x86_64
+#define helper_cmpunordpd_xmm helper_cmpunordpd_xmm_x86_64
+#define helper_cmpunordpd_ymm helper_cmpunordpd_ymm_x86_64
+#define helper_cmpunordps_xmm helper_cmpunordps_xmm_x86_64
+#define helper_cmpunordps_ymm helper_cmpunordps_ymm_x86_64
+#define helper_cvtdq2pd_xmm helper_cvtdq2pd_xmm_x86_64
+#define helper_cvtdq2pd_ymm helper_cvtdq2pd_ymm_x86_64
+#define helper_cvtdq2ps_xmm helper_cvtdq2ps_xmm_x86_64
+#define helper_cvtdq2ps_ymm helper_cvtdq2ps_ymm_x86_64
+#define helper_cvtpd2dq_xmm helper_cvtpd2dq_xmm_x86_64
+#define helper_cvtpd2dq_ymm helper_cvtpd2dq_ymm_x86_64
+#define helper_cvtpd2ps_xmm helper_cvtpd2ps_xmm_x86_64
+#define helper_cvtpd2ps_ymm helper_cvtpd2ps_ymm_x86_64
+#define helper_cvtps2dq_xmm helper_cvtps2dq_xmm_x86_64
+#define helper_cvtps2dq_ymm helper_cvtps2dq_ymm_x86_64
+#define helper_cvtps2pd_xmm helper_cvtps2pd_xmm_x86_64
+#define helper_cvtps2pd_ymm helper_cvtps2pd_ymm_x86_64
+#define helper_cvttpd2dq_xmm helper_cvttpd2dq_xmm_x86_64
+#define helper_cvttpd2dq_ymm helper_cvttpd2dq_ymm_x86_64
+#define helper_cvttps2dq_xmm helper_cvttps2dq_xmm_x86_64
+#define helper_cvttps2dq_ymm helper_cvttps2dq_ymm_x86_64
+#define helper_divpd_xmm helper_divpd_xmm_x86_64
+#define helper_divpd_ymm helper_divpd_ymm_x86_64
+#define helper_divps_xmm helper_divps_xmm_x86_64
+#define helper_divps_ymm helper_divps_ymm_x86_64
+#define helper_haddpd_xmm helper_haddpd_xmm_x86_64
+#define helper_haddpd_ymm helper_haddpd_ymm_x86_64
+#define helper_haddps_xmm helper_haddps_xmm_x86_64
+#define helper_haddps_ymm helper_haddps_ymm_x86_64
+#define helper_hsubpd_xmm helper_hsubpd_xmm_x86_64
+#define helper_hsubpd_ymm helper_hsubpd_ymm_x86_64
+#define helper_hsubps_xmm helper_hsubps_xmm_x86_64
+#define helper_hsubps_ymm helper_hsubps_ymm_x86_64
+#define helper_maxpd_xmm helper_maxpd_xmm_x86_64
+#define helper_maxpd_ymm helper_maxpd_ymm_x86_64
+#define helper_maxps_xmm helper_maxps_xmm_x86_64
+#define helper_maxps_ymm helper_maxps_ymm_x86_64
+#define helper_minpd_xmm helper_minpd_xmm_x86_64
+#define helper_minpd_ymm helper_minpd_ymm_x86_64
+#define helper_minps_xmm helper_minps_xmm_x86_64
+#define helper_minps_ymm helper_minps_ymm_x86_64
+#define helper_movmskpd_xmm helper_movmskpd_xmm_x86_64
+#define helper_movmskpd_ymm helper_movmskpd_ymm_x86_64
+#define helper_movmskps_xmm helper_movmskps_xmm_x86_64
+#define helper_movmskps_ymm helper_movmskps_ymm_x86_64
+#define helper_mulpd_xmm helper_mulpd_xmm_x86_64
+#define helper_mulpd_ymm helper_mulpd_ymm_x86_64
+#define helper_mulps_xmm helper_mulps_xmm_x86_64
+#define helper_mulps_ymm helper_mulps_ymm_x86_64
+#define helper_rcpps_xmm helper_rcpps_xmm_x86_64
+#define helper_rcpps_ymm helper_rcpps_ymm_x86_64
+#define helper_rsqrtps_xmm helper_rsqrtps_xmm_x86_64
+#define helper_rsqrtps_ymm helper_rsqrtps_ymm_x86_64
+#define helper_shufpd_xmm helper_shufpd_xmm_x86_64
+#define helper_shufpd_ymm helper_shufpd_ymm_x86_64
+#define helper_shufps_xmm helper_shufps_xmm_x86_64
+#define helper_shufps_ymm helper_shufps_ymm_x86_64
+#define helper_sqrtpd_xmm helper_sqrtpd_xmm_x86_64
+#define helper_sqrtpd_ymm helper_sqrtpd_ymm_x86_64
+#define helper_sqrtps_xmm helper_sqrtps_xmm_x86_64
+#define helper_sqrtps_ymm helper_sqrtps_ymm_x86_64
+#define helper_subpd_xmm helper_subpd_xmm_x86_64
+#define helper_subpd_ymm helper_subpd_ymm_x86_64
+#define helper_subps_xmm helper_subps_xmm_x86_64
+#define helper_subps_ymm helper_subps_ymm_x86_64
 #endif
diff --git a/symbols.sh b/symbols.sh
index bcedaea983..6ca25d24ae 100755
--- a/symbols.sh
+++ b/symbols.sh
@@ -329,6 +329,98 @@ float64_silence_nan \
 float16_squash_input_denormal \
 float32_squash_input_denormal \
 float64_squash_input_denormal \
+bfloat16_add \
+bfloat16_compare \
+bfloat16_compare_quiet \
+bfloat16_default_nan \
+bfloat16_div \
+bfloat16_is_quiet_nan \
+bfloat16_is_signaling_nan \
+bfloat16_max \
+bfloat16_maximum_number \
+bfloat16_maxnum \
+bfloat16_maxnummag \
+bfloat16_min \
+bfloat16_minimum_number \
+bfloat16_minnum \
+bfloat16_minnummag \
+bfloat16_mul \
+bfloat16_muladd \
+bfloat16_round_to_int \
+bfloat16_scalbn \
+bfloat16_silence_nan \
+bfloat16_sqrt \
+bfloat16_squash_input_denormal \
+bfloat16_sub \
+bfloat16_to_float32 \
+bfloat16_to_float64 \
+bfloat16_to_int16 \
+bfloat16_to_int16_round_to_zero \
+bfloat16_to_int16_scalbn \
+bfloat16_to_int32 \
+bfloat16_to_int32_round_to_zero \
+bfloat16_to_int32_scalbn \
+bfloat16_to_int64 \
+bfloat16_to_int64_round_to_zero \
+bfloat16_to_int64_scalbn \
+bfloat16_to_uint16 \
+bfloat16_to_uint16_round_to_zero \
+bfloat16_to_uint16_scalbn \
+bfloat16_to_uint32 \
+bfloat16_to_uint32_round_to_zero \
+bfloat16_to_uint32_scalbn \
+bfloat16_to_uint64 \
+bfloat16_to_uint64_round_to_zero \
+bfloat16_to_uint64_scalbn \
+float128_maximum_number \
+float128_max \
+float128_maxnum \
+float128_maxnummag \
+float128_min \
+float128_minimum_number \
+float128_minnum \
+float128_minnummag \
+float128_muladd \
+float128_to_int128 \
+float128_to_int128_round_to_zero \
+float128_to_uint128 \
+float128_to_uint128_round_to_zero \
+float16_maximum_number \
+float16_minimum_number \
+float16_to_int8 \
+float16_to_int8_scalbn \
+float16_to_uint8 \
+float16_to_uint8_scalbn \
+float32_maximum_number \
+float32_minimum_number \
+float32_to_bfloat16 \
+float64_maximum_number \
+float64_minimum_number \
+float64_to_bfloat16 \
+float64r32_add \
+float64r32_div \
+float64r32_mul \
+float64r32_muladd \
+float64r32_sqrt \
+float64r32_sub \
+floatx80_mod \
+floatx80_modrem \
+int128_to_float128 \
+int16_to_bfloat16 \
+int16_to_bfloat16_scalbn \
+int32_to_bfloat16 \
+int32_to_bfloat16_scalbn \
+int64_to_bfloat16 \
+int64_to_bfloat16_scalbn \
+int8_to_float16 \
+uint128_to_float128 \
+uint16_to_bfloat16 \
+uint16_to_bfloat16_scalbn \
+uint32_to_bfloat16 \
+uint32_to_bfloat16_scalbn \
+uint64_to_bfloat16 \
+uint64_to_bfloat16_scalbn \
+uint8_to_float16 \
 normalizeFloatx80Subnormal \
 roundAndPackFloatx80 \
 normalizeRoundAndPackFloatx80 \
@@ -1126,6 +1218,11 @@ helper_ctpop_i32 \
 helper_ctpop_i64 \
 helper_lookup_tb_ptr \
 helper_exit_atomic \
+helper_memset \
+helper_emu_stop \
+tcg_remove_ops_after \
+tcg_constant_vec_matching \
+tcg_gen_gvec_dup_imm \
 helper_gvec_add8 \
 helper_gvec_add16 \
 helper_gvec_add32 \
@@ -1289,6 +1386,680 @@ gen_helper_raise_exception \
 gen_helper_raise_interrupt \
 gen_helper_vfp_get_fpscr \
 gen_helper_vfp_set_fpscr \
+gen_helper_mve_vctp \
+gen_helper_mve_vpnot \
+gen_helper_mve_vpsel \
+gen_helper_mve_vdup \
+gen_helper_mve_vmovi \
+gen_helper_mve_vandi \
+gen_helper_mve_vorri \
+gen_helper_mve_vidupb \
+gen_helper_mve_viduph \
+gen_helper_mve_vidupw \
+gen_helper_mve_viwdupb \
+gen_helper_mve_viwduph \
+gen_helper_mve_viwdupw \
+gen_helper_mve_vdwdupb \
+gen_helper_mve_vdwduph \
+gen_helper_mve_vdwdupw \
+gen_helper_mve_vcmpeqb \
+gen_helper_mve_vcmpeqh \
+gen_helper_mve_vcmpeqw \
+gen_helper_mve_vcmpeq_scalarb \
+gen_helper_mve_vcmpeq_scalarh \
+gen_helper_mve_vcmpeq_scalarw \
+gen_helper_mve_vcmpneb \
+gen_helper_mve_vcmpneh \
+gen_helper_mve_vcmpnew \
+gen_helper_mve_vcmpne_scalarb \
+gen_helper_mve_vcmpne_scalarh \
+gen_helper_mve_vcmpne_scalarw \
+gen_helper_mve_vcmpcsb \
+gen_helper_mve_vcmpcsh \
+gen_helper_mve_vcmpcsw \
+gen_helper_mve_vcmpcs_scalarb \
+gen_helper_mve_vcmpcs_scalarh \
+gen_helper_mve_vcmpcs_scalarw \
+gen_helper_mve_vcmphib \
+gen_helper_mve_vcmphih \
+gen_helper_mve_vcmphiw \
+gen_helper_mve_vcmphi_scalarb \
+gen_helper_mve_vcmphi_scalarh \
+gen_helper_mve_vcmphi_scalarw \
+gen_helper_mve_vcmpgeb \
+gen_helper_mve_vcmpgeh \
+gen_helper_mve_vcmpgew \
+gen_helper_mve_vcmpge_scalarb \
+gen_helper_mve_vcmpge_scalarh \
+gen_helper_mve_vcmpge_scalarw \
+gen_helper_mve_vcmpltb \
+gen_helper_mve_vcmplth \
+gen_helper_mve_vcmpltw \
+gen_helper_mve_vcmplt_scalarb \
+gen_helper_mve_vcmplt_scalarh \
+gen_helper_mve_vcmplt_scalarw \
+gen_helper_mve_vcmpgtb \
+gen_helper_mve_vcmpgth \
+gen_helper_mve_vcmpgtw \
+gen_helper_mve_vcmpgt_scalarb \
+gen_helper_mve_vcmpgt_scalarh \
+gen_helper_mve_vcmpgt_scalarw \
+gen_helper_mve_vcmpleb \
+gen_helper_mve_vcmpleh \
+gen_helper_mve_vcmplew \
+gen_helper_mve_vcmple_scalarb \
+gen_helper_mve_vcmple_scalarh \
+gen_helper_mve_vcmple_scalarw \
+gen_helper_mve_vfcmpeqh \
+gen_helper_mve_vfcmpeqs \
+gen_helper_mve_vfcmpneh \
+gen_helper_mve_vfcmpnes \
+gen_helper_mve_vfcmpgeh \
+gen_helper_mve_vfcmpges \
+gen_helper_mve_vfcmplth \
+gen_helper_mve_vfcmplts \
+gen_helper_mve_vfcmpgth \
+gen_helper_mve_vfcmpgts \
+gen_helper_mve_vfcmpleh \
+gen_helper_mve_vfcmples \
+gen_helper_mve_vfcmpeq_scalarh \
+gen_helper_mve_vfcmpeq_scalars \
+gen_helper_mve_vfcmpne_scalarh \
+gen_helper_mve_vfcmpne_scalars \
+gen_helper_mve_vfcmpge_scalarh \
+gen_helper_mve_vfcmpge_scalars \
+gen_helper_mve_vfcmplt_scalarh \
+gen_helper_mve_vfcmplt_scalars \
+gen_helper_mve_vfcmpgt_scalarh \
+gen_helper_mve_vfcmpgt_scalars \
+gen_helper_mve_vfcmple_scalarh \
+gen_helper_mve_vfcmple_scalars \
+gen_helper_mve_vfabsh \
+gen_helper_mve_vfabss \
+gen_helper_mve_vfnegh \
+gen_helper_mve_vfnegs \
+gen_helper_mve_vldrb \
+gen_helper_mve_vldrh \
+gen_helper_mve_vldrw \
+gen_helper_mve_vldrb_sh \
+gen_helper_mve_vldrb_uh \
+gen_helper_mve_vldrb_sw \
+gen_helper_mve_vldrb_uw \
+gen_helper_mve_vldrh_sw \
+gen_helper_mve_vldrh_uw \
+gen_helper_mve_vstrb \
+gen_helper_mve_vstrh \
+gen_helper_mve_vstrw \
+gen_helper_mve_vstrb_h \
+gen_helper_mve_vstrb_w \
+gen_helper_mve_vstrh_w \
+gen_helper_mve_vldrb_sg_sh \
+gen_helper_mve_vldrb_sg_sw \
+gen_helper_mve_vldrh_sg_sw \
+gen_helper_mve_vldrb_sg_ub \
+gen_helper_mve_vldrb_sg_uh \
+gen_helper_mve_vldrb_sg_uw \
+gen_helper_mve_vldrh_sg_uh \
+gen_helper_mve_vldrh_sg_uw \
+gen_helper_mve_vldrw_sg_uw \
+gen_helper_mve_vldrd_sg_ud \
+gen_helper_mve_vldrh_sg_os_sw \
+gen_helper_mve_vldrh_sg_os_uh \
+gen_helper_mve_vldrh_sg_os_uw \
+gen_helper_mve_vldrw_sg_os_uw \
+gen_helper_mve_vldrd_sg_os_ud \
+gen_helper_mve_vstrb_sg_ub \
+gen_helper_mve_vstrb_sg_uh \
+gen_helper_mve_vstrb_sg_uw \
+gen_helper_mve_vstrh_sg_uh \
+gen_helper_mve_vstrh_sg_uw \
+gen_helper_mve_vstrw_sg_uw \
+gen_helper_mve_vstrd_sg_ud \
+gen_helper_mve_vstrh_sg_os_uh \
+gen_helper_mve_vstrh_sg_os_uw \
+gen_helper_mve_vstrw_sg_os_uw \
+gen_helper_mve_vstrd_sg_os_ud \
+gen_helper_mve_vldrw_sg_wb_uw \
+gen_helper_mve_vldrd_sg_wb_ud \
+gen_helper_mve_vstrw_sg_wb_uw \
+gen_helper_mve_vstrd_sg_wb_ud \
+gen_helper_mve_vld20b \
+gen_helper_mve_vld20h \
+gen_helper_mve_vld20w \
+gen_helper_mve_vld21b \
+gen_helper_mve_vld21h \
+gen_helper_mve_vld21w \
+gen_helper_mve_vld40b \
+gen_helper_mve_vld40h \
+gen_helper_mve_vld40w \
+gen_helper_mve_vld41b \
+gen_helper_mve_vld41h \
+gen_helper_mve_vld41w \
+gen_helper_mve_vld42b \
+gen_helper_mve_vld42h \
+gen_helper_mve_vld42w \
+gen_helper_mve_vld43b \
+gen_helper_mve_vld43h \
+gen_helper_mve_vld43w \
+gen_helper_mve_vst20b \
+gen_helper_mve_vst20h \
+gen_helper_mve_vst20w \
+gen_helper_mve_vst21b \
+gen_helper_mve_vst21h \
+gen_helper_mve_vst21w \
+gen_helper_mve_vst40b \
+gen_helper_mve_vst40h \
+gen_helper_mve_vst40w \
+gen_helper_mve_vst41b \
+gen_helper_mve_vst41h \
+gen_helper_mve_vst41w \
+gen_helper_mve_vst42b \
+gen_helper_mve_vst42h \
+gen_helper_mve_vst42w \
+gen_helper_mve_vst43b \
+gen_helper_mve_vst43h \
+gen_helper_mve_vst43w \
+gen_helper_mve_vand \
+gen_helper_mve_vbic \
+gen_helper_mve_vorr \
+gen_helper_mve_vorn \
+gen_helper_mve_veor \
+gen_helper_mve_vaddb \
+gen_helper_mve_vaddh \
+gen_helper_mve_vaddw \
+gen_helper_mve_vadd_scalarb \
+gen_helper_mve_vadd_scalarh \
+gen_helper_mve_vadd_scalarw \
+gen_helper_mve_vsubb \
+gen_helper_mve_vsubh \
+gen_helper_mve_vsubw \
+gen_helper_mve_vsub_scalarb \
+gen_helper_mve_vsub_scalarh \
+gen_helper_mve_vsub_scalarw \
+gen_helper_mve_vmulb \
+gen_helper_mve_vmulh \
+gen_helper_mve_vmulw \
+gen_helper_mve_vmul_scalarb \
+gen_helper_mve_vmul_scalarh \
+gen_helper_mve_vmul_scalarw \
+gen_helper_mve_vmulhsb \
+gen_helper_mve_vmulhsh \
+gen_helper_mve_vmulhsw \
+gen_helper_mve_vmulhub \
+gen_helper_mve_vmulhuh \
+gen_helper_mve_vmulhuw \
+gen_helper_mve_vrmulhsb \
+gen_helper_mve_vrmulhsh \
+gen_helper_mve_vrmulhsw \
+gen_helper_mve_vrmulhub \
+gen_helper_mve_vrmulhuh \
+gen_helper_mve_vrmulhuw \
+gen_helper_mve_vmullbsb \
+gen_helper_mve_vmullbsh \
+gen_helper_mve_vmullbsw \
+gen_helper_mve_vmullbub \
+gen_helper_mve_vmullbuh \
+gen_helper_mve_vmullbuw \
+gen_helper_mve_vmulltsb \
+gen_helper_mve_vmulltsh \
+gen_helper_mve_vmulltsw \
+gen_helper_mve_vmulltub \
+gen_helper_mve_vmulltuh \
+gen_helper_mve_vmulltuw \
+gen_helper_mve_vmullpbh \
+gen_helper_mve_vmullpth \
+gen_helper_mve_vmullpbw \
+gen_helper_mve_vmullptw \
+gen_helper_mve_vqdmullbh \
+gen_helper_mve_vqdmullbw \
+gen_helper_mve_vqdmullth \
+gen_helper_mve_vqdmulltw \
+gen_helper_mve_vqdmullb_scalarh \
+gen_helper_mve_vqdmullb_scalarw \
+gen_helper_mve_vqdmullt_scalarh \
+gen_helper_mve_vqdmullt_scalarw \
+gen_helper_mve_vcadd90b \
+gen_helper_mve_vcadd90h \
+gen_helper_mve_vcadd90w \
+gen_helper_mve_vcadd270b \
+gen_helper_mve_vcadd270h \
+gen_helper_mve_vcadd270w \
+gen_helper_mve_vhcadd90b \
+gen_helper_mve_vhcadd90h \
+gen_helper_mve_vhcadd90w \
+gen_helper_mve_vhcadd270b \
+gen_helper_mve_vhcadd270h \
+gen_helper_mve_vhcadd270w \
+gen_helper_mve_vmaxsb \
+gen_helper_mve_vmaxsh \
+gen_helper_mve_vmaxsw \
+gen_helper_mve_vmaxub \
+gen_helper_mve_vmaxuh \
+gen_helper_mve_vmaxuw \
+gen_helper_mve_vminsb \
+gen_helper_mve_vminsh \
+gen_helper_mve_vminsw \
+gen_helper_mve_vminub \
+gen_helper_mve_vminuh \
+gen_helper_mve_vminuw \
+gen_helper_mve_vabdsb \
+gen_helper_mve_vabdsh \
+gen_helper_mve_vabdsw \
+gen_helper_mve_vabdub \
+gen_helper_mve_vabduh \
+gen_helper_mve_vabduw \
+gen_helper_mve_vhaddsb \
+gen_helper_mve_vhaddsh \
+gen_helper_mve_vhaddsw \
+gen_helper_mve_vhaddub \
+gen_helper_mve_vhadduh \
+gen_helper_mve_vhadduw \
+gen_helper_mve_vhadds_scalarb \
+gen_helper_mve_vhadds_scalarh \
+gen_helper_mve_vhadds_scalarw \
+gen_helper_mve_vhaddu_scalarb \
+gen_helper_mve_vhaddu_scalarh \
+gen_helper_mve_vhaddu_scalarw \
+gen_helper_mve_vrhaddsb \
+gen_helper_mve_vrhaddsh \
+gen_helper_mve_vrhaddsw \
+gen_helper_mve_vrhaddub \
+gen_helper_mve_vrhadduh \
+gen_helper_mve_vrhadduw \
+gen_helper_mve_vadc \
+gen_helper_mve_vadci \
+gen_helper_mve_vsbc \
+gen_helper_mve_vsbci \
+gen_helper_mve_vhsubsb \
+gen_helper_mve_vhsubsh \
+gen_helper_mve_vhsubsw \
+gen_helper_mve_vhsubub \
+gen_helper_mve_vhsubuh \
+gen_helper_mve_vhsubuw \
+gen_helper_mve_vhsubs_scalarb \
+gen_helper_mve_vhsubs_scalarh \
+gen_helper_mve_vhsubs_scalarw \
+gen_helper_mve_vhsubu_scalarb \
+gen_helper_mve_vhsubu_scalarh \
+gen_helper_mve_vhsubu_scalarw \
+gen_helper_mve_vqaddsb \
+gen_helper_mve_vqaddsh \
+gen_helper_mve_vqaddsw \
+gen_helper_mve_vqaddub \
+gen_helper_mve_vqadduh \
+gen_helper_mve_vqadduw \
+gen_helper_mve_vqsubsb \
+gen_helper_mve_vqsubsh \
+gen_helper_mve_vqsubsw \
+gen_helper_mve_vqsubub \
+gen_helper_mve_vqsubuh \
+gen_helper_mve_vqsubuw \
+gen_helper_mve_vqdmulhb \
+gen_helper_mve_vqdmulhh \
+gen_helper_mve_vqdmulhw \
+gen_helper_mve_vqrdmulhb \
+gen_helper_mve_vqrdmulhh \
+gen_helper_mve_vqrdmulhw \
+gen_helper_mve_vqadds_scalarb \
+gen_helper_mve_vqadds_scalarh \
+gen_helper_mve_vqadds_scalarw \
+gen_helper_mve_vqaddu_scalarb \
+gen_helper_mve_vqaddu_scalarh \
+gen_helper_mve_vqaddu_scalarw \
+gen_helper_mve_vqsubs_scalarb \
+gen_helper_mve_vqsubs_scalarh \
+gen_helper_mve_vqsubs_scalarw \
+gen_helper_mve_vqsubu_scalarb \
+gen_helper_mve_vqsubu_scalarh \
+gen_helper_mve_vqsubu_scalarw \
+gen_helper_mve_vqdmulh_scalarb \
+gen_helper_mve_vqdmulh_scalarh \
+gen_helper_mve_vqdmulh_scalarw \
+gen_helper_mve_vqrdmulh_scalarb \
+gen_helper_mve_vqrdmulh_scalarh \
+gen_helper_mve_vqrdmulh_scalarw \
+gen_helper_mve_vmlab \
+gen_helper_mve_vmlah \
+gen_helper_mve_vmlaw \
+gen_helper_mve_vmlasb \
+gen_helper_mve_vmlash \
+gen_helper_mve_vmlasw \
+gen_helper_mve_vqdmlahb \
+gen_helper_mve_vqdmlahh \
+gen_helper_mve_vqdmlahw \
+gen_helper_mve_vqrdmlahb \
+gen_helper_mve_vqrdmlahh \
+gen_helper_mve_vqrdmlahw \
+gen_helper_mve_vqdmlashb \
+gen_helper_mve_vqdmlashh \
+gen_helper_mve_vqdmlashw \
+gen_helper_mve_vqrdmlashb \
+gen_helper_mve_vqrdmlashh \
+gen_helper_mve_vqrdmlashw \
+gen_helper_mve_vfaddh \
+gen_helper_mve_vfadds \
+gen_helper_mve_vfsubh \
+gen_helper_mve_vfsubs \
+gen_helper_mve_vfmulh \
+gen_helper_mve_vfmuls \
+gen_helper_mve_vfabdh \
+gen_helper_mve_vfabds \
+gen_helper_mve_vmaxnmh \
+gen_helper_mve_vmaxnms \
+gen_helper_mve_vminnmh \
+gen_helper_mve_vminnms \
+gen_helper_mve_vmaxnmah \
+gen_helper_mve_vmaxnmas \
+gen_helper_mve_vminnmah \
+gen_helper_mve_vminnmas \
+gen_helper_mve_vfcadd90h \
+gen_helper_mve_vfcadd90s \
+gen_helper_mve_vfcadd270h \
+gen_helper_mve_vfcadd270s \
+gen_helper_mve_vcmul0h \
+gen_helper_mve_vcmul0s \
+gen_helper_mve_vcmul90h \
+gen_helper_mve_vcmul90s \
+gen_helper_mve_vcmul180h \
+gen_helper_mve_vcmul180s \
+gen_helper_mve_vcmul270h \
+gen_helper_mve_vcmul270s \
+gen_helper_mve_vcmla0h \
+gen_helper_mve_vcmla0s \
+gen_helper_mve_vcmla90h \
+gen_helper_mve_vcmla90s \
+gen_helper_mve_vcmla180h \
+gen_helper_mve_vcmla180s \
+gen_helper_mve_vcmla270h \
+gen_helper_mve_vcmla270s \
+gen_helper_mve_vfmah \
+gen_helper_mve_vfmas \
+gen_helper_mve_vfmsh \
+gen_helper_mve_vfmss \
+gen_helper_mve_vfadd_scalarh \
+gen_helper_mve_vfadd_scalars \
+gen_helper_mve_vfsub_scalarh \
+gen_helper_mve_vfsub_scalars \
+gen_helper_mve_vfmul_scalarh \
+gen_helper_mve_vfmul_scalars \
+gen_helper_mve_vfma_scalarh \
+gen_helper_mve_vfma_scalars \
+gen_helper_mve_vfmas_scalarh \
+gen_helper_mve_vfmas_scalars \
+gen_helper_mve_vcvt_sh \
+gen_helper_mve_vcvt_uh \
+gen_helper_mve_vcvt_hs \
+gen_helper_mve_vcvt_hu \
+gen_helper_mve_vcvt_sf \
+gen_helper_mve_vcvt_uf \
+gen_helper_mve_vcvt_fs \
+gen_helper_mve_vcvt_fu \
+gen_helper_mve_vcvtb_sh \
+gen_helper_mve_vcvtt_sh \
+gen_helper_mve_vcvtb_hs \
+gen_helper_mve_vcvtt_hs \
+gen_helper_mve_vcvt_rm_sh \
+gen_helper_mve_vcvt_rm_uh \
+gen_helper_mve_vcvt_rm_ss \
+gen_helper_mve_vcvt_rm_us \
+gen_helper_mve_vrint_rm_h \
+gen_helper_mve_vrint_rm_s \
+gen_helper_mve_vrintx_h \
+gen_helper_mve_vrintx_s \
+gen_helper_mve_vshlsb \
+gen_helper_mve_vshlsh \
+gen_helper_mve_vshlsw \
+gen_helper_mve_vshlub \
+gen_helper_mve_vshluh \
+gen_helper_mve_vshluw \
+gen_helper_mve_vrshlsb \
+gen_helper_mve_vrshlsh \
+gen_helper_mve_vrshlsw \
+gen_helper_mve_vrshlub \
+gen_helper_mve_vrshluh \
+gen_helper_mve_vrshluw \
+gen_helper_mve_vqshlsb \
+gen_helper_mve_vqshlsh \
+gen_helper_mve_vqshlsw \
+gen_helper_mve_vqshlub \
+gen_helper_mve_vqshluh \
+gen_helper_mve_vqshluw \
+gen_helper_mve_vqrshlsb \
+gen_helper_mve_vqrshlsh \
+gen_helper_mve_vqrshlsw \
+gen_helper_mve_vqrshlub \
+gen_helper_mve_vqrshluh \
+gen_helper_mve_vqrshluw \
+gen_helper_mve_vqdmladhb \
+gen_helper_mve_vqdmladhh \
+gen_helper_mve_vqdmladhw \
+gen_helper_mve_vqdmladhxb \
+gen_helper_mve_vqdmladhxh \
+gen_helper_mve_vqdmladhxw \
+gen_helper_mve_vqrdmladhb \
+gen_helper_mve_vqrdmladhh \
+gen_helper_mve_vqrdmladhw \
+gen_helper_mve_vqrdmladhxb \
+gen_helper_mve_vqrdmladhxh \
+gen_helper_mve_vqrdmladhxw \
+gen_helper_mve_vqdmlsdhb \
+gen_helper_mve_vqdmlsdhh \
+gen_helper_mve_vqdmlsdhw \
+gen_helper_mve_vqdmlsdhxb \
+gen_helper_mve_vqdmlsdhxh \
+gen_helper_mve_vqdmlsdhxw \
+gen_helper_mve_vqrdmlsdhb \
+gen_helper_mve_vqrdmlsdhh \
+gen_helper_mve_vqrdmlsdhw \
+gen_helper_mve_vqrdmlsdhxb \
+gen_helper_mve_vqrdmlsdhxh \
+gen_helper_mve_vqrdmlsdhxw \
+gen_helper_mve_vbrsrb \
+gen_helper_mve_vbrsrh \
+gen_helper_mve_vbrsrw \
+gen_helper_mve_vshli_sb \
+gen_helper_mve_vshli_sh \
+gen_helper_mve_vshli_sw \
+gen_helper_mve_vshli_ub \
+gen_helper_mve_vshli_uh \
+gen_helper_mve_vshli_uw \
+gen_helper_mve_vrshli_sb \
+gen_helper_mve_vrshli_sh \
+gen_helper_mve_vrshli_sw \
+gen_helper_mve_vrshli_ub \
+gen_helper_mve_vrshli_uh \
+gen_helper_mve_vrshli_uw \
+gen_helper_mve_vqshli_sb \
+gen_helper_mve_vqshli_sh \
+gen_helper_mve_vqshli_sw \
+gen_helper_mve_vqshli_ub \
+gen_helper_mve_vqshli_uh \
+gen_helper_mve_vqshli_uw \
+gen_helper_mve_vqrshli_sb \
+gen_helper_mve_vqrshli_sh \
+gen_helper_mve_vqrshli_sw \
+gen_helper_mve_vqrshli_ub \
+gen_helper_mve_vqrshli_uh \
+gen_helper_mve_vqrshli_uw \
+gen_helper_mve_vqshlui_sb \
+gen_helper_mve_vqshlui_sh \
+gen_helper_mve_vqshlui_sw \
+gen_helper_mve_vshllbsb \
+gen_helper_mve_vshllbsh \
+gen_helper_mve_vshllbub \
+gen_helper_mve_vshllbuh \
+gen_helper_mve_vshlltsb \
+gen_helper_mve_vshlltsh \
+gen_helper_mve_vshlltub \
+gen_helper_mve_vshlltuh \
+gen_helper_mve_vshrnbb \
+gen_helper_mve_vshrnbh \
+gen_helper_mve_vshrntb \
+gen_helper_mve_vshrnth \
+gen_helper_mve_vrshrnbb \
+gen_helper_mve_vrshrnbh \
+gen_helper_mve_vrshrntb \
+gen_helper_mve_vrshrnth \
+gen_helper_mve_vqshrnb_sb \
+gen_helper_mve_vqshrnb_sh \
+gen_helper_mve_vqshrnt_sb \
+gen_helper_mve_vqshrnt_sh \
+gen_helper_mve_vqshrnb_ub \
+gen_helper_mve_vqshrnb_uh \
+gen_helper_mve_vqshrnt_ub \
+gen_helper_mve_vqshrnt_uh \
+gen_helper_mve_vqshrunbb \
+gen_helper_mve_vqshrunbh \
+gen_helper_mve_vqshruntb \
+gen_helper_mve_vqshrunth \
+gen_helper_mve_vqrshrnb_sb \
+gen_helper_mve_vqrshrnb_sh \
+gen_helper_mve_vqrshrnt_sb \
+gen_helper_mve_vqrshrnt_sh \
+gen_helper_mve_vqrshrnb_ub \
+gen_helper_mve_vqrshrnb_uh \
+gen_helper_mve_vqrshrnt_ub \
+gen_helper_mve_vqrshrnt_uh \
+gen_helper_mve_vqrshrunbb \
+gen_helper_mve_vqrshrunbh \
+gen_helper_mve_vqrshruntb \
+gen_helper_mve_vqrshrunth \
+gen_helper_mve_vmovnbb \
+gen_helper_mve_vmovnbh \
+gen_helper_mve_vmovntb \
+gen_helper_mve_vmovnth \
+gen_helper_mve_vqmovnbsb \
+gen_helper_mve_vqmovnbsh \
+gen_helper_mve_vqmovntsb \
+gen_helper_mve_vqmovntsh \
+gen_helper_mve_vqmovnbub \
+gen_helper_mve_vqmovnbuh \
+gen_helper_mve_vqmovntub \
+gen_helper_mve_vqmovntuh \
+gen_helper_mve_vqmovunbb \
+gen_helper_mve_vqmovunbh \
+gen_helper_mve_vqmovuntb \
+gen_helper_mve_vqmovunth \
+gen_helper_mve_sshrl \
+gen_helper_mve_ushll \
+gen_helper_mve_sqshll \
+gen_helper_mve_uqshll \
+gen_helper_mve_sqrshrl \
+gen_helper_mve_uqrshll \
+gen_helper_mve_sqrshrl48 \
+gen_helper_mve_uqrshll48 \
+gen_helper_mve_uqshl \
+gen_helper_mve_sqshl \
+gen_helper_mve_uqrshl \
+gen_helper_mve_sqrshr \
+gen_helper_mve_vshlc \
+gen_helper_mve_vsrib \
+gen_helper_mve_vsrih \
+gen_helper_mve_vsriw \
+gen_helper_mve_vslib \
+gen_helper_mve_vslih \
+gen_helper_mve_vsliw \
+gen_helper_mve_vclsb \
+gen_helper_mve_vclsh \
+gen_helper_mve_vclsw \
+gen_helper_mve_vclzb \
+gen_helper_mve_vclzh \
+gen_helper_mve_vclzw \
+gen_helper_mve_vrev16b \
+gen_helper_mve_vrev32b \
+gen_helper_mve_vrev32h \
+gen_helper_mve_vrev64b \
+gen_helper_mve_vrev64h \
+gen_helper_mve_vrev64w \
+gen_helper_mve_vmvn \
+gen_helper_mve_vabsb \
+gen_helper_mve_vabsh \
+gen_helper_mve_vabsw \
+gen_helper_mve_vnegb \
+gen_helper_mve_vnegh \
+gen_helper_mve_vnegw \
+gen_helper_mve_vmaxab \
+gen_helper_mve_vmaxah \
+gen_helper_mve_vmaxaw \
+gen_helper_mve_vminab \
+gen_helper_mve_vminah \
+gen_helper_mve_vminaw \
+gen_helper_mve_vqabsb \
+gen_helper_mve_vqabsh \
+gen_helper_mve_vqabsw \
+gen_helper_mve_vqnegb \
+gen_helper_mve_vqnegh \
+gen_helper_mve_vqnegw \
+gen_helper_mve_vmlaldavsh \
+gen_helper_mve_vmlaldavsw \
+gen_helper_mve_vmlaldavxsh \
+gen_helper_mve_vmlaldavxsw \
+gen_helper_mve_vmlaldavuh \
+gen_helper_mve_vmlaldavuw \
+gen_helper_mve_vmlsldavsh \
+gen_helper_mve_vmlsldavsw \
+gen_helper_mve_vmlsldavxsh \
+gen_helper_mve_vmlsldavxsw \
+gen_helper_mve_vrmlaldavhsw \
+gen_helper_mve_vrmlaldavhxsw \
+gen_helper_mve_vrmlaldavhuw \
+gen_helper_mve_vrmlsldavhsw \
+gen_helper_mve_vrmlsldavhxsw \
+gen_helper_mve_vmladavsb \
+gen_helper_mve_vmladavsh \
+gen_helper_mve_vmladavsw \
+gen_helper_mve_vmladavub \
+gen_helper_mve_vmladavuh \
+gen_helper_mve_vmladavuw \
+gen_helper_mve_vmlsdavb \
+gen_helper_mve_vmlsdavh \
+gen_helper_mve_vmlsdavw \
+gen_helper_mve_vmladavsxb \
+gen_helper_mve_vmladavsxh \
+gen_helper_mve_vmladavsxw \
+gen_helper_mve_vmlsdavxb \
+gen_helper_mve_vmlsdavxh \
+gen_helper_mve_vmlsdavxw \
+gen_helper_mve_vaddvsb \
+gen_helper_mve_vaddvsh \
+gen_helper_mve_vaddvsw \
+gen_helper_mve_vaddvub \
+gen_helper_mve_vaddvuh \
+gen_helper_mve_vaddvuw \
+gen_helper_mve_vmaxvsb \
+gen_helper_mve_vmaxvsh \
+gen_helper_mve_vmaxvsw \
+gen_helper_mve_vmaxvub \
+gen_helper_mve_vmaxvuh \
+gen_helper_mve_vmaxvuw \
+gen_helper_mve_vmaxavb \
+gen_helper_mve_vmaxavh \
+gen_helper_mve_vmaxavw \
+gen_helper_mve_vminvsb \
+gen_helper_mve_vminvsh \
+gen_helper_mve_vminvsw \
+gen_helper_mve_vminvub \
+gen_helper_mve_vminvuh \
+gen_helper_mve_vminvuw \
+gen_helper_mve_vminavb \
+gen_helper_mve_vminavh \
+gen_helper_mve_vminavw \
+gen_helper_mve_vmaxnmvh \
+gen_helper_mve_vmaxnmvs \
+gen_helper_mve_vminnmvh \
+gen_helper_mve_vminnmvs \
+gen_helper_mve_vmaxnmavh \
+gen_helper_mve_vmaxnmavs \
+gen_helper_mve_vminnmavh \
+gen_helper_mve_vminnmavs \
+gen_helper_mve_vaddlv_s \
+gen_helper_mve_vaddlv_u \
+gen_helper_mve_vabavsb \
+gen_helper_mve_vabavsh \
+gen_helper_mve_vabavsw \
+gen_helper_mve_vabavub \
+gen_helper_mve_vabavuh \
+gen_helper_mve_vabavuw \
 gen_helper_cpsr_read \
 gen_helper_cpsr_write \
 tlb_reset_dirty_by_vaddr \
@@ -1575,119 +2346,69 @@ helper_psadbw_xmm \
 helper_maskmov_xmm \
 helper_movl_mm_T0_xmm \
 helper_movq_mm_T0_xmm \
-helper_shufps \
-helper_shufpd \
 helper_pshufd_xmm \
 helper_pshuflw_xmm \
 helper_pshufhw_xmm \
-helper_addps \
 helper_addss \
-helper_addpd \
 helper_addsd \
-helper_subps \
 helper_subss \
-helper_subpd \
 helper_subsd \
-helper_mulps \
 helper_mulss \
-helper_mulpd \
 helper_mulsd \
-helper_divps \
 helper_divss \
-helper_divpd \
 helper_divsd \
-helper_minps \
 helper_minss \
-helper_minpd \
 helper_minsd \
-helper_maxps \
 helper_maxss \
-helper_maxpd \
 helper_maxsd \
-helper_sqrtps \
 helper_sqrtss \
-helper_sqrtpd \
 helper_sqrtsd \
-helper_cvtps2pd \
-helper_cvtpd2ps \
 helper_cvtss2sd \
 helper_cvtsd2ss \
-helper_cvtdq2ps \
-helper_cvtdq2pd \
 helper_cvtpi2ps \
 helper_cvtpi2pd \
 helper_cvtsi2ss \
 helper_cvtsi2sd \
 helper_cvtsq2ss \
 helper_cvtsq2sd \
-helper_cvtps2dq \
-helper_cvtpd2dq \
 helper_cvtps2pi \
 helper_cvtpd2pi \
 helper_cvtss2si \
 helper_cvtsd2si \
 helper_cvtss2sq \
 helper_cvtsd2sq \
-helper_cvttps2dq \
-helper_cvttpd2dq \
 helper_cvttps2pi \
 helper_cvttpd2pi \
 helper_cvttss2si \
 helper_cvttsd2si \
 helper_cvttss2sq \
 helper_cvttsd2sq \
-helper_rsqrtps \
 helper_rsqrtss \
-helper_rcpps \
 helper_rcpss \
 helper_extrq_r \
 helper_extrq_i \
 helper_insertq_r \
 helper_insertq_i \
-helper_haddps \
-helper_haddpd \
-helper_hsubps \
-helper_hsubpd \
-helper_addsubps \
-helper_addsubpd \
-helper_cmpeqps \
 helper_cmpeqss \
-helper_cmpeqpd \
 helper_cmpeqsd \
-helper_cmpltps \
 helper_cmpltss \
-helper_cmpltpd \
 helper_cmpltsd \
-helper_cmpleps \
 helper_cmpless \
-helper_cmplepd \
 helper_cmplesd \
-helper_cmpunordps \
 helper_cmpunordss \
-helper_cmpunordpd \
 helper_cmpunordsd \
-helper_cmpneqps \
 helper_cmpneqss \
-helper_cmpneqpd \
 helper_cmpneqsd \
-helper_cmpnltps \
 helper_cmpnltss \
-helper_cmpnltpd \
 helper_cmpnltsd \
-helper_cmpnleps \
 helper_cmpnless \
-helper_cmpnlepd \
 helper_cmpnlesd \
-helper_cmpordps \
 helper_cmpordss \
-helper_cmpordpd \
 helper_cmpordsd \
 helper_ucomiss \
 helper_comiss \
 helper_ucomisd \
 helper_comisd \
-helper_movmskps \
-helper_movmskpd \
 helper_pmovmskb_xmm \
 helper_packsswb_xmm \
 helper_packuswb_xmm \
@@ -1898,6 +2619,106 @@ x86_cpu_xsave_all_areas \
 x86_cpu_xrstor_all_areas \
 cpu_get_fp80 \
 cpu_set_fp80 \
+helper_addpd_xmm \
+helper_addpd_ymm \
+helper_addps_xmm \
+helper_addps_ymm \
+helper_addsubpd_xmm \
+helper_addsubpd_ymm \
+helper_addsubps_xmm \
+helper_addsubps_ymm \
+helper_cmpeqpd_xmm \
+helper_cmpeqpd_ymm \
+helper_cmpeqps_xmm \
+helper_cmpeqps_ymm \
+helper_cmplepd_xmm \
+helper_cmplepd_ymm \
+helper_cmpleps_xmm \
+helper_cmpleps_ymm \
+helper_cmpltpd_xmm \
+helper_cmpltpd_ymm \
+helper_cmpltps_xmm \
+helper_cmpltps_ymm \
+helper_cmpneqpd_xmm \
+helper_cmpneqpd_ymm \
+helper_cmpneqps_xmm \
+helper_cmpneqps_ymm \
+helper_cmpnlepd_xmm \
+helper_cmpnlepd_ymm \
+helper_cmpnleps_xmm \
+helper_cmpnleps_ymm \
+helper_cmpnltpd_xmm \
+helper_cmpnltpd_ymm \
+helper_cmpnltps_xmm \
+helper_cmpnltps_ymm \
+helper_cmpordpd_xmm \
+helper_cmpordpd_ymm \
+helper_cmpordps_xmm \
+helper_cmpordps_ymm \
+helper_cmpunordpd_xmm \
+helper_cmpunordpd_ymm \
+helper_cmpunordps_xmm \
+helper_cmpunordps_ymm \
+helper_cvtdq2pd_xmm \
+helper_cvtdq2pd_ymm \
+helper_cvtdq2ps_xmm \
+helper_cvtdq2ps_ymm \
+helper_cvtpd2dq_xmm \
+helper_cvtpd2dq_ymm \
+helper_cvtpd2ps_xmm \
+helper_cvtpd2ps_ymm \
+helper_cvtps2dq_xmm \
+helper_cvtps2dq_ymm \
+helper_cvtps2pd_xmm \
+helper_cvtps2pd_ymm \
+helper_cvttpd2dq_xmm \
+helper_cvttpd2dq_ymm \
+helper_cvttps2dq_xmm \
+helper_cvttps2dq_ymm \
+helper_divpd_xmm \
+helper_divpd_ymm \
+helper_divps_xmm \
+helper_divps_ymm \
+helper_haddpd_xmm \
+helper_haddpd_ymm \
+helper_haddps_xmm \
+helper_haddps_ymm \
+helper_hsubpd_xmm \
+helper_hsubpd_ymm \
+helper_hsubps_xmm \
+helper_hsubps_ymm \
+helper_maxpd_xmm \
+helper_maxpd_ymm \
+helper_maxps_xmm \
+helper_maxps_ymm \
+helper_minpd_xmm \
+helper_minpd_ymm \
+helper_minps_xmm \
+helper_minps_ymm \
+helper_movmskpd_xmm \
+helper_movmskpd_ymm \
+helper_movmskps_xmm \
+helper_movmskps_ymm \
+helper_mulpd_xmm \
+helper_mulpd_ymm \
+helper_mulps_xmm \
+helper_mulps_ymm \
+helper_rcpps_xmm \
+helper_rcpps_ymm \
+helper_rsqrtps_xmm \
+helper_rsqrtps_ymm \
+helper_shufpd_xmm \
+helper_shufpd_ymm \
+helper_shufps_xmm \
+helper_shufps_ymm \
+helper_sqrtpd_xmm \
+helper_sqrtpd_ymm \
+helper_sqrtps_xmm \
+helper_sqrtps_ymm \
+helper_subpd_xmm \
+helper_subpd_ymm \
+helper_subps_xmm \
+helper_subps_ymm \
 "
 
 arm_SYMBOLS="
@@ -1913,6 +2734,8 @@ arm_cpu_class_init \
 cpu_arm_init \
 helper_crypto_aese \
 helper_crypto_aesmc \
+helper_crypto_sve_aese \
+helper_crypto_sve_aesmc \
 helper_crypto_sha1_3reg \
 helper_crypto_sha1h \
 helper_crypto_sha1su1 \
@@ -1929,6 +2752,9 @@ helper_crypto_sm3partw2 \
 helper_crypto_sm3tt \
 helper_crypto_sm4e \
 helper_crypto_sm4ekey \
+helper_crypto_sve_sm4e \
+helper_crypto_sve_sm4ekey \
+helper_crypto_rax1 \
 helper_check_breakpoints \
 arm_debug_check_watchpoint \
 arm_debug_excp_handler \
@@ -2406,12 +3232,23 @@ helper_neon_qrdmlsh_s32 \
 helper_gvec_qrdmlsh_s32 \
 helper_gvec_sdot_b \
 helper_gvec_udot_b \
+helper_gvec_usdot_b \
 helper_gvec_sdot_h \
 helper_gvec_udot_h \
 helper_gvec_sdot_idx_b \
 helper_gvec_udot_idx_b \
+helper_gvec_sudot_idx_b \
+helper_gvec_usdot_idx_b \
 helper_gvec_sdot_idx_h \
 helper_gvec_udot_idx_h \
+helper_gvec_smmla_b \
+helper_gvec_ummla_b \
+helper_gvec_usmmla_b \
+helper_gvec_bfdot \
+helper_gvec_bfdot_idx \
+helper_gvec_bfmmla \
+helper_gvec_bfmlal \
+helper_gvec_bfmlal_idx \
 helper_gvec_fcaddh \
 helper_gvec_fcadds \
 helper_gvec_fcaddd \
@@ -2460,10 +3297,20 @@ helper_gvec_uqadd_d \
 helper_gvec_uqsub_d \
 helper_gvec_sqadd_d \
 helper_gvec_sqsub_d \
+helper_gvec_saba_b \
+helper_gvec_saba_h \
+helper_gvec_saba_s \
+helper_gvec_saba_d \
+helper_gvec_uaba_b \
+helper_gvec_uaba_h \
+helper_gvec_uaba_s \
+helper_gvec_uaba_d \
 helper_gvec_fmlal_a32 \
 helper_gvec_fmlal_a64 \
 helper_gvec_fmlal_idx_a32 \
 helper_gvec_fmlal_idx_a64 \
+helper_sve2_fmlal_zzzw_s \
+helper_sve2_fmlal_zzxw_s \
 helper_gvec_sshl_b \
 helper_gvec_sshl_h \
 helper_gvec_ushl_b \
@@ -2475,6 +3322,680 @@ helper_vfp_get_fpscr \
 vfp_get_fpscr \
 helper_vfp_set_fpscr \
 vfp_set_fpscr \
+helper_mve_vctp \
+helper_mve_vpnot \
+helper_mve_vpsel \
+helper_mve_vdup \
+helper_mve_vmovi \
+helper_mve_vandi \
+helper_mve_vorri \
+helper_mve_vidupb \
+helper_mve_viduph \
+helper_mve_vidupw \
+helper_mve_viwdupb \
+helper_mve_viwduph \
+helper_mve_viwdupw \
+helper_mve_vdwdupb \
+helper_mve_vdwduph \
+helper_mve_vdwdupw \
+helper_mve_vcmpeqb \
+helper_mve_vcmpeqh \
+helper_mve_vcmpeqw \
+helper_mve_vcmpeq_scalarb \
+helper_mve_vcmpeq_scalarh \
+helper_mve_vcmpeq_scalarw \
+helper_mve_vcmpneb \
+helper_mve_vcmpneh \
+helper_mve_vcmpnew \
+helper_mve_vcmpne_scalarb \
+helper_mve_vcmpne_scalarh \
+helper_mve_vcmpne_scalarw \
+helper_mve_vcmpcsb \
+helper_mve_vcmpcsh \
+helper_mve_vcmpcsw \
+helper_mve_vcmpcs_scalarb \
+helper_mve_vcmpcs_scalarh \
+helper_mve_vcmpcs_scalarw \
+helper_mve_vcmphib \
+helper_mve_vcmphih \
+helper_mve_vcmphiw \
+helper_mve_vcmphi_scalarb \
+helper_mve_vcmphi_scalarh \
+helper_mve_vcmphi_scalarw \
+helper_mve_vcmpgeb \
+helper_mve_vcmpgeh \
+helper_mve_vcmpgew \
+helper_mve_vcmpge_scalarb \
+helper_mve_vcmpge_scalarh \
+helper_mve_vcmpge_scalarw \
+helper_mve_vcmpltb \
+helper_mve_vcmplth \
+helper_mve_vcmpltw \
+helper_mve_vcmplt_scalarb \
+helper_mve_vcmplt_scalarh \
+helper_mve_vcmplt_scalarw \
+helper_mve_vcmpgtb \
+helper_mve_vcmpgth \
+helper_mve_vcmpgtw \
+helper_mve_vcmpgt_scalarb \
+helper_mve_vcmpgt_scalarh \
+helper_mve_vcmpgt_scalarw \
+helper_mve_vcmpleb \
+helper_mve_vcmpleh \
+helper_mve_vcmplew \
+helper_mve_vcmple_scalarb \
+helper_mve_vcmple_scalarh \
+helper_mve_vcmple_scalarw \
+helper_mve_vfcmpeqh \
+helper_mve_vfcmpeqs \
+helper_mve_vfcmpneh \
+helper_mve_vfcmpnes \
+helper_mve_vfcmpgeh \
+helper_mve_vfcmpges \
+helper_mve_vfcmplth \
+helper_mve_vfcmplts \
+helper_mve_vfcmpgth \
+helper_mve_vfcmpgts \
+helper_mve_vfcmpleh \
+helper_mve_vfcmples \
+helper_mve_vfcmpeq_scalarh \
+helper_mve_vfcmpeq_scalars \
+helper_mve_vfcmpne_scalarh \
+helper_mve_vfcmpne_scalars \
+helper_mve_vfcmpge_scalarh \
+helper_mve_vfcmpge_scalars \
+helper_mve_vfcmplt_scalarh \
+helper_mve_vfcmplt_scalars \
+helper_mve_vfcmpgt_scalarh \
+helper_mve_vfcmpgt_scalars \
+helper_mve_vfcmple_scalarh \
+helper_mve_vfcmple_scalars \
+helper_mve_vfabsh \
+helper_mve_vfabss \
+helper_mve_vfnegh \
+helper_mve_vfnegs \
+helper_mve_vldrb \
+helper_mve_vldrh \
+helper_mve_vldrw \
+helper_mve_vldrb_sh \
+helper_mve_vldrb_uh \
+helper_mve_vldrb_sw \
+helper_mve_vldrb_uw \
+helper_mve_vldrh_sw \
+helper_mve_vldrh_uw \
+helper_mve_vstrb \
+helper_mve_vstrh \
+helper_mve_vstrw \
+helper_mve_vstrb_h \
+helper_mve_vstrb_w \
+helper_mve_vstrh_w \
+helper_mve_vldrb_sg_sh \
+helper_mve_vldrb_sg_sw \
+helper_mve_vldrh_sg_sw \
+helper_mve_vldrb_sg_ub \
+helper_mve_vldrb_sg_uh \
+helper_mve_vldrb_sg_uw \
+helper_mve_vldrh_sg_uh \
+helper_mve_vldrh_sg_uw \
+helper_mve_vldrw_sg_uw \
+helper_mve_vldrd_sg_ud \
+helper_mve_vldrh_sg_os_sw \
+helper_mve_vldrh_sg_os_uh \
+helper_mve_vldrh_sg_os_uw \
+helper_mve_vldrw_sg_os_uw \
+helper_mve_vldrd_sg_os_ud \
+helper_mve_vstrb_sg_ub \
+helper_mve_vstrb_sg_uh \
+helper_mve_vstrb_sg_uw \
+helper_mve_vstrh_sg_uh \
+helper_mve_vstrh_sg_uw \
+helper_mve_vstrw_sg_uw \
+helper_mve_vstrd_sg_ud \
+helper_mve_vstrh_sg_os_uh \
+helper_mve_vstrh_sg_os_uw \
+helper_mve_vstrw_sg_os_uw \
+helper_mve_vstrd_sg_os_ud \
+helper_mve_vldrw_sg_wb_uw \
+helper_mve_vldrd_sg_wb_ud \
+helper_mve_vstrw_sg_wb_uw \
+helper_mve_vstrd_sg_wb_ud \
+helper_mve_vld20b \
+helper_mve_vld20h \
+helper_mve_vld20w \
+helper_mve_vld21b \
+helper_mve_vld21h \
+helper_mve_vld21w \
+helper_mve_vld40b \
+helper_mve_vld40h \
+helper_mve_vld40w \
+helper_mve_vld41b \
+helper_mve_vld41h \
+helper_mve_vld41w \
+helper_mve_vld42b \
+helper_mve_vld42h \
+helper_mve_vld42w \
+helper_mve_vld43b \
+helper_mve_vld43h \
+helper_mve_vld43w \
+helper_mve_vst20b \
+helper_mve_vst20h \
+helper_mve_vst20w \
+helper_mve_vst21b \
+helper_mve_vst21h \
+helper_mve_vst21w \
+helper_mve_vst40b \
+helper_mve_vst40h \
+helper_mve_vst40w \
+helper_mve_vst41b \
+helper_mve_vst41h \
+helper_mve_vst41w \
+helper_mve_vst42b \
+helper_mve_vst42h \
+helper_mve_vst42w \
+helper_mve_vst43b \
+helper_mve_vst43h \
+helper_mve_vst43w \
+helper_mve_vand \
+helper_mve_vbic \
+helper_mve_vorr \
+helper_mve_vorn \
+helper_mve_veor \
+helper_mve_vaddb \
+helper_mve_vaddh \
+helper_mve_vaddw \
+helper_mve_vadd_scalarb \
+helper_mve_vadd_scalarh \
+helper_mve_vadd_scalarw \
+helper_mve_vsubb \
+helper_mve_vsubh \
+helper_mve_vsubw \
+helper_mve_vsub_scalarb \
+helper_mve_vsub_scalarh \
+helper_mve_vsub_scalarw \
+helper_mve_vmulb \
+helper_mve_vmulh \
+helper_mve_vmulw \
+helper_mve_vmul_scalarb \
+helper_mve_vmul_scalarh \
+helper_mve_vmul_scalarw \
+helper_mve_vmulhsb \
+helper_mve_vmulhsh \
+helper_mve_vmulhsw \
+helper_mve_vmulhub \
+helper_mve_vmulhuh \
+helper_mve_vmulhuw \
+helper_mve_vrmulhsb \
+helper_mve_vrmulhsh \
+helper_mve_vrmulhsw \
+helper_mve_vrmulhub \
+helper_mve_vrmulhuh \
+helper_mve_vrmulhuw \
+helper_mve_vmullbsb \
+helper_mve_vmullbsh \
+helper_mve_vmullbsw \
+helper_mve_vmullbub \
+helper_mve_vmullbuh \
+helper_mve_vmullbuw \
+helper_mve_vmulltsb \
+helper_mve_vmulltsh \
+helper_mve_vmulltsw \
+helper_mve_vmulltub \
+helper_mve_vmulltuh \
+helper_mve_vmulltuw \
+helper_mve_vmullpbh \
+helper_mve_vmullpth \
+helper_mve_vmullpbw \
+helper_mve_vmullptw \
+helper_mve_vqdmullbh \
+helper_mve_vqdmullbw \
+helper_mve_vqdmullth \
+helper_mve_vqdmulltw \
+helper_mve_vqdmullb_scalarh \
+helper_mve_vqdmullb_scalarw \
+helper_mve_vqdmullt_scalarh \
+helper_mve_vqdmullt_scalarw \
+helper_mve_vcadd90b \
+helper_mve_vcadd90h \
+helper_mve_vcadd90w \
+helper_mve_vcadd270b \
+helper_mve_vcadd270h \
+helper_mve_vcadd270w \
+helper_mve_vhcadd90b \
+helper_mve_vhcadd90h \
+helper_mve_vhcadd90w \
+helper_mve_vhcadd270b \
+helper_mve_vhcadd270h \
+helper_mve_vhcadd270w \
+helper_mve_vmaxsb \
+helper_mve_vmaxsh \
+helper_mve_vmaxsw \
+helper_mve_vmaxub \
+helper_mve_vmaxuh \
+helper_mve_vmaxuw \
+helper_mve_vminsb \
+helper_mve_vminsh \
+helper_mve_vminsw \
+helper_mve_vminub \
+helper_mve_vminuh \
+helper_mve_vminuw \
+helper_mve_vabdsb \
+helper_mve_vabdsh \
+helper_mve_vabdsw \
+helper_mve_vabdub \
+helper_mve_vabduh \
+helper_mve_vabduw \
+helper_mve_vhaddsb \
+helper_mve_vhaddsh \
+helper_mve_vhaddsw \
+helper_mve_vhaddub \
+helper_mve_vhadduh \
+helper_mve_vhadduw \
+helper_mve_vhadds_scalarb \
+helper_mve_vhadds_scalarh \
+helper_mve_vhadds_scalarw \
+helper_mve_vhaddu_scalarb \
+helper_mve_vhaddu_scalarh \
+helper_mve_vhaddu_scalarw \
+helper_mve_vrhaddsb \
+helper_mve_vrhaddsh \
+helper_mve_vrhaddsw \
+helper_mve_vrhaddub \
+helper_mve_vrhadduh \
+helper_mve_vrhadduw \
+helper_mve_vadc \
+helper_mve_vadci \
+helper_mve_vsbc \
+helper_mve_vsbci \
+helper_mve_vhsubsb \
+helper_mve_vhsubsh \
+helper_mve_vhsubsw \
+helper_mve_vhsubub \
+helper_mve_vhsubuh \
+helper_mve_vhsubuw \
+helper_mve_vhsubs_scalarb \
+helper_mve_vhsubs_scalarh \
+helper_mve_vhsubs_scalarw \
+helper_mve_vhsubu_scalarb \
+helper_mve_vhsubu_scalarh \
+helper_mve_vhsubu_scalarw \
+helper_mve_vqaddsb \
+helper_mve_vqaddsh \
+helper_mve_vqaddsw \
+helper_mve_vqaddub \
+helper_mve_vqadduh \
+helper_mve_vqadduw \
+helper_mve_vqsubsb \
+helper_mve_vqsubsh \
+helper_mve_vqsubsw \
+helper_mve_vqsubub \
+helper_mve_vqsubuh \
+helper_mve_vqsubuw \
+helper_mve_vqdmulhb \
+helper_mve_vqdmulhh \
+helper_mve_vqdmulhw \
+helper_mve_vqrdmulhb \
+helper_mve_vqrdmulhh \
+helper_mve_vqrdmulhw \
+helper_mve_vqadds_scalarb \
+helper_mve_vqadds_scalarh \
+helper_mve_vqadds_scalarw \
+helper_mve_vqaddu_scalarb \
+helper_mve_vqaddu_scalarh \
+helper_mve_vqaddu_scalarw \
+helper_mve_vqsubs_scalarb \
+helper_mve_vqsubs_scalarh \
+helper_mve_vqsubs_scalarw \
+helper_mve_vqsubu_scalarb \
+helper_mve_vqsubu_scalarh \
+helper_mve_vqsubu_scalarw \
+helper_mve_vqdmulh_scalarb \
+helper_mve_vqdmulh_scalarh \
+helper_mve_vqdmulh_scalarw \
+helper_mve_vqrdmulh_scalarb \
+helper_mve_vqrdmulh_scalarh \
+helper_mve_vqrdmulh_scalarw \
+helper_mve_vmlab \
+helper_mve_vmlah \
+helper_mve_vmlaw \
+helper_mve_vmlasb \
+helper_mve_vmlash \
+helper_mve_vmlasw \
+helper_mve_vqdmlahb \
+helper_mve_vqdmlahh \
+helper_mve_vqdmlahw \
+helper_mve_vqrdmlahb \
+helper_mve_vqrdmlahh \
+helper_mve_vqrdmlahw \
+helper_mve_vqdmlashb \
+helper_mve_vqdmlashh \
+helper_mve_vqdmlashw \
+helper_mve_vqrdmlashb \
+helper_mve_vqrdmlashh \
+helper_mve_vqrdmlashw \
+helper_mve_vfaddh \
+helper_mve_vfadds \
+helper_mve_vfsubh \
+helper_mve_vfsubs \
+helper_mve_vfmulh \
+helper_mve_vfmuls \
+helper_mve_vfabdh \
+helper_mve_vfabds \
+helper_mve_vmaxnmh \
+helper_mve_vmaxnms \
+helper_mve_vminnmh \
+helper_mve_vminnms \
+helper_mve_vmaxnmah \
+helper_mve_vmaxnmas \
+helper_mve_vminnmah \
+helper_mve_vminnmas \
+helper_mve_vfcadd90h \
+helper_mve_vfcadd90s \
+helper_mve_vfcadd270h \
+helper_mve_vfcadd270s \
+helper_mve_vcmul0h \
+helper_mve_vcmul0s \
+helper_mve_vcmul90h \
+helper_mve_vcmul90s \
+helper_mve_vcmul180h \
+helper_mve_vcmul180s \
+helper_mve_vcmul270h \
+helper_mve_vcmul270s \
+helper_mve_vcmla0h \
+helper_mve_vcmla0s \
+helper_mve_vcmla90h \
+helper_mve_vcmla90s \
+helper_mve_vcmla180h \
+helper_mve_vcmla180s \
+helper_mve_vcmla270h \
+helper_mve_vcmla270s \
+helper_mve_vfmah \
+helper_mve_vfmas \
+helper_mve_vfmsh \
+helper_mve_vfmss \
+helper_mve_vfadd_scalarh \
+helper_mve_vfadd_scalars \
+helper_mve_vfsub_scalarh \
+helper_mve_vfsub_scalars \
+helper_mve_vfmul_scalarh \
+helper_mve_vfmul_scalars \
+helper_mve_vfma_scalarh \
+helper_mve_vfma_scalars \
+helper_mve_vfmas_scalarh \
+helper_mve_vfmas_scalars \
+helper_mve_vcvt_sh \
+helper_mve_vcvt_uh \
+helper_mve_vcvt_hs \
+helper_mve_vcvt_hu \
+helper_mve_vcvt_sf \
+helper_mve_vcvt_uf \
+helper_mve_vcvt_fs \
+helper_mve_vcvt_fu \
+helper_mve_vcvtb_sh \
+helper_mve_vcvtt_sh \
+helper_mve_vcvtb_hs \
+helper_mve_vcvtt_hs \
+helper_mve_vcvt_rm_sh \
+helper_mve_vcvt_rm_uh \
+helper_mve_vcvt_rm_ss \
+helper_mve_vcvt_rm_us \
+helper_mve_vrint_rm_h \
+helper_mve_vrint_rm_s \
+helper_mve_vrintx_h \
+helper_mve_vrintx_s \
+helper_mve_vshlsb \
+helper_mve_vshlsh \
+helper_mve_vshlsw \
+helper_mve_vshlub \
+helper_mve_vshluh \
+helper_mve_vshluw \
+helper_mve_vrshlsb \
+helper_mve_vrshlsh \
+helper_mve_vrshlsw \
+helper_mve_vrshlub \
+helper_mve_vrshluh \
+helper_mve_vrshluw \
+helper_mve_vqshlsb \
+helper_mve_vqshlsh \
+helper_mve_vqshlsw \
+helper_mve_vqshlub \
+helper_mve_vqshluh \
+helper_mve_vqshluw \
+helper_mve_vqrshlsb \
+helper_mve_vqrshlsh \
+helper_mve_vqrshlsw \
+helper_mve_vqrshlub \
+helper_mve_vqrshluh \
+helper_mve_vqrshluw \
+helper_mve_vqdmladhb \
+helper_mve_vqdmladhh \
+helper_mve_vqdmladhw \
+helper_mve_vqdmladhxb \
+helper_mve_vqdmladhxh \
+helper_mve_vqdmladhxw \
+helper_mve_vqrdmladhb \
+helper_mve_vqrdmladhh \
+helper_mve_vqrdmladhw \
+helper_mve_vqrdmladhxb \
+helper_mve_vqrdmladhxh \
+helper_mve_vqrdmladhxw \
+helper_mve_vqdmlsdhb \
+helper_mve_vqdmlsdhh \
+helper_mve_vqdmlsdhw \
+helper_mve_vqdmlsdhxb \
+helper_mve_vqdmlsdhxh \
+helper_mve_vqdmlsdhxw \
+helper_mve_vqrdmlsdhb \
+helper_mve_vqrdmlsdhh \
+helper_mve_vqrdmlsdhw \
+helper_mve_vqrdmlsdhxb \
+helper_mve_vqrdmlsdhxh \
+helper_mve_vqrdmlsdhxw \
+helper_mve_vbrsrb \
+helper_mve_vbrsrh \
+helper_mve_vbrsrw \
+helper_mve_vshli_sb \
+helper_mve_vshli_sh \
+helper_mve_vshli_sw \
+helper_mve_vshli_ub \
+helper_mve_vshli_uh \
+helper_mve_vshli_uw \
+helper_mve_vrshli_sb \
+helper_mve_vrshli_sh \
+helper_mve_vrshli_sw \
+helper_mve_vrshli_ub \
+helper_mve_vrshli_uh \
+helper_mve_vrshli_uw \
+helper_mve_vqshli_sb \
+helper_mve_vqshli_sh \
+helper_mve_vqshli_sw \
+helper_mve_vqshli_ub \
+helper_mve_vqshli_uh \
+helper_mve_vqshli_uw \
+helper_mve_vqrshli_sb \
+helper_mve_vqrshli_sh \
+helper_mve_vqrshli_sw \
+helper_mve_vqrshli_ub \
+helper_mve_vqrshli_uh \
+helper_mve_vqrshli_uw \
+helper_mve_vqshlui_sb \
+helper_mve_vqshlui_sh \
+helper_mve_vqshlui_sw \
+helper_mve_vshllbsb \
+helper_mve_vshllbsh \
+helper_mve_vshllbub \
+helper_mve_vshllbuh \
+helper_mve_vshlltsb \
+helper_mve_vshlltsh \
+helper_mve_vshlltub \
+helper_mve_vshlltuh \
+helper_mve_vshrnbb \
+helper_mve_vshrnbh \
+helper_mve_vshrntb \
+helper_mve_vshrnth \
+helper_mve_vrshrnbb \
+helper_mve_vrshrnbh \
+helper_mve_vrshrntb \
+helper_mve_vrshrnth \
+helper_mve_vqshrnb_sb \
+helper_mve_vqshrnb_sh \
+helper_mve_vqshrnt_sb \
+helper_mve_vqshrnt_sh \
+helper_mve_vqshrnb_ub \
+helper_mve_vqshrnb_uh \
+helper_mve_vqshrnt_ub \
+helper_mve_vqshrnt_uh \
+helper_mve_vqshrunbb \
+helper_mve_vqshrunbh \
+helper_mve_vqshruntb \
+helper_mve_vqshrunth \
+helper_mve_vqrshrnb_sb \
+helper_mve_vqrshrnb_sh \
+helper_mve_vqrshrnt_sb \
+helper_mve_vqrshrnt_sh \
+helper_mve_vqrshrnb_ub \
+helper_mve_vqrshrnb_uh \
+helper_mve_vqrshrnt_ub \
+helper_mve_vqrshrnt_uh \
+helper_mve_vqrshrunbb \
+helper_mve_vqrshrunbh \
+helper_mve_vqrshruntb \
+helper_mve_vqrshrunth \
+helper_mve_vmovnbb \
+helper_mve_vmovnbh \
+helper_mve_vmovntb \
+helper_mve_vmovnth \
+helper_mve_vqmovnbsb \
+helper_mve_vqmovnbsh \
+helper_mve_vqmovntsb \
+helper_mve_vqmovntsh \
+helper_mve_vqmovnbub \
+helper_mve_vqmovnbuh \
+helper_mve_vqmovntub \
+helper_mve_vqmovntuh \
+helper_mve_vqmovunbb \
+helper_mve_vqmovunbh \
+helper_mve_vqmovuntb \
+helper_mve_vqmovunth \
+helper_mve_sshrl \
+helper_mve_ushll \
+helper_mve_sqshll \
+helper_mve_uqshll \
+helper_mve_sqrshrl \
+helper_mve_uqrshll \
+helper_mve_sqrshrl48 \
+helper_mve_uqrshll48 \
+helper_mve_uqshl \
+helper_mve_sqshl \
+helper_mve_uqrshl \
+helper_mve_sqrshr \
+helper_mve_vshlc \
+helper_mve_vsrib \
+helper_mve_vsrih \
+helper_mve_vsriw \
+helper_mve_vslib \
+helper_mve_vslih \
+helper_mve_vsliw \
+helper_mve_vclsb \
+helper_mve_vclsh \
+helper_mve_vclsw \
+helper_mve_vclzb \
+helper_mve_vclzh \
+helper_mve_vclzw \
+helper_mve_vrev16b \
+helper_mve_vrev32b \
+helper_mve_vrev32h \
+helper_mve_vrev64b \
+helper_mve_vrev64h \
+helper_mve_vrev64w \
+helper_mve_vmvn \
+helper_mve_vabsb \
+helper_mve_vabsh \
+helper_mve_vabsw \
+helper_mve_vnegb \
+helper_mve_vnegh \
+helper_mve_vnegw \
+helper_mve_vmaxab \
+helper_mve_vmaxah \
+helper_mve_vmaxaw \
+helper_mve_vminab \
+helper_mve_vminah \
+helper_mve_vminaw \
+helper_mve_vqabsb \
+helper_mve_vqabsh \
+helper_mve_vqabsw \
+helper_mve_vqnegb \
+helper_mve_vqnegh \
+helper_mve_vqnegw \
+helper_mve_vmlaldavsh \
+helper_mve_vmlaldavsw \
+helper_mve_vmlaldavxsh \
+helper_mve_vmlaldavxsw \
+helper_mve_vmlaldavuh \
+helper_mve_vmlaldavuw \
+helper_mve_vmlsldavsh \
+helper_mve_vmlsldavsw \
+helper_mve_vmlsldavxsh \
+helper_mve_vmlsldavxsw \
+helper_mve_vrmlaldavhsw \
+helper_mve_vrmlaldavhxsw \
+helper_mve_vrmlaldavhuw \
+helper_mve_vrmlsldavhsw \
+helper_mve_vrmlsldavhxsw \
+helper_mve_vmladavsb \
+helper_mve_vmladavsh \
+helper_mve_vmladavsw \
+helper_mve_vmladavub \
+helper_mve_vmladavuh \
+helper_mve_vmladavuw \
+helper_mve_vmlsdavb \
+helper_mve_vmlsdavh \
+helper_mve_vmlsdavw \
+helper_mve_vmladavsxb \
+helper_mve_vmladavsxh \
+helper_mve_vmladavsxw \
+helper_mve_vmlsdavxb \
+helper_mve_vmlsdavxh \
+helper_mve_vmlsdavxw \
+helper_mve_vaddvsb \
+helper_mve_vaddvsh \
+helper_mve_vaddvsw \
+helper_mve_vaddvub \
+helper_mve_vaddvuh \
+helper_mve_vaddvuw \
+helper_mve_vmaxvsb \
+helper_mve_vmaxvsh \
+helper_mve_vmaxvsw \
+helper_mve_vmaxvub \
+helper_mve_vmaxvuh \
+helper_mve_vmaxvuw \
+helper_mve_vmaxavb \
+helper_mve_vmaxavh \
+helper_mve_vmaxavw \
+helper_mve_vminvsb \
+helper_mve_vminvsh \
+helper_mve_vminvsw \
+helper_mve_vminvub \
+helper_mve_vminvuh \
+helper_mve_vminvuw \
+helper_mve_vminavb \
+helper_mve_vminavh \
+helper_mve_vminavw \
+helper_mve_vmaxnmvh \
+helper_mve_vmaxnmvs \
+helper_mve_vminnmvh \
+helper_mve_vminnmvs \
+helper_mve_vmaxnmavh \
+helper_mve_vmaxnmavs \
+helper_mve_vminnmavh \
+helper_mve_vminnmavs \
+helper_mve_vaddlv_s \
+helper_mve_vaddlv_u \
+helper_mve_vabavsb \
+helper_mve_vabavsh \
+helper_mve_vabavsw \
+helper_mve_vabavub \
+helper_mve_vabavuh \
+helper_mve_vabavuw \
 helper_vfp_adds \
 helper_vfp_addd \
 helper_vfp_subs \
@@ -2553,10 +4074,14 @@ helper_vfp_touls_round_to_zero \
 helper_vfp_touls \
 helper_vfp_uqtos \
 helper_vfp_touqs \
+helper_vfp_shtoh \
+helper_vfp_uhtoh \
 helper_vfp_sltoh \
 helper_vfp_ultoh \
 helper_vfp_sqtoh \
 helper_vfp_uqtoh \
+helper_vfp_toshh_round_to_zero \
+helper_vfp_touhh_round_to_zero \
 helper_vfp_toshh \
 helper_vfp_touhh \
 helper_vfp_toslh \
@@ -2569,6 +4094,8 @@ helper_vfp_fcvt_f16_to_f32 \
 helper_vfp_fcvt_f32_to_f16 \
 helper_vfp_fcvt_f16_to_f64 \
 helper_vfp_fcvt_f64_to_f16 \
+helper_bfcvt \
+helper_bfcvt_pair \
 helper_recps_f32 \
 helper_rsqrts_f32 \
 helper_recpe_f16 \
@@ -2581,8 +4108,10 @@ helper_recpe_u32 \
 helper_rsqrte_u32 \
 helper_vfp_muladds \
 helper_vfp_muladdd \
+helper_rinth_exact \
 helper_rints_exact \
 helper_rintd_exact \
+helper_rinth \
 helper_rints \
 helper_rintd \
 arm_rmode_to_sf \
@@ -2621,6 +4150,8 @@ arm_cpu_class_init \
 cpu_arm_init \
 helper_crypto_aese \
 helper_crypto_aesmc \
+helper_crypto_sve_aese \
+helper_crypto_sve_aesmc \
 helper_crypto_sha1_3reg \
 helper_crypto_sha1h \
 helper_crypto_sha1su1 \
@@ -2637,6 +4168,9 @@ helper_crypto_sm3partw2 \
 helper_crypto_sm3tt \
 helper_crypto_sm4e \
 helper_crypto_sm4ekey \
+helper_crypto_sve_sm4e \
+helper_crypto_sve_sm4ekey \
+helper_crypto_rax1 \
 helper_check_breakpoints \
 arm_debug_check_watchpoint \
 arm_debug_excp_handler \
@@ -2647,6 +4181,7 @@ helper_rbit64 \
 helper_msr_i_spsel \
 helper_msr_i_daifset \
 helper_msr_i_daifclear \
+helper_set_svcr \
 helper_vfp_cmph_a64 \
 helper_vfp_cmpeh_a64 \
 helper_vfp_cmps_a64 \
@@ -3165,6 +4700,18 @@ helper_pacib \
 helper_pacda \
 helper_pacdb \
 helper_pacga \
+helper_irg \
+helper_addsubg \
+helper_ldg \
+helper_stg \
+helper_stg_stub \
+helper_st2g \
+helper_st2g_stub \
+helper_ldgm \
+helper_stgm \
+helper_stzgm_tags \
+helper_mte_check \
+helper_mte_check_zva \
 helper_autia \
 helper_autib \
 helper_autda \
@@ -3466,6 +5013,14 @@ helper_sve_tbl_b \
 helper_sve_tbl_h \
 helper_sve_tbl_s \
 helper_sve_tbl_d \
+helper_sve2_tbl_b \
+helper_sve2_tbl_h \
+helper_sve2_tbl_s \
+helper_sve2_tbl_d \
+helper_sve2_tbx_b \
+helper_sve2_tbx_h \
+helper_sve2_tbx_s \
+helper_sve2_tbx_d \
 helper_sve_sunpk_h \
 helper_sve_sunpk_s \
 helper_sve_sunpk_d \
@@ -3489,6 +5044,9 @@ helper_sve_trn_b \
 helper_sve_trn_h \
 helper_sve_trn_s \
 helper_sve_trn_d \
+helper_sve2_zip_q \
+helper_sve2_uzp_q \
+helper_sve2_trn_q \
 helper_sve_compact_s \
 helper_sve_compact_d \
 helper_sve_last_active_element \
@@ -3497,6 +5055,106 @@ helper_sve_sel_zpzz_b \
 helper_sve_sel_zpzz_h \
 helper_sve_sel_zpzz_s \
 helper_sve_sel_zpzz_d \
+helper_sve_sel_zpzz_q \
+helper_sme_zero \
+helper_sme_mova_cz_b \
+helper_sme_mova_cz_h \
+helper_sme_mova_cz_s \
+helper_sme_mova_cz_d \
+helper_sme_mova_cz_q \
+helper_sme_mova_zc_b \
+helper_sme_mova_zc_h \
+helper_sme_mova_zc_s \
+helper_sme_mova_zc_d \
+helper_sme_mova_zc_q \
+helper_sme_ld1b_h \
+helper_sme_ld1b_v \
+helper_sme_ld1h_le_h \
+helper_sme_ld1h_le_v \
+helper_sme_ld1h_be_h \
+helper_sme_ld1h_be_v \
+helper_sme_ld1s_le_h \
+helper_sme_ld1s_le_v \
+helper_sme_ld1s_be_h \
+helper_sme_ld1s_be_v \
+helper_sme_ld1d_le_h \
+helper_sme_ld1d_le_v \
+helper_sme_ld1d_be_h \
+helper_sme_ld1d_be_v \
+helper_sme_ld1q_le_h \
+helper_sme_ld1q_le_v \
+helper_sme_ld1q_be_h \
+helper_sme_ld1q_be_v \
+helper_sme_ld1b_h_mte \
+helper_sme_ld1b_v_mte \
+helper_sme_ld1h_le_h_mte \
+helper_sme_ld1h_le_v_mte \
+helper_sme_ld1h_be_h_mte \
+helper_sme_ld1h_be_v_mte \
+helper_sme_ld1s_le_h_mte \
+helper_sme_ld1s_le_v_mte \
+helper_sme_ld1s_be_h_mte \
+helper_sme_ld1s_be_v_mte \
+helper_sme_ld1d_le_h_mte \
+helper_sme_ld1d_le_v_mte \
+helper_sme_ld1d_be_h_mte \
+helper_sme_ld1d_be_v_mte \
+helper_sme_ld1q_le_h_mte \
+helper_sme_ld1q_le_v_mte \
+helper_sme_ld1q_be_h_mte \
+helper_sme_ld1q_be_v_mte \
+helper_sme_st1b_h \
+helper_sme_st1b_v \
+helper_sme_st1h_le_h \
+helper_sme_st1h_le_v \
+helper_sme_st1h_be_h \
+helper_sme_st1h_be_v \
+helper_sme_st1s_le_h \
+helper_sme_st1s_le_v \
+helper_sme_st1s_be_h \
+helper_sme_st1s_be_v \
+helper_sme_st1d_le_h \
+helper_sme_st1d_le_v \
+helper_sme_st1d_be_h \
+helper_sme_st1d_be_v \
+helper_sme_st1q_le_h \
+helper_sme_st1q_le_v \
+helper_sme_st1q_be_h \
+helper_sme_st1q_be_v \
+helper_sme_st1b_h_mte \
+helper_sme_st1b_v_mte \
+helper_sme_st1h_le_h_mte \
+helper_sme_st1h_le_v_mte \
+helper_sme_st1h_be_h_mte \
+helper_sme_st1h_be_v_mte \
+helper_sme_st1s_le_h_mte \
+helper_sme_st1s_le_v_mte \
+helper_sme_st1s_be_h_mte \
+helper_sme_st1s_be_v_mte \
+helper_sme_st1d_le_h_mte \
+helper_sme_st1d_le_v_mte \
+helper_sme_st1d_be_h_mte \
+helper_sme_st1d_be_v_mte \
+helper_sme_st1q_le_h_mte \
+helper_sme_st1q_le_v_mte \
+helper_sme_st1q_be_h_mte \
+helper_sme_st1q_be_v_mte \
+helper_sme_addha_s \
+helper_sme_addva_s \
+helper_sme_addha_d \
+helper_sme_addva_d \
+helper_sme_fmopa_h \
+helper_sme_fmopa_s \
+helper_sme_fmopa_d \
+helper_sme_bfmopa \
+helper_sme_smopa_s \
+helper_sme_umopa_s \
+helper_sme_sumopa_s \
+helper_sme_usmopa_s \
+helper_sme_smopa_d \
+helper_sme_umopa_d \
+helper_sme_sumopa_d \
+helper_sme_usmopa_d \
 helper_sve_cmpeq_ppzz_b \
 helper_sve_cmpeq_ppzz_h \
 helper_sve_cmpeq_ppzz_s \
@@ -3688,6 +5346,13 @@ helper_sve_fcvt_dh \
 helper_sve_fcvt_hd \
 helper_sve_fcvt_ds \
 helper_sve_fcvt_sd \
+helper_sve2_fcvtnt_sh \
+helper_sve2_fcvtnt_ds \
+helper_sve2_fcvtlt_hs \
+helper_sve2_fcvtlt_sd \
+helper_flogb_h \
+helper_flogb_s \
+helper_flogb_d \
 helper_sve_fcvtzs_hh \
 helper_sve_fcvtzs_hs \
 helper_sve_fcvtzs_ss \
@@ -3834,6 +5499,52 @@ helper_sve_ld3dd_le_r \
 helper_sve_ld3dd_be_r \
 helper_sve_ld4dd_le_r \
 helper_sve_ld4dd_be_r \
+helper_sve_ld1bb_r_mte \
+helper_sve_ld1bhu_r_mte \
+helper_sve_ld1bhs_r_mte \
+helper_sve_ld1bsu_r_mte \
+helper_sve_ld1bss_r_mte \
+helper_sve_ld1bdu_r_mte \
+helper_sve_ld1bds_r_mte \
+helper_sve_ld1hh_le_r_mte \
+helper_sve_ld1hh_be_r_mte \
+helper_sve_ld1hsu_le_r_mte \
+helper_sve_ld1hsu_be_r_mte \
+helper_sve_ld1hss_le_r_mte \
+helper_sve_ld1hss_be_r_mte \
+helper_sve_ld1hdu_le_r_mte \
+helper_sve_ld1hdu_be_r_mte \
+helper_sve_ld1hds_le_r_mte \
+helper_sve_ld1hds_be_r_mte \
+helper_sve_ld1ss_le_r_mte \
+helper_sve_ld1ss_be_r_mte \
+helper_sve_ld1sdu_le_r_mte \
+helper_sve_ld1sdu_be_r_mte \
+helper_sve_ld1sds_le_r_mte \
+helper_sve_ld1sds_be_r_mte \
+helper_sve_ld1dd_le_r_mte \
+helper_sve_ld1dd_be_r_mte \
+helper_sve_ld2bb_r_mte \
+helper_sve_ld3bb_r_mte \
+helper_sve_ld4bb_r_mte \
+helper_sve_ld2hh_le_r_mte \
+helper_sve_ld2hh_be_r_mte \
+helper_sve_ld3hh_le_r_mte \
+helper_sve_ld3hh_be_r_mte \
+helper_sve_ld4hh_le_r_mte \
+helper_sve_ld4hh_be_r_mte \
+helper_sve_ld2ss_le_r_mte \
+helper_sve_ld2ss_be_r_mte \
+helper_sve_ld3ss_le_r_mte \
+helper_sve_ld3ss_be_r_mte \
+helper_sve_ld4ss_le_r_mte \
+helper_sve_ld4ss_be_r_mte \
+helper_sve_ld2dd_le_r_mte \
+helper_sve_ld2dd_be_r_mte \
+helper_sve_ld3dd_le_r_mte \
+helper_sve_ld3dd_be_r_mte \
+helper_sve_ld4dd_le_r_mte \
+helper_sve_ld4dd_be_r_mte \
 helper_sve_ldff1bb_r \
 helper_sve_ldnf1bb_r \
 helper_sve_ldff1bhu_r \
@@ -3884,6 +5595,56 @@ helper_sve_ldff1dd_le_r \
 helper_sve_ldnf1dd_le_r \
 helper_sve_ldff1dd_be_r \
 helper_sve_ldnf1dd_be_r \
+helper_sve_ldff1bb_r_mte \
+helper_sve_ldnf1bb_r_mte \
+helper_sve_ldff1bhu_r_mte \
+helper_sve_ldnf1bhu_r_mte \
+helper_sve_ldff1bhs_r_mte \
+helper_sve_ldnf1bhs_r_mte \
+helper_sve_ldff1bsu_r_mte \
+helper_sve_ldnf1bsu_r_mte \
+helper_sve_ldff1bss_r_mte \
+helper_sve_ldnf1bss_r_mte \
+helper_sve_ldff1bdu_r_mte \
+helper_sve_ldnf1bdu_r_mte \
+helper_sve_ldff1bds_r_mte \
+helper_sve_ldnf1bds_r_mte \
+helper_sve_ldff1hh_le_r_mte \
+helper_sve_ldnf1hh_le_r_mte \
+helper_sve_ldff1hh_be_r_mte \
+helper_sve_ldnf1hh_be_r_mte \
+helper_sve_ldff1hsu_le_r_mte \
+helper_sve_ldnf1hsu_le_r_mte \
+helper_sve_ldff1hsu_be_r_mte \
+helper_sve_ldnf1hsu_be_r_mte \
+helper_sve_ldff1hss_le_r_mte \
+helper_sve_ldnf1hss_le_r_mte \
+helper_sve_ldff1hss_be_r_mte \
+helper_sve_ldnf1hss_be_r_mte \
+helper_sve_ldff1hdu_le_r_mte \
+helper_sve_ldnf1hdu_le_r_mte \
+helper_sve_ldff1hdu_be_r_mte \
+helper_sve_ldnf1hdu_be_r_mte \
+helper_sve_ldff1hds_le_r_mte \
+helper_sve_ldnf1hds_le_r_mte \
+helper_sve_ldff1hds_be_r_mte \
+helper_sve_ldnf1hds_be_r_mte \
+helper_sve_ldff1ss_le_r_mte \
+helper_sve_ldnf1ss_le_r_mte \
+helper_sve_ldff1ss_be_r_mte \
+helper_sve_ldnf1ss_be_r_mte \
+helper_sve_ldff1sdu_le_r_mte \
+helper_sve_ldnf1sdu_le_r_mte \
+helper_sve_ldff1sdu_be_r_mte \
+helper_sve_ldnf1sdu_be_r_mte \
+helper_sve_ldff1sds_le_r_mte \
+helper_sve_ldnf1sds_le_r_mte \
+helper_sve_ldff1sds_be_r_mte \
+helper_sve_ldnf1sds_be_r_mte \
+helper_sve_ldff1dd_le_r_mte \
+helper_sve_ldnf1dd_le_r_mte \
+helper_sve_ldff1dd_be_r_mte \
+helper_sve_ldnf1dd_be_r_mte \
 helper_sve_st1bb_r \
 helper_sve_st1bh_r \
 helper_sve_st1bs_r \
@@ -3921,6 +5682,43 @@ helper_sve_st3dd_le_r \
 helper_sve_st3dd_be_r \
 helper_sve_st4dd_le_r \
 helper_sve_st4dd_be_r \
+helper_sve_st1bb_r_mte \
+helper_sve_st1bh_r_mte \
+helper_sve_st1bs_r_mte \
+helper_sve_st1bd_r_mte \
+helper_sve_st2bb_r_mte \
+helper_sve_st3bb_r_mte \
+helper_sve_st4bb_r_mte \
+helper_sve_st1hh_le_r_mte \
+helper_sve_st1hh_be_r_mte \
+helper_sve_st1hs_le_r_mte \
+helper_sve_st1hs_be_r_mte \
+helper_sve_st1hd_le_r_mte \
+helper_sve_st1hd_be_r_mte \
+helper_sve_st2hh_le_r_mte \
+helper_sve_st2hh_be_r_mte \
+helper_sve_st3hh_le_r_mte \
+helper_sve_st3hh_be_r_mte \
+helper_sve_st4hh_le_r_mte \
+helper_sve_st4hh_be_r_mte \
+helper_sve_st1ss_le_r_mte \
+helper_sve_st1ss_be_r_mte \
+helper_sve_st1sd_le_r_mte \
+helper_sve_st1sd_be_r_mte \
+helper_sve_st2ss_le_r_mte \
+helper_sve_st2ss_be_r_mte \
+helper_sve_st3ss_le_r_mte \
+helper_sve_st3ss_be_r_mte \
+helper_sve_st4ss_le_r_mte \
+helper_sve_st4ss_be_r_mte \
+helper_sve_st1dd_le_r_mte \
+helper_sve_st1dd_be_r_mte \
+helper_sve_st2dd_le_r_mte \
+helper_sve_st2dd_be_r_mte \
+helper_sve_st3dd_le_r_mte \
+helper_sve_st3dd_be_r_mte \
+helper_sve_st4dd_le_r_mte \
+helper_sve_st4dd_be_r_mte \
 helper_sve_ldbsu_zsu \
 helper_sve_ldbsu_zss \
 helper_sve_ldbdu_zsu \
@@ -3973,6 +5771,58 @@ helper_sve_lddd_le_zd \
 helper_sve_lddd_be_zsu \
 helper_sve_lddd_be_zss \
 helper_sve_lddd_be_zd \
+helper_sve_ldbsu_zsu_mte \
+helper_sve_ldbsu_zss_mte \
+helper_sve_ldbdu_zsu_mte \
+helper_sve_ldbdu_zss_mte \
+helper_sve_ldbdu_zd_mte \
+helper_sve_ldbss_zsu_mte \
+helper_sve_ldbss_zss_mte \
+helper_sve_ldbds_zsu_mte \
+helper_sve_ldbds_zss_mte \
+helper_sve_ldbds_zd_mte \
+helper_sve_ldhsu_le_zsu_mte \
+helper_sve_ldhsu_le_zss_mte \
+helper_sve_ldhdu_le_zsu_mte \
+helper_sve_ldhdu_le_zss_mte \
+helper_sve_ldhdu_le_zd_mte \
+helper_sve_ldhsu_be_zsu_mte \
+helper_sve_ldhsu_be_zss_mte \
+helper_sve_ldhdu_be_zsu_mte \
+helper_sve_ldhdu_be_zss_mte \
+helper_sve_ldhdu_be_zd_mte \
+helper_sve_ldhss_le_zsu_mte \
+helper_sve_ldhss_le_zss_mte \
+helper_sve_ldhds_le_zsu_mte \
+helper_sve_ldhds_le_zss_mte \
+helper_sve_ldhds_le_zd_mte \
+helper_sve_ldhss_be_zsu_mte \
+helper_sve_ldhss_be_zss_mte \
+helper_sve_ldhds_be_zsu_mte \
+helper_sve_ldhds_be_zss_mte \
+helper_sve_ldhds_be_zd_mte \
+helper_sve_ldss_le_zsu_mte \
+helper_sve_ldss_le_zss_mte \
+helper_sve_ldsdu_le_zsu_mte \
+helper_sve_ldsdu_le_zss_mte \
+helper_sve_ldsdu_le_zd_mte \
+helper_sve_ldss_be_zsu_mte \
+helper_sve_ldss_be_zss_mte \
+helper_sve_ldsdu_be_zsu_mte \
+helper_sve_ldsdu_be_zss_mte \
+helper_sve_ldsdu_be_zd_mte \
+helper_sve_ldsds_le_zsu_mte \
+helper_sve_ldsds_le_zss_mte \
+helper_sve_ldsds_le_zd_mte \
+helper_sve_ldsds_be_zsu_mte \
+helper_sve_ldsds_be_zss_mte \
+helper_sve_ldsds_be_zd_mte \
+helper_sve_lddd_le_zsu_mte \
+helper_sve_lddd_le_zss_mte \
+helper_sve_lddd_le_zd_mte \
+helper_sve_lddd_be_zsu_mte \
+helper_sve_lddd_be_zss_mte \
+helper_sve_lddd_be_zd_mte \
 helper_sve_ldffbsu_zsu \
 helper_sve_ldffbsu_zss \
 helper_sve_ldffbdu_zsu \
@@ -4025,6 +5875,58 @@ helper_sve_ldffdd_le_zd \
 helper_sve_ldffdd_be_zsu \
 helper_sve_ldffdd_be_zss \
 helper_sve_ldffdd_be_zd \
+helper_sve_ldffbsu_zsu_mte \
+helper_sve_ldffbsu_zss_mte \
+helper_sve_ldffbdu_zsu_mte \
+helper_sve_ldffbdu_zss_mte \
+helper_sve_ldffbdu_zd_mte \
+helper_sve_ldffbss_zsu_mte \
+helper_sve_ldffbss_zss_mte \
+helper_sve_ldffbds_zsu_mte \
+helper_sve_ldffbds_zss_mte \
+helper_sve_ldffbds_zd_mte \
+helper_sve_ldffhsu_le_zsu_mte \
+helper_sve_ldffhsu_le_zss_mte \
+helper_sve_ldffhdu_le_zsu_mte \
+helper_sve_ldffhdu_le_zss_mte \
+helper_sve_ldffhdu_le_zd_mte \
+helper_sve_ldffhsu_be_zsu_mte \
+helper_sve_ldffhsu_be_zss_mte \
+helper_sve_ldffhdu_be_zsu_mte \
+helper_sve_ldffhdu_be_zss_mte \
+helper_sve_ldffhdu_be_zd_mte \
+helper_sve_ldffhss_le_zsu_mte \
+helper_sve_ldffhss_le_zss_mte \
+helper_sve_ldffhds_le_zsu_mte \
+helper_sve_ldffhds_le_zss_mte \
+helper_sve_ldffhds_le_zd_mte \
+helper_sve_ldffhss_be_zsu_mte \
+helper_sve_ldffhss_be_zss_mte \
+helper_sve_ldffhds_be_zsu_mte \
+helper_sve_ldffhds_be_zss_mte \
+helper_sve_ldffhds_be_zd_mte \
+helper_sve_ldffss_le_zsu_mte \
+helper_sve_ldffss_le_zss_mte \
+helper_sve_ldffsdu_le_zsu_mte \
+helper_sve_ldffsdu_le_zss_mte \
+helper_sve_ldffsdu_le_zd_mte \
+helper_sve_ldffss_be_zsu_mte \
+helper_sve_ldffss_be_zss_mte \
+helper_sve_ldffsdu_be_zsu_mte \
+helper_sve_ldffsdu_be_zss_mte \
+helper_sve_ldffsdu_be_zd_mte \
+helper_sve_ldffsds_le_zsu_mte \
+helper_sve_ldffsds_le_zss_mte \
+helper_sve_ldffsds_le_zd_mte \
+helper_sve_ldffsds_be_zsu_mte \
+helper_sve_ldffsds_be_zss_mte \
+helper_sve_ldffsds_be_zd_mte \
+helper_sve_ldffdd_le_zsu_mte \
+helper_sve_ldffdd_le_zss_mte \
+helper_sve_ldffdd_le_zd_mte \
+helper_sve_ldffdd_be_zsu_mte \
+helper_sve_ldffdd_be_zss_mte \
+helper_sve_ldffdd_be_zd_mte \
 helper_sve_stbs_zsu \
 helper_sve_sths_le_zsu \
 helper_sve_sths_be_zsu \
@@ -4056,6 +5958,37 @@ helper_sve_stsd_le_zd \
 helper_sve_stsd_be_zd \
 helper_sve_stdd_le_zd \
 helper_sve_stdd_be_zd \
+helper_sve_stbs_zsu_mte \
+helper_sve_sths_le_zsu_mte \
+helper_sve_sths_be_zsu_mte \
+helper_sve_stss_le_zsu_mte \
+helper_sve_stss_be_zsu_mte \
+helper_sve_stbs_zss_mte \
+helper_sve_sths_le_zss_mte \
+helper_sve_sths_be_zss_mte \
+helper_sve_stss_le_zss_mte \
+helper_sve_stss_be_zss_mte \
+helper_sve_stbd_zsu_mte \
+helper_sve_sthd_le_zsu_mte \
+helper_sve_sthd_be_zsu_mte \
+helper_sve_stsd_le_zsu_mte \
+helper_sve_stsd_be_zsu_mte \
+helper_sve_stdd_le_zsu_mte \
+helper_sve_stdd_be_zsu_mte \
+helper_sve_stbd_zss_mte \
+helper_sve_sthd_le_zss_mte \
+helper_sve_sthd_be_zss_mte \
+helper_sve_stsd_le_zss_mte \
+helper_sve_stsd_be_zss_mte \
+helper_sve_stdd_le_zss_mte \
+helper_sve_stdd_be_zss_mte \
+helper_sve_stbd_zd_mte \
+helper_sve_sthd_le_zd_mte \
+helper_sve_sthd_be_zd_mte \
+helper_sve_stsd_le_zd_mte \
+helper_sve_stsd_be_zd_mte \
+helper_sve_stdd_le_zd_mte \
+helper_sve_stdd_be_zd_mte \
 arm_cpu_do_unaligned_access \
 arm_cpu_do_transaction_failed \
 arm_cpu_tlb_fill \
@@ -4096,12 +6029,23 @@ helper_neon_qrdmlsh_s32 \
 helper_gvec_qrdmlsh_s32 \
 helper_gvec_sdot_b \
 helper_gvec_udot_b \
+helper_gvec_usdot_b \
 helper_gvec_sdot_h \
 helper_gvec_udot_h \
 helper_gvec_sdot_idx_b \
 helper_gvec_udot_idx_b \
+helper_gvec_sudot_idx_b \
+helper_gvec_usdot_idx_b \
 helper_gvec_sdot_idx_h \
 helper_gvec_udot_idx_h \
+helper_gvec_smmla_b \
+helper_gvec_ummla_b \
+helper_gvec_usmmla_b \
+helper_gvec_bfdot \
+helper_gvec_bfdot_idx \
+helper_gvec_bfmmla \
+helper_gvec_bfmlal \
+helper_gvec_bfmlal_idx \
 helper_gvec_fcaddh \
 helper_gvec_fcadds \
 helper_gvec_fcaddd \
@@ -4156,10 +6100,20 @@ helper_gvec_uqadd_d \
 helper_gvec_uqsub_d \
 helper_gvec_sqadd_d \
 helper_gvec_sqsub_d \
+helper_gvec_saba_b \
+helper_gvec_saba_h \
+helper_gvec_saba_s \
+helper_gvec_saba_d \
+helper_gvec_uaba_b \
+helper_gvec_uaba_h \
+helper_gvec_uaba_s \
+helper_gvec_uaba_d \
 helper_gvec_fmlal_a32 \
 helper_gvec_fmlal_a64 \
 helper_gvec_fmlal_idx_a32 \
 helper_gvec_fmlal_idx_a64 \
+helper_sve2_fmlal_zzzw_s \
+helper_sve2_fmlal_zzxw_s \
 helper_gvec_sshl_b \
 helper_gvec_sshl_h \
 helper_gvec_ushl_b \
@@ -4167,11 +6121,1103 @@ helper_gvec_ushl_h \
 helper_gvec_pmul_b \
 helper_gvec_pmull_q \
 helper_neon_pmull_h \
+helper_sve2_sqabs_b \
+helper_sve2_sqabs_h \
+helper_sve2_sqabs_s \
+helper_sve2_sqabs_d \
+helper_sve2_sqneg_b \
+helper_sve2_sqneg_h \
+helper_sve2_sqneg_s \
+helper_sve2_sqneg_d \
+helper_sve2_urecpe_s \
+helper_sve2_ursqrte_s \
+helper_sve2_sadalp_zpzz_h \
+helper_sve2_sadalp_zpzz_s \
+helper_sve2_sadalp_zpzz_d \
+helper_sve2_uadalp_zpzz_h \
+helper_sve2_uadalp_zpzz_s \
+helper_sve2_uadalp_zpzz_d \
+helper_sve2_shadd_zpzz_b \
+helper_sve2_shadd_zpzz_h \
+helper_sve2_shadd_zpzz_s \
+helper_sve2_shadd_zpzz_d \
+helper_sve2_uhadd_zpzz_b \
+helper_sve2_uhadd_zpzz_h \
+helper_sve2_uhadd_zpzz_s \
+helper_sve2_uhadd_zpzz_d \
+helper_sve2_srhadd_zpzz_b \
+helper_sve2_srhadd_zpzz_h \
+helper_sve2_srhadd_zpzz_s \
+helper_sve2_srhadd_zpzz_d \
+helper_sve2_urhadd_zpzz_b \
+helper_sve2_urhadd_zpzz_h \
+helper_sve2_urhadd_zpzz_s \
+helper_sve2_urhadd_zpzz_d \
+helper_sve2_shsub_zpzz_b \
+helper_sve2_shsub_zpzz_h \
+helper_sve2_shsub_zpzz_s \
+helper_sve2_shsub_zpzz_d \
+helper_sve2_uhsub_zpzz_b \
+helper_sve2_uhsub_zpzz_h \
+helper_sve2_uhsub_zpzz_s \
+helper_sve2_uhsub_zpzz_d \
+helper_sve2_addp_zpzz_b \
+helper_sve2_addp_zpzz_h \
+helper_sve2_addp_zpzz_s \
+helper_sve2_addp_zpzz_d \
+helper_sve2_smaxp_zpzz_b \
+helper_sve2_smaxp_zpzz_h \
+helper_sve2_smaxp_zpzz_s \
+helper_sve2_smaxp_zpzz_d \
+helper_sve2_umaxp_zpzz_b \
+helper_sve2_umaxp_zpzz_h \
+helper_sve2_umaxp_zpzz_s \
+helper_sve2_umaxp_zpzz_d \
+helper_sve2_sminp_zpzz_b \
+helper_sve2_sminp_zpzz_h \
+helper_sve2_sminp_zpzz_s \
+helper_sve2_sminp_zpzz_d \
+helper_sve2_uminp_zpzz_b \
+helper_sve2_uminp_zpzz_h \
+helper_sve2_uminp_zpzz_s \
+helper_sve2_uminp_zpzz_d \
+helper_sve2_faddp_zpzz_h \
+helper_sve2_faddp_zpzz_s \
+helper_sve2_faddp_zpzz_d \
+helper_sve2_fmaxnmp_zpzz_h \
+helper_sve2_fmaxnmp_zpzz_s \
+helper_sve2_fmaxnmp_zpzz_d \
+helper_sve2_fminnmp_zpzz_h \
+helper_sve2_fminnmp_zpzz_s \
+helper_sve2_fminnmp_zpzz_d \
+helper_sve2_fmaxp_zpzz_h \
+helper_sve2_fmaxp_zpzz_s \
+helper_sve2_fmaxp_zpzz_d \
+helper_sve2_fminp_zpzz_h \
+helper_sve2_fminp_zpzz_s \
+helper_sve2_fminp_zpzz_d \
+helper_sve2_srshl_zpzz_b \
+helper_sve2_srshl_zpzz_h \
+helper_sve2_srshl_zpzz_s \
+helper_sve2_srshl_zpzz_d \
+helper_sve2_urshl_zpzz_b \
+helper_sve2_urshl_zpzz_h \
+helper_sve2_urshl_zpzz_s \
+helper_sve2_urshl_zpzz_d \
+helper_sve2_sqshl_zpzz_b \
+helper_sve2_sqshl_zpzz_h \
+helper_sve2_sqshl_zpzz_s \
+helper_sve2_sqshl_zpzz_d \
+helper_sve2_uqshl_zpzz_b \
+helper_sve2_uqshl_zpzz_h \
+helper_sve2_uqshl_zpzz_s \
+helper_sve2_uqshl_zpzz_d \
+helper_sve2_sqrshl_zpzz_b \
+helper_sve2_sqrshl_zpzz_h \
+helper_sve2_sqrshl_zpzz_s \
+helper_sve2_sqrshl_zpzz_d \
+helper_sve2_uqrshl_zpzz_b \
+helper_sve2_uqrshl_zpzz_h \
+helper_sve2_uqrshl_zpzz_s \
+helper_sve2_uqrshl_zpzz_d \
+helper_sve2_sqadd_zpzz_b \
+helper_sve2_sqadd_zpzz_h \
+helper_sve2_sqadd_zpzz_s \
+helper_sve2_sqadd_zpzz_d \
+helper_sve2_uqadd_zpzz_b \
+helper_sve2_uqadd_zpzz_h \
+helper_sve2_uqadd_zpzz_s \
+helper_sve2_uqadd_zpzz_d \
+helper_sve2_sqsub_zpzz_b \
+helper_sve2_sqsub_zpzz_h \
+helper_sve2_sqsub_zpzz_s \
+helper_sve2_sqsub_zpzz_d \
+helper_sve2_uqsub_zpzz_b \
+helper_sve2_uqsub_zpzz_h \
+helper_sve2_uqsub_zpzz_s \
+helper_sve2_uqsub_zpzz_d \
+helper_sve2_suqadd_zpzz_b \
+helper_sve2_suqadd_zpzz_h \
+helper_sve2_suqadd_zpzz_s \
+helper_sve2_suqadd_zpzz_d \
+helper_sve2_usqadd_zpzz_b \
+helper_sve2_usqadd_zpzz_h \
+helper_sve2_usqadd_zpzz_s \
+helper_sve2_usqadd_zpzz_d \
+helper_sve2_eor3 \
+helper_sve2_bcax \
+helper_sve2_bsl1n \
+helper_sve2_bsl2n \
+helper_sve2_nbsl \
+helper_sve2_xar_b \
+helper_sve2_xar_h \
+helper_sve2_xar_s \
+helper_sve2_xar_d \
+helper_fmmla_s \
+helper_fmmla_d \
 helper_sve2_pmull_h \
+helper_sve2_pmull_d \
+helper_sve2_eoril_b \
+helper_sve2_eoril_h \
+helper_sve2_eoril_s \
+helper_sve2_eoril_d \
+helper_sve2_bext_b \
+helper_sve2_bext_h \
+helper_sve2_bext_s \
+helper_sve2_bext_d \
+helper_sve2_bdep_b \
+helper_sve2_bdep_h \
+helper_sve2_bdep_s \
+helper_sve2_bdep_d \
+helper_sve2_bgrp_b \
+helper_sve2_bgrp_h \
+helper_sve2_bgrp_s \
+helper_sve2_bgrp_d \
+helper_sve2_cadd_b \
+helper_sve2_cadd_h \
+helper_sve2_cadd_s \
+helper_sve2_cadd_d \
+helper_sve2_sqcadd_b \
+helper_sve2_sqcadd_h \
+helper_sve2_sqcadd_s \
+helper_sve2_sqcadd_d \
+helper_sve2_smulh_zzz_b \
+helper_sve2_smulh_zzz_h \
+helper_sve2_smulh_zzz_s \
+helper_sve2_smulh_zzz_d \
+helper_sve2_umulh_zzz_b \
+helper_sve2_umulh_zzz_h \
+helper_sve2_umulh_zzz_s \
+helper_sve2_umulh_zzz_d \
+helper_sve2_sqdmulh_b \
+helper_sve2_sqdmulh_h \
+helper_sve2_sqdmulh_s \
+helper_sve2_sqdmulh_d \
+helper_sve2_sqrdmulh_b \
+helper_sve2_sqrdmulh_h \
+helper_sve2_sqrdmulh_s \
+helper_sve2_sqrdmulh_d \
+helper_sve2_sqrdmlah_b \
+helper_sve2_sqrdmlah_h \
+helper_sve2_sqrdmlah_s \
+helper_sve2_sqrdmlah_d \
+helper_sve2_sqrdmlsh_b \
+helper_sve2_sqrdmlsh_h \
+helper_sve2_sqrdmlsh_s \
+helper_sve2_sqrdmlsh_d \
+helper_sve2_cmla_zzzz_b \
+helper_sve2_cmla_zzzz_h \
+helper_sve2_cmla_zzzz_s \
+helper_sve2_cmla_zzzz_d \
+helper_sve2_sqrdcmlah_zzzz_b \
+helper_sve2_sqrdcmlah_zzzz_h \
+helper_sve2_sqrdcmlah_zzzz_s \
+helper_sve2_sqrdcmlah_zzzz_d \
+helper_sve2_cdot_zzzz_s \
+helper_sve2_cdot_zzzz_d \
+helper_sve2_mul_idx_h \
+helper_sve2_mul_idx_s \
+helper_sve2_mul_idx_d \
+helper_sve2_sqdmulh_idx_h \
+helper_sve2_sqdmulh_idx_s \
+helper_sve2_sqdmulh_idx_d \
+helper_sve2_sqrdmulh_idx_h \
+helper_sve2_sqrdmulh_idx_s \
+helper_sve2_sqrdmulh_idx_d \
+helper_sve2_sqrdmlah_idx_h \
+helper_sve2_sqrdmlah_idx_s \
+helper_sve2_sqrdmlah_idx_d \
+helper_sve2_sqrdmlsh_idx_h \
+helper_sve2_sqrdmlsh_idx_s \
+helper_sve2_sqrdmlsh_idx_d \
+helper_sve2_cmla_idx_h \
+helper_sve2_cmla_idx_s \
+helper_sve2_sqrdcmlah_idx_h \
+helper_sve2_sqrdcmlah_idx_s \
+helper_sve2_cdot_idx_s \
+helper_sve2_cdot_idx_d \
+helper_sve_bfcvt \
+helper_sve_bfcvtnt \
+helper_sve2_saddl_h \
+helper_sve2_saddl_s \
+helper_sve2_saddl_d \
+helper_sve2_uaddl_h \
+helper_sve2_uaddl_s \
+helper_sve2_uaddl_d \
+helper_sve2_ssubl_h \
+helper_sve2_ssubl_s \
+helper_sve2_ssubl_d \
+helper_sve2_usubl_h \
+helper_sve2_usubl_s \
+helper_sve2_usubl_d \
+helper_sve2_sabdl_h \
+helper_sve2_sabdl_s \
+helper_sve2_sabdl_d \
+helper_sve2_uabdl_h \
+helper_sve2_uabdl_s \
+helper_sve2_uabdl_d \
+helper_sve2_smull_zzz_h \
+helper_sve2_smull_zzz_s \
+helper_sve2_smull_zzz_d \
+helper_sve2_umull_zzz_h \
+helper_sve2_umull_zzz_s \
+helper_sve2_umull_zzz_d \
+helper_sve2_sqdmull_zzz_h \
+helper_sve2_sqdmull_zzz_s \
+helper_sve2_sqdmull_zzz_d \
+helper_sve2_smull_idx_s \
+helper_sve2_smull_idx_d \
+helper_sve2_umull_idx_s \
+helper_sve2_umull_idx_d \
+helper_sve2_sqdmull_idx_s \
+helper_sve2_sqdmull_idx_d \
+helper_sve2_sabal_h \
+helper_sve2_sabal_s \
+helper_sve2_sabal_d \
+helper_sve2_uabal_h \
+helper_sve2_uabal_s \
+helper_sve2_uabal_d \
+helper_sve2_smlal_zzzw_h \
+helper_sve2_smlal_zzzw_s \
+helper_sve2_smlal_zzzw_d \
+helper_sve2_umlal_zzzw_h \
+helper_sve2_umlal_zzzw_s \
+helper_sve2_umlal_zzzw_d \
+helper_sve2_smlsl_zzzw_h \
+helper_sve2_smlsl_zzzw_s \
+helper_sve2_smlsl_zzzw_d \
+helper_sve2_umlsl_zzzw_h \
+helper_sve2_umlsl_zzzw_s \
+helper_sve2_umlsl_zzzw_d \
+helper_sve2_sqdmlal_zzzw_h \
+helper_sve2_sqdmlal_zzzw_s \
+helper_sve2_sqdmlal_zzzw_d \
+helper_sve2_sqdmlsl_zzzw_h \
+helper_sve2_sqdmlsl_zzzw_s \
+helper_sve2_sqdmlsl_zzzw_d \
+helper_sve2_smlal_idx_s \
+helper_sve2_smlal_idx_d \
+helper_sve2_umlal_idx_s \
+helper_sve2_umlal_idx_d \
+helper_sve2_smlsl_idx_s \
+helper_sve2_smlsl_idx_d \
+helper_sve2_umlsl_idx_s \
+helper_sve2_umlsl_idx_d \
+helper_sve2_sqdmlal_idx_s \
+helper_sve2_sqdmlal_idx_d \
+helper_sve2_sqdmlsl_idx_s \
+helper_sve2_sqdmlsl_idx_d \
+helper_sve2_saddw_h \
+helper_sve2_saddw_s \
+helper_sve2_saddw_d \
+helper_sve2_uaddw_h \
+helper_sve2_uaddw_s \
+helper_sve2_uaddw_d \
+helper_sve2_ssubw_h \
+helper_sve2_ssubw_s \
+helper_sve2_ssubw_d \
+helper_sve2_usubw_h \
+helper_sve2_usubw_s \
+helper_sve2_usubw_d \
+helper_sve2_sshll_h \
+helper_sve2_sshll_s \
+helper_sve2_sshll_d \
+helper_sve2_ushll_h \
+helper_sve2_ushll_s \
+helper_sve2_ushll_d \
+helper_sve2_ssra_b \
+helper_sve2_ssra_h \
+helper_sve2_ssra_s \
+helper_sve2_ssra_d \
+helper_sve2_usra_b \
+helper_sve2_usra_h \
+helper_sve2_usra_s \
+helper_sve2_usra_d \
+helper_sve2_srsra_b \
+helper_sve2_srsra_h \
+helper_sve2_srsra_s \
+helper_sve2_srsra_d \
+helper_sve2_ursra_b \
+helper_sve2_ursra_h \
+helper_sve2_ursra_s \
+helper_sve2_ursra_d \
+helper_sve2_sqshrunb_h \
+helper_sve2_sqshrunb_s \
+helper_sve2_sqshrunb_d \
+helper_sve2_sqshrunt_h \
+helper_sve2_sqshrunt_s \
+helper_sve2_sqshrunt_d \
+helper_sve2_sqrshrunb_h \
+helper_sve2_sqrshrunb_s \
+helper_sve2_sqrshrunb_d \
+helper_sve2_sqrshrunt_h \
+helper_sve2_sqrshrunt_s \
+helper_sve2_sqrshrunt_d \
+helper_sve2_shrnb_h \
+helper_sve2_shrnb_s \
+helper_sve2_shrnb_d \
+helper_sve2_shrnt_h \
+helper_sve2_shrnt_s \
+helper_sve2_shrnt_d \
+helper_sve2_rshrnb_h \
+helper_sve2_rshrnb_s \
+helper_sve2_rshrnb_d \
+helper_sve2_rshrnt_h \
+helper_sve2_rshrnt_s \
+helper_sve2_rshrnt_d \
+helper_sve2_sqshrnb_h \
+helper_sve2_sqshrnb_s \
+helper_sve2_sqshrnb_d \
+helper_sve2_sqshrnt_h \
+helper_sve2_sqshrnt_s \
+helper_sve2_sqshrnt_d \
+helper_sve2_sqrshrnb_h \
+helper_sve2_sqrshrnb_s \
+helper_sve2_sqrshrnb_d \
+helper_sve2_sqrshrnt_h \
+helper_sve2_sqrshrnt_s \
+helper_sve2_sqrshrnt_d \
+helper_sve2_uqshrnb_h \
+helper_sve2_uqshrnb_s \
+helper_sve2_uqshrnb_d \
+helper_sve2_uqshrnt_h \
+helper_sve2_uqshrnt_s \
+helper_sve2_uqshrnt_d \
+helper_sve2_uqrshrnb_h \
+helper_sve2_uqrshrnb_s \
+helper_sve2_uqrshrnb_d \
+helper_sve2_uqrshrnt_h \
+helper_sve2_uqrshrnt_s \
+helper_sve2_uqrshrnt_d \
+helper_sve2_addhnb_h \
+helper_sve2_addhnb_s \
+helper_sve2_addhnb_d \
+helper_sve2_addhnt_h \
+helper_sve2_addhnt_s \
+helper_sve2_addhnt_d \
+helper_sve2_raddhnb_h \
+helper_sve2_raddhnb_s \
+helper_sve2_raddhnb_d \
+helper_sve2_raddhnt_h \
+helper_sve2_raddhnt_s \
+helper_sve2_raddhnt_d \
+helper_sve2_subhnb_h \
+helper_sve2_subhnb_s \
+helper_sve2_subhnb_d \
+helper_sve2_subhnt_h \
+helper_sve2_subhnt_s \
+helper_sve2_subhnt_d \
+helper_sve2_rsubhnb_h \
+helper_sve2_rsubhnb_s \
+helper_sve2_rsubhnb_d \
+helper_sve2_rsubhnt_h \
+helper_sve2_rsubhnt_s \
+helper_sve2_rsubhnt_d \
+helper_sve2_sqxtnb_h \
+helper_sve2_sqxtnb_s \
+helper_sve2_sqxtnb_d \
+helper_sve2_sqxtnt_h \
+helper_sve2_sqxtnt_s \
+helper_sve2_sqxtnt_d \
+helper_sve2_uqxtnb_h \
+helper_sve2_uqxtnb_s \
+helper_sve2_uqxtnb_d \
+helper_sve2_uqxtnt_h \
+helper_sve2_uqxtnt_s \
+helper_sve2_uqxtnt_d \
+helper_sve2_sqxtunb_h \
+helper_sve2_sqxtunb_s \
+helper_sve2_sqxtunb_d \
+helper_sve2_sqxtunt_h \
+helper_sve2_sqxtunt_s \
+helper_sve2_sqxtunt_d \
+helper_sve2_match_ppzz_b \
+helper_sve2_match_ppzz_h \
+helper_sve2_nmatch_ppzz_b \
+helper_sve2_nmatch_ppzz_h \
+helper_sve2_histcnt_s \
+helper_sve2_histcnt_d \
+helper_sve2_histseg \
+helper_sve2_adcl_s \
+helper_sve2_adcl_d \
 helper_vfp_get_fpscr \
 vfp_get_fpscr \
 helper_vfp_set_fpscr \
 vfp_set_fpscr \
+helper_mve_vctp \
+helper_mve_vpnot \
+helper_mve_vpsel \
+helper_mve_vdup \
+helper_mve_vmovi \
+helper_mve_vandi \
+helper_mve_vorri \
+helper_mve_vidupb \
+helper_mve_viduph \
+helper_mve_vidupw \
+helper_mve_viwdupb \
+helper_mve_viwduph \
+helper_mve_viwdupw \
+helper_mve_vdwdupb \
+helper_mve_vdwduph \
+helper_mve_vdwdupw \
+helper_mve_vcmpeqb \
+helper_mve_vcmpeqh \
+helper_mve_vcmpeqw \
+helper_mve_vcmpeq_scalarb \
+helper_mve_vcmpeq_scalarh \
+helper_mve_vcmpeq_scalarw \
+helper_mve_vcmpneb \
+helper_mve_vcmpneh \
+helper_mve_vcmpnew \
+helper_mve_vcmpne_scalarb \
+helper_mve_vcmpne_scalarh \
+helper_mve_vcmpne_scalarw \
+helper_mve_vcmpcsb \
+helper_mve_vcmpcsh \
+helper_mve_vcmpcsw \
+helper_mve_vcmpcs_scalarb \
+helper_mve_vcmpcs_scalarh \
+helper_mve_vcmpcs_scalarw \
+helper_mve_vcmphib \
+helper_mve_vcmphih \
+helper_mve_vcmphiw \
+helper_mve_vcmphi_scalarb \
+helper_mve_vcmphi_scalarh \
+helper_mve_vcmphi_scalarw \
+helper_mve_vcmpgeb \
+helper_mve_vcmpgeh \
+helper_mve_vcmpgew \
+helper_mve_vcmpge_scalarb \
+helper_mve_vcmpge_scalarh \
+helper_mve_vcmpge_scalarw \
+helper_mve_vcmpltb \
+helper_mve_vcmplth \
+helper_mve_vcmpltw \
+helper_mve_vcmplt_scalarb \
+helper_mve_vcmplt_scalarh \
+helper_mve_vcmplt_scalarw \
+helper_mve_vcmpgtb \
+helper_mve_vcmpgth \
+helper_mve_vcmpgtw \
+helper_mve_vcmpgt_scalarb \
+helper_mve_vcmpgt_scalarh \
+helper_mve_vcmpgt_scalarw \
+helper_mve_vcmpleb \
+helper_mve_vcmpleh \
+helper_mve_vcmplew \
+helper_mve_vcmple_scalarb \
+helper_mve_vcmple_scalarh \
+helper_mve_vcmple_scalarw \
+helper_mve_vfcmpeqh \
+helper_mve_vfcmpeqs \
+helper_mve_vfcmpneh \
+helper_mve_vfcmpnes \
+helper_mve_vfcmpgeh \
+helper_mve_vfcmpges \
+helper_mve_vfcmplth \
+helper_mve_vfcmplts \
+helper_mve_vfcmpgth \
+helper_mve_vfcmpgts \
+helper_mve_vfcmpleh \
+helper_mve_vfcmples \
+helper_mve_vfcmpeq_scalarh \
+helper_mve_vfcmpeq_scalars \
+helper_mve_vfcmpne_scalarh \
+helper_mve_vfcmpne_scalars \
+helper_mve_vfcmpge_scalarh \
+helper_mve_vfcmpge_scalars \
+helper_mve_vfcmplt_scalarh \
+helper_mve_vfcmplt_scalars \
+helper_mve_vfcmpgt_scalarh \
+helper_mve_vfcmpgt_scalars \
+helper_mve_vfcmple_scalarh \
+helper_mve_vfcmple_scalars \
+helper_mve_vfabsh \
+helper_mve_vfabss \
+helper_mve_vfnegh \
+helper_mve_vfnegs \
+helper_mve_vldrb \
+helper_mve_vldrh \
+helper_mve_vldrw \
+helper_mve_vldrb_sh \
+helper_mve_vldrb_uh \
+helper_mve_vldrb_sw \
+helper_mve_vldrb_uw \
+helper_mve_vldrh_sw \
+helper_mve_vldrh_uw \
+helper_mve_vstrb \
+helper_mve_vstrh \
+helper_mve_vstrw \
+helper_mve_vstrb_h \
+helper_mve_vstrb_w \
+helper_mve_vstrh_w \
+helper_mve_vldrb_sg_sh \
+helper_mve_vldrb_sg_sw \
+helper_mve_vldrh_sg_sw \
+helper_mve_vldrb_sg_ub \
+helper_mve_vldrb_sg_uh \
+helper_mve_vldrb_sg_uw \
+helper_mve_vldrh_sg_uh \
+helper_mve_vldrh_sg_uw \
+helper_mve_vldrw_sg_uw \
+helper_mve_vldrd_sg_ud \
+helper_mve_vldrh_sg_os_sw \
+helper_mve_vldrh_sg_os_uh \
+helper_mve_vldrh_sg_os_uw \
+helper_mve_vldrw_sg_os_uw \
+helper_mve_vldrd_sg_os_ud \
+helper_mve_vstrb_sg_ub \
+helper_mve_vstrb_sg_uh \
+helper_mve_vstrb_sg_uw \
+helper_mve_vstrh_sg_uh \
+helper_mve_vstrh_sg_uw \
+helper_mve_vstrw_sg_uw \
+helper_mve_vstrd_sg_ud \
+helper_mve_vstrh_sg_os_uh \
+helper_mve_vstrh_sg_os_uw \
+helper_mve_vstrw_sg_os_uw \
+helper_mve_vstrd_sg_os_ud \
+helper_mve_vldrw_sg_wb_uw \
+helper_mve_vldrd_sg_wb_ud \
+helper_mve_vstrw_sg_wb_uw \
+helper_mve_vstrd_sg_wb_ud \
+helper_mve_vld20b \
+helper_mve_vld20h \
+helper_mve_vld20w \
+helper_mve_vld21b \
+helper_mve_vld21h \
+helper_mve_vld21w \
+helper_mve_vld40b \
+helper_mve_vld40h \
+helper_mve_vld40w \
+helper_mve_vld41b \
+helper_mve_vld41h \
+helper_mve_vld41w \
+helper_mve_vld42b \
+helper_mve_vld42h \
+helper_mve_vld42w \
+helper_mve_vld43b \
+helper_mve_vld43h \
+helper_mve_vld43w \
+helper_mve_vst20b \
+helper_mve_vst20h \
+helper_mve_vst20w \
+helper_mve_vst21b \
+helper_mve_vst21h \
+helper_mve_vst21w \
+helper_mve_vst40b \
+helper_mve_vst40h \
+helper_mve_vst40w \
+helper_mve_vst41b \
+helper_mve_vst41h \
+helper_mve_vst41w \
+helper_mve_vst42b \
+helper_mve_vst42h \
+helper_mve_vst42w \
+helper_mve_vst43b \
+helper_mve_vst43h \
+helper_mve_vst43w \
+helper_mve_vand \
+helper_mve_vbic \
+helper_mve_vorr \
+helper_mve_vorn \
+helper_mve_veor \
+helper_mve_vaddb \
+helper_mve_vaddh \
+helper_mve_vaddw \
+helper_mve_vadd_scalarb \
+helper_mve_vadd_scalarh \
+helper_mve_vadd_scalarw \
+helper_mve_vsubb \
+helper_mve_vsubh \
+helper_mve_vsubw \
+helper_mve_vsub_scalarb \
+helper_mve_vsub_scalarh \
+helper_mve_vsub_scalarw \
+helper_mve_vmulb \
+helper_mve_vmulh \
+helper_mve_vmulw \
+helper_mve_vmul_scalarb \
+helper_mve_vmul_scalarh \
+helper_mve_vmul_scalarw \
+helper_mve_vmulhsb \
+helper_mve_vmulhsh \
+helper_mve_vmulhsw \
+helper_mve_vmulhub \
+helper_mve_vmulhuh \
+helper_mve_vmulhuw \
+helper_mve_vrmulhsb \
+helper_mve_vrmulhsh \
+helper_mve_vrmulhsw \
+helper_mve_vrmulhub \
+helper_mve_vrmulhuh \
+helper_mve_vrmulhuw \
+helper_mve_vmullbsb \
+helper_mve_vmullbsh \
+helper_mve_vmullbsw \
+helper_mve_vmullbub \
+helper_mve_vmullbuh \
+helper_mve_vmullbuw \
+helper_mve_vmulltsb \
+helper_mve_vmulltsh \
+helper_mve_vmulltsw \
+helper_mve_vmulltub \
+helper_mve_vmulltuh \
+helper_mve_vmulltuw \
+helper_mve_vmullpbh \
+helper_mve_vmullpth \
+helper_mve_vmullpbw \
+helper_mve_vmullptw \
+helper_mve_vqdmullbh \
+helper_mve_vqdmullbw \
+helper_mve_vqdmullth \
+helper_mve_vqdmulltw \
+helper_mve_vqdmullb_scalarh \
+helper_mve_vqdmullb_scalarw \
+helper_mve_vqdmullt_scalarh \
+helper_mve_vqdmullt_scalarw \
+helper_mve_vcadd90b \
+helper_mve_vcadd90h \
+helper_mve_vcadd90w \
+helper_mve_vcadd270b \
+helper_mve_vcadd270h \
+helper_mve_vcadd270w \
+helper_mve_vhcadd90b \
+helper_mve_vhcadd90h \
+helper_mve_vhcadd90w \
+helper_mve_vhcadd270b \
+helper_mve_vhcadd270h \
+helper_mve_vhcadd270w \
+helper_mve_vmaxsb \
+helper_mve_vmaxsh \
+helper_mve_vmaxsw \
+helper_mve_vmaxub \
+helper_mve_vmaxuh \
+helper_mve_vmaxuw \
+helper_mve_vminsb \
+helper_mve_vminsh \
+helper_mve_vminsw \
+helper_mve_vminub \
+helper_mve_vminuh \
+helper_mve_vminuw \
+helper_mve_vabdsb \
+helper_mve_vabdsh \
+helper_mve_vabdsw \
+helper_mve_vabdub \
+helper_mve_vabduh \
+helper_mve_vabduw \
+helper_mve_vhaddsb \
+helper_mve_vhaddsh \
+helper_mve_vhaddsw \
+helper_mve_vhaddub \
+helper_mve_vhadduh \
+helper_mve_vhadduw \
+helper_mve_vhadds_scalarb \
+helper_mve_vhadds_scalarh \
+helper_mve_vhadds_scalarw \
+helper_mve_vhaddu_scalarb \
+helper_mve_vhaddu_scalarh \
+helper_mve_vhaddu_scalarw \
+helper_mve_vrhaddsb \
+helper_mve_vrhaddsh \
+helper_mve_vrhaddsw \
+helper_mve_vrhaddub \
+helper_mve_vrhadduh \
+helper_mve_vrhadduw \
+helper_mve_vadc \
+helper_mve_vadci \
+helper_mve_vsbc \
+helper_mve_vsbci \
+helper_mve_vhsubsb \
+helper_mve_vhsubsh \
+helper_mve_vhsubsw \
+helper_mve_vhsubub \
+helper_mve_vhsubuh \
+helper_mve_vhsubuw \
+helper_mve_vhsubs_scalarb \
+helper_mve_vhsubs_scalarh \
+helper_mve_vhsubs_scalarw \
+helper_mve_vhsubu_scalarb \
+helper_mve_vhsubu_scalarh \
+helper_mve_vhsubu_scalarw \
+helper_mve_vqaddsb \
+helper_mve_vqaddsh \
+helper_mve_vqaddsw \
+helper_mve_vqaddub \
+helper_mve_vqadduh \
+helper_mve_vqadduw \
+helper_mve_vqsubsb \
+helper_mve_vqsubsh \
+helper_mve_vqsubsw \
+helper_mve_vqsubub \
+helper_mve_vqsubuh \
+helper_mve_vqsubuw \
+helper_mve_vqdmulhb \
+helper_mve_vqdmulhh \
+helper_mve_vqdmulhw \
+helper_mve_vqrdmulhb \
+helper_mve_vqrdmulhh \
+helper_mve_vqrdmulhw \
+helper_mve_vqadds_scalarb \
+helper_mve_vqadds_scalarh \
+helper_mve_vqadds_scalarw \
+helper_mve_vqaddu_scalarb \
+helper_mve_vqaddu_scalarh \
+helper_mve_vqaddu_scalarw \
+helper_mve_vqsubs_scalarb \
+helper_mve_vqsubs_scalarh \
+helper_mve_vqsubs_scalarw \
+helper_mve_vqsubu_scalarb \
+helper_mve_vqsubu_scalarh \
+helper_mve_vqsubu_scalarw \
+helper_mve_vqdmulh_scalarb \
+helper_mve_vqdmulh_scalarh \
+helper_mve_vqdmulh_scalarw \
+helper_mve_vqrdmulh_scalarb \
+helper_mve_vqrdmulh_scalarh \
+helper_mve_vqrdmulh_scalarw \
+helper_mve_vmlab \
+helper_mve_vmlah \
+helper_mve_vmlaw \
+helper_mve_vmlasb \
+helper_mve_vmlash \
+helper_mve_vmlasw \
+helper_mve_vqdmlahb \
+helper_mve_vqdmlahh \
+helper_mve_vqdmlahw \
+helper_mve_vqrdmlahb \
+helper_mve_vqrdmlahh \
+helper_mve_vqrdmlahw \
+helper_mve_vqdmlashb \
+helper_mve_vqdmlashh \
+helper_mve_vqdmlashw \
+helper_mve_vqrdmlashb \
+helper_mve_vqrdmlashh \
+helper_mve_vqrdmlashw \
+helper_mve_vfaddh \
+helper_mve_vfadds \
+helper_mve_vfsubh \
+helper_mve_vfsubs \
+helper_mve_vfmulh \
+helper_mve_vfmuls \
+helper_mve_vfabdh \
+helper_mve_vfabds \
+helper_mve_vmaxnmh \
+helper_mve_vmaxnms \
+helper_mve_vminnmh \
+helper_mve_vminnms \
+helper_mve_vmaxnmah \
+helper_mve_vmaxnmas \
+helper_mve_vminnmah \
+helper_mve_vminnmas \
+helper_mve_vfcadd90h \
+helper_mve_vfcadd90s \
+helper_mve_vfcadd270h \
+helper_mve_vfcadd270s \
+helper_mve_vcmul0h \
+helper_mve_vcmul0s \
+helper_mve_vcmul90h \
+helper_mve_vcmul90s \
+helper_mve_vcmul180h \
+helper_mve_vcmul180s \
+helper_mve_vcmul270h \
+helper_mve_vcmul270s \
+helper_mve_vcmla0h \
+helper_mve_vcmla0s \
+helper_mve_vcmla90h \
+helper_mve_vcmla90s \
+helper_mve_vcmla180h \
+helper_mve_vcmla180s \
+helper_mve_vcmla270h \
+helper_mve_vcmla270s \
+helper_mve_vfmah \
+helper_mve_vfmas \
+helper_mve_vfmsh \
+helper_mve_vfmss \
+helper_mve_vfadd_scalarh \
+helper_mve_vfadd_scalars \
+helper_mve_vfsub_scalarh \
+helper_mve_vfsub_scalars \
+helper_mve_vfmul_scalarh \
+helper_mve_vfmul_scalars \
+helper_mve_vfma_scalarh \
+helper_mve_vfma_scalars \
+helper_mve_vfmas_scalarh \
+helper_mve_vfmas_scalars \
+helper_mve_vcvt_sh \
+helper_mve_vcvt_uh \
+helper_mve_vcvt_hs \
+helper_mve_vcvt_hu \
+helper_mve_vcvt_sf \
+helper_mve_vcvt_uf \
+helper_mve_vcvt_fs \
+helper_mve_vcvt_fu \
+helper_mve_vcvtb_sh \
+helper_mve_vcvtt_sh \
+helper_mve_vcvtb_hs \
+helper_mve_vcvtt_hs \
+helper_mve_vcvt_rm_sh \
+helper_mve_vcvt_rm_uh \
+helper_mve_vcvt_rm_ss \
+helper_mve_vcvt_rm_us \
+helper_mve_vrint_rm_h \
+helper_mve_vrint_rm_s \
+helper_mve_vrintx_h \
+helper_mve_vrintx_s \
+helper_mve_vshlsb \
+helper_mve_vshlsh \
+helper_mve_vshlsw \
+helper_mve_vshlub \
+helper_mve_vshluh \
+helper_mve_vshluw \
+helper_mve_vrshlsb \
+helper_mve_vrshlsh \
+helper_mve_vrshlsw \
+helper_mve_vrshlub \
+helper_mve_vrshluh \
+helper_mve_vrshluw \
+helper_mve_vqshlsb \
+helper_mve_vqshlsh \
+helper_mve_vqshlsw \
+helper_mve_vqshlub \
+helper_mve_vqshluh \
+helper_mve_vqshluw \
+helper_mve_vqrshlsb \
+helper_mve_vqrshlsh \
+helper_mve_vqrshlsw \
+helper_mve_vqrshlub \
+helper_mve_vqrshluh \
+helper_mve_vqrshluw \
+helper_mve_vqdmladhb \
+helper_mve_vqdmladhh \
+helper_mve_vqdmladhw \
+helper_mve_vqdmladhxb \
+helper_mve_vqdmladhxh \
+helper_mve_vqdmladhxw \
+helper_mve_vqrdmladhb \
+helper_mve_vqrdmladhh \
+helper_mve_vqrdmladhw \
+helper_mve_vqrdmladhxb \
+helper_mve_vqrdmladhxh \
+helper_mve_vqrdmladhxw \
+helper_mve_vqdmlsdhb \
+helper_mve_vqdmlsdhh \
+helper_mve_vqdmlsdhw \
+helper_mve_vqdmlsdhxb \
+helper_mve_vqdmlsdhxh \
+helper_mve_vqdmlsdhxw \
+helper_mve_vqrdmlsdhb \
+helper_mve_vqrdmlsdhh \
+helper_mve_vqrdmlsdhw \
+helper_mve_vqrdmlsdhxb \
+helper_mve_vqrdmlsdhxh \
+helper_mve_vqrdmlsdhxw \
+helper_mve_vbrsrb \
+helper_mve_vbrsrh \
+helper_mve_vbrsrw \
+helper_mve_vshli_sb \
+helper_mve_vshli_sh \
+helper_mve_vshli_sw \
+helper_mve_vshli_ub \
+helper_mve_vshli_uh \
+helper_mve_vshli_uw \
+helper_mve_vrshli_sb \
+helper_mve_vrshli_sh \
+helper_mve_vrshli_sw \
+helper_mve_vrshli_ub \
+helper_mve_vrshli_uh \
+helper_mve_vrshli_uw \
+helper_mve_vqshli_sb \
+helper_mve_vqshli_sh \
+helper_mve_vqshli_sw \
+helper_mve_vqshli_ub \
+helper_mve_vqshli_uh \
+helper_mve_vqshli_uw \
+helper_mve_vqrshli_sb \
+helper_mve_vqrshli_sh \
+helper_mve_vqrshli_sw \
+helper_mve_vqrshli_ub \
+helper_mve_vqrshli_uh \
+helper_mve_vqrshli_uw \
+helper_mve_vqshlui_sb \
+helper_mve_vqshlui_sh \
+helper_mve_vqshlui_sw \
+helper_mve_vshllbsb \
+helper_mve_vshllbsh \
+helper_mve_vshllbub \
+helper_mve_vshllbuh \
+helper_mve_vshlltsb \
+helper_mve_vshlltsh \
+helper_mve_vshlltub \
+helper_mve_vshlltuh \
+helper_mve_vshrnbb \
+helper_mve_vshrnbh \
+helper_mve_vshrntb \
+helper_mve_vshrnth \
+helper_mve_vrshrnbb \
+helper_mve_vrshrnbh \
+helper_mve_vrshrntb \
+helper_mve_vrshrnth \
+helper_mve_vqshrnb_sb \
+helper_mve_vqshrnb_sh \
+helper_mve_vqshrnt_sb \
+helper_mve_vqshrnt_sh \
+helper_mve_vqshrnb_ub \
+helper_mve_vqshrnb_uh \
+helper_mve_vqshrnt_ub \
+helper_mve_vqshrnt_uh \
+helper_mve_vqshrunbb \
+helper_mve_vqshrunbh \
+helper_mve_vqshruntb \
+helper_mve_vqshrunth \
+helper_mve_vqrshrnb_sb \
+helper_mve_vqrshrnb_sh \
+helper_mve_vqrshrnt_sb \
+helper_mve_vqrshrnt_sh \
+helper_mve_vqrshrnb_ub \
+helper_mve_vqrshrnb_uh \
+helper_mve_vqrshrnt_ub \
+helper_mve_vqrshrnt_uh \
+helper_mve_vqrshrunbb \
+helper_mve_vqrshrunbh \
+helper_mve_vqrshruntb \
+helper_mve_vqrshrunth \
+helper_mve_vmovnbb \
+helper_mve_vmovnbh \
+helper_mve_vmovntb \
+helper_mve_vmovnth \
+helper_mve_vqmovnbsb \
+helper_mve_vqmovnbsh \
+helper_mve_vqmovntsb \
+helper_mve_vqmovntsh \
+helper_mve_vqmovnbub \
+helper_mve_vqmovnbuh \
+helper_mve_vqmovntub \
+helper_mve_vqmovntuh \
+helper_mve_vqmovunbb \
+helper_mve_vqmovunbh \
+helper_mve_vqmovuntb \
+helper_mve_vqmovunth \
+helper_mve_sshrl \
+helper_mve_ushll \
+helper_mve_sqshll \
+helper_mve_uqshll \
+helper_mve_sqrshrl \
+helper_mve_uqrshll \
+helper_mve_sqrshrl48 \
+helper_mve_uqrshll48 \
+helper_mve_uqshl \
+helper_mve_sqshl \
+helper_mve_uqrshl \
+helper_mve_sqrshr \
+helper_mve_vshlc \
+helper_mve_vsrib \
+helper_mve_vsrih \
+helper_mve_vsriw \
+helper_mve_vslib \
+helper_mve_vslih \
+helper_mve_vsliw \
+helper_mve_vclsb \
+helper_mve_vclsh \
+helper_mve_vclsw \
+helper_mve_vclzb \
+helper_mve_vclzh \
+helper_mve_vclzw \
+helper_mve_vrev16b \
+helper_mve_vrev32b \
+helper_mve_vrev32h \
+helper_mve_vrev64b \
+helper_mve_vrev64h \
+helper_mve_vrev64w \
+helper_mve_vmvn \
+helper_mve_vabsb \
+helper_mve_vabsh \
+helper_mve_vabsw \
+helper_mve_vnegb \
+helper_mve_vnegh \
+helper_mve_vnegw \
+helper_mve_vmaxab \
+helper_mve_vmaxah \
+helper_mve_vmaxaw \
+helper_mve_vminab \
+helper_mve_vminah \
+helper_mve_vminaw \
+helper_mve_vqabsb \
+helper_mve_vqabsh \
+helper_mve_vqabsw \
+helper_mve_vqnegb \
+helper_mve_vqnegh \
+helper_mve_vqnegw \
+helper_mve_vmlaldavsh \
+helper_mve_vmlaldavsw \
+helper_mve_vmlaldavxsh \
+helper_mve_vmlaldavxsw \
+helper_mve_vmlaldavuh \
+helper_mve_vmlaldavuw \
+helper_mve_vmlsldavsh \
+helper_mve_vmlsldavsw \
+helper_mve_vmlsldavxsh \
+helper_mve_vmlsldavxsw \
+helper_mve_vrmlaldavhsw \
+helper_mve_vrmlaldavhxsw \
+helper_mve_vrmlaldavhuw \
+helper_mve_vrmlsldavhsw \
+helper_mve_vrmlsldavhxsw \
+helper_mve_vmladavsb \
+helper_mve_vmladavsh \
+helper_mve_vmladavsw \
+helper_mve_vmladavub \
+helper_mve_vmladavuh \
+helper_mve_vmladavuw \
+helper_mve_vmlsdavb \
+helper_mve_vmlsdavh \
+helper_mve_vmlsdavw \
+helper_mve_vmladavsxb \
+helper_mve_vmladavsxh \
+helper_mve_vmladavsxw \
+helper_mve_vmlsdavxb \
+helper_mve_vmlsdavxh \
+helper_mve_vmlsdavxw \
+helper_mve_vaddvsb \
+helper_mve_vaddvsh \
+helper_mve_vaddvsw \
+helper_mve_vaddvub \
+helper_mve_vaddvuh \
+helper_mve_vaddvuw \
+helper_mve_vmaxvsb \
+helper_mve_vmaxvsh \
+helper_mve_vmaxvsw \
+helper_mve_vmaxvub \
+helper_mve_vmaxvuh \
+helper_mve_vmaxvuw \
+helper_mve_vmaxavb \
+helper_mve_vmaxavh \
+helper_mve_vmaxavw \
+helper_mve_vminvsb \
+helper_mve_vminvsh \
+helper_mve_vminvsw \
+helper_mve_vminvub \
+helper_mve_vminvuh \
+helper_mve_vminvuw \
+helper_mve_vminavb \
+helper_mve_vminavh \
+helper_mve_vminavw \
+helper_mve_vmaxnmvh \
+helper_mve_vmaxnmvs \
+helper_mve_vminnmvh \
+helper_mve_vminnmvs \
+helper_mve_vmaxnmavh \
+helper_mve_vmaxnmavs \
+helper_mve_vminnmavh \
+helper_mve_vminnmavs \
+helper_mve_vaddlv_s \
+helper_mve_vaddlv_u \
+helper_mve_vabavsb \
+helper_mve_vabavsh \
+helper_mve_vabavsw \
+helper_mve_vabavub \
+helper_mve_vabavuh \
+helper_mve_vabavuw \
 helper_vfp_adds \
 helper_vfp_addd \
 helper_vfp_subs \
@@ -4250,10 +7296,14 @@ helper_vfp_touls_round_to_zero \
 helper_vfp_touls \
 helper_vfp_uqtos \
 helper_vfp_touqs \
+helper_vfp_shtoh \
+helper_vfp_uhtoh \
 helper_vfp_sltoh \
 helper_vfp_ultoh \
 helper_vfp_sqtoh \
 helper_vfp_uqtoh \
+helper_vfp_toshh_round_to_zero \
+helper_vfp_touhh_round_to_zero \
 helper_vfp_toshh \
 helper_vfp_touhh \
 helper_vfp_toslh \
@@ -4266,6 +7316,8 @@ helper_vfp_fcvt_f16_to_f32 \
 helper_vfp_fcvt_f32_to_f16 \
 helper_vfp_fcvt_f16_to_f64 \
 helper_vfp_fcvt_f64_to_f16 \
+helper_bfcvt \
+helper_bfcvt_pair \
 helper_recps_f32 \
 helper_rsqrts_f32 \
 helper_recpe_f16 \
@@ -4278,8 +7330,10 @@ helper_recpe_u32 \
 helper_rsqrte_u32 \
 helper_vfp_muladds \
 helper_vfp_muladdd \
+helper_rinth_exact \
 helper_rints_exact \
 helper_rintd_exact \
+helper_rinth \
 helper_rints \
 helper_rintd \
 arm_rmode_to_sf \
@@ -4333,14 +7387,19 @@ riscv_csrrw_debug \
 riscv_cpu_get_fflags \
 riscv_cpu_set_fflags \
 helper_set_rounding_mode \
+helper_set_rod_rounding_mode \
 helper_fmadd_s \
 helper_fmadd_d \
+helper_fmadd_h \
 helper_fmsub_s \
 helper_fmsub_d \
+helper_fmsub_h \
 helper_fnmsub_s \
 helper_fnmsub_d \
+helper_fnmsub_h \
 helper_fnmadd_s \
 helper_fnmadd_d \
+helper_fnmadd_h \
 helper_fadd_s \
 helper_fsub_s \
 helper_fmul_s \
@@ -4356,6 +7415,25 @@ helper_fcvt_wu_s \
 helper_fcvt_s_w \
 helper_fcvt_s_wu \
 helper_fclass_s \
+helper_fadd_h \
+helper_fsub_h \
+helper_fmul_h \
+helper_fdiv_h \
+helper_fmin_h \
+helper_fmax_h \
+helper_fsqrt_h \
+helper_fle_h \
+helper_flt_h \
+helper_feq_h \
+helper_fcvt_s_h \
+helper_fcvt_h_s \
+helper_fcvt_d_h \
+helper_fcvt_h_d \
+helper_fcvt_w_h \
+helper_fcvt_wu_h \
+helper_fcvt_h_w \
+helper_fcvt_h_wu \
+helper_fclass_h \
 helper_fadd_d \
 helper_fsub_d \
 helper_fmul_d \
@@ -4379,15 +7457,965 @@ helper_uc_riscv_exit \
 helper_csrrw \
 helper_csrrs \
 helper_csrrc \
+helper_vsetvl \
+helper_vle8_v \
+helper_vle16_v \
+helper_vle32_v \
+helper_vle64_v \
+helper_vse8_v \
+helper_vse16_v \
+helper_vse32_v \
+helper_vse64_v \
+helper_vlse8_v \
+helper_vlse16_v \
+helper_vlse32_v \
+helper_vlse64_v \
+helper_vsse8_v \
+helper_vsse16_v \
+helper_vsse32_v \
+helper_vsse64_v \
+helper_vlxei8_8_v \
+helper_vlxei8_16_v \
+helper_vlxei8_32_v \
+helper_vlxei8_64_v \
+helper_vlxei16_8_v \
+helper_vlxei16_16_v \
+helper_vlxei16_32_v \
+helper_vlxei16_64_v \
+helper_vlxei32_8_v \
+helper_vlxei32_16_v \
+helper_vlxei32_32_v \
+helper_vlxei32_64_v \
+helper_vlxei64_8_v \
+helper_vlxei64_16_v \
+helper_vlxei64_32_v \
+helper_vlxei64_64_v \
+helper_vsxei8_8_v \
+helper_vsxei8_16_v \
+helper_vsxei8_32_v \
+helper_vsxei8_64_v \
+helper_vsxei16_8_v \
+helper_vsxei16_16_v \
+helper_vsxei16_32_v \
+helper_vsxei16_64_v \
+helper_vsxei32_8_v \
+helper_vsxei32_16_v \
+helper_vsxei32_32_v \
+helper_vsxei32_64_v \
+helper_vsxei64_8_v \
+helper_vsxei64_16_v \
+helper_vsxei64_32_v \
+helper_vsxei64_64_v \
+helper_vle8ff_v \
+helper_vle16ff_v \
+helper_vle32ff_v \
+helper_vle64ff_v \
+helper_vl1re8_v \
+helper_vl1re16_v \
+helper_vl1re32_v \
+helper_vl1re64_v \
+helper_vl2re8_v \
+helper_vl2re16_v \
+helper_vl2re32_v \
+helper_vl2re64_v \
+helper_vl4re8_v \
+helper_vl4re16_v \
+helper_vl4re32_v \
+helper_vl4re64_v \
+helper_vl8re8_v \
+helper_vl8re16_v \
+helper_vl8re32_v \
+helper_vl8re64_v \
+helper_vs1r_v \
+helper_vs2r_v \
+helper_vs4r_v \
+helper_vs8r_v \
+helper_vlm_v \
+helper_vsm_v \
+helper_vadd_vv_b \
+helper_vadd_vv_h \
+helper_vadd_vv_w \
+helper_vadd_vv_d \
+helper_vsub_vv_b \
+helper_vsub_vv_h \
+helper_vsub_vv_w \
+helper_vsub_vv_d \
+helper_vfadd_vv_h \
+helper_vfadd_vv_w \
+helper_vfadd_vv_d \
+helper_vfsub_vv_h \
+helper_vfsub_vv_w \
+helper_vfsub_vv_d \
+helper_vfmul_vv_h \
+helper_vfmul_vv_w \
+helper_vfmul_vv_d \
+helper_vfdiv_vv_h \
+helper_vfdiv_vv_w \
+helper_vfdiv_vv_d \
+helper_vfwadd_vv_h \
+helper_vfwadd_vv_w \
+helper_vfwsub_vv_h \
+helper_vfwsub_vv_w \
+helper_vfwadd_wv_h \
+helper_vfwadd_wv_w \
+helper_vfwsub_wv_h \
+helper_vfwsub_wv_w \
+helper_vfwmul_vv_h \
+helper_vfwmul_vv_w \
+helper_vfmacc_vv_h \
+helper_vfmacc_vv_w \
+helper_vfmacc_vv_d \
+helper_vfnmacc_vv_h \
+helper_vfnmacc_vv_w \
+helper_vfnmacc_vv_d \
+helper_vfmsac_vv_h \
+helper_vfmsac_vv_w \
+helper_vfmsac_vv_d \
+helper_vfnmsac_vv_h \
+helper_vfnmsac_vv_w \
+helper_vfnmsac_vv_d \
+helper_vfmadd_vv_h \
+helper_vfmadd_vv_w \
+helper_vfmadd_vv_d \
+helper_vfnmadd_vv_h \
+helper_vfnmadd_vv_w \
+helper_vfnmadd_vv_d \
+helper_vfmsub_vv_h \
+helper_vfmsub_vv_w \
+helper_vfmsub_vv_d \
+helper_vfnmsub_vv_h \
+helper_vfnmsub_vv_w \
+helper_vfnmsub_vv_d \
+helper_vfwmacc_vv_h \
+helper_vfwmacc_vv_w \
+helper_vfwnmacc_vv_h \
+helper_vfwnmacc_vv_w \
+helper_vfwmsac_vv_h \
+helper_vfwmsac_vv_w \
+helper_vfwnmsac_vv_h \
+helper_vfwnmsac_vv_w \
+helper_vfmin_vv_h \
+helper_vfmin_vv_w \
+helper_vfmin_vv_d \
+helper_vfmax_vv_h \
+helper_vfmax_vv_w \
+helper_vfmax_vv_d \
+helper_vfsgnj_vv_h \
+helper_vfsgnj_vv_w \
+helper_vfsgnj_vv_d \
+helper_vfsgnjn_vv_h \
+helper_vfsgnjn_vv_w \
+helper_vfsgnjn_vv_d \
+helper_vfsgnjx_vv_h \
+helper_vfsgnjx_vv_w \
+helper_vfsgnjx_vv_d \
+helper_vadc_vvm_b \
+helper_vadc_vvm_h \
+helper_vadc_vvm_w \
+helper_vadc_vvm_d \
+helper_vsbc_vvm_b \
+helper_vsbc_vvm_h \
+helper_vsbc_vvm_w \
+helper_vsbc_vvm_d \
+helper_vmadc_vvm_b \
+helper_vmadc_vvm_h \
+helper_vmadc_vvm_w \
+helper_vmadc_vvm_d \
+helper_vmsbc_vvm_b \
+helper_vmsbc_vvm_h \
+helper_vmsbc_vvm_w \
+helper_vmsbc_vvm_d \
+helper_vand_vv_b \
+helper_vand_vv_h \
+helper_vand_vv_w \
+helper_vand_vv_d \
+helper_vor_vv_b \
+helper_vor_vv_h \
+helper_vor_vv_w \
+helper_vor_vv_d \
+helper_vxor_vv_b \
+helper_vxor_vv_h \
+helper_vxor_vv_w \
+helper_vxor_vv_d \
+helper_vminu_vv_b \
+helper_vminu_vv_h \
+helper_vminu_vv_w \
+helper_vminu_vv_d \
+helper_vmin_vv_b \
+helper_vmin_vv_h \
+helper_vmin_vv_w \
+helper_vmin_vv_d \
+helper_vmaxu_vv_b \
+helper_vmaxu_vv_h \
+helper_vmaxu_vv_w \
+helper_vmaxu_vv_d \
+helper_vmax_vv_b \
+helper_vmax_vv_h \
+helper_vmax_vv_w \
+helper_vmax_vv_d \
+helper_vmseq_vv_b \
+helper_vmseq_vv_h \
+helper_vmseq_vv_w \
+helper_vmseq_vv_d \
+helper_vmsne_vv_b \
+helper_vmsne_vv_h \
+helper_vmsne_vv_w \
+helper_vmsne_vv_d \
+helper_vmsltu_vv_b \
+helper_vmsltu_vv_h \
+helper_vmsltu_vv_w \
+helper_vmsltu_vv_d \
+helper_vmslt_vv_b \
+helper_vmslt_vv_h \
+helper_vmslt_vv_w \
+helper_vmslt_vv_d \
+helper_vmsleu_vv_b \
+helper_vmsleu_vv_h \
+helper_vmsleu_vv_w \
+helper_vmsleu_vv_d \
+helper_vmsle_vv_b \
+helper_vmsle_vv_h \
+helper_vmsle_vv_w \
+helper_vmsle_vv_d \
+helper_vmfeq_vv_h \
+helper_vmfeq_vv_w \
+helper_vmfeq_vv_d \
+helper_vmfne_vv_h \
+helper_vmfne_vv_w \
+helper_vmfne_vv_d \
+helper_vmflt_vv_h \
+helper_vmflt_vv_w \
+helper_vmflt_vv_d \
+helper_vmfle_vv_h \
+helper_vmfle_vv_w \
+helper_vmfle_vv_d \
+helper_vfsqrt_v_h \
+helper_vfsqrt_v_w \
+helper_vfsqrt_v_d \
+helper_vfrsqrt7_v_h \
+helper_vfrsqrt7_v_w \
+helper_vfrsqrt7_v_d \
+helper_vfrec7_v_h \
+helper_vfrec7_v_w \
+helper_vfrec7_v_d \
+helper_vfcvt_xu_f_v_h \
+helper_vfcvt_xu_f_v_w \
+helper_vfcvt_xu_f_v_d \
+helper_vfcvt_x_f_v_h \
+helper_vfcvt_x_f_v_w \
+helper_vfcvt_x_f_v_d \
+helper_vfcvt_f_xu_v_h \
+helper_vfcvt_f_xu_v_w \
+helper_vfcvt_f_xu_v_d \
+helper_vfcvt_f_x_v_h \
+helper_vfcvt_f_x_v_w \
+helper_vfcvt_f_x_v_d \
+helper_vfwcvt_xu_f_v_h \
+helper_vfwcvt_xu_f_v_w \
+helper_vfwcvt_x_f_v_h \
+helper_vfwcvt_x_f_v_w \
+helper_vfwcvt_f_xu_v_b \
+helper_vfwcvt_f_xu_v_h \
+helper_vfwcvt_f_xu_v_w \
+helper_vfwcvt_f_x_v_b \
+helper_vfwcvt_f_x_v_h \
+helper_vfwcvt_f_x_v_w \
+helper_vfwcvt_f_f_v_h \
+helper_vfwcvt_f_f_v_w \
+helper_vfncvt_xu_f_w_b \
+helper_vfncvt_xu_f_w_h \
+helper_vfncvt_xu_f_w_w \
+helper_vfncvt_x_f_w_b \
+helper_vfncvt_x_f_w_h \
+helper_vfncvt_x_f_w_w \
+helper_vfncvt_f_xu_w_h \
+helper_vfncvt_f_xu_w_w \
+helper_vfncvt_f_x_w_h \
+helper_vfncvt_f_x_w_w \
+helper_vfncvt_f_f_w_h \
+helper_vfncvt_f_f_w_w \
+helper_vfclass_v_h \
+helper_vfclass_v_w \
+helper_vfclass_v_d \
+helper_vmand_mm \
+helper_vmnand_mm \
+helper_vmandn_mm \
+helper_vmxor_mm \
+helper_vmor_mm \
+helper_vmnor_mm \
+helper_vmorn_mm \
+helper_vmxnor_mm \
+helper_vcpop_m \
+helper_vfirst_m \
+helper_vmsbf_m \
+helper_vmsif_m \
+helper_vmsof_m \
+helper_viota_m_b \
+helper_viota_m_h \
+helper_viota_m_w \
+helper_viota_m_d \
+helper_vid_v_b \
+helper_vid_v_h \
+helper_vid_v_w \
+helper_vid_v_d \
+helper_vredsum_vs_b \
+helper_vredsum_vs_h \
+helper_vredsum_vs_w \
+helper_vredsum_vs_d \
+helper_vredand_vs_b \
+helper_vredand_vs_h \
+helper_vredand_vs_w \
+helper_vredand_vs_d \
+helper_vredor_vs_b \
+helper_vredor_vs_h \
+helper_vredor_vs_w \
+helper_vredor_vs_d \
+helper_vredxor_vs_b \
+helper_vredxor_vs_h \
+helper_vredxor_vs_w \
+helper_vredxor_vs_d \
+helper_vredminu_vs_b \
+helper_vredminu_vs_h \
+helper_vredminu_vs_w \
+helper_vredminu_vs_d \
+helper_vredmin_vs_b \
+helper_vredmin_vs_h \
+helper_vredmin_vs_w \
+helper_vredmin_vs_d \
+helper_vredmaxu_vs_b \
+helper_vredmaxu_vs_h \
+helper_vredmaxu_vs_w \
+helper_vredmaxu_vs_d \
+helper_vredmax_vs_b \
+helper_vredmax_vs_h \
+helper_vredmax_vs_w \
+helper_vredmax_vs_d \
+helper_vwredsumu_vs_b \
+helper_vwredsumu_vs_h \
+helper_vwredsumu_vs_w \
+helper_vwredsum_vs_b \
+helper_vwredsum_vs_h \
+helper_vwredsum_vs_w \
+helper_vfredusum_vs_h \
+helper_vfredusum_vs_w \
+helper_vfredusum_vs_d \
+helper_vfredosum_vs_h \
+helper_vfredosum_vs_w \
+helper_vfredosum_vs_d \
+helper_vfredmin_vs_h \
+helper_vfredmin_vs_w \
+helper_vfredmin_vs_d \
+helper_vfredmax_vs_h \
+helper_vfredmax_vs_w \
+helper_vfredmax_vs_d \
+helper_vfwredusum_vs_h \
+helper_vfwredusum_vs_w \
+helper_vfwredosum_vs_h \
+helper_vfwredosum_vs_w \
+helper_vadd_vx_b \
+helper_vadd_vx_h \
+helper_vadd_vx_w \
+helper_vadd_vx_d \
+helper_vsub_vx_b \
+helper_vsub_vx_h \
+helper_vsub_vx_w \
+helper_vsub_vx_d \
+helper_vrsub_vx_b \
+helper_vrsub_vx_h \
+helper_vrsub_vx_w \
+helper_vrsub_vx_d \
+helper_vfadd_vf_h \
+helper_vfadd_vf_w \
+helper_vfadd_vf_d \
+helper_vfsub_vf_h \
+helper_vfsub_vf_w \
+helper_vfsub_vf_d \
+helper_vfrsub_vf_h \
+helper_vfrsub_vf_w \
+helper_vfrsub_vf_d \
+helper_vfmul_vf_h \
+helper_vfmul_vf_w \
+helper_vfmul_vf_d \
+helper_vfdiv_vf_h \
+helper_vfdiv_vf_w \
+helper_vfdiv_vf_d \
+helper_vfrdiv_vf_h \
+helper_vfrdiv_vf_w \
+helper_vfrdiv_vf_d \
+helper_vfwadd_vf_h \
+helper_vfwadd_vf_w \
+helper_vfwsub_vf_h \
+helper_vfwsub_vf_w \
+helper_vfwadd_wf_h \
+helper_vfwadd_wf_w \
+helper_vfwsub_wf_h \
+helper_vfwsub_wf_w \
+helper_vfwmul_vf_h \
+helper_vfwmul_vf_w \
+helper_vfmacc_vf_h \
+helper_vfmacc_vf_w \
+helper_vfmacc_vf_d \
+helper_vfnmacc_vf_h \
+helper_vfnmacc_vf_w \
+helper_vfnmacc_vf_d \
+helper_vfmsac_vf_h \
+helper_vfmsac_vf_w \
+helper_vfmsac_vf_d \
+helper_vfnmsac_vf_h \
+helper_vfnmsac_vf_w \
+helper_vfnmsac_vf_d \
+helper_vfmadd_vf_h \
+helper_vfmadd_vf_w \
+helper_vfmadd_vf_d \
+helper_vfnmadd_vf_h \
+helper_vfnmadd_vf_w \
+helper_vfnmadd_vf_d \
+helper_vfmsub_vf_h \
+helper_vfmsub_vf_w \
+helper_vfmsub_vf_d \
+helper_vfnmsub_vf_h \
+helper_vfnmsub_vf_w \
+helper_vfnmsub_vf_d \
+helper_vfwmacc_vf_h \
+helper_vfwmacc_vf_w \
+helper_vfwnmacc_vf_h \
+helper_vfwnmacc_vf_w \
+helper_vfwmsac_vf_h \
+helper_vfwmsac_vf_w \
+helper_vfwnmsac_vf_h \
+helper_vfwnmsac_vf_w \
+helper_vfmin_vf_h \
+helper_vfmin_vf_w \
+helper_vfmin_vf_d \
+helper_vfmax_vf_h \
+helper_vfmax_vf_w \
+helper_vfmax_vf_d \
+helper_vfsgnj_vf_h \
+helper_vfsgnj_vf_w \
+helper_vfsgnj_vf_d \
+helper_vfsgnjn_vf_h \
+helper_vfsgnjn_vf_w \
+helper_vfsgnjn_vf_d \
+helper_vfsgnjx_vf_h \
+helper_vfsgnjx_vf_w \
+helper_vfsgnjx_vf_d \
+helper_vmfeq_vf_h \
+helper_vmfeq_vf_w \
+helper_vmfeq_vf_d \
+helper_vmfne_vf_h \
+helper_vmfne_vf_w \
+helper_vmfne_vf_d \
+helper_vmflt_vf_h \
+helper_vmflt_vf_w \
+helper_vmflt_vf_d \
+helper_vmfle_vf_h \
+helper_vmfle_vf_w \
+helper_vmfle_vf_d \
+helper_vmfgt_vf_h \
+helper_vmfgt_vf_w \
+helper_vmfgt_vf_d \
+helper_vmfge_vf_h \
+helper_vmfge_vf_w \
+helper_vmfge_vf_d \
+helper_vfmerge_vfm_h \
+helper_vfmerge_vfm_w \
+helper_vfmerge_vfm_d \
+helper_vslideup_vx_b \
+helper_vslideup_vx_h \
+helper_vslideup_vx_w \
+helper_vslideup_vx_d \
+helper_vslidedown_vx_b \
+helper_vslidedown_vx_h \
+helper_vslidedown_vx_w \
+helper_vslidedown_vx_d \
+helper_vslide1up_vx_b \
+helper_vslide1up_vx_h \
+helper_vslide1up_vx_w \
+helper_vslide1up_vx_d \
+helper_vslide1down_vx_b \
+helper_vslide1down_vx_h \
+helper_vslide1down_vx_w \
+helper_vslide1down_vx_d \
+helper_vrgather_vv_b \
+helper_vrgather_vv_h \
+helper_vrgather_vv_w \
+helper_vrgather_vv_d \
+helper_vrgatherei16_vv_b \
+helper_vrgatherei16_vv_h \
+helper_vrgatherei16_vv_w \
+helper_vrgatherei16_vv_d \
+helper_vrgather_vx_b \
+helper_vrgather_vx_h \
+helper_vrgather_vx_w \
+helper_vrgather_vx_d \
+helper_vcompress_vm_b \
+helper_vcompress_vm_h \
+helper_vcompress_vm_w \
+helper_vcompress_vm_d \
+helper_vmvr_v \
+helper_vfslide1up_vf_h \
+helper_vfslide1up_vf_w \
+helper_vfslide1up_vf_d \
+helper_vfslide1down_vf_h \
+helper_vfslide1down_vf_w \
+helper_vfslide1down_vf_d \
+helper_vadc_vxm_b \
+helper_vadc_vxm_h \
+helper_vadc_vxm_w \
+helper_vadc_vxm_d \
+helper_vsbc_vxm_b \
+helper_vsbc_vxm_h \
+helper_vsbc_vxm_w \
+helper_vsbc_vxm_d \
+helper_vmadc_vxm_b \
+helper_vmadc_vxm_h \
+helper_vmadc_vxm_w \
+helper_vmadc_vxm_d \
+helper_vmsbc_vxm_b \
+helper_vmsbc_vxm_h \
+helper_vmsbc_vxm_w \
+helper_vmsbc_vxm_d \
+helper_vand_vx_b \
+helper_vand_vx_h \
+helper_vand_vx_w \
+helper_vand_vx_d \
+helper_vor_vx_b \
+helper_vor_vx_h \
+helper_vor_vx_w \
+helper_vor_vx_d \
+helper_vxor_vx_b \
+helper_vxor_vx_h \
+helper_vxor_vx_w \
+helper_vxor_vx_d \
+helper_vsaddu_vv_b \
+helper_vsaddu_vv_h \
+helper_vsaddu_vv_w \
+helper_vsaddu_vv_d \
+helper_vsaddu_vx_b \
+helper_vsaddu_vx_h \
+helper_vsaddu_vx_w \
+helper_vsaddu_vx_d \
+helper_vsadd_vv_b \
+helper_vsadd_vv_h \
+helper_vsadd_vv_w \
+helper_vsadd_vv_d \
+helper_vsadd_vx_b \
+helper_vsadd_vx_h \
+helper_vsadd_vx_w \
+helper_vsadd_vx_d \
+helper_vssubu_vv_b \
+helper_vssubu_vv_h \
+helper_vssubu_vv_w \
+helper_vssubu_vv_d \
+helper_vssubu_vx_b \
+helper_vssubu_vx_h \
+helper_vssubu_vx_w \
+helper_vssubu_vx_d \
+helper_vssub_vv_b \
+helper_vssub_vv_h \
+helper_vssub_vv_w \
+helper_vssub_vv_d \
+helper_vssub_vx_b \
+helper_vssub_vx_h \
+helper_vssub_vx_w \
+helper_vssub_vx_d \
+helper_vaadd_vv_b \
+helper_vaadd_vv_h \
+helper_vaadd_vv_w \
+helper_vaadd_vv_d \
+helper_vaadd_vx_b \
+helper_vaadd_vx_h \
+helper_vaadd_vx_w \
+helper_vaadd_vx_d \
+helper_vaaddu_vv_b \
+helper_vaaddu_vv_h \
+helper_vaaddu_vv_w \
+helper_vaaddu_vv_d \
+helper_vaaddu_vx_b \
+helper_vaaddu_vx_h \
+helper_vaaddu_vx_w \
+helper_vaaddu_vx_d \
+helper_vasub_vv_b \
+helper_vasub_vv_h \
+helper_vasub_vv_w \
+helper_vasub_vv_d \
+helper_vasub_vx_b \
+helper_vasub_vx_h \
+helper_vasub_vx_w \
+helper_vasub_vx_d \
+helper_vasubu_vv_b \
+helper_vasubu_vv_h \
+helper_vasubu_vv_w \
+helper_vasubu_vv_d \
+helper_vasubu_vx_b \
+helper_vasubu_vx_h \
+helper_vasubu_vx_w \
+helper_vasubu_vx_d \
+helper_vsmul_vv_b \
+helper_vsmul_vv_h \
+helper_vsmul_vv_w \
+helper_vsmul_vv_d \
+helper_vsmul_vx_b \
+helper_vsmul_vx_h \
+helper_vsmul_vx_w \
+helper_vsmul_vx_d \
+helper_vssrl_vv_b \
+helper_vssrl_vv_h \
+helper_vssrl_vv_w \
+helper_vssrl_vv_d \
+helper_vssrl_vx_b \
+helper_vssrl_vx_h \
+helper_vssrl_vx_w \
+helper_vssrl_vx_d \
+helper_vssra_vv_b \
+helper_vssra_vv_h \
+helper_vssra_vv_w \
+helper_vssra_vv_d \
+helper_vssra_vx_b \
+helper_vssra_vx_h \
+helper_vssra_vx_w \
+helper_vssra_vx_d \
+helper_vnclip_wv_b \
+helper_vnclip_wv_h \
+helper_vnclip_wv_w \
+helper_vnclip_wx_b \
+helper_vnclip_wx_h \
+helper_vnclip_wx_w \
+helper_vnclipu_wv_b \
+helper_vnclipu_wv_h \
+helper_vnclipu_wv_w \
+helper_vnclipu_wx_b \
+helper_vnclipu_wx_h \
+helper_vnclipu_wx_w \
+helper_vminu_vx_b \
+helper_vminu_vx_h \
+helper_vminu_vx_w \
+helper_vminu_vx_d \
+helper_vmin_vx_b \
+helper_vmin_vx_h \
+helper_vmin_vx_w \
+helper_vmin_vx_d \
+helper_vmaxu_vx_b \
+helper_vmaxu_vx_h \
+helper_vmaxu_vx_w \
+helper_vmaxu_vx_d \
+helper_vmax_vx_b \
+helper_vmax_vx_h \
+helper_vmax_vx_w \
+helper_vmax_vx_d \
+helper_vwaddu_vv_b \
+helper_vwaddu_vv_h \
+helper_vwaddu_vv_w \
+helper_vwsubu_vv_b \
+helper_vwsubu_vv_h \
+helper_vwsubu_vv_w \
+helper_vwadd_vv_b \
+helper_vwadd_vv_h \
+helper_vwadd_vv_w \
+helper_vwsub_vv_b \
+helper_vwsub_vv_h \
+helper_vwsub_vv_w \
+helper_vwaddu_vx_b \
+helper_vwaddu_vx_h \
+helper_vwaddu_vx_w \
+helper_vwsubu_vx_b \
+helper_vwsubu_vx_h \
+helper_vwsubu_vx_w \
+helper_vwadd_vx_b \
+helper_vwadd_vx_h \
+helper_vwadd_vx_w \
+helper_vwsub_vx_b \
+helper_vwsub_vx_h \
+helper_vwsub_vx_w \
+helper_vwaddu_wv_b \
+helper_vwaddu_wv_h \
+helper_vwaddu_wv_w \
+helper_vwsubu_wv_b \
+helper_vwsubu_wv_h \
+helper_vwsubu_wv_w \
+helper_vwadd_wv_b \
+helper_vwadd_wv_h \
+helper_vwadd_wv_w \
+helper_vwsub_wv_b \
+helper_vwsub_wv_h \
+helper_vwsub_wv_w \
+helper_vwaddu_wx_b \
+helper_vwaddu_wx_h \
+helper_vwaddu_wx_w \
+helper_vwsubu_wx_b \
+helper_vwsubu_wx_h \
+helper_vwsubu_wx_w \
+helper_vwadd_wx_b \
+helper_vwadd_wx_h \
+helper_vwadd_wx_w \
+helper_vwsub_wx_b \
+helper_vwsub_wx_h \
+helper_vwsub_wx_w \
+helper_vwmul_vv_b \
+helper_vwmul_vv_h \
+helper_vwmul_vv_w \
+helper_vwmulu_vv_b \
+helper_vwmulu_vv_h \
+helper_vwmulu_vv_w \
+helper_vwmulsu_vv_b \
+helper_vwmulsu_vv_h \
+helper_vwmulsu_vv_w \
+helper_vwmul_vx_b \
+helper_vwmul_vx_h \
+helper_vwmul_vx_w \
+helper_vwmulu_vx_b \
+helper_vwmulu_vx_h \
+helper_vwmulu_vx_w \
+helper_vwmulsu_vx_b \
+helper_vwmulsu_vx_h \
+helper_vwmulsu_vx_w \
+helper_vmacc_vv_b \
+helper_vmacc_vv_h \
+helper_vmacc_vv_w \
+helper_vmacc_vv_d \
+helper_vnmsac_vv_b \
+helper_vnmsac_vv_h \
+helper_vnmsac_vv_w \
+helper_vnmsac_vv_d \
+helper_vmadd_vv_b \
+helper_vmadd_vv_h \
+helper_vmadd_vv_w \
+helper_vmadd_vv_d \
+helper_vnmsub_vv_b \
+helper_vnmsub_vv_h \
+helper_vnmsub_vv_w \
+helper_vnmsub_vv_d \
+helper_vmacc_vx_b \
+helper_vmacc_vx_h \
+helper_vmacc_vx_w \
+helper_vmacc_vx_d \
+helper_vnmsac_vx_b \
+helper_vnmsac_vx_h \
+helper_vnmsac_vx_w \
+helper_vnmsac_vx_d \
+helper_vmadd_vx_b \
+helper_vmadd_vx_h \
+helper_vmadd_vx_w \
+helper_vmadd_vx_d \
+helper_vnmsub_vx_b \
+helper_vnmsub_vx_h \
+helper_vnmsub_vx_w \
+helper_vnmsub_vx_d \
+helper_vwmaccu_vv_b \
+helper_vwmaccu_vv_h \
+helper_vwmaccu_vv_w \
+helper_vwmacc_vv_b \
+helper_vwmacc_vv_h \
+helper_vwmacc_vv_w \
+helper_vwmaccsu_vv_b \
+helper_vwmaccsu_vv_h \
+helper_vwmaccsu_vv_w \
+helper_vwmaccu_vx_b \
+helper_vwmaccu_vx_h \
+helper_vwmaccu_vx_w \
+helper_vwmacc_vx_b \
+helper_vwmacc_vx_h \
+helper_vwmacc_vx_w \
+helper_vwmaccsu_vx_b \
+helper_vwmaccsu_vx_h \
+helper_vwmaccsu_vx_w \
+helper_vwmaccus_vx_b \
+helper_vwmaccus_vx_h \
+helper_vwmaccus_vx_w \
+helper_vmul_vv_b \
+helper_vmul_vv_h \
+helper_vmul_vv_w \
+helper_vmul_vv_d \
+helper_vmulh_vv_b \
+helper_vmulh_vv_h \
+helper_vmulh_vv_w \
+helper_vmulh_vv_d \
+helper_vmulhu_vv_b \
+helper_vmulhu_vv_h \
+helper_vmulhu_vv_w \
+helper_vmulhu_vv_d \
+helper_vmulhsu_vv_b \
+helper_vmulhsu_vv_h \
+helper_vmulhsu_vv_w \
+helper_vmulhsu_vv_d \
+helper_vmul_vx_b \
+helper_vmul_vx_h \
+helper_vmul_vx_w \
+helper_vmul_vx_d \
+helper_vmulh_vx_b \
+helper_vmulh_vx_h \
+helper_vmulh_vx_w \
+helper_vmulh_vx_d \
+helper_vmulhu_vx_b \
+helper_vmulhu_vx_h \
+helper_vmulhu_vx_w \
+helper_vmulhu_vx_d \
+helper_vmulhsu_vx_b \
+helper_vmulhsu_vx_h \
+helper_vmulhsu_vx_w \
+helper_vmulhsu_vx_d \
+helper_vdivu_vv_b \
+helper_vdivu_vv_h \
+helper_vdivu_vv_w \
+helper_vdivu_vv_d \
+helper_vdiv_vv_b \
+helper_vdiv_vv_h \
+helper_vdiv_vv_w \
+helper_vdiv_vv_d \
+helper_vremu_vv_b \
+helper_vremu_vv_h \
+helper_vremu_vv_w \
+helper_vremu_vv_d \
+helper_vrem_vv_b \
+helper_vrem_vv_h \
+helper_vrem_vv_w \
+helper_vrem_vv_d \
+helper_vdivu_vx_b \
+helper_vdivu_vx_h \
+helper_vdivu_vx_w \
+helper_vdivu_vx_d \
+helper_vdiv_vx_b \
+helper_vdiv_vx_h \
+helper_vdiv_vx_w \
+helper_vdiv_vx_d \
+helper_vremu_vx_b \
+helper_vremu_vx_h \
+helper_vremu_vx_w \
+helper_vremu_vx_d \
+helper_vrem_vx_b \
+helper_vrem_vx_h \
+helper_vrem_vx_w \
+helper_vrem_vx_d \
+helper_vsll_vv_b \
+helper_vsll_vv_h \
+helper_vsll_vv_w \
+helper_vsll_vv_d \
+helper_vsrl_vv_b \
+helper_vsrl_vv_h \
+helper_vsrl_vv_w \
+helper_vsrl_vv_d \
+helper_vsra_vv_b \
+helper_vsra_vv_h \
+helper_vsra_vv_w \
+helper_vsra_vv_d \
+helper_vsll_vx_b \
+helper_vsll_vx_h \
+helper_vsll_vx_w \
+helper_vsll_vx_d \
+helper_vsrl_vx_b \
+helper_vsrl_vx_h \
+helper_vsrl_vx_w \
+helper_vsrl_vx_d \
+helper_vsra_vx_b \
+helper_vsra_vx_h \
+helper_vsra_vx_w \
+helper_vsra_vx_d \
+helper_vnsrl_wv_b \
+helper_vnsrl_wv_h \
+helper_vnsrl_wv_w \
+helper_vnsra_wv_b \
+helper_vnsra_wv_h \
+helper_vnsra_wv_w \
+helper_vnsrl_wx_b \
+helper_vnsrl_wx_h \
+helper_vnsrl_wx_w \
+helper_vnsra_wx_b \
+helper_vnsra_wx_h \
+helper_vnsra_wx_w \
+helper_vzext_vf2_h \
+helper_vzext_vf2_w \
+helper_vzext_vf2_d \
+helper_vzext_vf4_w \
+helper_vzext_vf4_d \
+helper_vzext_vf8_d \
+helper_vsext_vf2_h \
+helper_vsext_vf2_w \
+helper_vsext_vf2_d \
+helper_vsext_vf4_w \
+helper_vsext_vf4_d \
+helper_vsext_vf8_d \
+helper_vmseq_vx_b \
+helper_vmseq_vx_h \
+helper_vmseq_vx_w \
+helper_vmseq_vx_d \
+helper_vmsne_vx_b \
+helper_vmsne_vx_h \
+helper_vmsne_vx_w \
+helper_vmsne_vx_d \
+helper_vmsltu_vx_b \
+helper_vmsltu_vx_h \
+helper_vmsltu_vx_w \
+helper_vmsltu_vx_d \
+helper_vmslt_vx_b \
+helper_vmslt_vx_h \
+helper_vmslt_vx_w \
+helper_vmslt_vx_d \
+helper_vmsleu_vx_b \
+helper_vmsleu_vx_h \
+helper_vmsleu_vx_w \
+helper_vmsleu_vx_d \
+helper_vmsle_vx_b \
+helper_vmsle_vx_h \
+helper_vmsle_vx_w \
+helper_vmsle_vx_d \
+helper_vmsgtu_vx_b \
+helper_vmsgtu_vx_h \
+helper_vmsgtu_vx_w \
+helper_vmsgtu_vx_d \
+helper_vmsgt_vx_b \
+helper_vmsgt_vx_h \
+helper_vmsgt_vx_w \
+helper_vmsgt_vx_d \
+helper_vmv_v_v_b \
+helper_vmv_v_v_h \
+helper_vmv_v_v_w \
+helper_vmv_v_v_d \
+helper_vmv_v_x_b \
+helper_vmv_v_x_h \
+helper_vmv_v_x_w \
+helper_vmv_v_x_d \
+helper_vmerge_vvm_b \
+helper_vmerge_vvm_h \
+helper_vmerge_vvm_w \
+helper_vmerge_vvm_d \
+helper_vmerge_vxm_b \
+helper_vmerge_vxm_h \
+helper_vmerge_vxm_w \
+helper_vmerge_vxm_d \
+helper_clmul \
+helper_clmulr \
+helper_brev8 \
+helper_unzip \
+helper_zip \
+helper_xperm4 \
+helper_xperm8 \
+helper_aes32esmi \
+helper_aes32esi \
+helper_aes32dsmi \
+helper_aes32dsi \
+helper_aes64esm \
+helper_aes64es \
+helper_aes64ds \
+helper_aes64dsm \
+helper_aes64ks2 \
+helper_aes64ks1i \
+helper_aes64im \
+helper_sm4ed \
+helper_sm4ks \
 helper_sret \
 helper_mret \
 helper_wfi \
 helper_tlb_flush \
+helper_hyp_tlb_flush \
+helper_hyp_gvma_tlb_flush \
+helper_hyp_hlvx_hu \
+helper_hyp_hlvx_wu \
 pmp_hart_has_privs \
 pmpcfg_csr_write \
 pmpcfg_csr_read \
 pmpaddr_csr_write \
 pmpaddr_csr_read \
+riscv_cpu_vector_enabled \
 gen_intermediate_code \
 riscv_translate_init \
 restore_state_to_opc \
@@ -4400,7 +8428,15 @@ helper_fcvt_l_d \
 helper_fcvt_lu_d \
 helper_fcvt_d_l \
 helper_fcvt_d_lu \
+helper_fcvt_l_h \
+helper_fcvt_lu_h \
+helper_fcvt_h_l \
+helper_fcvt_h_lu \
 gen_helper_tlb_flush \
+gen_helper_hyp_tlb_flush \
+gen_helper_hyp_gvma_tlb_flush \
+gen_helper_hyp_hlvx_hu \
+gen_helper_hyp_hlvx_wu \
 riscv_fpr_regnames \
 riscv_int_regnames \
 "
@@ -4554,7 +8590,10 @@ helper_evpe \
 helper_dvp \
 helper_evp \
 cpu_mips_get_random \
+cpu_mips_get_count \
 cpu_mips_init \
+cpu_mips_store_count \
+cpu_mips_store_compare \
 helper_absq_s_ph \
 helper_absq_s_qb \
 helper_absq_s_w \
@@ -5868,6 +9907,18 @@ helper_vprtybw \
 helper_vprtybd \
 helper_vprtybq \
 helper_vmuluwm \
+helper_VDIVSQ \
+helper_VDIVUQ \
+helper_VDIVESD \
+helper_VDIVEUD \
+helper_VDIVESQ \
+helper_VDIVEUQ \
+helper_VMODSQ \
+helper_VMODUQ \
+helper_VSTRIBL \
+helper_VSTRIBR \
+helper_VSTRIHL \
+helper_VSTRIHR \
 helper_vaddfp \
 helper_vsubfp \
 helper_vminfp \
@@ -6015,6 +10066,10 @@ helper_vextuwlx \
 helper_vextubrx \
 helper_vextuhrx \
 helper_vextuwrx \
+helper_VEXTDUBVLX \
+helper_VEXTDUHVLX \
+helper_VEXTDUWVLX \
+helper_VEXTDDVLX \
 helper_vslv \
 helper_vsrv \
 helper_vsldoi \
@@ -6023,6 +10078,10 @@ helper_vinsertb \
 helper_vinserth \
 helper_vinsertw \
 helper_vinsertd \
+helper_VINSBLX \
+helper_VINSHLX \
+helper_VINSWLX \
+helper_VINSDLX \
 helper_vextractub \
 helper_vextractuh \
 helper_vextractuw \
@@ -6418,6 +10477,14 @@ helper_xsmadddp \
 helper_xsmsubdp \
 helper_xsnmadddp \
 helper_xsnmsubdp \
+helper_XSMADDQP \
+helper_XSMADDQPO \
+helper_XSMSUBQP \
+helper_XSMSUBQPO \
+helper_XSNMADDQP \
+helper_XSNMADDQPO \
+helper_XSNMSUBQP \
+helper_XSNMSUBQPO \
 helper_xsmaddsp \
 helper_xsmsubsp \
 helper_xsnmaddsp \
@@ -6434,6 +10501,9 @@ helper_xscmpeqdp \
 helper_xscmpgedp \
 helper_xscmpgtdp \
 helper_xscmpnedp \
+helper_XSCMPEQQP \
+helper_XSCMPGEQP \
+helper_XSCMPGTQP \
 helper_xscmpexpdp \
 helper_xscmpexpqp \
 helper_xscmpodp \
@@ -6450,6 +10520,8 @@ helper_xsmaxcdp \
 helper_xsmincdp \
 helper_xsmaxjdp \
 helper_xsminjdp \
+helper_XSMAXCQP \
+helper_XSMINCQP \
 helper_xvcmpeqdp \
 helper_xvcmpgedp \
 helper_xvcmpgtdp \
@@ -6467,7 +10539,98 @@ helper_xscvdphp \
 helper_xscvhpdp \
 helper_xvcvsphp \
 helper_xvcvhpsp \
+helper_XVCVSPBF16 \
+helper_XVCVBF16SPN \
+helper_dadd \
+helper_daddq \
+helper_dsub \
+helper_dsubq \
+helper_dmul \
+helper_dmulq \
+helper_ddiv \
+helper_ddivq \
+helper_dcmpo \
+helper_dcmpoq \
+helper_dcmpu \
+helper_dcmpuq \
+helper_dtstdc \
+helper_dtstdcq \
+helper_dtstdg \
+helper_dtstdgq \
+helper_dtstex \
+helper_dtstexq \
+helper_dtstsf \
+helper_dtstsfq \
+helper_dtstsfi \
+helper_dtstsfiq \
+helper_dquai \
+helper_dquaiq \
+helper_dqua \
+helper_dquaq \
+helper_drrnd \
+helper_drrndq \
+helper_drintx \
+helper_drintxq \
+helper_drintn \
+helper_drintnq \
+helper_dctdp \
+helper_dctqpq \
+helper_drsp \
+helper_drdpq \
+helper_dcffix \
+helper_dcffixq \
+helper_DCFFIXQQ \
+helper_dctfix \
+helper_dctfixq \
+helper_DCTFIXQQ \
+helper_ddedpd \
+helper_ddedpdq \
+helper_denbcd \
+helper_denbcdq \
+helper_dxex \
+helper_dxexq \
+helper_diex \
+helper_diexq \
+helper_dscri \
+helper_dscriq \
+helper_dscli \
+helper_dscliq \
+helper_CDTBCD \
+helper_CBCDTD \
+helper_XVI4GER8 \
+helper_XVI4GER8PP \
+helper_XVI8GER4 \
+helper_XVI8GER4PP \
+helper_XVI8GER4SPP \
+helper_XVI16GER2 \
+helper_XVI16GER2S \
+helper_XVI16GER2PP \
+helper_XVI16GER2SPP \
+helper_XVBF16GER2 \
+helper_XVBF16GER2PP \
+helper_XVBF16GER2PN \
+helper_XVBF16GER2NP \
+helper_XVBF16GER2NN \
+helper_XVF16GER2 \
+helper_XVF16GER2PP \
+helper_XVF16GER2PN \
+helper_XVF16GER2NP \
+helper_XVF16GER2NN \
+helper_XVF32GER \
+helper_XVF32GERPP \
+helper_XVF32GERPN \
+helper_XVF32GERNP \
+helper_XVF32GERNN \
+helper_XVF64GER \
+helper_XVF64GERPP \
+helper_XVF64GERPN \
+helper_XVF64GERNP \
+helper_XVF64GERNN \
 helper_xscvqpdp \
+helper_XSCVQPUQZ \
+helper_XSCVQPSQZ \
+helper_XSCVUQQP \
+helper_XSCVSQQP \
 helper_xscvdpspn \
 helper_xscvspdpn \
 helper_xscvdpsxds \
@@ -6518,6 +10681,23 @@ helper_xvrspiz \
 helper_xsrsp \
 helper_xxperm \
 helper_xxpermr \
+helper_XXPERMX \
+helper_XXGENPCVBM_be_exp \
+helper_XXGENPCVBM_be_comp \
+helper_XXGENPCVBM_le_exp \
+helper_XXGENPCVBM_le_comp \
+helper_XXGENPCVHM_be_exp \
+helper_XXGENPCVHM_be_comp \
+helper_XXGENPCVHM_le_exp \
+helper_XXGENPCVHM_le_comp \
+helper_XXGENPCVWM_be_exp \
+helper_XXGENPCVWM_be_comp \
+helper_XXGENPCVWM_le_exp \
+helper_XXGENPCVWM_le_comp \
+helper_XXGENPCVDM_be_exp \
+helper_XXGENPCVDM_be_comp \
+helper_XXGENPCVDM_le_exp \
+helper_XXGENPCVDM_le_comp \
 helper_xvxsigsp \
 helper_xvtstdcdp \
 helper_xvtstdcsp \
diff --git a/tests/unit/acutest.h b/tests/unit/acutest.h
index 8ec7d9892a..a8caab15b8 100644
--- a/tests/unit/acutest.h
+++ b/tests/unit/acutest.h
@@ -270,6 +270,14 @@
 #include <string.h>
 #include <setjmp.h>
 
+#ifdef _MSC_VER
+#include <crtdbg.h>
+#endif
+
+#if defined(_MSC_VER) && defined(_RTC)
+#include <rtcapi.h>
+#endif
+
 #if defined(unix) || defined(__unix__) || defined(__unix) ||                   \
     defined(__APPLE__) || defined(__HAIKU__)
 #define ACUTEST_UNIX_ 1
@@ -332,6 +340,57 @@
 extern "C" {
 #endif
 
+#ifdef _MSC_VER
+static void acutest_msvc_debug_report_init_(void)
+{
+    static const int report_types[] = {
+        _CRT_WARN, _CRT_ERROR, _CRT_ASSERT,
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(report_types) / sizeof(report_types[0]); i++) {
+        _CrtSetReportMode(report_types[i], _CRTDBG_MODE_FILE);
+        _CrtSetReportFile(report_types[i], _CRTDBG_FILE_STDERR);
+    }
+}
+#endif
+
+#if defined(_MSC_VER) && defined(_RTC)
+static int __cdecl
+acutest_rtc_error_(int errnum, const char *filename, int lineno,
+                   const char *module, const char *format, ...)
+{
+    va_list args;
+    const char *description = _RTC_GetErrDesc((_RTC_ErrorNumber)errnum);
+
+    fprintf(stderr, "MSVC runtime check failure %d", errnum);
+    if (description != NULL) {
+        fprintf(stderr, " (%s)", description);
+    }
+    if (filename != NULL) {
+        fprintf(stderr, " at %s:%d", filename, lineno);
+    }
+    if (module != NULL) {
+        fprintf(stderr, " in %s", module);
+    }
+    fprintf(stderr, ": ");
+    va_start(args, format);
+    if (format != NULL) {
+        vfprintf(stderr, format, args);
+    }
+    va_end(args);
+    fprintf(stderr, "\n");
+    fflush(stderr);
+    exit(3);
+    return 0;
+}
+
+static void acutest_msvc_runtime_check_init_(void)
+{
+    _RTC_SetErrorFunc(acutest_rtc_error_);
+}
+#endif
+
 #ifdef _MSC_VER
 /* In the multi-platform code like ours, we cannot use the non-standard
  * "safe" functions from Microsoft C lib like e.g. sprintf_s() instead of
@@ -1772,6 +1831,10 @@ int main(int argc, char **argv)
     SetUnhandledExceptionFilter(acutest_seh_exception_filter_);
 #ifdef _MSC_VER
     _set_abort_behavior(0, _WRITE_ABORT_MSG);
+    acutest_msvc_debug_report_init_();
+#ifdef _RTC
+    acutest_msvc_runtime_check_init_();
+#endif
 #endif
 #endif
 
diff --git a/tests/unit/test_arm.c b/tests/unit/test_arm.c
index b2b299b06c..c74a97494f 100644
--- a/tests/unit/test_arm.c
+++ b/tests/unit/test_arm.c
@@ -1,4 +1,9 @@
 #include "unicorn_test.h"
+#include "uc_priv.h"
+#define NEED_CPU_H
+#include "target/arm/cpu.h"
+#undef NEED_CPU_H
+#include <string.h>
 
 const uint64_t code_start = 0x1000;
 const uint64_t code_len = 0x4000;
@@ -16,6 +21,314 @@ typedef struct _WFI_HOOK_INSN_RESULT {
     bool called;
 } WFI_HOOK_INSN_RESULT;
 
+static void test_arm_emit32(uint8_t *code, int offset, uint32_t insn)
+{
+    code[offset] = (uint8_t)insn;
+    code[offset + 1] = (uint8_t)(insn >> 8);
+    code[offset + 2] = (uint8_t)(insn >> 16);
+    code[offset + 3] = (uint8_t)(insn >> 24);
+}
+
+static void test_arm_enable_vfp(uc_engine *uc)
+{
+    uint32_t cpacr = 0xfU << 20;
+    uint32_t fpexc = 1U << 30;
+
+    OK(uc_reg_write(uc, UC_ARM_REG_C1_C0_2, &cpacr));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPEXC, &fpexc));
+}
+
+static uint32_t test_arm_id_isar6_read(uc_engine *uc)
+{
+    uc_arm_cp_reg reg = {
+        .cp = 15,
+        .is64 = 0,
+        .sec = 0,
+        .crn = 0,
+        .crm = 2,
+        .opc1 = 0,
+        .opc2 = 7,
+        .val = 0,
+    };
+
+    OK(uc_reg_read(uc, UC_ARM_REG_CP_REG, &reg));
+    return (uint32_t)reg.val;
+}
+
+static void test_arm_i8mm_q_run(uint32_t insn, const uint8_t *initial,
+                                const uint8_t *n, const uint8_t *m,
+                                const uint8_t *expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    size_t i;
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM_MAX);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_i8mm_expect_exception(uint32_t insn, uc_cpu_arm cpu)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), cpu);
+    test_arm_enable_vfp(uc);
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0) ==
+               UC_ERR_INSN_INVALID);
+    OK(uc_close(uc));
+}
+
+static void test_arm_i8mm(void)
+{
+    uc_engine *uc;
+    const uint32_t id_isar6_i8mm = 0xfU << 24;
+    const uint8_t initial[16] = {
+        0x10, 0x00, 0x00, 0x00, 0xf0, 0xff, 0xff, 0xff,
+        0x80, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff,
+    };
+    const uint8_t n[16] = {
+        0x01, 0x7f, 0x80, 0xff, 0x10, 0xf0, 0x22, 0xdd,
+        0x40, 0xc0, 0x55, 0xaa, 0x7e, 0x82, 0x01, 0xff,
+    };
+    const uint8_t m[16] = {
+        0x02, 0xfe, 0x03, 0xfd, 0x80, 0x7f, 0x04, 0xfc,
+        0x11, 0xef, 0x66, 0x99, 0x08, 0xf8, 0x01, 0xff,
+    };
+    const uint8_t exp_usdot[16] = {
+        0x97, 0xfd, 0xff, 0xff, 0x14, 0x6c, 0x00, 0x00,
+        0x78, 0xd5, 0xff, 0xff, 0x61, 0xfe, 0xff, 0xff,
+    };
+    const uint8_t exp_usdot_idx[16] = {
+        0x95, 0x3c, 0x00, 0x00, 0x14, 0x6c, 0x00, 0x00,
+        0x6c, 0x3e, 0x00, 0x00, 0x05, 0xfd, 0xff, 0xff,
+    };
+    const uint8_t exp_sudot_idx[16] = {
+        0x95, 0x3c, 0x00, 0x00, 0x14, 0xde, 0xff, 0xff,
+        0x6c, 0xad, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff,
+    };
+    const uint8_t exp_smmla[16] = {
+        0xbb, 0xee, 0xff, 0xff, 0x3e, 0xc6, 0xff, 0xff,
+        0x07, 0x86, 0xff, 0xff, 0x59, 0x54, 0x00, 0x00,
+    };
+    const uint8_t exp_ummla[16] = {
+        0xbb, 0xd4, 0x02, 0x00, 0x3e, 0x07, 0x03, 0x00,
+        0x07, 0xe3, 0x02, 0x00, 0x59, 0xbe, 0x02, 0x00,
+    };
+    const uint8_t exp_usmmla[16] = {
+        0xbb, 0x69, 0x00, 0x00, 0x3e, 0xbc, 0xff, 0xff,
+        0x07, 0xfc, 0xff, 0xff, 0x59, 0xd3, 0xff, 0xff,
+    };
+
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, "\x00\xf0\x20\xe3", 4,
+                    UC_CPU_ARM_MAX);
+    TEST_CHECK(((test_arm_id_isar6_read(uc) & id_isar6_i8mm) >> 24) == 1);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, "\x00\xf0\x20\xe3", 4,
+                    UC_CPU_ARM_CORTEX_A15);
+    TEST_CHECK((test_arm_id_isar6_read(uc) & id_isar6_i8mm) == 0);
+    OK(uc_close(uc));
+
+    test_arm_i8mm_q_run(0xfca20d44, initial, n, m, exp_usdot);
+    test_arm_i8mm_q_run(0xfe820d64, initial, n, m, exp_usdot_idx);
+    test_arm_i8mm_q_run(0xfe820d74, initial, n, m, exp_sudot_idx);
+    test_arm_i8mm_q_run(0xfc220c44, initial, n, m, exp_smmla);
+    test_arm_i8mm_q_run(0xfc220c54, initial, n, m, exp_ummla);
+    test_arm_i8mm_q_run(0xfca20c44, initial, n, m, exp_usmmla);
+
+    test_arm_i8mm_expect_exception(0xfca20d44, UC_CPU_ARM_CORTEX_A15);
+}
+
+static void test_arm_bf16_scalar_convert(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t s0 = 0;
+    uint32_t s1 = 0x3fc00000u;
+
+    test_arm_emit32(code, 0, 0xeeb30960);
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM_MAX);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_S0, &s0));
+    TEST_CHECK(s0 == 0x00003fc0u);
+    OK(uc_close(uc));
+}
+
+static void test_arm_bf16_vector_convert_run(uint32_t insn,
+                                             const uint16_t *initial,
+                                             const uint32_t *source,
+                                             const uint16_t *expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint16_t *got = (const uint16_t *)q0;
+    size_t i;
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM_MAX);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, source, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_bf16_q_run(uint32_t insn, const uint32_t *initial,
+                                const uint16_t *n, const uint16_t *m,
+                                const uint32_t *expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint32_t *got = (const uint32_t *)q0;
+    size_t i;
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM_MAX);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_bf16(void)
+{
+    uc_engine *uc;
+    const uint32_t id_isar6_bf16 = 0xfU << 20;
+    const uint16_t init_h[8] = {
+        0x1111, 0x2222, 0x3333, 0x4444,
+        0xaaaa, 0xbbbb, 0xcccc, 0xdddd,
+    };
+    const uint32_t fp32_source[4] = {
+        0x3f800000u, 0xc0000000u, 0x40400000u, 0x40800000u,
+    };
+    const uint16_t exp_bfcvt[8] = {
+        0x3f80, 0xc000, 0x4040, 0x4080,
+        0xaaaa, 0xbbbb, 0xcccc, 0xdddd,
+    };
+    const uint32_t init_s[4] = {
+        0x41200000u, 0x41a00000u, 0x41f00000u, 0x42200000u,
+    };
+    const uint16_t n_pair[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x3f80, 0x3f80, 0x4000, 0x4000,
+    };
+    const uint16_t m_pair[8] = {
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+        0x4000, 0x4040, 0x4080, 0x40a0,
+    };
+    const uint32_t exp_bfdot[4] = {
+        0x41d80000u, 0x42920000u, 0x420c0000u, 0x42680000u,
+    };
+    const uint32_t exp_bfdot_idx[4] = {
+        0x41d80000u, 0x426c0000u, 0x42240000u, 0x42780000u,
+    };
+    const uint16_t n_mmla[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+    };
+    const uint16_t m_mmla[8] = {
+        0x3f80, 0x3f80, 0x4000, 0x4000,
+        0x4040, 0x4040, 0x4080, 0x4080,
+    };
+    const uint32_t exp_bfmmla[4] = {
+        0x41d80000u, 0x42640000u, 0x428e0000u, 0x43050000u,
+    };
+    const uint16_t n_long[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+    };
+    const uint16_t m_long[8] = {
+        0x4000, 0x4040, 0x4080, 0x40a0,
+        0x40c0, 0x40e0, 0x4100, 0x4110,
+    };
+    const uint32_t exp_bfmlalb[4] = {
+        0x41400000u, 0x42000000u, 0x42700000u, 0x42c00000u,
+    };
+    const uint32_t exp_bfmlalt[4] = {
+        0x41800000u, 0x42200000u, 0x42900000u, 0x42e00000u,
+    };
+    const uint32_t exp_bfmlalb_idx[4] = {
+        0x41400000u, 0x41d00000u, 0x42200000u, 0x42580000u,
+    };
+    const uint32_t exp_bfmlalt_idx[4] = {
+        0x41600000u, 0x41e00000u, 0x42280000u, 0x42600000u,
+    };
+
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, "\x00\xf0\x20\xe3", 4,
+                    UC_CPU_ARM_MAX);
+    TEST_CHECK(((test_arm_id_isar6_read(uc) & id_isar6_bf16) >> 20) == 1);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM, UC_MODE_ARM, "\x00\xf0\x20\xe3", 4,
+                    UC_CPU_ARM_CORTEX_A15);
+    TEST_CHECK((test_arm_id_isar6_read(uc) & id_isar6_bf16) == 0);
+    OK(uc_close(uc));
+
+    test_arm_bf16_scalar_convert();
+    test_arm_bf16_vector_convert_run(0xf3b60642, init_h, fp32_source,
+                                     exp_bfcvt);
+    test_arm_bf16_q_run(0xfc020d44, init_s, n_pair, m_pair, exp_bfdot);
+    test_arm_bf16_q_run(0xfe020d44, init_s, n_pair, m_pair, exp_bfdot_idx);
+    test_arm_bf16_q_run(0xfc020c44, init_s, n_mmla, m_mmla, exp_bfmmla);
+    test_arm_bf16_q_run(0xfc320814, init_s, n_long, m_long, exp_bfmlalb);
+    test_arm_bf16_q_run(0xfc320854, init_s, n_long, m_long, exp_bfmlalt);
+    test_arm_bf16_q_run(0xfe320814, init_s, n_long, m_long,
+                        exp_bfmlalb_idx);
+    test_arm_bf16_q_run(0xfe320854, init_s, n_long, m_long,
+                        exp_bfmlalt_idx);
+
+    test_arm_i8mm_expect_exception(0xeeb30960, UC_CPU_ARM_CORTEX_A15);
+    test_arm_i8mm_expect_exception(0xfc020d44, UC_CPU_ARM_CORTEX_A15);
+    test_arm_i8mm_expect_exception(0xfc030c44, UC_CPU_ARM_MAX);
+}
+
 static void test_arm_nop(void)
 {
     uc_engine *uc;
@@ -261,6 +574,10988 @@ static void test_arm_m_control(void)
     OK(uc_close(uc));
 }
 
+static uint32_t test_arm_m33_sau_tt_query(uc_engine *uc, uint32_t address,
+                                          uint8_t op)
+{
+    uint32_t result = 0;
+    uint8_t code[] = {
+        0x41, 0xe8, 0x00, 0xf0, /* tt-family r0, r1 */
+    };
+
+    code[2] = op << 6;
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &result));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &address));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R0, &result));
+
+    return result;
+}
+
+static void test_arm_m33_sau_region(CPUARMState *env, uint32_t region,
+                                    uint32_t base, uint32_t limit,
+                                    bool non_secure_callable)
+{
+    env->sau.rbar[region] = base & ~0x1fU;
+    env->sau.rlar[region] = (limit & ~0x1fU) | 1U;
+    if (non_secure_callable) {
+        env->sau.rlar[region] |= 2U;
+    }
+}
+
+static void test_arm_m33_sau_tt(void)
+{
+    uc_engine *uc;
+    ARMCPU *cpu;
+    CPUARMState *env;
+    uint32_t result;
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M33));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    if (!TEST_CHECK(cpu->sau_sregion >= 2)) {
+        OK(uc_close(uc));
+        return;
+    }
+
+    env->sau.ctrl = 0;
+    memset(env->sau.rbar, 0, sizeof(*env->sau.rbar) * cpu->sau_sregion);
+    memset(env->sau.rlar, 0, sizeof(*env->sau.rlar) * cpu->sau_sregion);
+
+    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
+    TEST_CHECK(result & (1U << 22));
+    TEST_CHECK(!(result & (1U << 17)));
+
+    env->sau.ctrl = 2;
+    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
+    TEST_CHECK(!(result & (1U << 22)));
+    TEST_CHECK(!(result & (1U << 17)));
+
+    env->sau.ctrl = 1;
+    test_arm_m33_sau_region(env, 0, 0x2000, 0x2fff, false);
+    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
+    TEST_CHECK(!(result & (1U << 22)));
+    TEST_CHECK(result & (1U << 17));
+    TEST_CHECK(((result >> 8) & 0xff) == 0);
+
+    test_arm_m33_sau_region(env, 1, 0x3000, 0x3fff, true);
+    result = test_arm_m33_sau_tt_query(uc, 0x3000, 0);
+    TEST_CHECK(result & (1U << 22));
+    TEST_CHECK(result & (1U << 17));
+    TEST_CHECK(((result >> 8) & 0xff) == 1);
+
+    test_arm_m33_sau_region(env, 1, 0x2000, 0x2fff, true);
+    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
+    TEST_CHECK(result & (1U << 22));
+    TEST_CHECK(!(result & (1U << 17)));
+
+    result = test_arm_m33_sau_tt_query(uc, 0xe000e010, 0);
+    TEST_CHECK(result & (1U << 22));
+
+    result = test_arm_m33_sau_tt_query(uc, 0xe000e010, 2);
+    TEST_CHECK(!(result & (1U << 22)));
+
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_pmsav8_pxn(void)
+{
+    uc_engine *uc;
+    ARMCPU *cpu;
+    CPUARMState *env;
+    uint32_t r0 = 0;
+    uc_err err;
+    const uint8_t code[] = {
+        0x2a, 0x20, /* movs r0, #42 */
+    };
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M55));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.mpu_ctrl[M_REG_S] = R_V7M_MPU_CTRL_ENABLE_MASK;
+    env->pmsav8.rbar[M_REG_S][0] = (uint32_t)code_start | (1U << 1);
+    env->pmsav8.rlar[M_REG_S][0] =
+        ((uint32_t)(code_start + code_len - 1) & ~0x1fU) | (1U << 4) | 1U;
+
+    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_reg_read(uc, UC_ARM_REG_R0, &r0));
+    TEST_CHECK(r0 == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_id(void)
+{
+    uc_engine *uc;
+    uint32_t mvfr0, mvfr1, mvfr2;
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M55));
+
+    OK(uc_reg_read(uc, UC_ARM_REG_MVFR0, &mvfr0));
+    OK(uc_reg_read(uc, UC_ARM_REG_MVFR1, &mvfr1));
+    OK(uc_reg_read(uc, UC_ARM_REG_MVFR2, &mvfr2));
+
+    TEST_CHECK(mvfr0 == 0x10110221);
+    TEST_CHECK(mvfr1 == 0x12100211);
+    TEST_CHECK(mvfr2 == 0x00000040);
+    TEST_CHECK(((mvfr1 >> 8) & 0xf) == 2);
+    TEST_CHECK(((mvfr1 >> 20) & 0xf) == 1);
+
+    OK(uc_close(uc));
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M33));
+
+    OK(uc_reg_read(uc, UC_ARM_REG_MVFR1, &mvfr1));
+    TEST_CHECK(((mvfr1 >> 8) & 0xf) == 0);
+    TEST_CHECK(((mvfr1 >> 20) & 0xf) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_vpr_public_reg(void)
+{
+    uc_engine *uc;
+    uc_context *ctx;
+    uint32_t vpr = 0x00abcdef;
+    uint32_t read_vpr = 0;
+    uint32_t changed_vpr = 0x00123456;
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M55));
+
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &read_vpr));
+    TEST_CHECK(read_vpr == vpr);
+
+    OK(uc_context_alloc(uc, &ctx));
+    OK(uc_context_save(uc, ctx));
+    OK(uc_context_reg_read(ctx, UC_ARM_REG_VPR, &read_vpr));
+    TEST_CHECK(read_vpr == vpr);
+
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &changed_vpr));
+    OK(uc_context_restore(uc, ctx));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &read_vpr));
+    TEST_CHECK(read_vpr == vpr);
+
+    OK(uc_context_free(ctx));
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_vpr_sysreg(void)
+{
+    uc_engine *uc;
+    uint8_t code[16];
+    uint32_t vpr = 0x00abcdef;
+    uint32_t p0 = 0x1357;
+    uint32_t r1 = 0;
+    uint32_t r3 = 0;
+    uint32_t read_vpr = 0;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);  /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x1a10eefc);  /* vmrs r1,vpr */
+    test_arm_emit32(code, 8, 0x2a10eeed);  /* vmsr p0,r2 */
+    test_arm_emit32(code, 12, 0x3a10eefd); /* vmrs r3,p0 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &p0));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_read(uc, UC_ARM_REG_R3, &r3));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &read_vpr));
+    TEST_CHECK(r1 == vpr);
+    TEST_CHECK(r3 == p0);
+    TEST_CHECK(read_vpr == ((vpr & 0x00ff0000) | p0));
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_fpscr_ltpsize(void)
+{
+    const uint32_t ltpsize_shift = 16;
+    const uint32_t ltpsize_mask = 7U << ltpsize_shift;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint32_t fpscr = 5U << ltpsize_shift;
+    uint32_t read_fpscr = 0;
+    uint32_t r1 = 0;
+
+    test_arm_emit32(code, 0, 0x0a10eee1); /* vmsr fpscr,r0 */
+    test_arm_emit32(code, 4, 0x1a10eef1); /* vmrs r1,fpscr */
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M55));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &read_fpscr));
+    TEST_CHECK((read_fpscr & ltpsize_mask) == fpscr);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    fpscr = 6U << ltpsize_shift;
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &fpscr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    TEST_CHECK((r1 & ltpsize_mask) == fpscr);
+    OK(uc_close(uc));
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M33));
+    fpscr = 3U << ltpsize_shift;
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &read_fpscr));
+    TEST_CHECK((read_fpscr & ltpsize_mask) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_fpscr_nzcvqc_sysreg(void)
+{
+    const uint32_t fpcr_ahp = 1U << 26;
+    const uint32_t fpcr_qc = 1U << 27;
+    const uint32_t fpcr_v = 1U << 28;
+    const uint32_t fpcr_c = 1U << 29;
+    const uint32_t fpcr_z = 1U << 30;
+    const uint32_t fpcr_n = 1U << 31;
+    const uint32_t nzcvqc_mask =
+        fpcr_n | fpcr_z | fpcr_c | fpcr_v | fpcr_qc;
+    const uint32_t preserved = fpcr_ahp | (5U << 16);
+    const uint32_t set_qc = fpcr_n | fpcr_c | fpcr_qc | 0x0000ffff;
+    const uint32_t clear_qc = fpcr_z | fpcr_v | 0x0000ffff;
+    const uint32_t expected_final =
+        preserved | (clear_qc & (fpcr_n | fpcr_z | fpcr_c | fpcr_v));
+    uc_engine *uc;
+    uint8_t code[24];
+    uint32_t r1 = 0;
+    uint32_t r3 = 0;
+    uint32_t r5 = 0;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x4a10eee1);  /* vmsr fpscr,r4 */
+    test_arm_emit32(code, 4, 0x0a10eee2);  /* vmsr fpscr_nzcvqc,r0 */
+    test_arm_emit32(code, 8, 0x1a10eef2);  /* vmrs r1,fpscr_nzcvqc */
+    test_arm_emit32(code, 12, 0x2a10eee2); /* vmsr fpscr_nzcvqc,r2 */
+    test_arm_emit32(code, 16, 0x3a10eef2); /* vmrs r3,fpscr_nzcvqc */
+    test_arm_emit32(code, 20, 0x5a10eef1); /* vmrs r5,fpscr */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &set_qc));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &clear_qc));
+    OK(uc_reg_write(uc, UC_ARM_REG_R4, &preserved));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_read(uc, UC_ARM_REG_R3, &r3));
+    OK(uc_reg_read(uc, UC_ARM_REG_R5, &r5));
+    TEST_CHECK(r1 == (set_qc & nzcvqc_mask));
+    TEST_CHECK(r3 == (clear_qc & nzcvqc_mask));
+    TEST_CHECK_((r5 & (nzcvqc_mask | fpcr_ahp | (7U << 16))) ==
+                expected_final, "fpscr=0x%08x expected=0x%08x", r5,
+                expected_final);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)(code + 4), 4, UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &set_qc));
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_vctp(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t r0 = 3;
+    uint32_t vpr = 0;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0xe801f020); /* vctp.32 r0 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &r0));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    TEST_CHECK_(vpr == 0x00000fff, "vpr=0x%08x", vpr);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_eci(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t eci_reserved = 3U << 12;
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t r0 = 3;
+    uint32_t epsr;
+    uint32_t vpr = 0;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0xe801f020); /* vctp.32 r0 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &r0));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    TEST_CHECK_(vpr == 0x00000f00, "vpr=0x%08x", vpr);
+    TEST_CHECK_((epsr & (0xfc00 | (3U << 25))) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    epsr = xpsr_t | eci_reserved;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_vpst_vpnot(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint32_t epsr;
+    uint32_t vpr;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x6f4dfe71); /* vpst 0xb */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    vpr = 0x00001357;
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    TEST_CHECK_(vpr == 0x00bb1357, "vpr=0x%08x", vpr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0xcf4dfe31); /* vpst 0x6 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    vpr = 0x00001357;
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, (code_start + 4) | 1, code_start + sizeof(code), 0,
+                    0));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    TEST_CHECK_(vpr == 0x00601357, "vpr=0x%08x", vpr);
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x0f4dfe31); /* vpnot */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    vpr = 0x0000a55a;
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    TEST_CHECK_(vpr == 0x00005aa5, "vpr=0x%08x", vpr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f4dfe31); /* vpnot */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4,
+                    UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x6f4dfe71); /* vpst 0xb */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4,
+                    UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static uint32_t test_arm_load_le(const uint8_t *data, unsigned size)
+{
+    uint32_t ret = 0;
+    unsigned i;
+
+    for (i = 0; i < size; i++) {
+        ret |= (uint32_t)data[i] << (i * 8);
+    }
+
+    return ret;
+}
+
+static void test_arm_store_le(uint8_t *data, unsigned size, uint32_t value)
+{
+    unsigned i;
+
+    for (i = 0; i < size; i++) {
+        data[i] = (uint8_t)(value >> (i * 8));
+    }
+}
+
+static uint64_t test_arm_load_le64(const uint8_t *data)
+{
+    uint64_t ret = 0;
+    unsigned i;
+
+    for (i = 0; i < 8; i++) {
+        ret |= (uint64_t)data[i] << (i * 8);
+    }
+
+    return ret;
+}
+
+static void test_arm_store_le64(uint8_t *data, uint64_t value)
+{
+    unsigned i;
+
+    for (i = 0; i < 8; i++) {
+        data[i] = (uint8_t)(value >> (i * 8));
+    }
+}
+
+static void test_arm_m_profile_activate_fp_context(uc_engine *uc)
+{
+    const uint32_t control_fpca = 1U << 2;
+    const uint32_t control_sfpa = 1U << 3;
+    ARMCPU *cpu = ARM_CPU(uc->cpu);
+    CPUARMState *env = &cpu->env;
+
+    env->v7m.control[M_REG_S] |= control_fpca | control_sfpa;
+    env->v7m.cpacr[M_REG_S] |= 3U << 20;
+    env->v7m.fpccr[M_REG_S] &= ~R_V7M_FPCCR_LSPACT_MASK;
+    env->v7m.fpccr[M_REG_NS] &= ~R_V7M_FPCCR_LSPACT_MASK;
+    FIELD_DP32(env->hflags, TBFLAG_M32, LSPACT, 0, env->hflags);
+    FIELD_DP32(env->hflags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 0,
+               env->hflags);
+    FIELD_DP32(env->hflags, TBFLAG_M32, FPCCR_S_WRONG, 0, env->hflags);
+}
+
+static void test_arm_m_profile_disable_lazy_fp(uc_engine *uc)
+{
+    ARMCPU *cpu = ARM_CPU(uc->cpu);
+    CPUARMState *env = &cpu->env;
+
+    env->v7m.fpccr[M_REG_S] &= ~R_V7M_FPCCR_LSPEN_MASK;
+}
+
+static void test_arm_m55_sysreg_mem(void)
+{
+    const uint32_t fpcr_ahp = 1U << 26;
+    const uint32_t fpcr_qc = 1U << 27;
+    const uint32_t fpcr_v = 1U << 28;
+    const uint32_t fpcr_c = 1U << 29;
+    const uint32_t fpcr_z = 1U << 30;
+    const uint32_t fpcr_n = 1U << 31;
+    const uint32_t ltpsize_mask = 7U << 16;
+    const uint32_t nzcvqc_mask =
+        fpcr_n | fpcr_z | fpcr_c | fpcr_v | fpcr_qc;
+    const uint64_t data_addr = code_start + 0x1000;
+    const uint32_t initial_vpr = 0x00ab1357;
+    const uint32_t p0_load = 0x0000a55a;
+    const uint32_t loaded_vpr = 0x00cd2468;
+    const uint32_t preserved = fpcr_ahp | (5U << 16);
+    const uint32_t fpscr_initial =
+        preserved | fpcr_n | fpcr_c | fpcr_qc | 0x1234;
+    const uint32_t fpscr_load = fpcr_z | fpcr_v | 0xffff;
+    const uint32_t fpscr_expected = preserved | fpcr_z | fpcr_v;
+    uc_engine *uc;
+    uint8_t code[44];
+    uint8_t mem[32] = { 0 };
+    uint32_t r1 = (uint32_t)data_addr;
+    uint32_t r2 = (uint32_t)data_addr + 16;
+    uint32_t r3 = 0;
+    uint32_t r4 = 0;
+    uint32_t r6 = 0;
+    uint32_t read_vpr = 0;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);  /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0xaf80edc1);  /* vstr p0,[r1,#0] */
+    test_arm_emit32(code, 8, 0x8f81edc1);  /* vstr vpr,[r1,#4] */
+    test_arm_emit32(code, 12, 0xaf82edd1); /* vldr p0,[r1,#8] */
+    test_arm_emit32(code, 16, 0x3a10eefc); /* vmrs r3,vpr */
+    test_arm_emit32(code, 20, 0x8f81ecf2); /* vldr vpr,[r2],#4 */
+    test_arm_emit32(code, 24, 0x4a10eefc); /* vmrs r4,vpr */
+    test_arm_emit32(code, 28, 0x5a10eee1); /* vmsr fpscr,r5 */
+    test_arm_emit32(code, 32, 0x4f85ed81); /* vstr fpscr_nzcvqc,[r1,#20] */
+    test_arm_emit32(code, 36, 0x4f86ed91); /* vldr fpscr_nzcvqc,[r1,#24] */
+    test_arm_emit32(code, 40, 0x6a10eef1); /* vmrs r6,fpscr */
+
+    test_arm_store_le(mem + 8, 4, p0_load);
+    test_arm_store_le(mem + 16, 4, loaded_vpr);
+    test_arm_store_le(mem + 24, 4, fpscr_load);
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &initial_vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R5, &fpscr_initial));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+
+    memset(mem, 0, sizeof(mem));
+    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_read(uc, UC_ARM_REG_R3, &r3));
+    OK(uc_reg_read(uc, UC_ARM_REG_R4, &r4));
+    OK(uc_reg_read(uc, UC_ARM_REG_R6, &r6));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &read_vpr));
+
+    TEST_CHECK(test_arm_load_le(mem, 4) == (initial_vpr & 0xffff));
+    TEST_CHECK(test_arm_load_le(mem + 4, 4) == initial_vpr);
+    TEST_CHECK(test_arm_load_le(mem + 20, 4) ==
+               (fpscr_initial & nzcvqc_mask));
+    TEST_CHECK(r2 == (uint32_t)data_addr + 20);
+    TEST_CHECK(r3 == ((initial_vpr & 0x00ff0000) |
+                      (p0_load & 0xffff)));
+    TEST_CHECK(r4 == loaded_vpr);
+    TEST_CHECK(read_vpr == loaded_vpr);
+    TEST_CHECK((r6 & (fpcr_ahp | ltpsize_mask | nzcvqc_mask)) ==
+               fpscr_expected);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code + 4, 4, UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_vlldm_vlstm(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint64_t data_addr = code_start + 0x1000;
+    uc_engine *uc;
+    uint8_t code[4];
+    uint8_t mem[0x48] = { 0 };
+    uint32_t r1 = (uint32_t)data_addr;
+    uint32_t s0 = 0x11223344;
+    uint32_t s1 = 0x55667788;
+    uint32_t fpscr = 0x90000000;
+    uint32_t vpr = 0x00abcdef;
+    uint32_t epsr;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x0a00ec21); /* vlstm r1, t1 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    test_arm_m_profile_disable_lazy_fp(uc);
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    memset(mem, 0, sizeof(mem));
+    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    TEST_CHECK(test_arm_load_le(mem, 4) == s0);
+    TEST_CHECK(test_arm_load_le(mem + 4, 4) == s1);
+    TEST_CHECK((test_arm_load_le(mem + 0x40, 4) & 0xf0000000) ==
+               (fpscr & 0xf0000000));
+    TEST_CHECK(test_arm_load_le(mem + 0x44, 4) == vpr);
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0, "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    memset(mem, 0, sizeof(mem));
+    s0 = 0x89abcdef;
+    s1 = 0x76543210;
+    fpscr = 0x60000000;
+    vpr = 0x00123456;
+    test_arm_emit32(code, 0, 0x0a80ec21); /* vlstm r1, t2 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    test_arm_m_profile_disable_lazy_fp(uc);
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    memset(mem, 0, sizeof(mem));
+    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
+    TEST_CHECK(test_arm_load_le(mem, 4) == s0);
+    TEST_CHECK(test_arm_load_le(mem + 4, 4) == s1);
+    TEST_CHECK((test_arm_load_le(mem + 0x40, 4) & 0xf0000000) ==
+               (fpscr & 0xf0000000));
+    TEST_CHECK(test_arm_load_le(mem + 0x44, 4) == vpr);
+    OK(uc_close(uc));
+
+    memset(mem, 0, sizeof(mem));
+    test_arm_store_le(mem, 4, 0x0badc0de);
+    test_arm_store_le(mem + 4, 4, 0xf00d1234);
+    test_arm_store_le(mem + 0x40, 4, 0xa0000000);
+    test_arm_store_le(mem + 0x44, 4, 0x00fedcba);
+    s0 = 0;
+    s1 = 0;
+    fpscr = 0;
+    vpr = 0;
+    test_arm_emit32(code, 0, 0x0a80ec31); /* vlldm r1, t2 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    test_arm_m_profile_disable_lazy_fp(uc);
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_read(uc, UC_ARM_REG_S1, &s1));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    TEST_CHECK(s0 == 0x0badc0de);
+    TEST_CHECK(s1 == 0xf00d1234);
+    TEST_CHECK((fpscr & 0xf0000000) == 0xa0000000);
+    TEST_CHECK(vpr == 0x00fedcba);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_fpcxt_sysreg(void)
+{
+    const uint32_t control_fpca = 1U << 2;
+    const uint32_t control_sfpa = 1U << 3;
+    const uint32_t fpcr_v = 1U << 28;
+    const uint32_t fpcr_c = 1U << 29;
+    const uint32_t fpcr_z = 1U << 30;
+    const uint32_t fpcr_n = 1U << 31;
+    const uint32_t nzcv_mask = fpcr_n | fpcr_z | fpcr_c | fpcr_v;
+    const uint64_t data_addr = code_start + 0x1000;
+    const uint32_t fpdscr_ns = 0x00070000;
+    uc_engine *uc;
+    ARMCPU *cpu;
+    CPUARMState *env;
+    uint8_t code[8];
+    uint8_t mem[16] = { 0 };
+    uint32_t fpscr_initial = 0x0a040000 | fpcr_n | fpcr_c;
+    uint32_t fpcxt_write = 0x86040000;
+    uint32_t expected_write = fpcxt_write & ~nzcv_mask;
+    uint32_t expected_read;
+    uint32_t r1 = 0;
+    uint32_t r2 = fpcxt_write;
+    uint32_t r3 = 0;
+    uint32_t fpscr = fpscr_initial;
+    uint32_t rbase = (uint32_t)data_addr;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x3a10eeff); /* vmrs r3,fpcxt_s */
+    test_arm_emit32(code, 4, 0x2a10eeef); /* vmsr fpcxt_s,r2 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    expected_read = (fpscr & ~nzcv_mask) | 0x80000000;
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R3, &r3));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(r3 == expected_read,
+                "r3=0x%08x", r3);
+    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) != 0,
+                "control=0x%08x", env->v7m.control[M_REG_S]);
+    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_write,
+                "fpscr=0x%08x expected=0x%08x", fpscr, expected_write);
+    TEST_CHECK_((fpscr & nzcv_mask) == 0, "fpscr=0x%08x", fpscr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x1a10eefe); /* vmrs r1,fpcxt_ns */
+    test_arm_emit32(code, 4, 0x0a10eeee); /* vmsr fpcxt_ns,r0 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
+    env->v7m.fpccr[M_REG_NS] |= R_V7M_FPCCR_ASPEN_MASK;
+    env->v7m.control[M_REG_S] &= ~control_fpca;
+    fpscr = 0x01030000;
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    expected_read = fpscr & ~nzcv_mask;
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &fpcxt_write));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(r1 == fpdscr_ns, "r1=0x%08x", r1);
+    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_read,
+                "fpscr=0x%08x", fpscr);
+    TEST_CHECK_((env->v7m.control[M_REG_S] & control_fpca) == 0,
+                "control=0x%08x", env->v7m.control[M_REG_S]);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x1a10eefe); /* vmrs r1,fpcxt_ns */
+    test_arm_emit32(code, 4, 0x0a10eeee); /* vmsr fpcxt_ns,r0 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
+    env->v7m.control[M_REG_S] &= ~control_sfpa;
+    fpscr = fpscr_initial;
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    expected_read = fpscr & ~nzcv_mask;
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &fpcxt_write));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(r1 == expected_read, "r1=0x%08x", r1);
+    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) != 0,
+                "control=0x%08x", env->v7m.control[M_REG_S]);
+    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_write,
+                "fpscr=0x%08x expected=0x%08x", fpscr, expected_write);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0xef80edc1); /* vstr fpcxt_s,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
+    fpscr = fpscr_initial;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &rbase));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    expected_read = (fpscr & ~nzcv_mask) | 0x80000000;
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(test_arm_load_le(mem, 4) == expected_read,
+                "stored=0x%08x", test_arm_load_le(mem, 4));
+    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) == 0,
+                "control=0x%08x", env->v7m.control[M_REG_S]);
+    TEST_CHECK_((fpscr & ~nzcv_mask) == fpdscr_ns,
+                "fpscr=0x%08x", fpscr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0xef80edd1); /* vldr fpcxt_s,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    fpscr = 0;
+    memset(mem, 0, sizeof(mem));
+    test_arm_store_le(mem, 4, fpcxt_write);
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &rbase));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) != 0,
+                "control=0x%08x", env->v7m.control[M_REG_S]);
+    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_write,
+                "fpscr=0x%08x expected=0x%08x", fpscr, expected_write);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0xcf80edc1); /* vstr fpcxt_ns,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
+    env->v7m.control[M_REG_S] &= ~control_sfpa;
+    fpscr = fpscr_initial;
+    memset(mem, 0, sizeof(mem));
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &rbase));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    expected_read = fpscr & ~nzcv_mask;
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(test_arm_load_le(mem, 4) == expected_read,
+                "stored=0x%08x", test_arm_load_le(mem, 4));
+    TEST_CHECK_((fpscr & ~nzcv_mask) == fpdscr_ns,
+                "fpscr=0x%08x", fpscr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0xcf81ecf2); /* vldr fpcxt_ns,[r2],#4 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.fpccr[M_REG_NS] |= R_V7M_FPCCR_ASPEN_MASK;
+    env->v7m.control[M_REG_S] &= ~control_fpca;
+    fpscr = 0x04460000;
+    r2 = (uint32_t)data_addr;
+    test_arm_store_le(mem, 4, fpcxt_write);
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
+    TEST_CHECK_(r2 == (uint32_t)data_addr, "initial r2=0x%08x", r2);
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    expected_read = fpscr & ~nzcv_mask;
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(r2 == (uint32_t)data_addr + 4, "r2=0x%08x", r2);
+    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_read,
+                "fpscr=0x%08x", fpscr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0xcf81ece2); /* vstr fpcxt_ns,[r2],#4 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
+    env->v7m.fpccr[M_REG_NS] |= R_V7M_FPCCR_ASPEN_MASK;
+    env->v7m.control[M_REG_S] &= ~control_fpca;
+    fpscr = 0x04460000;
+    r2 = (uint32_t)data_addr;
+    memset(mem, 0, sizeof(mem));
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    expected_read = fpscr & ~nzcv_mask;
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(r2 == (uint32_t)data_addr + 4, "r2=0x%08x", r2);
+    TEST_CHECK_(test_arm_load_le(mem, 4) == fpdscr_ns,
+                "stored=0x%08x", test_arm_load_le(mem, 4));
+    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_read,
+                "fpscr=0x%08x", fpscr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x1a10eefe); /* vmrs r1,fpcxt_ns */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
+    cpu = ARM_CPU(uc->cpu);
+    env = &cpu->env;
+    env->v7m.control[M_REG_S] |= control_fpca;
+    env->v7m.fpccr[M_REG_NS] &= ~R_V7M_FPCCR_ASPEN_MASK;
+    env->v7m.cpacr[M_REG_S] &= ~(3U << 20);
+    env->v7m.cpacr[M_REG_NS] &= ~(3U << 20);
+    FIELD_DP32(env->hflags, TBFLAG_ANY, FPEXC_EL, 1, env->hflags);
+    FIELD_DP32(env->hflags, TBFLAG_M32, LSPACT, 0, env->hflags);
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_vscclrm(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t active_vpr = 0x00abcdef;
+    const uint32_t inactive_vpr = 0x00123456;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint32_t s0 = 0x11111111;
+    uint32_t s1 = 0x22222222;
+    uint32_t s2 = 0x33333333;
+    uint32_t s3 = 0x44444444;
+    uint32_t s4 = 0x55555555;
+    uint32_t vpr;
+    uint32_t epsr;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x0a01ec9f); /* vscclrm s0,#1 */
+    test_arm_emit32(code, 4, 0x1b02ec9f); /* vscclrm d1,#1 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    test_arm_m_profile_activate_fp_context(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
+    OK(uc_reg_write(uc, UC_ARM_REG_S2, &s2));
+    OK(uc_reg_write(uc, UC_ARM_REG_S3, &s3));
+    OK(uc_reg_write(uc, UC_ARM_REG_S4, &s4));
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &active_vpr));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_read(uc, UC_ARM_REG_S1, &s1));
+    OK(uc_reg_read(uc, UC_ARM_REG_S2, &s2));
+    OK(uc_reg_read(uc, UC_ARM_REG_S3, &s3));
+    OK(uc_reg_read(uc, UC_ARM_REG_S4, &s4));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    TEST_CHECK(s0 == 0);
+    TEST_CHECK(s1 == 0x22222222);
+    TEST_CHECK(s2 == 0);
+    TEST_CHECK(s3 == 0);
+    TEST_CHECK(s4 == 0x55555555);
+    TEST_CHECK(vpr == 0);
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0, "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    s0 = 0x77777777;
+    test_arm_emit32(code, 0, 0x0a01ec9f); /* vscclrm s0,#1 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &inactive_vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_S0, &s0));
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    TEST_CHECK(s0 == 0x77777777);
+    TEST_CHECK(vpr == inactive_vpr);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_mve_expected_logic(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        const uint8_t *n, const uint8_t *m,
+                                        char op, uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i++) {
+        uint8_t value;
+
+        switch (op) {
+        case '&':
+            value = n[i] & m[i];
+            break;
+        case 'b':
+            value = n[i] & ~m[i];
+            break;
+        case '|':
+            value = n[i] | m[i];
+            break;
+        case 'o':
+            value = n[i] | ~m[i];
+            break;
+        case '^':
+            value = n[i] ^ m[i];
+            break;
+        default:
+            value = 0;
+            break;
+        }
+        if (mask & (1U << i)) {
+            expected[i] = value;
+        }
+    }
+}
+
+static void test_arm_mve_expected_addsub(uint8_t *expected,
+                                         const uint8_t *initial,
+                                         const uint8_t *n,
+                                         const uint8_t *m,
+                                         unsigned esize, bool sub,
+                                         uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        uint32_t value = sub ? lhs - rhs : lhs + rhs;
+        uint8_t lane[4];
+        unsigned b;
+
+        test_arm_store_le(lane, esize, value);
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = lane[b];
+            }
+        }
+    }
+}
+
+static void test_arm_mve_expected_mul(uint8_t *expected,
+                                      const uint8_t *initial,
+                                      const uint8_t *n, const uint8_t *m,
+                                      unsigned esize, uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        uint32_t value = lhs * rhs;
+        uint8_t lane[4];
+        unsigned b;
+
+        test_arm_store_le(lane, esize, value);
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = lane[b];
+            }
+        }
+    }
+}
+
+static void test_arm_mve_expected_scalar_2op(uint8_t *expected,
+                                             const uint8_t *initial,
+                                             const uint8_t *n,
+                                             uint32_t scalar,
+                                             unsigned esize, char op,
+                                             uint16_t mask)
+{
+    uint32_t smask = esize == 4 ? UINT32_MAX : ((1U << (esize * 8)) - 1);
+    uint32_t rhs = scalar & smask;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t value;
+        uint8_t lane[4];
+        unsigned b;
+
+        switch (op) {
+        case '+':
+            value = lhs + rhs;
+            break;
+        case '-':
+            value = lhs - rhs;
+            break;
+        case '*':
+            value = lhs * rhs;
+            break;
+        default:
+            value = 0;
+            break;
+        }
+
+        test_arm_store_le(lane, esize, value);
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = lane[b];
+            }
+        }
+    }
+}
+
+static uint32_t test_arm_reverse_bits(uint32_t value, unsigned bits)
+{
+    uint32_t result = 0;
+    unsigned bit;
+
+    for (bit = 0; bit < bits; bit++) {
+        result <<= 1;
+        result |= (value >> bit) & 1U;
+    }
+
+    return result;
+}
+
+static int64_t test_arm_sign_extend(uint32_t value, unsigned bits)
+{
+    uint64_t sign = 1ULL << (bits - 1);
+    uint64_t mask = bits == 32 ? UINT32_MAX : ((1ULL << bits) - 1);
+
+    value &= (uint32_t)mask;
+    return (int64_t)((value ^ sign) - sign);
+}
+
+static void test_arm_mve_expected_mulh(uint8_t *expected,
+                                       const uint8_t *initial,
+                                       const uint8_t *n,
+                                       const uint8_t *m,
+                                       unsigned esize, bool is_signed,
+                                       bool rounded, uint16_t mask)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        uint32_t result;
+        unsigned b;
+
+        if (is_signed) {
+            int64_t slhs = test_arm_sign_extend(lhs, bits);
+            int64_t srhs = test_arm_sign_extend(rhs, bits);
+            int64_t sresult = slhs * srhs;
+
+            if (rounded) {
+                sresult += (int64_t)(1ULL << (bits - 1));
+            }
+            result = (uint32_t)(sresult >> bits);
+        } else {
+            uint64_t uresult = (uint64_t)lhs * rhs;
+
+            if (rounded) {
+                uresult += 1ULL << (bits - 1);
+            }
+            result = (uint32_t)(uresult >> bits);
+        }
+
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = (uint8_t)(result >> (b * 8));
+            }
+        }
+    }
+}
+
+static void test_arm_mve_store_masked(uint8_t *expected, size_t offset,
+                                      unsigned size, uint64_t value,
+                                      uint16_t mask)
+{
+    uint8_t lane[8];
+    unsigned b;
+
+    if (size == 8) {
+        test_arm_store_le64(lane, value);
+    } else {
+        test_arm_store_le(lane, size, (uint32_t)value);
+    }
+    for (b = 0; b < size; b++) {
+        if (mask & (1U << (offset + b))) {
+            expected[offset + b] = lane[b];
+        }
+    }
+}
+
+static void test_arm_mve_expected_vbrsr(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        const uint8_t *n, uint32_t scalar,
+                                        unsigned esize, uint16_t mask)
+{
+    unsigned bits = esize * 8;
+    uint32_t count = scalar & 0xff;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t value = 0;
+
+        if (count != 0) {
+            value = test_arm_reverse_bits(test_arm_load_le(n + i, esize),
+                                          bits);
+            if (count < bits) {
+                value >>= bits - count;
+            }
+        }
+        test_arm_mve_store_masked(expected, i, esize, value, mask);
+    }
+}
+
+static void test_arm_mve_expected_scalar_acc(uint8_t *expected,
+                                             const uint8_t *initial,
+                                             const uint8_t *n,
+                                             uint32_t scalar, unsigned esize,
+                                             bool sub, uint16_t mask)
+{
+    uint64_t smask = esize == 4 ? UINT32_MAX : ((1ULL << (esize * 8)) - 1);
+    uint64_t m = scalar & smask;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint64_t d = test_arm_load_le(initial + i, esize);
+        uint64_t value = test_arm_load_le(n + i, esize);
+        uint64_t result = sub ? value * d + m : value * m + d;
+
+        test_arm_mve_store_masked(expected, i, esize, result, mask);
+    }
+}
+
+static void test_arm_mve_expected_vmull(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        const uint8_t *n,
+                                        const uint8_t *m,
+                                        unsigned esize, bool top,
+                                        bool is_signed, uint16_t mask)
+{
+    unsigned bits = esize * 8;
+    unsigned lesize = esize * 2;
+    size_t le;
+
+    memcpy(expected, initial, 16);
+    for (le = 0; le < 16 / lesize; le++) {
+        size_t src = le * 2 + (top ? 1 : 0);
+        size_t dst = le * lesize;
+        uint32_t lhs = test_arm_load_le(n + src * esize, esize);
+        uint32_t rhs = test_arm_load_le(m + src * esize, esize);
+        uint64_t result;
+
+        if (is_signed) {
+            int64_t slhs = test_arm_sign_extend(lhs, bits);
+            int64_t srhs = test_arm_sign_extend(rhs, bits);
+
+            result = (uint64_t)(slhs * srhs);
+        } else {
+            result = (uint64_t)lhs * rhs;
+        }
+        test_arm_mve_store_masked(expected, dst, lesize, result, mask);
+    }
+}
+
+static uint64_t test_arm_mve_poly_mul(uint32_t lhs, uint32_t rhs,
+                                      unsigned bits)
+{
+    uint64_t result = 0;
+    unsigned bit;
+
+    for (bit = 0; bit < bits; bit++) {
+        if (lhs & (1U << bit)) {
+            result ^= (uint64_t)rhs << bit;
+        }
+    }
+
+    return result;
+}
+
+static void test_arm_mve_expected_vmullp(uint8_t *expected,
+                                         const uint8_t *initial,
+                                         const uint8_t *n,
+                                         const uint8_t *m,
+                                         unsigned bits, bool top,
+                                         uint16_t mask)
+{
+    unsigned esize = bits / 8;
+    unsigned lesize = esize * 2;
+    size_t le;
+
+    memcpy(expected, initial, 16);
+    for (le = 0; le < 16 / lesize; le++) {
+        size_t src = le * 2 + (top ? 1 : 0);
+        size_t dst = le * lesize;
+        uint32_t lhs = test_arm_load_le(n + src * esize, esize);
+        uint32_t rhs = test_arm_load_le(m + src * esize, esize);
+        uint64_t result = test_arm_mve_poly_mul(lhs, rhs, bits);
+
+        test_arm_mve_store_masked(expected, dst, lesize, result, mask);
+    }
+}
+
+static void test_arm_mve_expected_qdmull(uint8_t *expected,
+                                         const uint8_t *initial,
+                                         const uint8_t *n,
+                                         const uint8_t *m,
+                                         unsigned esize, bool top,
+                                         uint16_t mask, bool *qc)
+{
+    const int64_t int64_min = -0x7fffffffffffffffLL - 1;
+    const int64_t int64_max = 0x7fffffffffffffffLL;
+    unsigned bits = esize * 8;
+    unsigned lesize = esize * 2;
+    unsigned satmask = esize == 2 ? (top ? (1U << 2) : 1U) :
+        ((1U << 4) | 1U);
+    int64_t min = lesize == 8 ? int64_min : -(1LL << (lesize * 8 - 1));
+    int64_t max = lesize == 8 ? int64_max : (1LL << (lesize * 8 - 1)) - 1;
+    size_t le;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (le = 0; le < 16 / lesize; le++) {
+        size_t src = le * 2 + (top ? 1 : 0);
+        size_t dst = le * lesize;
+        int64_t lhs = test_arm_sign_extend(test_arm_load_le(n + src * esize,
+                                                            esize), bits);
+        int64_t rhs = test_arm_sign_extend(test_arm_load_le(m + src * esize,
+                                                            esize), bits);
+        int64_t product = lhs * rhs;
+        int64_t result;
+        bool saturated = false;
+
+        if (esize == 4) {
+            if (product > int64_max / 2) {
+                result = int64_max;
+                saturated = true;
+            } else if (product < int64_min / 2) {
+                result = int64_min;
+                saturated = true;
+            } else {
+                result = product * 2;
+            }
+        } else {
+            result = product * 2;
+            if (result > max) {
+                result = max;
+                saturated = true;
+            } else if (result < min) {
+                result = min;
+                saturated = true;
+            }
+        }
+
+        if (saturated && ((mask >> dst) & satmask)) {
+            *qc = true;
+        }
+        test_arm_mve_store_masked(expected, dst, lesize, (uint64_t)result,
+                                  mask);
+    }
+}
+
+static void test_arm_mve_expected_qdmlah(uint8_t *expected,
+                                         const uint8_t *initial,
+                                         const uint8_t *n, uint32_t scalar,
+                                         unsigned esize, bool sub,
+                                         bool rounded, uint16_t mask,
+                                         bool *qc)
+{
+    const int64_t int64_min = -0x7fffffffffffffffLL - 1;
+    const int64_t int64_max = 0x7fffffffffffffffLL;
+    unsigned bits = esize * 8;
+    int64_t min = bits == 32 ? int64_min : -(1LL << (bits * 2 - 1));
+    int64_t max = bits == 32 ? int64_max : (1LL << (bits * 2 - 1)) - 1;
+    int64_t m = test_arm_sign_extend(scalar, bits);
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (i = 0; i < 16; i += esize) {
+        int64_t d = test_arm_sign_extend(test_arm_load_le(initial + i, esize),
+                                         bits);
+        int64_t value = test_arm_sign_extend(test_arm_load_le(n + i, esize),
+                                             bits);
+        int64_t product_lhs = value;
+        int64_t product_rhs = sub ? d : m;
+        int64_t addend = sub ? m : d;
+        int64_t result = product_lhs * product_rhs * 2;
+        bool saturated = false;
+
+        result += addend * (1LL << bits);
+        if (rounded) {
+            result += 1LL << (bits - 1);
+        }
+        if (result > max) {
+            result = max;
+            saturated = true;
+        } else if (result < min) {
+            result = min;
+            saturated = true;
+        }
+        result >>= bits;
+        if (saturated && (mask & (1U << i))) {
+            *qc = true;
+        }
+        test_arm_mve_store_masked(expected, i, esize, (uint64_t)result, mask);
+    }
+}
+
+static void test_arm_mve_expected_minmax(uint8_t *expected,
+                                         const uint8_t *initial,
+                                         const uint8_t *n,
+                                         const uint8_t *m,
+                                         unsigned esize, bool is_signed,
+                                         bool is_min, uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        uint32_t result;
+        unsigned b;
+
+        if (is_signed) {
+            int64_t slhs = test_arm_sign_extend(lhs, esize * 8);
+            int64_t srhs = test_arm_sign_extend(rhs, esize * 8);
+            int64_t sresult = is_min ?
+                (slhs <= srhs ? slhs : srhs) :
+                (slhs >= srhs ? slhs : srhs);
+
+            result = (uint32_t)sresult;
+        } else {
+            result = is_min ?
+                (lhs <= rhs ? lhs : rhs) :
+                (lhs >= rhs ? lhs : rhs);
+        }
+
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = (uint8_t)(result >> (b * 8));
+            }
+        }
+    }
+}
+
+static void test_arm_mve_expected_abd(uint8_t *expected,
+                                      const uint8_t *initial,
+                                      const uint8_t *n,
+                                      const uint8_t *m,
+                                      unsigned esize, bool is_signed,
+                                      uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        uint32_t result;
+        unsigned b;
+
+        if (is_signed) {
+            int64_t slhs = test_arm_sign_extend(lhs, esize * 8);
+            int64_t srhs = test_arm_sign_extend(rhs, esize * 8);
+            uint64_t diff = slhs >= srhs ? slhs - srhs : srhs - slhs;
+
+            result = (uint32_t)diff;
+        } else {
+            result = lhs >= rhs ? lhs - rhs : rhs - lhs;
+        }
+
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = (uint8_t)(result >> (b * 8));
+            }
+        }
+    }
+}
+
+static void test_arm_mve_expected_halving(uint8_t *expected,
+                                          const uint8_t *initial,
+                                          const uint8_t *n,
+                                          const uint8_t *m,
+                                          unsigned esize, bool is_signed,
+                                          bool sub, bool rounded,
+                                          uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        uint32_t result;
+        unsigned b;
+
+        if (is_signed) {
+            int64_t slhs = test_arm_sign_extend(lhs, esize * 8);
+            int64_t srhs = test_arm_sign_extend(rhs, esize * 8);
+            int64_t sresult = sub ? slhs - srhs : slhs + srhs;
+
+            if (rounded) {
+                sresult++;
+            }
+            sresult >>= 1;
+            result = (uint32_t)sresult;
+        } else {
+            uint64_t uresult = sub ? (uint64_t)lhs - rhs :
+                (uint64_t)lhs + rhs;
+
+            if (rounded) {
+                uresult++;
+            }
+            uresult >>= 1;
+            result = (uint32_t)uresult;
+        }
+
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = (uint8_t)(result >> (b * 8));
+            }
+        }
+    }
+}
+
+static void test_arm_mve_expected_cadd(uint8_t *expected,
+                                       const uint8_t *initial,
+                                       const uint8_t *n,
+                                       const uint8_t *m,
+                                       unsigned esize, bool rot270,
+                                       bool halving, uint16_t mask)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize * 2) {
+        int64_t n0 = test_arm_sign_extend(test_arm_load_le(n + i, esize),
+                                          bits);
+        int64_t n1 = test_arm_sign_extend(test_arm_load_le(n + i + esize,
+                                                           esize), bits);
+        int64_t m0 = test_arm_sign_extend(test_arm_load_le(m + i, esize),
+                                          bits);
+        int64_t m1 = test_arm_sign_extend(test_arm_load_le(m + i + esize,
+                                                           esize), bits);
+        int64_t r0 = rot270 ? n0 + m1 : n0 - m1;
+        int64_t r1 = rot270 ? n1 - m0 : n1 + m0;
+
+        if (halving) {
+            r0 >>= 1;
+            r1 >>= 1;
+        }
+        test_arm_mve_store_masked(expected, i, esize, (uint64_t)r0, mask);
+        test_arm_mve_store_masked(expected, i + esize, esize,
+                                  (uint64_t)r1, mask);
+    }
+}
+
+static void test_arm_mve_expected_qaddsub(uint8_t *expected,
+                                          const uint8_t *initial,
+                                          const uint8_t *n,
+                                          const uint8_t *m,
+                                          unsigned esize, bool is_signed,
+                                          bool sub, uint16_t mask, bool *qc)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        uint32_t result;
+        bool saturated = false;
+        unsigned b;
+
+        if (is_signed) {
+            int64_t min = -(1LL << (bits - 1));
+            int64_t max = (1LL << (bits - 1)) - 1;
+            int64_t slhs = test_arm_sign_extend(lhs, bits);
+            int64_t srhs = test_arm_sign_extend(rhs, bits);
+            int64_t sresult = sub ? slhs - srhs : slhs + srhs;
+
+            if (sresult > max) {
+                sresult = max;
+                saturated = true;
+            } else if (sresult < min) {
+                sresult = min;
+                saturated = true;
+            }
+            result = (uint32_t)sresult;
+        } else {
+            uint64_t max = bits == 32 ? UINT32_MAX : ((1ULL << bits) - 1);
+            uint64_t uresult;
+
+            if (sub && lhs < rhs) {
+                uresult = 0;
+                saturated = true;
+            } else {
+                uresult = sub ? (uint64_t)lhs - rhs : (uint64_t)lhs + rhs;
+                if (uresult > max) {
+                    uresult = max;
+                    saturated = true;
+                }
+            }
+            result = (uint32_t)uresult;
+        }
+
+        if (saturated && (mask & (1U << i))) {
+            *qc = true;
+        }
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = (uint8_t)(result >> (b * 8));
+            }
+        }
+    }
+}
+
+static void test_arm_mve_expected_shift(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        const uint8_t *values,
+                                        const uint8_t *shifts,
+                                        unsigned esize, bool is_signed,
+                                        bool rounded, uint16_t mask)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t value = test_arm_load_le(values + i, esize);
+        int8_t shift = (int8_t)test_arm_load_le(shifts + i, esize);
+        uint32_t result;
+        unsigned b;
+
+        if (is_signed) {
+            int64_t svalue = test_arm_sign_extend(value, bits);
+            int64_t sresult;
+
+            if (shift <= -(int)bits) {
+                sresult = rounded ? 0 : (svalue < 0 ? -1 : 0);
+            } else if (shift < 0) {
+                if (rounded) {
+                    svalue >>= -shift - 1;
+                    sresult = (svalue >> 1) + (svalue & 1);
+                } else {
+                    sresult = svalue >> -shift;
+                }
+            } else if (shift < (int)bits) {
+                sresult = (uint32_t)value << shift;
+            } else {
+                sresult = 0;
+            }
+            result = (uint32_t)sresult;
+        } else {
+            uint64_t uvalue = value;
+            uint64_t uresult;
+
+            if (shift <= -((int)bits + rounded)) {
+                uresult = 0;
+            } else if (shift < 0) {
+                if (rounded) {
+                    uvalue >>= -shift - 1;
+                    uresult = (uvalue >> 1) + (uvalue & 1);
+                } else {
+                    uresult = uvalue >> -shift;
+                }
+            } else if (shift < (int)bits) {
+                uresult = uvalue << shift;
+            } else {
+                uresult = 0;
+            }
+            result = (uint32_t)uresult;
+        }
+
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = (uint8_t)(result >> (b * 8));
+            }
+        }
+    }
+}
+
+static uint32_t test_arm_mve_expected_qshift_scalar_lane(uint32_t src,
+                                                        unsigned bits,
+                                                        int8_t shift,
+                                                        bool is_signed,
+                                                        bool rounded,
+                                                        bool *saturated);
+
+static void test_arm_mve_expected_qshift(uint8_t *expected,
+                                         const uint8_t *initial,
+                                         const uint8_t *values,
+                                         const uint8_t *shifts,
+                                         unsigned esize, bool is_signed,
+                                         bool rounded, uint16_t pred,
+                                         bool *qc);
+
+static void test_arm_mve_fill_scalar_shift(uint8_t *shifts, unsigned esize,
+                                           uint32_t rm)
+{
+    size_t i;
+
+    for (i = 0; i < 16; i += esize) {
+        test_arm_store_le(shifts + i, esize, rm);
+    }
+}
+
+static void test_arm_mve_expected_qdmulh(uint8_t *expected,
+                                          const uint8_t *initial,
+                                          const uint8_t *n,
+                                          const uint8_t *m,
+                                          unsigned esize, bool rounded,
+                                         uint16_t mask, bool *qc)
+{
+    unsigned bits = esize * 8;
+    int64_t min = -(1LL << (bits - 1));
+    int64_t max = (1LL << (bits - 1)) - 1;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (i = 0; i < 16; i += esize) {
+        int64_t lhs = test_arm_sign_extend(test_arm_load_le(n + i, esize),
+                                           bits);
+        int64_t rhs = test_arm_sign_extend(test_arm_load_le(m + i, esize),
+                                           bits);
+        int64_t result = lhs * rhs;
+        bool saturated = false;
+        unsigned b;
+
+        if (rounded) {
+            result += 1LL << (bits - 2);
+        }
+        result >>= bits - 1;
+        if (result > max) {
+            result = max;
+            saturated = true;
+        } else if (result < min) {
+            result = min;
+            saturated = true;
+        }
+        if (saturated && (mask & (1U << i))) {
+            *qc = true;
+        }
+        for (b = 0; b < esize; b++) {
+            if (mask & (1U << (i + b))) {
+                expected[i + b] = (uint8_t)((uint64_t)result >> (b * 8));
+            }
+        }
+    }
+}
+
+static bool test_arm_sadd64_overflow(int64_t lhs, int64_t rhs, int64_t *ret)
+{
+    uint64_t ulhs = (uint64_t)lhs;
+    uint64_t urhs = (uint64_t)rhs;
+    uint64_t ures = ulhs + urhs;
+
+    *ret = (int64_t)ures;
+    return ((~(ulhs ^ urhs) & (ulhs ^ ures)) >> 63) != 0;
+}
+
+static bool test_arm_ssub64_overflow(int64_t lhs, int64_t rhs, int64_t *ret)
+{
+    uint64_t ulhs = (uint64_t)lhs;
+    uint64_t urhs = (uint64_t)rhs;
+    uint64_t ures = ulhs - urhs;
+
+    *ret = (int64_t)ures;
+    return (((ulhs ^ urhs) & (ulhs ^ ures)) >> 63) != 0;
+}
+
+static int32_t test_arm_mve_expected_qdmladh_lane(
+    int64_t a, int64_t b, int64_t c, int64_t d, unsigned esize,
+    bool subtract, bool rounded, bool *saturated)
+{
+    unsigned bits = esize * 8;
+
+    *saturated = false;
+    if (esize == 4) {
+        int64_t m1 = a * b;
+        int64_t m2 = c * d;
+        int64_t result;
+
+        if ((subtract ? test_arm_ssub64_overflow(m1, m2, &result) :
+             test_arm_sadd64_overflow(m1, m2, &result)) ||
+            test_arm_sadd64_overflow(result,
+                                     (int64_t)rounded << 30, &result) ||
+            test_arm_sadd64_overflow(result, result, &result)) {
+            *saturated = true;
+            return result < 0 ? INT32_MAX : INT32_MIN;
+        }
+        return (int32_t)(result >> 32);
+    } else {
+        int64_t min = -(1LL << (bits * 2 - 1));
+        int64_t max = (1LL << (bits * 2 - 1)) - 1;
+        int64_t result = subtract ? a * b - c * d : a * b + c * d;
+
+        result = result * 2 + ((int64_t)rounded << (bits - 1));
+        if (result > max) {
+            result = max;
+            *saturated = true;
+        } else if (result < min) {
+            result = min;
+            *saturated = true;
+        }
+        return (int32_t)(result >> bits);
+    }
+}
+
+static void test_arm_mve_expected_qdmladh(uint8_t *expected,
+                                          const uint8_t *initial,
+                                          const uint8_t *n,
+                                          const uint8_t *m,
+                                          unsigned esize, bool subtract,
+                                          bool exchange, bool rounded,
+                                          uint16_t mask, bool *qc)
+{
+    unsigned bits = esize * 8;
+    size_t e;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (e = 0; e < 16 / esize; e++) {
+        size_t off = e * esize;
+        bool saturated;
+        int64_t a;
+        int64_t b;
+        int64_t c;
+        int64_t d;
+        int32_t result;
+
+        if ((e & 1) != exchange) {
+            continue;
+        }
+        a = test_arm_sign_extend(test_arm_load_le(n + off, esize), bits);
+        b = test_arm_sign_extend(
+            test_arm_load_le(m + (e - exchange) * esize, esize), bits);
+        c = test_arm_sign_extend(
+            test_arm_load_le(n + (e + (1 - 2 * exchange)) * esize, esize),
+            bits);
+        d = test_arm_sign_extend(
+            test_arm_load_le(m + (e + (1 - exchange)) * esize, esize),
+            bits);
+        result = test_arm_mve_expected_qdmladh_lane(
+            a, b, c, d, esize, subtract, rounded, &saturated);
+        test_arm_mve_store_masked(expected, off, esize, (uint64_t)result,
+                                  mask);
+        if (saturated && (mask & (1U << off))) {
+            *qc = true;
+        }
+    }
+}
+
+static void test_arm_m55_mve_2op_run(uint32_t insn, const uint8_t *initial,
+                                     const uint8_t *n, const uint8_t *m,
+                                     const uint8_t *expected, bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t vpr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, insn);
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_2op_run_qc(uint32_t insn,
+                                        const uint8_t *initial,
+                                        const uint8_t *n, const uint8_t *m,
+                                        const uint8_t *expected, bool eci,
+                                        bool expected_qc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_qc = 1U << 27;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    uint32_t vpr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(((fpscr & fpscr_qc) != 0) == expected_qc,
+                "fpscr=0x%08x expected_qc=%d",
+                fpscr, expected_qc);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_2op_expect_error(uint32_t insn, uc_cpu_arm cpu,
+                                              uc_err expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uc_err err;
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code), cpu);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == expected,
+                "insn=0x%08x cpu=%d err=%u expected=%u",
+                insn, (int)cpu, (unsigned)err, (unsigned)expected);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_scalar_2op_run(uint32_t insn,
+                                            const uint8_t *initial,
+                                            const uint8_t *n, uint32_t rm,
+                                            const uint8_t *expected,
+                                            bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t vpr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &rm));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_scalar_2op_run_qc(uint32_t insn,
+                                               const uint8_t *initial,
+                                               const uint8_t *n, uint32_t rm,
+                                               uint32_t vpr,
+                                               const uint8_t *expected,
+                                               bool eci, bool initial_qc,
+                                               bool expected_qc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_qc = 1U << 27;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &rm));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci || initial_qc) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        if (initial_qc) {
+            OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+            fpscr |= fpscr_qc;
+            OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+        }
+        epsr = xpsr_t | eci_a0a1;
+        if (eci) {
+            OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        }
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        if (eci) {
+            OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+            TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                        "epsr=0x%08x", epsr);
+        }
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(((fpscr & fpscr_qc) != 0) == expected_qc,
+                "fpscr=0x%08x expected_qc=%d",
+                fpscr, expected_qc);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_ldst_expect_error(uint32_t insn,
+                                               uc_cpu_arm cpu,
+                                               uc_err expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uc_err err;
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code), cpu);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == expected, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_logic(void)
+{
+    const uint8_t initial[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    const uint8_t n[16] = {
+        0xf0, 0x0f, 0xaa, 0x55, 0x33, 0xcc, 0x80, 0x7f,
+        0x01, 0xfe, 0x5a, 0xa5, 0x3c, 0xc3, 0x99, 0x66,
+    };
+    const uint8_t m[16] = {
+        0x0f, 0x33, 0x55, 0xaa, 0xf0, 0x0f, 0x7f, 0x80,
+        0xff, 0x10, 0xa5, 0x5a, 0xc3, 0x3c, 0x66, 0x99,
+    };
+    uint8_t expected[16];
+
+    test_arm_mve_expected_logic(expected, initial, n, m, '&', 0xffff);
+    test_arm_m55_mve_2op_run(0x0154ef02, initial, n, m, expected, false);
+
+    test_arm_mve_expected_logic(expected, initial, n, m, 'b', 0xffff);
+    test_arm_m55_mve_2op_run(0x0154ef12, initial, n, m, expected, false);
+
+    test_arm_mve_expected_logic(expected, initial, n, m, '|', 0xffff);
+    test_arm_m55_mve_2op_run(0x0154ef22, initial, n, m, expected, false);
+
+    test_arm_mve_expected_logic(expected, initial, n, m, 'o', 0xffff);
+    test_arm_m55_mve_2op_run(0x0154ef32, initial, n, m, expected, false);
+
+    test_arm_mve_expected_logic(expected, initial, n, m, '^', 0xffff);
+    test_arm_m55_mve_2op_run(0x0154ff02, initial, n, m, expected, false);
+
+    test_arm_mve_expected_logic(expected, initial, n, m, '^', 0xff00);
+    test_arm_m55_mve_2op_run(0x0154ff02, initial, n, m, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0154ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0154ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_add_sub(void)
+{
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t n[16] = {
+        0xff, 0x00, 0x7f, 0x80, 0x34, 0x12, 0x78, 0x56,
+        0xef, 0xcd, 0xab, 0x89, 0x01, 0x00, 0x00, 0x80,
+    };
+    const uint8_t m[16] = {
+        0x02, 0x03, 0x81, 0x80, 0x02, 0x01, 0x08, 0x07,
+        0x11, 0x22, 0x33, 0x44, 0xff, 0xff, 0xff, 0x7f,
+    };
+    uint8_t expected[16];
+
+    test_arm_mve_expected_addsub(expected, initial, n, m, 1, false, 0xffff);
+    test_arm_m55_mve_2op_run(0x0844ef02, initial, n, m, expected, false);
+
+    test_arm_mve_expected_addsub(expected, initial, n, m, 2, false, 0xffff);
+    test_arm_m55_mve_2op_run(0x0844ef12, initial, n, m, expected, false);
+
+    test_arm_mve_expected_addsub(expected, initial, n, m, 4, false, 0xffff);
+    test_arm_m55_mve_2op_run(0x0844ef22, initial, n, m, expected, false);
+
+    test_arm_mve_expected_addsub(expected, initial, n, m, 1, true, 0xffff);
+    test_arm_m55_mve_2op_run(0x0844ff02, initial, n, m, expected, false);
+
+    test_arm_mve_expected_addsub(expected, initial, n, m, 2, true, 0xffff);
+    test_arm_m55_mve_2op_run(0x0844ff12, initial, n, m, expected, false);
+
+    test_arm_mve_expected_addsub(expected, initial, n, m, 4, true, 0xffff);
+    test_arm_m55_mve_2op_run(0x0844ff22, initial, n, m, expected, false);
+
+    test_arm_mve_expected_mul(expected, initial, n, m, 1, 0xffff);
+    test_arm_m55_mve_2op_run(0x0954ef02, initial, n, m, expected, false);
+
+    test_arm_mve_expected_mul(expected, initial, n, m, 2, 0xffff);
+    test_arm_m55_mve_2op_run(0x0954ef12, initial, n, m, expected, false);
+
+    test_arm_mve_expected_mul(expected, initial, n, m, 4, 0xffff);
+    test_arm_m55_mve_2op_run(0x0954ef22, initial, n, m, expected, false);
+
+    test_arm_mve_expected_addsub(expected, initial, n, m, 4, false, 0xff00);
+    test_arm_m55_mve_2op_run(0x0844ef22, initial, n, m, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0844ef62, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0954ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0954ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0844ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_scalar_2op(void)
+{
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t n[16] = {
+        0xff, 0x00, 0x7f, 0x80, 0x34, 0x12, 0x78, 0x56,
+        0xef, 0xcd, 0xab, 0x89, 0x01, 0x00, 0x00, 0x80,
+    };
+    const uint32_t scalar = 0x80010203;
+    uint8_t expected[16];
+
+    test_arm_mve_expected_scalar_2op(expected, initial, n, scalar, 1, '+',
+                                     0xffff);
+    test_arm_m55_mve_scalar_2op_run(0x0f43ee03, initial, n, scalar,
+                                    expected, false);
+
+    test_arm_mve_expected_scalar_2op(expected, initial, n, scalar, 2, '+',
+                                     0xffff);
+    test_arm_m55_mve_scalar_2op_run(0x0f43ee13, initial, n, scalar,
+                                    expected, false);
+
+    test_arm_mve_expected_scalar_2op(expected, initial, n, scalar, 4, '-',
+                                     0xffff);
+    test_arm_m55_mve_scalar_2op_run(0x1f43ee23, initial, n, scalar,
+                                    expected, false);
+
+    test_arm_mve_expected_scalar_2op(expected, initial, n, scalar, 1, '*',
+                                     0xffff);
+    test_arm_m55_mve_scalar_2op_run(0x1e63ee03, initial, n, scalar,
+                                    expected, false);
+
+    test_arm_mve_expected_scalar_2op(expected, initial, n, scalar, 2, '*',
+                                     0xffff);
+    test_arm_m55_mve_scalar_2op_run(0x1e63ee13, initial, n, scalar,
+                                    expected, false);
+
+    test_arm_mve_expected_scalar_2op(expected, initial, n, scalar, 4, '*',
+                                     0xffff);
+    test_arm_m55_mve_scalar_2op_run(0x1e63ee23, initial, n, scalar,
+                                    expected, false);
+
+    test_arm_mve_expected_scalar_2op(expected, initial, n, scalar, 4, '+',
+                                     0xff00);
+    test_arm_m55_mve_scalar_2op_run(0x0f43ee23, initial, n, scalar,
+                                    expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0f43ee43, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f4dee03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f4fee03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f43ee03, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_vbrsr(void)
+{
+    static const uint32_t vbrsr_insns[] = {
+        0x1e63fe03, 0x1e63fe13, 0x1e63fe23,
+    };
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x01, 0x55, 0xaa, 0x34, 0x12, 0xef, 0xcd,
+        0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        unsigned esize = esizes[ncase];
+        unsigned bits = esize * 8;
+
+        test_arm_mve_expected_vbrsr(expected, initial, n, bits - 1, esize,
+                                    0xffff);
+        test_arm_m55_mve_scalar_2op_run(vbrsr_insns[ncase], initial, n,
+                                        bits - 1, expected, false);
+
+        test_arm_mve_expected_vbrsr(expected, initial, n, bits, esize,
+                                    0xffff);
+        test_arm_m55_mve_scalar_2op_run(vbrsr_insns[ncase], initial, n,
+                                        bits, expected, false);
+
+        test_arm_mve_expected_vbrsr(expected, initial, n, bits + 1, esize,
+                                    0xffff);
+        test_arm_m55_mve_scalar_2op_run(vbrsr_insns[ncase], initial, n,
+                                        bits + 1, expected, false);
+    }
+
+    test_arm_mve_expected_vbrsr(expected, initial, n, 0, 1, 0xffff);
+    test_arm_m55_mve_scalar_2op_run(vbrsr_insns[0], initial, n, 0,
+                                    expected, false);
+
+    test_arm_mve_expected_vbrsr(expected, initial, n, 0x100, 2, 0xffff);
+    test_arm_m55_mve_scalar_2op_run(vbrsr_insns[1], initial, n, 0x100,
+                                    expected, false);
+
+    test_arm_mve_expected_vbrsr(expected, initial, n, 5, 4, 0x00f0);
+    test_arm_m55_mve_scalar_2op_run_qc(vbrsr_insns[2], initial, n, 5,
+                                       0x001100f0, expected, false, false,
+                                       false);
+
+    test_arm_mve_expected_vbrsr(expected, initial, n, 5, 4, 0xff00);
+    test_arm_m55_mve_scalar_2op_run(vbrsr_insns[2], initial, n, 5,
+                                    expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x1e63fe43, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x1e6dfe03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x1e6ffe03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(vbrsr_insns[0],
+                                      UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_scalar_halving_sat(void)
+{
+    static const uint32_t vhadds_insns[] = {
+        0x0f43ee02, 0x0f43ee12, 0x0f43ee22,
+    };
+    static const uint32_t vhaddu_insns[] = {
+        0x0f43fe02, 0x0f43fe12, 0x0f43fe22,
+    };
+    static const uint32_t vhsubs_insns[] = {
+        0x1f43ee02, 0x1f43ee12, 0x1f43ee22,
+    };
+    static const uint32_t vhsubu_insns[] = {
+        0x1f43fe02, 0x1f43fe12, 0x1f43fe22,
+    };
+    static const uint32_t vqadds_insns[] = {
+        0x0f63ee02, 0x0f63ee12, 0x0f63ee22,
+    };
+    static const uint32_t vqaddu_insns[] = {
+        0x0f63fe02, 0x0f63fe12, 0x0f63fe22,
+    };
+    static const uint32_t vqsubs_insns[] = {
+        0x1f63ee02, 0x1f63ee12, 0x1f63ee22,
+    };
+    static const uint32_t vqsubu_insns[] = {
+        0x1f63fe02, 0x1f63fe12, 0x1f63fe22,
+    };
+    static const uint32_t vqdmulh_insns[] = {
+        0x0e63ee03, 0x0e63ee13, 0x0e63ee23,
+    };
+    static const uint32_t vqrdmulh_insns[] = {
+        0x0e63fe03, 0x0e63fe13, 0x0e63fe23,
+    };
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t n[16] = {
+        0x81, 0x7f, 0x12, 0xf0, 0x55, 0xaa, 0x01, 0x80,
+        0xfe, 0x10, 0x33, 0xcc, 0x00, 0x00, 0x00, 0x80,
+    };
+    const uint32_t scalar = 0x8001807f;
+    const uint32_t sat_scalar = 0x80008080;
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t scalar_lanes[16];
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        unsigned esize = esizes[ncase];
+
+        test_arm_mve_fill_scalar_shift(scalar_lanes, esize, scalar);
+        test_arm_mve_expected_halving(expected, initial, n, scalar_lanes,
+                                      esize, true, false, false, 0xffff);
+        test_arm_m55_mve_scalar_2op_run(vhadds_insns[ncase], initial, n,
+                                        scalar, expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, scalar_lanes,
+                                      esize, false, false, false, 0xffff);
+        test_arm_m55_mve_scalar_2op_run(vhaddu_insns[ncase], initial, n,
+                                        scalar, expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, scalar_lanes,
+                                      esize, true, true, false, 0xffff);
+        test_arm_m55_mve_scalar_2op_run(vhsubs_insns[ncase], initial, n,
+                                        scalar, expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, scalar_lanes,
+                                      esize, false, true, false, 0xffff);
+        test_arm_m55_mve_scalar_2op_run(vhsubu_insns[ncase], initial, n,
+                                        scalar, expected, false);
+
+        test_arm_mve_fill_scalar_shift(scalar_lanes, esize, sat_scalar);
+        test_arm_mve_expected_qaddsub(expected, initial, n, scalar_lanes,
+                                      esize, true, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqadds_insns[ncase], initial, n,
+                                           sat_scalar, 0, expected, false,
+                                           false, qc);
+
+        test_arm_mve_expected_qaddsub(expected, initial, n, scalar_lanes,
+                                      esize, false, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqaddu_insns[ncase], initial, n,
+                                           sat_scalar, 0, expected, false,
+                                           false, qc);
+
+        test_arm_mve_expected_qaddsub(expected, initial, n, scalar_lanes,
+                                      esize, true, true, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqsubs_insns[ncase], initial, n,
+                                           sat_scalar, 0, expected, false,
+                                           false, qc);
+
+        test_arm_mve_expected_qaddsub(expected, initial, n, scalar_lanes,
+                                      esize, false, true, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqsubu_insns[ncase], initial, n,
+                                           sat_scalar, 0, expected, false,
+                                           false, qc);
+
+        test_arm_mve_expected_qdmulh(expected, initial, n, scalar_lanes,
+                                     esize, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqdmulh_insns[ncase], initial, n,
+                                           sat_scalar, 0, expected, false,
+                                           false, qc);
+
+        test_arm_mve_expected_qdmulh(expected, initial, n, scalar_lanes,
+                                     esize, true, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqrdmulh_insns[ncase], initial, n,
+                                           sat_scalar, 0, expected, false,
+                                           false, qc);
+    }
+
+    test_arm_mve_fill_scalar_shift(scalar_lanes, 4, sat_scalar);
+    test_arm_mve_expected_qaddsub(expected, initial, n, scalar_lanes, 4,
+                                  true, false, 0xff00, &qc);
+    test_arm_m55_mve_scalar_2op_run_qc(0x0f63ee22, initial, n, sat_scalar,
+                                       0, expected, true, false, qc);
+
+    test_arm_mve_expected_qaddsub(expected, initial, n, scalar_lanes, 4,
+                                  true, false, 0xffff, &qc);
+    test_arm_m55_mve_scalar_2op_run_qc(0x0f63ee22, initial, n, sat_scalar,
+                                       0, expected, false, true, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0f43ee42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f4dee02, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f4fee02, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f43ee02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_mulh(void)
+{
+    static const uint32_t vmulhs_insns[] = {
+        0x0e05ee03, 0x0e05ee13, 0x0e05ee23,
+    };
+    static const uint32_t vmulhu_insns[] = {
+        0x0e05fe03, 0x0e05fe13, 0x0e05fe23,
+    };
+    static const uint32_t vrmulhs_insns[] = {
+        0x1e05ee03, 0x1e05ee13, 0x1e05ee23,
+    };
+    static const uint32_t vrmulhu_insns[] = {
+        0x1e05fe03, 0x1e05fe13, 0x1e05fe23,
+    };
+    const uint8_t initial[16] = {
+        0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+        0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x7f, 0xff, 0x00, 0x34, 0x12, 0x00, 0x80,
+        0xfe, 0xff, 0x00, 0x40, 0x01, 0x00, 0xff, 0x7f,
+    };
+    const uint8_t m[16] = {
+        0x7f, 0x80, 0x01, 0xff, 0x35, 0x12, 0xff, 0x7f,
+        0x02, 0x00, 0xff, 0xbf, 0x00, 0x80, 0x00, 0x80,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_mulh(expected, initial, n, m, esizes[ncase],
+                                   true, false, 0xffff);
+        test_arm_m55_mve_2op_run(vmulhs_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_mulh(expected, initial, n, m, esizes[ncase],
+                                   false, false, 0xffff);
+        test_arm_m55_mve_2op_run(vmulhu_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_mulh(expected, initial, n, m, esizes[ncase],
+                                   true, true, 0xffff);
+        test_arm_m55_mve_2op_run(vrmulhs_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_mulh(expected, initial, n, m, esizes[ncase],
+                                   false, true, 0xffff);
+        test_arm_m55_mve_2op_run(vrmulhu_insns[ncase], initial, n, m,
+                                 expected, false);
+    }
+
+    test_arm_mve_expected_mulh(expected, initial, n, m, 4, true, true,
+                               0xff00);
+    test_arm_m55_mve_2op_run(0x1e05ee23, initial, n, m, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0e05ee43, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0e05ee03, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_vmull(void)
+{
+    static const uint32_t vmullbs_insns[] = {
+        0x0e04ee03, 0x0e04ee13, 0x0e04ee23,
+    };
+    static const uint32_t vmullbu_insns[] = {
+        0x0e04fe03, 0x0e04fe13, 0x0e04fe23,
+    };
+    static const uint32_t vmullts_insns[] = {
+        0x1e04ee03, 0x1e04ee13, 0x1e04ee23,
+    };
+    static const uint32_t vmulltu_insns[] = {
+        0x1e04fe03, 0x1e04fe13, 0x1e04fe23,
+    };
+    const uint8_t initial[16] = {
+        0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+        0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x7f, 0xff, 0x00, 0x34, 0x12, 0x00, 0x80,
+        0xfe, 0xff, 0x00, 0x40, 0x01, 0x00, 0xff, 0x7f,
+    };
+    const uint8_t m[16] = {
+        0x7f, 0x80, 0x01, 0xff, 0x35, 0x12, 0xff, 0x7f,
+        0x02, 0x00, 0xff, 0xbf, 0x00, 0x80, 0x00, 0x80,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_vmull(expected, initial, n, m, esizes[ncase],
+                                    false, true, 0xffff);
+        test_arm_m55_mve_2op_run(vmullbs_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_vmull(expected, initial, n, m, esizes[ncase],
+                                    false, false, 0xffff);
+        test_arm_m55_mve_2op_run(vmullbu_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_vmull(expected, initial, n, m, esizes[ncase],
+                                    true, true, 0xffff);
+        test_arm_m55_mve_2op_run(vmullts_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_vmull(expected, initial, n, m, esizes[ncase],
+                                    true, false, 0xffff);
+        test_arm_m55_mve_2op_run(vmulltu_insns[ncase], initial, n, m,
+                                 expected, false);
+    }
+
+    test_arm_mve_expected_vmullp(expected, initial, n, m, 8, false, 0xffff);
+    test_arm_m55_mve_2op_run(0x0e04ee33, initial, n, m, expected, false);
+
+    test_arm_mve_expected_vmullp(expected, initial, n, m, 16, false, 0xffff);
+    test_arm_m55_mve_2op_run(0x0e04fe33, initial, n, m, expected, false);
+
+    test_arm_mve_expected_vmullp(expected, initial, n, m, 8, true, 0xffff);
+    test_arm_m55_mve_2op_run(0x1e04ee33, initial, n, m, expected, false);
+
+    test_arm_mve_expected_vmullp(expected, initial, n, m, 16, true, 0xffff);
+    test_arm_m55_mve_2op_run(0x1e04fe33, initial, n, m, expected, false);
+
+    test_arm_mve_expected_vmull(expected, initial, n, m, 4, true, true,
+                                0xff00);
+    test_arm_m55_mve_2op_run(0x1e04ee23, initial, n, m, expected, true);
+
+    test_arm_mve_expected_vmull(expected, initial, initial, m, 1, false,
+                                true, 0xffff);
+    test_arm_m55_mve_2op_run(0x0e04ee01, initial, n, m, expected, false);
+
+    test_arm_mve_expected_vmull(expected, initial, n, initial, 1, false,
+                                true, 0xffff);
+    test_arm_m55_mve_2op_run(0x0e00ee03, initial, n, m, expected, false);
+
+    test_arm_m55_mve_2op_expect_error(0x0e04ee43, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0e04ee03, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_minmax(void)
+{
+    static const uint32_t vmaxs_insns[] = {
+        0x0644ef02, 0x0644ef12, 0x0644ef22,
+    };
+    static const uint32_t vmaxu_insns[] = {
+        0x0644ff02, 0x0644ff12, 0x0644ff22,
+    };
+    static const uint32_t vmins_insns[] = {
+        0x0654ef02, 0x0654ef12, 0x0654ef22,
+    };
+    static const uint32_t vminu_insns[] = {
+        0x0654ff02, 0x0654ff12, 0x0654ff22,
+    };
+    const uint8_t initial[16] = {
+        0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+        0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x7f, 0xff, 0x00, 0x34, 0x12, 0x00, 0x80,
+        0xfe, 0xff, 0x00, 0x40, 0x01, 0x00, 0xff, 0x7f,
+    };
+    const uint8_t m[16] = {
+        0x7f, 0x80, 0x01, 0xff, 0x35, 0x12, 0xff, 0x7f,
+        0x02, 0x00, 0xff, 0xbf, 0x00, 0x80, 0x00, 0x80,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_minmax(expected, initial, n, m,
+                                     esizes[ncase], true, false, 0xffff);
+        test_arm_m55_mve_2op_run(vmaxs_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_minmax(expected, initial, n, m,
+                                     esizes[ncase], false, false, 0xffff);
+        test_arm_m55_mve_2op_run(vmaxu_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_minmax(expected, initial, n, m,
+                                     esizes[ncase], true, true, 0xffff);
+        test_arm_m55_mve_2op_run(vmins_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_minmax(expected, initial, n, m,
+                                     esizes[ncase], false, true, 0xffff);
+        test_arm_m55_mve_2op_run(vminu_insns[ncase], initial, n, m,
+                                 expected, false);
+    }
+
+    test_arm_mve_expected_minmax(expected, initial, n, m, 4, false, false,
+                                 0xff00);
+    test_arm_m55_mve_2op_run(0x0644ff22, initial, n, m, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0644ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0644ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0644ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_vabd(void)
+{
+    static const uint32_t vabds_insns[] = {
+        0x0744ef02, 0x0744ef12, 0x0744ef22,
+    };
+    static const uint32_t vabdu_insns[] = {
+        0x0744ff02, 0x0744ff12, 0x0744ff22,
+    };
+    const uint8_t initial[16] = {
+        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+        0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x7f, 0xff, 0x00, 0x34, 0x12, 0x00, 0x80,
+        0xfe, 0xff, 0x00, 0x40, 0x01, 0x00, 0xff, 0x7f,
+    };
+    const uint8_t m[16] = {
+        0x7f, 0x80, 0x01, 0xff, 0x35, 0x12, 0xff, 0x7f,
+        0x02, 0x00, 0xff, 0xbf, 0x00, 0x80, 0x00, 0x80,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_abd(expected, initial, n, m, esizes[ncase],
+                                  true, 0xffff);
+        test_arm_m55_mve_2op_run(vabds_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_abd(expected, initial, n, m, esizes[ncase],
+                                  false, 0xffff);
+        test_arm_m55_mve_2op_run(vabdu_insns[ncase], initial, n, m,
+                                 expected, false);
+    }
+
+    test_arm_mve_expected_abd(expected, initial, n, m, 4, true, 0xff00);
+    test_arm_m55_mve_2op_run(0x0744ef22, initial, n, m, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0744ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0744ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0744ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_halving(void)
+{
+    static const uint32_t vhadds_insns[] = {
+        0x0044ef02, 0x0044ef12, 0x0044ef22,
+    };
+    static const uint32_t vhaddu_insns[] = {
+        0x0044ff02, 0x0044ff12, 0x0044ff22,
+    };
+    static const uint32_t vrhadds_insns[] = {
+        0x0144ef02, 0x0144ef12, 0x0144ef22,
+    };
+    static const uint32_t vrhaddu_insns[] = {
+        0x0144ff02, 0x0144ff12, 0x0144ff22,
+    };
+    static const uint32_t vhsubs_insns[] = {
+        0x0244ef02, 0x0244ef12, 0x0244ef22,
+    };
+    static const uint32_t vhsubu_insns[] = {
+        0x0244ff02, 0x0244ff12, 0x0244ff22,
+    };
+    const uint8_t initial[16] = {
+        0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+        0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x7f, 0xff, 0x00, 0x34, 0x12, 0x00, 0x80,
+        0xfe, 0xff, 0x00, 0x40, 0x01, 0x00, 0xff, 0x7f,
+    };
+    const uint8_t m[16] = {
+        0x7f, 0x80, 0x01, 0xff, 0x35, 0x12, 0xff, 0x7f,
+        0x02, 0x00, 0xff, 0xbf, 0x00, 0x80, 0x00, 0x80,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_halving(expected, initial, n, m,
+                                      esizes[ncase], true, false, false,
+                                      0xffff);
+        test_arm_m55_mve_2op_run(vhadds_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, m,
+                                      esizes[ncase], false, false, false,
+                                      0xffff);
+        test_arm_m55_mve_2op_run(vhaddu_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, m,
+                                      esizes[ncase], true, false, true,
+                                      0xffff);
+        test_arm_m55_mve_2op_run(vrhadds_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, m,
+                                      esizes[ncase], false, false, true,
+                                      0xffff);
+        test_arm_m55_mve_2op_run(vrhaddu_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, m,
+                                      esizes[ncase], true, true, false,
+                                      0xffff);
+        test_arm_m55_mve_2op_run(vhsubs_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_halving(expected, initial, n, m,
+                                      esizes[ncase], false, true, false,
+                                      0xffff);
+        test_arm_m55_mve_2op_run(vhsubu_insns[ncase], initial, n, m,
+                                 expected, false);
+    }
+
+    test_arm_mve_expected_halving(expected, initial, n, m, 4, false, true,
+                                  false, 0xff00);
+    test_arm_m55_mve_2op_run(0x0244ff22, initial, n, m, expected, true);
+
+    test_arm_mve_expected_halving(expected, initial, n, m, 4, true, false,
+                                  true, 0xff00);
+    test_arm_m55_mve_2op_run(0x0144ef22, initial, n, m, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0044ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0144ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0044ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0144ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0044ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0144ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_qaddsub(void)
+{
+    static const uint32_t vqadds_insns[] = {
+        0x0054ef02, 0x0054ef12, 0x0054ef22,
+    };
+    static const uint32_t vqaddu_insns[] = {
+        0x0054ff02, 0x0054ff12, 0x0054ff22,
+    };
+    static const uint32_t vqsubs_insns[] = {
+        0x0254ef02, 0x0254ef12, 0x0254ef22,
+    };
+    static const uint32_t vqsubu_insns[] = {
+        0x0254ff02, 0x0254ff12, 0x0254ff22,
+    };
+    const uint8_t initial[16] = {
+        0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+        0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+    };
+    const uint8_t n[16] = {
+        0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x80,
+        0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t m[16] = {
+        0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+        0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_qaddsub(expected, initial, n, m,
+                                      esizes[ncase], true, false, 0xffff,
+                                      &qc);
+        test_arm_m55_mve_2op_run_qc(vqadds_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qaddsub(expected, initial, n, m,
+                                      esizes[ncase], false, false, 0xffff,
+                                      &qc);
+        test_arm_m55_mve_2op_run_qc(vqaddu_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qaddsub(expected, initial, n, m,
+                                      esizes[ncase], true, true, 0xffff,
+                                      &qc);
+        test_arm_m55_mve_2op_run_qc(vqsubs_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qaddsub(expected, initial, n, m,
+                                      esizes[ncase], false, true, 0xffff,
+                                      &qc);
+        test_arm_m55_mve_2op_run_qc(vqsubu_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+    }
+
+    test_arm_mve_expected_qaddsub(expected, initial, n, m, 4, true, false,
+                                  0xff00, &qc);
+    test_arm_m55_mve_2op_run_qc(0x0054ef22, initial, n, m, expected, true,
+                                qc);
+
+    test_arm_m55_mve_2op_expect_error(0x0054ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0054ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0054ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_shift(void)
+{
+    static const uint32_t vshls_insns[] = {
+        0x0444ef02, 0x0444ef12, 0x0444ef22,
+    };
+    static const uint32_t vshlu_insns[] = {
+        0x0444ff02, 0x0444ff12, 0x0444ff22,
+    };
+    static const uint32_t vrshls_insns[] = {
+        0x0544ef02, 0x0544ef12, 0x0544ef22,
+    };
+    static const uint32_t vrshlu_insns[] = {
+        0x0544ff02, 0x0544ff12, 0x0544ff22,
+    };
+    static const uint32_t vqshls_insns[] = {
+        0x0454ef02, 0x0454ef12, 0x0454ef22,
+    };
+    static const uint32_t vqshlu_insns[] = {
+        0x0454ff02, 0x0454ff12, 0x0454ff22,
+    };
+    static const uint32_t vqrshls_insns[] = {
+        0x0554ef02, 0x0554ef12, 0x0554ef22,
+    };
+    static const uint32_t vqrshlu_insns[] = {
+        0x0554ff02, 0x0554ff12, 0x0554ff22,
+    };
+    const uint8_t initial[16] = {
+        0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+        0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+    };
+    const uint8_t values[16] = {
+        0x81, 0x7f, 0x40, 0x03, 0xff, 0x80, 0x55, 0xaa,
+        0x34, 0x12, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x80,
+    };
+    const uint8_t shifts[16] = {
+        0x01, 0xff, 0xfe, 0x08, 0xf8, 0x00, 0x03, 0xfd,
+        0x10, 0xf0, 0x04, 0xfc, 0x20, 0xe0, 0x07, 0xf9,
+    };
+    const uint8_t skip_values[16] = {
+        0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xc0,
+        0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    };
+    const uint8_t skip_shifts[16] = {
+        0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+        0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_shift(expected, initial, values, shifts,
+                                    esizes[ncase], true, false, 0xffff);
+        test_arm_m55_mve_2op_run(vshls_insns[ncase], initial, shifts,
+                                 values, expected, false);
+
+        test_arm_mve_expected_shift(expected, initial, values, shifts,
+                                    esizes[ncase], false, false, 0xffff);
+        test_arm_m55_mve_2op_run(vshlu_insns[ncase], initial, shifts,
+                                 values, expected, false);
+
+        test_arm_mve_expected_shift(expected, initial, values, shifts,
+                                    esizes[ncase], true, true, 0xffff);
+        test_arm_m55_mve_2op_run(vrshls_insns[ncase], initial, shifts,
+                                 values, expected, false);
+
+        test_arm_mve_expected_shift(expected, initial, values, shifts,
+                                    esizes[ncase], false, true, 0xffff);
+        test_arm_m55_mve_2op_run(vrshlu_insns[ncase], initial, shifts,
+                                 values, expected, false);
+
+        test_arm_mve_expected_qshift(expected, initial, values, shifts,
+                                     esizes[ncase], true, false, 0xffff,
+                                     &qc);
+        test_arm_m55_mve_2op_run_qc(vqshls_insns[ncase], initial, shifts,
+                                    values, expected, false, qc);
+
+        test_arm_mve_expected_qshift(expected, initial, values, shifts,
+                                     esizes[ncase], false, false, 0xffff,
+                                     &qc);
+        test_arm_m55_mve_2op_run_qc(vqshlu_insns[ncase], initial, shifts,
+                                    values, expected, false, qc);
+
+        test_arm_mve_expected_qshift(expected, initial, values, shifts,
+                                     esizes[ncase], true, true, 0xffff,
+                                     &qc);
+        test_arm_m55_mve_2op_run_qc(vqrshls_insns[ncase], initial, shifts,
+                                    values, expected, false, qc);
+
+        test_arm_mve_expected_qshift(expected, initial, values, shifts,
+                                     esizes[ncase], false, true, 0xffff,
+                                     &qc);
+        test_arm_m55_mve_2op_run_qc(vqrshlu_insns[ncase], initial, shifts,
+                                    values, expected, false, qc);
+    }
+
+    test_arm_mve_expected_shift(expected, initial, values, shifts, 4, false,
+                                true, 0xff00);
+    test_arm_m55_mve_2op_run(0x0544ff22, initial, shifts, values, expected,
+                             true);
+
+    test_arm_mve_expected_qshift(expected, initial, skip_values, skip_shifts,
+                                 4, true, false, 0xff00, &qc);
+    test_arm_m55_mve_2op_run_qc(0x0454ef22, initial, skip_shifts,
+                                skip_values, expected, true, qc);
+
+    test_arm_m55_mve_2op_expect_error(0x0444ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0444ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0444ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0454ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0454ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0454ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_qdmulh(void)
+{
+    static const uint32_t vqdmulh_insns[] = {
+        0x0b44ef02, 0x0b44ef12, 0x0b44ef22,
+    };
+    static const uint32_t vqrdmulh_insns[] = {
+        0x0b44ff02, 0x0b44ff12, 0x0b44ff22,
+    };
+    const uint8_t initial[16] = {
+        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+        0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x7f, 0xff, 0x7f, 0x00, 0x80, 0x00, 0x40,
+        0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, 0x7f,
+    };
+    const uint8_t m[16] = {
+        0x80, 0x7f, 0xff, 0x7f, 0x00, 0x80, 0x00, 0x40,
+        0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, 0x7f,
+    };
+    const uint8_t eci_n[16] = {
+        0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+        0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x20,
+    };
+    const uint8_t eci_m[16] = {
+        0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+        0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x20,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_qdmulh(expected, initial, n, m,
+                                     esizes[ncase], false, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqdmulh_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmulh(expected, initial, n, m,
+                                     esizes[ncase], true, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqrdmulh_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+    }
+
+    test_arm_mve_expected_qdmulh(expected, initial, eci_n, eci_m, 4, false,
+                                 0xff00, &qc);
+    test_arm_m55_mve_2op_run_qc(0x0b44ef22, initial, eci_n, eci_m,
+                                expected, true, qc);
+
+    test_arm_m55_mve_2op_expect_error(0x0b44ef32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0b44ef42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0b44ef02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_qdmladh(void)
+{
+    static const uint32_t vqdmladh_insns[] = {
+        0x0e04ee02, 0x0e04ee12, 0x0e04ee22,
+    };
+    static const uint32_t vqdmladhx_insns[] = {
+        0x1e04ee02, 0x1e04ee12, 0x1e04ee22,
+    };
+    static const uint32_t vqrdmladh_insns[] = {
+        0x0e05ee02, 0x0e05ee12, 0x0e05ee22,
+    };
+    static const uint32_t vqrdmladhx_insns[] = {
+        0x1e05ee02, 0x1e05ee12, 0x1e05ee22,
+    };
+    static const uint32_t vqdmlsdh_insns[] = {
+        0x0e04fe02, 0x0e04fe12, 0x0e04fe22,
+    };
+    static const uint32_t vqdmlsdhx_insns[] = {
+        0x1e04fe02, 0x1e04fe12, 0x1e04fe22,
+    };
+    static const uint32_t vqrdmlsdh_insns[] = {
+        0x0e05fe02, 0x0e05fe12, 0x0e05fe22,
+    };
+    static const uint32_t vqrdmlsdhx_insns[] = {
+        0x1e05fe02, 0x1e05fe12, 0x1e05fe22,
+    };
+    const uint8_t initial[16] = {
+        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+        0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+    };
+    const uint8_t n[16] = {
+        0x7f, 0x7f, 0x08, 0x00, 0x80, 0x80, 0x02, 0x00,
+        0x10, 0x00, 0x20, 0x00, 0xff, 0xff, 0x7f, 0x00,
+    };
+    const uint8_t m[16] = {
+        0x7f, 0x7f, 0x08, 0x00, 0x80, 0x80, 0x02, 0x00,
+        0x04, 0x00, 0x08, 0x00, 0xff, 0xff, 0x01, 0x00,
+    };
+    const uint8_t sat_n[16] = {
+        0x7f, 0x7f, 0x7f, 0x7f, 0x00, 0x00, 0x00, 0x40,
+        0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    };
+    const uint8_t sat_m[16] = {
+        0x7f, 0x7f, 0x7f, 0x7f, 0x00, 0x00, 0x00, 0x40,
+        0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    };
+    const uint8_t eci_n[16] = {
+        0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+        0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00,
+    };
+    const uint8_t eci_m[16] = {
+        0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f,
+        0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        unsigned esize = esizes[ncase];
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      false, false, false, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqdmladh_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      false, true, false, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqdmladhx_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      false, false, true, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqrdmladh_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      false, true, true, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqrdmladhx_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      true, false, false, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqdmlsdh_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      true, true, false, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqdmlsdhx_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      true, false, true, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqrdmlsdh_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmladh(expected, initial, n, m, esize,
+                                      true, true, true, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqrdmlsdhx_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+    }
+
+    test_arm_mve_expected_qdmladh(expected, initial, sat_n, sat_m, 1,
+                                  false, false, false, 0xffff, &qc);
+    test_arm_m55_mve_2op_run_qc(0x0e04ee02, initial, sat_n, sat_m,
+                                expected, false, qc);
+
+    test_arm_mve_expected_qdmladh(expected, initial, eci_n, eci_m, 4,
+                                  false, false, false, 0xff00, &qc);
+    test_arm_m55_mve_2op_run_qc(0x0e04ee22, initial, eci_n, eci_m,
+                                expected, true, qc);
+
+    test_arm_m55_mve_2op_expect_error(0x0e04ee42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0e04ee02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_qdmull(void)
+{
+    static const uint32_t vqdmullb_insns[] = {
+        0x0f05ee32, 0x0f05fe32,
+    };
+    static const uint32_t vqdmullt_insns[] = {
+        0x1f05ee32, 0x1f05fe32,
+    };
+    const uint8_t initial[16] = {
+        0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+        0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+    };
+    const uint8_t n[16] = {
+        0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+        0x00, 0x80, 0x00, 0x40, 0xff, 0xff, 0xff, 0x7f,
+    };
+    const uint8_t m[16] = {
+        0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
+        0x00, 0x80, 0x00, 0x40, 0xff, 0xff, 0xff, 0x7f,
+    };
+    const uint8_t eci_n[16] = {
+        0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x20,
+    };
+    const uint8_t eci_m[16] = {
+        0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x20,
+    };
+    const unsigned esizes[] = { 2, 4 };
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 2; ncase++) {
+        test_arm_mve_expected_qdmull(expected, initial, n, m, esizes[ncase],
+                                     false, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqdmullb_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+
+        test_arm_mve_expected_qdmull(expected, initial, n, m, esizes[ncase],
+                                     true, 0xffff, &qc);
+        test_arm_m55_mve_2op_run_qc(vqdmullt_insns[ncase], initial, n, m,
+                                    expected, false, qc);
+    }
+
+    test_arm_mve_expected_qdmull(expected, initial, eci_n, eci_m, 4, false,
+                                 0xff00, &qc);
+    test_arm_m55_mve_2op_run_qc(0x0f05fe32, initial, eci_n, eci_m, expected,
+                                true, qc);
+
+    test_arm_mve_expected_qdmull(expected, initial, initial, m, 2, false,
+                                 0xffff, &qc);
+    test_arm_m55_mve_2op_run_qc(0x0f05ee30, initial, n, m, expected, false,
+                                qc);
+
+    test_arm_m55_mve_2op_expect_error(0x0f05ee72, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f05fe30, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f01fe32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f05ee32, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_scalar_qdmull(void)
+{
+    static const uint32_t vqdmullb_insns[] = {
+        0x0f63ee32, 0x0f63fe32,
+    };
+    static const uint32_t vqdmullt_insns[] = {
+        0x1f63ee32, 0x1f63fe32,
+    };
+    const uint8_t initial[16] = {
+        0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+        0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+    };
+    const uint8_t n[16] = {
+        0x00, 0x80, 0x00, 0x80, 0xff, 0x7f, 0xff, 0x7f,
+        0x00, 0x00, 0x00, 0x80, 0xff, 0xff, 0xff, 0x7f,
+    };
+    const uint32_t scalars[] = {
+        0x80008000, 0x80000000,
+    };
+    const unsigned esizes[] = { 2, 4 };
+    uint8_t scalar_lanes[16];
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 2; ncase++) {
+        unsigned esize = esizes[ncase];
+        uint32_t scalar = scalars[ncase];
+
+        test_arm_mve_fill_scalar_shift(scalar_lanes, esize, scalar);
+        test_arm_mve_expected_qdmull(expected, initial, n, scalar_lanes,
+                                     esize, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqdmullb_insns[ncase], initial,
+                                           n, scalar, 0, expected, false,
+                                           false, qc);
+
+        test_arm_mve_expected_qdmull(expected, initial, n, scalar_lanes,
+                                     esize, true, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqdmullt_insns[ncase], initial,
+                                           n, scalar, 0, expected, false,
+                                           false, qc);
+    }
+
+    test_arm_mve_fill_scalar_shift(scalar_lanes, 4, 0x80000000);
+    test_arm_mve_expected_qdmull(expected, initial, n, scalar_lanes, 4,
+                                 false, 0xff00, &qc);
+    test_arm_m55_mve_scalar_2op_run_qc(0x0f63fe32, initial, n, 0x80000000,
+                                       0, expected, true, false, qc);
+
+    test_arm_mve_fill_scalar_shift(scalar_lanes, 2, 0x80008000);
+    test_arm_mve_expected_qdmull(expected, initial, initial, scalar_lanes, 2,
+                                 false, 0xffff, &qc);
+    test_arm_m55_mve_scalar_2op_run_qc(0x0f63ee30, initial, n, 0x80008000,
+                                       0, expected, false, false, qc);
+
+    test_arm_m55_mve_2op_expect_error(0x0f6dee32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f6fee32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f63ee72, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0fe3ee30, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f63fe30, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f63ee32, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_scalar_acc(void)
+{
+    static const uint32_t vmla_insns[] = {
+        0x0e43ee03, 0x0e43ee13, 0x0e43ee23,
+    };
+    static const uint32_t vmla_alias_insns[] = {
+        0x0e43fe03, 0x0e43fe13, 0x0e43fe23,
+    };
+    static const uint32_t vmlas_insns[] = {
+        0x1e43ee03, 0x1e43ee13, 0x1e43ee23,
+    };
+    static const uint32_t vmlas_alias_insns[] = {
+        0x1e43fe03, 0x1e43fe13, 0x1e43fe23,
+    };
+    static const uint32_t vqdmlah_insns[] = {
+        0x0e63ee02, 0x0e63ee12, 0x0e63ee22,
+    };
+    static const uint32_t vqrdmlah_insns[] = {
+        0x0e43ee02, 0x0e43ee12, 0x0e43ee22,
+    };
+    static const uint32_t vqdmlash_insns[] = {
+        0x1e63ee02, 0x1e63ee12, 0x1e63ee22,
+    };
+    static const uint32_t vqrdmlash_insns[] = {
+        0x1e43ee02, 0x1e43ee12, 0x1e43ee22,
+    };
+    const uint8_t initial[16] = {
+        0xf0, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
+        0x20, 0x21, 0x22, 0x23, 0x30, 0x31, 0x32, 0x33,
+    };
+    const uint8_t n[16] = {
+        0x7f, 0x80, 0x01, 0xff, 0x34, 0x12, 0x00, 0x80,
+        0x02, 0x00, 0x00, 0x40, 0xff, 0xff, 0xff, 0x7f,
+    };
+    const uint8_t q_initial[16] = {
+        0xff, 0x7f, 0x7f, 0x00, 0x02, 0x00, 0x03, 0x00,
+        0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    };
+    const uint8_t q_n[16] = {
+        0x00, 0x80, 0x80, 0x01, 0x01, 0x00, 0x02, 0x00,
+        0x01, 0x00, 0x00, 0x40, 0x02, 0x00, 0x00, 0x20,
+    };
+    const uint32_t scalar = 0x40004040;
+    const uint32_t sat_scalar = 0x80008080;
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        unsigned esize = esizes[ncase];
+
+        test_arm_mve_expected_scalar_acc(expected, initial, n, scalar,
+                                         esize, false, 0xffff);
+        test_arm_m55_mve_scalar_2op_run(vmla_insns[ncase], initial, n,
+                                        scalar, expected, false);
+        test_arm_m55_mve_scalar_2op_run(vmla_alias_insns[ncase], initial,
+                                        n, scalar, expected, false);
+
+        test_arm_mve_expected_scalar_acc(expected, initial, n, scalar,
+                                         esize, true, 0xffff);
+        test_arm_m55_mve_scalar_2op_run(vmlas_insns[ncase], initial, n,
+                                        scalar, expected, false);
+        test_arm_m55_mve_scalar_2op_run(vmlas_alias_insns[ncase], initial,
+                                        n, scalar, expected, false);
+
+        test_arm_mve_expected_qdmlah(expected, q_initial, q_n, sat_scalar,
+                                     esize, false, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqdmlah_insns[ncase], q_initial,
+                                           q_n, sat_scalar, 0, expected,
+                                           false, false, qc);
+
+        test_arm_mve_expected_qdmlah(expected, q_initial, q_n, scalar,
+                                     esize, false, true, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqrdmlah_insns[ncase], q_initial,
+                                           q_n, scalar, 0, expected, false,
+                                           false, qc);
+
+        test_arm_mve_expected_qdmlah(expected, q_initial, q_n, sat_scalar,
+                                     esize, true, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqdmlash_insns[ncase], q_initial,
+                                           q_n, sat_scalar, 0, expected,
+                                           false, false, qc);
+
+        test_arm_mve_expected_qdmlah(expected, q_initial, q_n, scalar,
+                                     esize, true, true, 0xffff, &qc);
+        test_arm_m55_mve_scalar_2op_run_qc(vqrdmlash_insns[ncase],
+                                           q_initial, q_n, scalar, 0,
+                                           expected, false, false, qc);
+    }
+
+    test_arm_mve_expected_scalar_acc(expected, initial, n, scalar, 4,
+                                     false, 0xff00);
+    test_arm_m55_mve_scalar_2op_run(vmla_insns[2], initial, n, scalar,
+                                    expected, true);
+
+    test_arm_mve_expected_qdmlah(expected, q_initial, q_n, sat_scalar, 4,
+                                 false, false, 0xff00, &qc);
+    test_arm_m55_mve_scalar_2op_run_qc(vqdmlah_insns[2], q_initial, q_n,
+                                       sat_scalar, 0, expected, true, false,
+                                       qc);
+
+    test_arm_mve_expected_qdmlah(expected, q_initial, q_n, scalar, 4,
+                                 false, true, 0xffff, &qc);
+    test_arm_m55_mve_scalar_2op_run_qc(vqrdmlah_insns[2], q_initial, q_n,
+                                       scalar, 0, expected, false, true,
+                                       true);
+
+    test_arm_m55_mve_2op_expect_error(0x0e63ee32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x1e63ee32, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0e43ee43, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0ec3ee01, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0e4dee03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0e4fee03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0e43ee03, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_cadd(void)
+{
+    static const uint32_t vhcadd90_insns[] = {
+        0x0f04ee02, 0x0f04ee12, 0x0f04ee22,
+    };
+    static const uint32_t vhcadd270_insns[] = {
+        0x1f04ee02, 0x1f04ee12, 0x1f04ee22,
+    };
+    static const uint32_t vcadd90_insns[] = {
+        0x0f04fe02, 0x0f04fe12, 0x0f04fe22,
+    };
+    static const uint32_t vcadd270_insns[] = {
+        0x1f04fe02, 0x1f04fe12, 0x1f04fe22,
+    };
+    const uint8_t initial[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint8_t n[16] = {
+        0x80, 0x7f, 0x01, 0xff, 0x34, 0x12, 0x00, 0x80,
+        0xfe, 0xff, 0x00, 0x40, 0x11, 0x22, 0xef, 0xdd,
+    };
+    const uint8_t m[16] = {
+        0x7f, 0x80, 0x02, 0xfe, 0x35, 0x12, 0xff, 0x7f,
+        0x02, 0x00, 0xff, 0xbf, 0x44, 0x33, 0xbc, 0xaa,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        test_arm_mve_expected_cadd(expected, initial, n, m, esizes[ncase],
+                                   false, true, 0xffff);
+        test_arm_m55_mve_2op_run(vhcadd90_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_cadd(expected, initial, n, m, esizes[ncase],
+                                   true, true, 0xffff);
+        test_arm_m55_mve_2op_run(vhcadd270_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_cadd(expected, initial, n, m, esizes[ncase],
+                                   false, false, 0xffff);
+        test_arm_m55_mve_2op_run(vcadd90_insns[ncase], initial, n, m,
+                                 expected, false);
+
+        test_arm_mve_expected_cadd(expected, initial, n, m, esizes[ncase],
+                                   true, false, 0xffff);
+        test_arm_m55_mve_2op_run(vcadd270_insns[ncase], initial, n, m,
+                                 expected, false);
+    }
+
+    test_arm_mve_expected_cadd(expected, initial, n, m, 4, true, true,
+                               0xff00);
+    test_arm_m55_mve_2op_run(0x1f04ee22, initial, n, m, expected, true);
+
+    test_arm_mve_expected_cadd(expected, initial, n, initial, 4, false,
+                               false, 0xffff);
+    test_arm_m55_mve_2op_run(0x0f00fe22, initial, n, m, expected, false);
+
+    test_arm_m55_mve_2op_expect_error(0x0f04fe42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f04fe02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_mve_expected_vdup(uint8_t *expected,
+                                       const uint8_t *initial,
+                                       uint32_t value, unsigned esize,
+                                       uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        test_arm_mve_store_masked(expected, i, esize, value, mask);
+    }
+}
+
+static uint64_t test_arm_mve_dup_const32(uint32_t value)
+{
+    return ((uint64_t)value << 32) | value;
+}
+
+static uint64_t test_arm_mve_asimd_imm_const(uint32_t imm, unsigned cmode,
+                                             bool op)
+{
+    switch (cmode) {
+    case 0:
+    case 1:
+        break;
+    case 2:
+    case 3:
+        imm <<= 8;
+        break;
+    case 4:
+    case 5:
+        imm <<= 16;
+        break;
+    case 6:
+    case 7:
+        imm <<= 24;
+        break;
+    case 8:
+    case 9:
+        imm |= imm << 16;
+        break;
+    case 10:
+    case 11:
+        imm = (imm << 8) | (imm << 24);
+        break;
+    case 12:
+        imm = (imm << 8) | 0xff;
+        break;
+    case 13:
+        imm = (imm << 16) | 0xffff;
+        break;
+    case 14:
+        if (op) {
+            uint64_t imm64 = 0;
+            unsigned n;
+
+            for (n = 0; n < 8; n++) {
+                if (imm & (1U << n)) {
+                    imm64 |= 0xffULL << (n * 8);
+                }
+            }
+            return imm64;
+        }
+        imm |= (imm << 8) | (imm << 16) | (imm << 24);
+        break;
+    case 15:
+        TEST_CHECK(!op);
+        imm = ((imm & 0x80) << 24) | ((imm & 0x3f) << 19) |
+              ((imm & 0x40) ? (0x1f << 25) : (1 << 30));
+        break;
+    default:
+        TEST_CHECK(false);
+        break;
+    }
+    if (op) {
+        imm = ~imm;
+    }
+    return test_arm_mve_dup_const32(imm);
+}
+
+static uint32_t test_arm_mve_vimm_1r_insn(unsigned qd, uint32_t imm,
+                                          unsigned cmode, bool op)
+{
+    uint32_t view = 0xef800050;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((imm >> 7) & 1) << 28;
+    view |= ((imm >> 4) & 7) << 16;
+    view |= imm & 0xf;
+    view |= (cmode & 0xf) << 8;
+    if (op) {
+        view |= 1U << 5;
+    }
+    return (view << 16) | (view >> 16);
+}
+
+static void test_arm_mve_expected_vimm(uint8_t *expected,
+                                       const uint8_t *initial,
+                                       uint64_t imm, char op,
+                                       uint16_t mask)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += 8) {
+        uint64_t lhs = test_arm_load_le64(initial + i);
+        uint64_t result;
+
+        switch (op) {
+        case 'm':
+            result = imm;
+            break;
+        case '&':
+            result = lhs & imm;
+            break;
+        case '|':
+            result = lhs | imm;
+            break;
+        default:
+            TEST_CHECK(false);
+            result = 0;
+            break;
+        }
+        test_arm_mve_store_masked(expected, i, 8, result, mask);
+    }
+}
+
+static void test_arm_m55_mve_vimm_run(uint32_t insn, const uint8_t *initial,
+                                      uint32_t vpr,
+                                      const uint8_t *expected, bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_vimm(void)
+{
+    const uint8_t initial[16] = {
+        0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87,
+        0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f,
+    };
+    uint8_t expected[16];
+    uint64_t imm;
+    uint32_t insn;
+
+    insn = test_arm_mve_vimm_1r_insn(0, 0x5a, 0, false);
+    imm = test_arm_mve_asimd_imm_const(0x5a, 0, false);
+    test_arm_mve_expected_vimm(expected, initial, imm, 'm', 0xffff);
+    test_arm_m55_mve_vimm_run(insn, initial, 0, expected, false);
+
+    insn = test_arm_mve_vimm_1r_insn(0, 0x24, 2, true);
+    imm = test_arm_mve_asimd_imm_const(0x24, 2, true);
+    test_arm_mve_expected_vimm(expected, initial, imm, 'm', 0xffff);
+    test_arm_m55_mve_vimm_run(insn, initial, 0, expected, false);
+
+    insn = test_arm_mve_vimm_1r_insn(0, 0x33, 1, false);
+    imm = test_arm_mve_asimd_imm_const(0x33, 1, false);
+    test_arm_mve_expected_vimm(expected, initial, imm, '|', 0xffff);
+    test_arm_m55_mve_vimm_run(insn, initial, 0, expected, false);
+
+    insn = test_arm_mve_vimm_1r_insn(0, 0x12, 3, true);
+    imm = test_arm_mve_asimd_imm_const(0x12, 3, true);
+    test_arm_mve_expected_vimm(expected, initial, imm, '&', 0xffff);
+    test_arm_m55_mve_vimm_run(insn, initial, 0, expected, false);
+
+    insn = test_arm_mve_vimm_1r_insn(0, 0x6c, 0, false);
+    imm = test_arm_mve_asimd_imm_const(0x6c, 0, false);
+    test_arm_mve_expected_vimm(expected, initial, imm, 'm', 0x00f0);
+    test_arm_m55_mve_vimm_run(insn, initial, 0x001100f0, expected, false);
+
+    insn = test_arm_mve_vimm_1r_insn(0, 0x77, 1, false);
+    imm = test_arm_mve_asimd_imm_const(0x77, 1, false);
+    test_arm_mve_expected_vimm(expected, initial, imm, '|', 0xff00);
+    test_arm_m55_mve_vimm_run(insn, initial, 0, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vimm_1r_insn(0, 0x01, 15, true),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vimm_1r_insn(8, 0x5a, 0, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vimm_1r_insn(0, 0x5a, 0, false),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_INSN_INVALID);
+}
+
+typedef enum test_arm_mve_shift_imm_kind {
+    TEST_ARM_MVE_SHIFT_IMM_SHL,
+    TEST_ARM_MVE_SHIFT_IMM_SHR_S,
+    TEST_ARM_MVE_SHIFT_IMM_SHR_U,
+    TEST_ARM_MVE_SHIFT_IMM_RSHR_S,
+    TEST_ARM_MVE_SHIFT_IMM_RSHR_U,
+    TEST_ARM_MVE_SHIFT_IMM_SRI,
+    TEST_ARM_MVE_SHIFT_IMM_SLI,
+    TEST_ARM_MVE_SHIFT_IMM_QSHL_S,
+    TEST_ARM_MVE_SHIFT_IMM_QSHL_U,
+    TEST_ARM_MVE_SHIFT_IMM_QSHL_SU,
+} test_arm_mve_shift_imm_kind;
+
+static uint32_t test_arm_mve_view_to_t32(uint32_t view)
+{
+    return (view << 16) | (view >> 16);
+}
+
+static uint32_t test_arm_mve_2op_insn(uint32_t base, unsigned qd,
+                                      unsigned qn, unsigned qm)
+{
+    uint32_t view = base;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static void test_arm_mve_expected_carry(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        const uint8_t *n, const uint8_t *m,
+                                        bool subtract, uint32_t carry_in,
+                                        uint16_t mask, uint32_t *carry_out)
+{
+    unsigned lane;
+
+    memcpy(expected, initial, 16);
+    for (lane = 0; lane < 4; lane++, mask >>= 4) {
+        uint16_t lane_mask = mask & 0xf;
+        uint64_t value = carry_in;
+        unsigned byte;
+
+        value += test_arm_load_le(n + lane * 4, 4);
+        value += test_arm_load_le(m + lane * 4, 4) ^
+            (subtract ? UINT32_MAX : 0);
+        if (lane_mask & 1) {
+            carry_in = (uint32_t)(value >> 32);
+        }
+        for (byte = 0; byte < 4; byte++) {
+            if (lane_mask & (1U << byte)) {
+                expected[lane * 4 + byte] =
+                    (uint8_t)((uint32_t)value >> (byte * 8));
+            }
+        }
+    }
+    *carry_out = carry_in;
+}
+
+static void test_arm_m55_mve_carry_run(uint32_t insn,
+                                       const uint8_t *initial,
+                                       const uint8_t *n, const uint8_t *m,
+                                       uint32_t fpscr_in, uint32_t vpr,
+                                       const uint8_t *expected,
+                                       uint32_t expected_c, bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_c = 1U << 29;
+    const uint32_t fpscr_nzv = (1U << 31) | (1U << 30) | (1U << 28);
+    uc_engine *uc;
+    uint8_t code[8] = { 0 };
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = fpscr_in;
+    uint32_t target_offset = 0;
+    bool seed_c = (fpscr_in & fpscr_c) != 0;
+    size_t i;
+
+    if (seed_c) {
+        test_arm_emit32(code, 0, test_arm_mve_2op_insn(0xee301f00, 3, 1, 2));
+        target_offset = 4;
+    }
+    test_arm_emit32(code, target_offset, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    if (seed_c) {
+        uint32_t zero = 0;
+
+        OK(uc_reg_write(uc, UC_ARM_REG_VPR, &zero));
+        OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &zero));
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    } else {
+        OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    }
+    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
+
+    if (eci) {
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + target_offset) | 1,
+                        code_start + target_offset + 4, 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, (code_start + target_offset) | 1,
+                        code_start + target_offset + 4, 0, 0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK((fpscr & fpscr_c) == (expected_c ? fpscr_c : 0));
+    TEST_CHECK((fpscr & fpscr_nzv) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_carry(void)
+{
+    const uint32_t vadc_base = 0xee300f00;
+    const uint32_t vadci_base = 0xee301f00;
+    const uint32_t vsbc_base = 0xfe300f00;
+    const uint32_t vsbci_base = 0xfe301f00;
+    const uint32_t fpscr_c = 1U << 29;
+    const uint32_t fpscr_nzv = (1U << 31) | (1U << 30) | (1U << 28);
+    const uint8_t initial[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint8_t n[16] = {
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+        0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x80,
+    };
+    const uint8_t m[16] = {
+        0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+        0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
+    };
+    uint8_t expected[16];
+    uint32_t carry_out;
+    uint32_t insn;
+
+    insn = test_arm_mve_2op_insn(vadc_base, 0, 1, 2);
+    test_arm_mve_expected_carry(expected, initial, n, m, false, 1, 0xffff,
+                                &carry_out);
+    test_arm_m55_mve_carry_run(insn, initial, n, m, fpscr_nzv | fpscr_c, 0,
+                               expected, carry_out, false);
+
+    insn = test_arm_mve_2op_insn(vadci_base, 0, 1, 2);
+    test_arm_mve_expected_carry(expected, initial, n, m, false, 0, 0xffff,
+                                &carry_out);
+    test_arm_m55_mve_carry_run(insn, initial, n, m, fpscr_nzv | fpscr_c, 0,
+                               expected, carry_out, false);
+
+    insn = test_arm_mve_2op_insn(vsbc_base, 0, 1, 2);
+    test_arm_mve_expected_carry(expected, initial, n, m, true, 1, 0xffff,
+                                &carry_out);
+    test_arm_m55_mve_carry_run(insn, initial, n, m, fpscr_nzv | fpscr_c, 0,
+                               expected, carry_out, false);
+
+    insn = test_arm_mve_2op_insn(vsbci_base, 0, 1, 2);
+    test_arm_mve_expected_carry(expected, initial, n, m, true, 1, 0xffff,
+                                &carry_out);
+    test_arm_m55_mve_carry_run(insn, initial, n, m, fpscr_nzv, 0, expected,
+                               carry_out, false);
+
+    insn = test_arm_mve_2op_insn(vadc_base, 0, 1, 2);
+    test_arm_mve_expected_carry(expected, initial, n, m, false, 1, 0x00ff,
+                                &carry_out);
+    test_arm_m55_mve_carry_run(insn, initial, n, m, fpscr_nzv | fpscr_c,
+                               0x00ff00ff, expected, carry_out, false);
+
+    insn = test_arm_mve_2op_insn(vadci_base, 0, 1, 2);
+    test_arm_mve_expected_carry(expected, initial, n, m, false, 1, 0xff00,
+                                &carry_out);
+    test_arm_m55_mve_carry_run(insn, initial, n, m, fpscr_nzv | fpscr_c, 0,
+                               expected, carry_out, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_2op_insn(vadc_base, 8, 1, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_2op_insn(vadc_base, 0, 1, 2),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static uint32_t test_arm_mve_fp_scalar_insn(uint32_t base, bool fp16,
+                                            unsigned qd, unsigned qn,
+                                            unsigned rm)
+{
+    uint32_t view = base;
+
+    if (fp16) {
+        view |= 1U << 28;
+    }
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= rm & 15;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static void test_arm_m55_mve_fp_scalar_run(uint32_t insn,
+                                           const uint8_t *initial,
+                                           const uint8_t *n, uint32_t rm,
+                                           uint32_t vpr,
+                                           const uint8_t *expected,
+                                           bool eci, bool expected_ioc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_ioc = 1U;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &rm));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(((fpscr & fpscr_ioc) != 0) == expected_ioc,
+                "insn=0x%08x fpscr=0x%08x expected_ioc=%u",
+                insn, fpscr, expected_ioc ? 1 : 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_fp_scalar(void)
+{
+    const uint32_t vadd_base = 0xee300f40;
+    const uint32_t vsub_base = 0xee301f40;
+    const uint32_t vmul_base = 0xee310e60;
+    const uint32_t vfma_base = 0xee310e40;
+    const uint32_t vfmas_base = 0xee311e40;
+    const uint8_t initial_f32[16] = {
+        0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0xc0,
+        0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x80, 0xc0,
+    };
+    const uint8_t n_f32[16] = {
+        0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x40, 0x40,
+        0x00, 0x00, 0xc0, 0xbf, 0x00, 0x00, 0x00, 0xbf,
+    };
+    const uint8_t vadd_f32[16] = {
+        0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0xa0, 0x40,
+        0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0xc0, 0x3f,
+    };
+    const uint8_t vsub_f32[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f,
+        0x00, 0x00, 0x60, 0xc0, 0x00, 0x00, 0x20, 0xc0,
+    };
+    const uint8_t vmul_f32[16] = {
+        0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0xc0, 0x40,
+        0x00, 0x00, 0x40, 0xc0, 0x00, 0x00, 0x80, 0xbf,
+    };
+    const uint8_t vfma_f32[16] = {
+        0x00, 0x00, 0xa0, 0x40, 0x00, 0x00, 0x80, 0x40,
+        0x00, 0x00, 0x20, 0xc0, 0x00, 0x00, 0xa0, 0xc0,
+    };
+    const uint8_t vfmas_f32[16] = {
+        0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0x80, 0xc0,
+        0x00, 0x00, 0xa0, 0x3f, 0x00, 0x00, 0x80, 0x40,
+    };
+    const uint8_t vfma_f32_eci[16] = {
+        0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0xc0,
+        0x00, 0x00, 0x20, 0xc0, 0x00, 0x00, 0xa0, 0xc0,
+    };
+    const uint8_t initial_f16[16] = {
+        0x00, 0x3c, 0x00, 0xc0, 0x00, 0x38, 0x00, 0xc4,
+        0x00, 0x3e, 0x00, 0xbc, 0x00, 0x00, 0x00, 0x42,
+    };
+    const uint8_t n_f16[16] = {
+        0x00, 0x40, 0x00, 0x42, 0x00, 0xbe, 0x00, 0xb8,
+        0x00, 0x44, 0x00, 0xc2, 0x00, 0x3c, 0x00, 0x34,
+    };
+    const uint8_t vadd_f16[16] = {
+        0x00, 0x44, 0x00, 0x45, 0x00, 0x38, 0x00, 0x3e,
+        0x00, 0x46, 0x00, 0xbc, 0x00, 0x42, 0x80, 0x40,
+    };
+    const uint8_t vsub_f16[16] = {
+        0x00, 0x00, 0x00, 0x3c, 0x00, 0xc3, 0x00, 0xc1,
+        0x00, 0x40, 0x00, 0xc5, 0x00, 0xbc, 0x00, 0xbf,
+    };
+    const uint8_t vmul_f16[16] = {
+        0x00, 0x44, 0x00, 0x46, 0x00, 0xc2, 0x00, 0xbc,
+        0x00, 0x48, 0x00, 0xc6, 0x00, 0x40, 0x00, 0x38,
+    };
+    const uint8_t vfma_f16[16] = {
+        0x00, 0x45, 0x00, 0x44, 0x00, 0xc1, 0x00, 0xc5,
+        0xc0, 0x48, 0x00, 0xc7, 0x00, 0x40, 0x00, 0x43,
+    };
+    const uint8_t vfmas_f16[16] = {
+        0x00, 0x44, 0x00, 0xc4, 0x00, 0x3d, 0x00, 0x44,
+        0x00, 0x48, 0x00, 0x45, 0x00, 0x40, 0x80, 0x41,
+    };
+    const uint8_t vmul_f16_vpr[16] = {
+        0x00, 0x3c, 0x00, 0xc0, 0x00, 0xc2, 0x00, 0xbc,
+        0x00, 0x3e, 0x00, 0xbc, 0x00, 0x00, 0x00, 0x42,
+    };
+    const struct {
+        uint32_t base;
+        const uint8_t *expected_f32;
+        const uint8_t *expected_f16;
+    } cases[] = {
+        { vadd_base, vadd_f32, vadd_f16 },
+        { vsub_base, vsub_f32, vsub_f16 },
+        { vmul_base, vmul_f32, vmul_f16 },
+        { vfma_base, vfma_f32, vfma_f16 },
+        { vfmas_base, vfmas_f32, vfmas_f16 },
+    };
+    uint32_t scalar_f32 = 0x40000000;
+    uint32_t scalar_f16 = 0x4000;
+    size_t i;
+
+    for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+        test_arm_m55_mve_fp_scalar_run(
+            test_arm_mve_fp_scalar_insn(cases[i].base, false, 0, 1, 3),
+            initial_f32, n_f32, scalar_f32, 0, cases[i].expected_f32,
+            false, false);
+        test_arm_m55_mve_fp_scalar_run(
+            test_arm_mve_fp_scalar_insn(cases[i].base, true, 0, 1, 3),
+            initial_f16, n_f16, scalar_f16, 0, cases[i].expected_f16,
+            false, false);
+    }
+
+    test_arm_m55_mve_fp_scalar_run(
+        test_arm_mve_fp_scalar_insn(vmul_base, true, 0, 1, 3),
+        initial_f16, n_f16, scalar_f16, 0x001100f0, vmul_f16_vpr,
+        false, false);
+    test_arm_m55_mve_fp_scalar_run(
+        test_arm_mve_fp_scalar_insn(vfma_base, false, 0, 1, 3),
+        initial_f32, n_f32, scalar_f32, 0, vfma_f32_eci, true, false);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_scalar_insn(vadd_base, false, 0, 1, 13),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_scalar_insn(vadd_base, false, 0, 1, 15),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_scalar_insn(vadd_base, false, 8, 1, 3),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_scalar_insn(vadd_base, false, 0, 8, 3),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_scalar_insn(vadd_base, false, 0, 1, 3),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static uint32_t test_arm_mve_vshift_imm_insn(unsigned qd, unsigned qm,
+                                             uint32_t base, unsigned size,
+                                             unsigned shift, bool right)
+{
+    unsigned bits = 8U << size;
+    uint32_t encoded = right ? bits - shift : shift;
+    uint32_t view = base;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    switch (size) {
+    case 0:
+        view |= 1U << 19;
+        view |= (encoded & 7) << 16;
+        break;
+    case 1:
+        view |= 1U << 20;
+        view |= (encoded & 15) << 16;
+        break;
+    case 2:
+        view |= 1U << 21;
+        view |= (encoded & 31) << 16;
+        break;
+    default:
+        TEST_CHECK(false);
+        break;
+    }
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_element_mask(unsigned bits)
+{
+    return bits == 32 ? UINT32_MAX : (1U << bits) - 1;
+}
+
+static uint32_t test_arm_mve_expected_shift_imm_lane(uint32_t dst,
+                                                     uint32_t src,
+                                                     unsigned bits,
+                                                     unsigned shift,
+                                                     test_arm_mve_shift_imm_kind kind)
+{
+    uint32_t mask = test_arm_mve_element_mask(bits);
+    uint64_t wide;
+    int64_t sval;
+    int64_t stmp;
+    uint32_t field_mask;
+
+    src &= mask;
+    dst &= mask;
+    switch (kind) {
+    case TEST_ARM_MVE_SHIFT_IMM_SHL:
+        return (uint32_t)((uint64_t)src << shift) & mask;
+    case TEST_ARM_MVE_SHIFT_IMM_SHR_S:
+        sval = test_arm_sign_extend(src, bits);
+        if (shift >= bits) {
+            return sval < 0 ? mask : 0;
+        }
+        return (uint32_t)(sval >> shift) & mask;
+    case TEST_ARM_MVE_SHIFT_IMM_SHR_U:
+        return shift >= bits ? 0 : src >> shift;
+    case TEST_ARM_MVE_SHIFT_IMM_RSHR_S:
+        sval = test_arm_sign_extend(src, bits);
+        stmp = sval >> (shift - 1);
+        return (uint32_t)((stmp >> 1) + (stmp & 1)) & mask;
+    case TEST_ARM_MVE_SHIFT_IMM_RSHR_U:
+        wide = src + (1ULL << (shift - 1));
+        return (uint32_t)(wide >> shift) & mask;
+    case TEST_ARM_MVE_SHIFT_IMM_SRI:
+        if (shift == bits) {
+            return dst;
+        }
+        field_mask = mask >> shift;
+        return ((src >> shift) & field_mask) | (dst & ~field_mask);
+    case TEST_ARM_MVE_SHIFT_IMM_SLI:
+        field_mask = (mask << shift) & mask;
+        return (((uint32_t)((uint64_t)src << shift)) & field_mask) |
+               (dst & ~field_mask);
+    default:
+        TEST_CHECK(false);
+        return 0;
+    }
+}
+
+static void test_arm_mve_expected_shift_imm(uint8_t *expected,
+                                            const uint8_t *initial,
+                                            const uint8_t *source,
+                                            unsigned esize,
+                                            unsigned shift,
+                                            test_arm_mve_shift_imm_kind kind,
+                                            uint16_t pred)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        uint32_t dst = test_arm_load_le(initial + i, esize);
+        uint32_t src = test_arm_load_le(source + i, esize);
+        uint32_t result;
+
+        result = test_arm_mve_expected_shift_imm_lane(dst, src, bits,
+                                                      shift, kind);
+        test_arm_mve_store_masked(expected, i, esize, result, pred);
+    }
+}
+
+static uint32_t test_arm_mve_sat_u(uint64_t value, unsigned bits,
+                                   bool *saturated)
+{
+    uint64_t max = bits == 32 ? UINT32_MAX : (1ULL << bits) - 1;
+
+    if (value > max) {
+        *saturated = true;
+        return (uint32_t)max;
+    }
+    return (uint32_t)value;
+}
+
+static uint32_t test_arm_mve_sat_s(int64_t value, unsigned bits,
+                                   bool *saturated)
+{
+    int64_t min = -(1LL << (bits - 1));
+    int64_t max = (1LL << (bits - 1)) - 1;
+
+    if (value > max) {
+        *saturated = true;
+        return (uint32_t)max;
+    } else if (value < min) {
+        *saturated = true;
+        return (uint32_t)min;
+    }
+    return (uint32_t)value;
+}
+
+static uint32_t test_arm_mve_expected_qshift_imm_lane(
+    uint32_t src, unsigned bits, unsigned shift,
+    test_arm_mve_shift_imm_kind kind, bool *saturated)
+{
+    uint32_t mask = test_arm_mve_element_mask(bits);
+    int64_t sval;
+
+    src &= mask;
+    *saturated = false;
+    switch (kind) {
+    case TEST_ARM_MVE_SHIFT_IMM_QSHL_S:
+        sval = test_arm_sign_extend(src, bits);
+        return test_arm_mve_sat_s(sval * (1LL << shift), bits, saturated) &
+               mask;
+    case TEST_ARM_MVE_SHIFT_IMM_QSHL_U:
+        return test_arm_mve_sat_u((uint64_t)src << shift, bits, saturated) &
+               mask;
+    case TEST_ARM_MVE_SHIFT_IMM_QSHL_SU:
+        sval = test_arm_sign_extend(src, bits);
+        if (sval < 0) {
+            *saturated = true;
+            return 0;
+        }
+        return test_arm_mve_sat_u((uint64_t)sval << shift, bits,
+                                  saturated) & mask;
+    default:
+        TEST_CHECK(false);
+        return 0;
+    }
+}
+
+static void test_arm_mve_expected_qshift_imm(
+    uint8_t *expected, const uint8_t *initial, const uint8_t *source,
+    unsigned esize, unsigned shift, test_arm_mve_shift_imm_kind kind,
+    uint16_t pred, bool *qc)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (i = 0; i < 16; i += esize) {
+        uint32_t src = test_arm_load_le(source + i, esize);
+        bool saturated;
+        uint32_t result;
+
+        result = test_arm_mve_expected_qshift_imm_lane(src, bits, shift,
+                                                       kind, &saturated);
+        test_arm_mve_store_masked(expected, i, esize, result, pred);
+        if (saturated && (pred & (1U << i))) {
+            *qc = true;
+        }
+    }
+}
+
+static void test_arm_m55_mve_shift_imm_run_qc_init(uint32_t insn,
+                                                   const uint8_t *initial,
+                                                   const uint8_t *source,
+                                                   uint32_t vpr,
+                                                   const uint8_t *expected,
+                                                   bool eci,
+                                                   bool initial_qc,
+                                                   bool expected_qc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_qc = 1U << 27;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, source, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci || initial_qc) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        if (initial_qc) {
+            OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+            fpscr |= fpscr_qc;
+            OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+        }
+        epsr = xpsr_t | eci_a0a1;
+        if (eci) {
+            OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        }
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        if (eci) {
+            OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+            TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                        "epsr=0x%08x", epsr);
+        }
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(((fpscr & fpscr_qc) != 0) == expected_qc,
+                "fpscr=0x%08x expected_qc=%d",
+                fpscr, expected_qc);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_shift_imm_run_qc(uint32_t insn,
+                                              const uint8_t *initial,
+                                              const uint8_t *source,
+                                              uint32_t vpr,
+                                              const uint8_t *expected,
+                                              bool eci,
+                                              bool expected_qc)
+{
+    test_arm_m55_mve_shift_imm_run_qc_init(insn, initial, source, vpr,
+                                           expected, eci, false,
+                                           expected_qc);
+}
+
+static void test_arm_m55_mve_shift_imm_run(uint32_t insn,
+                                           const uint8_t *initial,
+                                           const uint8_t *source,
+                                           uint32_t vpr,
+                                           const uint8_t *expected,
+                                           bool eci)
+{
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, vpr, expected,
+                                      eci, false);
+}
+
+static void test_arm_m55_mve_shift_imm(void)
+{
+    const uint8_t initial[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    const uint8_t source[16] = {
+        0x81, 0x7f, 0x12, 0xf0, 0x55, 0xaa, 0x01, 0x80,
+        0xfe, 0x10, 0x33, 0xcc, 0x09, 0x90, 0x44, 0x22,
+    };
+    uint8_t expected[16];
+    uint32_t insn;
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xef800550, 0, 3, false);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 1, 3,
+                                    TEST_ARM_MVE_SHIFT_IMM_SHL, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xef800050, 1, 4, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 2, 4,
+                                    TEST_ARM_MVE_SHIFT_IMM_SHR_S, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800050, 2, 8, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 4, 8,
+                                    TEST_ARM_MVE_SHIFT_IMM_SHR_U, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800250, 0, 2, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 1, 2,
+                                    TEST_ARM_MVE_SHIFT_IMM_RSHR_U,
+                                    0x00f0);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0x001100f0,
+                                   expected, false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xef800250, 2, 8, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 4, 8,
+                                    TEST_ARM_MVE_SHIFT_IMM_RSHR_S,
+                                    0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xef800050, 0, 8, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 1, 8,
+                                    TEST_ARM_MVE_SHIFT_IMM_SHR_S, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800050, 2, 32, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 4, 32,
+                                    TEST_ARM_MVE_SHIFT_IMM_SHR_U, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xef800250, 1, 16, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 2, 16,
+                                    TEST_ARM_MVE_SHIFT_IMM_RSHR_S,
+                                    0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800250, 0, 8, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 1, 8,
+                                    TEST_ARM_MVE_SHIFT_IMM_RSHR_U,
+                                    0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800450, 1, 4, true);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 2, 4,
+                                    TEST_ARM_MVE_SHIFT_IMM_SRI, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800550, 2, 8, false);
+    test_arm_mve_expected_shift_imm(expected, initial, source, 4, 8,
+                                    TEST_ARM_MVE_SHIFT_IMM_SLI, 0xff00);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(8, 1, 0xef800550, 0, 3, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(0, 1, 0xef800550, 0, 3, false),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_qshift_imm(void)
+{
+    const uint8_t initial[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    const uint8_t source[16] = {
+        0x81, 0x7f, 0x12, 0xf0, 0x55, 0xaa, 0x01, 0x80,
+        0xfe, 0x10, 0x33, 0xcc, 0x09, 0x90, 0x44, 0x22,
+    };
+    const uint8_t eci_source[16] = {
+        0x7f, 0x80, 0x70, 0x90, 0x60, 0xa0, 0x50, 0xb0,
+        0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+    };
+    uint8_t expected[16];
+    uint32_t insn;
+    bool qc;
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xef800750, 0, 2, false);
+    test_arm_mve_expected_qshift_imm(expected, initial, source, 1, 2,
+                                     TEST_ARM_MVE_SHIFT_IMM_QSHL_S,
+                                     0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800750, 1, 4, false);
+    test_arm_mve_expected_qshift_imm(expected, initial, source, 2, 4,
+                                     TEST_ARM_MVE_SHIFT_IMM_QSHL_U,
+                                     0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800650, 2, 8, false);
+    test_arm_mve_expected_qshift_imm(expected, initial, source, 4, 8,
+                                     TEST_ARM_MVE_SHIFT_IMM_QSHL_SU,
+                                     0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xff800750, 0, 7, false);
+    test_arm_mve_expected_qshift_imm(expected, initial, source, 1, 7,
+                                     TEST_ARM_MVE_SHIFT_IMM_QSHL_U,
+                                     0x00f0, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0x001100f0,
+                                      expected, false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xef800750, 0, 2, false);
+    test_arm_mve_expected_qshift_imm(expected, initial, eci_source, 1, 2,
+                                     TEST_ARM_MVE_SHIFT_IMM_QSHL_S,
+                                     0xff00, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, eci_source, 0,
+                                      expected, true, qc);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(8, 1, 0xef800750, 0, 2, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(0, 1, 0xef800750, 0, 2, false),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_INSN_INVALID);
+}
+
+static uint32_t test_arm_mve_vshift_scalar_insn(unsigned qd, uint32_t base,
+                                                unsigned size, unsigned rm)
+{
+    uint32_t view = base;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= (size & 3) << 18;
+    view |= rm & 15;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_expected_qshift_scalar_lane(uint32_t src,
+                                                        unsigned bits,
+                                                        int8_t shift,
+                                                        bool is_signed,
+                                                        bool rounded,
+                                                        bool *saturated)
+{
+    uint32_t mask = test_arm_mve_element_mask(bits);
+    int64_t sval;
+    int64_t sresult;
+    uint64_t uvalue;
+    uint64_t uresult;
+
+    src &= mask;
+    *saturated = false;
+    if (is_signed) {
+        sval = test_arm_sign_extend(src, bits);
+        if (shift <= -(int)bits) {
+            sresult = rounded ? 0 : (sval < 0 ? -1 : 0);
+        } else if (shift < 0) {
+            if (rounded) {
+                sval >>= -shift - 1;
+                sresult = (sval >> 1) + (sval & 1);
+            } else {
+                sresult = sval >> -shift;
+            }
+        } else if (shift < (int)bits) {
+            return test_arm_mve_sat_s(sval * (1LL << shift), bits,
+                                      saturated) & mask;
+        } else if (sval == 0) {
+            sresult = 0;
+        } else {
+            *saturated = true;
+            return (sval < 0 ? 1U << (bits - 1) :
+                    (1U << (bits - 1)) - 1) & mask;
+        }
+        return (uint32_t)sresult & mask;
+    }
+
+    uvalue = src;
+    if (shift <= -((int)bits + rounded)) {
+        uresult = 0;
+    } else if (shift < 0) {
+        if (rounded) {
+            uvalue >>= -shift - 1;
+            uresult = (uvalue >> 1) + (uvalue & 1);
+        } else {
+            uresult = uvalue >> -shift;
+        }
+    } else if (shift < (int)bits) {
+        return test_arm_mve_sat_u(uvalue << shift, bits, saturated) & mask;
+    } else if (uvalue == 0) {
+        uresult = 0;
+    } else {
+        *saturated = true;
+        return mask;
+    }
+    return (uint32_t)uresult & mask;
+}
+
+static void test_arm_mve_expected_qshift(uint8_t *expected,
+                                         const uint8_t *initial,
+                                         const uint8_t *values,
+                                         const uint8_t *shifts,
+                                         unsigned esize, bool is_signed,
+                                         bool rounded, uint16_t pred,
+                                         bool *qc)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (i = 0; i < 16; i += esize) {
+        uint32_t src = test_arm_load_le(values + i, esize);
+        int8_t shift = (int8_t)test_arm_load_le(shifts + i, esize);
+        bool saturated;
+        uint32_t result;
+
+        result = test_arm_mve_expected_qshift_scalar_lane(src, bits, shift,
+                                                          is_signed, rounded,
+                                                          &saturated);
+        test_arm_mve_store_masked(expected, i, esize, result, pred);
+        if (saturated && (pred & (1U << i))) {
+            *qc = true;
+        }
+    }
+}
+
+static void test_arm_mve_expected_qshift_scalar(uint8_t *expected,
+                                                const uint8_t *initial,
+                                                const uint8_t *source,
+                                                unsigned esize, uint32_t rm,
+                                                bool is_signed,
+                                                bool rounded, uint16_t pred,
+                                                bool *qc)
+{
+    unsigned bits = esize * 8;
+    int8_t shift = (int8_t)rm;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (i = 0; i < 16; i += esize) {
+        uint32_t src = test_arm_load_le(source + i, esize);
+        bool saturated;
+        uint32_t result;
+
+        result = test_arm_mve_expected_qshift_scalar_lane(src, bits, shift,
+                                                          is_signed, rounded,
+                                                          &saturated);
+        test_arm_mve_store_masked(expected, i, esize, result, pred);
+        if (saturated && (pred & (1U << i))) {
+            *qc = true;
+        }
+    }
+}
+
+static void test_arm_m55_mve_scalar_shift_run_qc_init(uint32_t insn,
+                                                      const uint8_t *initial,
+                                                      uint32_t rm,
+                                                      uint32_t vpr,
+                                                      const uint8_t *expected,
+                                                      bool eci,
+                                                      bool initial_qc,
+                                                      bool expected_qc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_qc = 1U << 27;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &rm));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci || initial_qc) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        if (initial_qc) {
+            OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+            fpscr |= fpscr_qc;
+            OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+        }
+        epsr = xpsr_t | eci_a0a1;
+        if (eci) {
+            OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        }
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        if (eci) {
+            OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+            TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                        "epsr=0x%08x", epsr);
+        }
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(((fpscr & fpscr_qc) != 0) == expected_qc,
+                "fpscr=0x%08x expected_qc=%d",
+                fpscr, expected_qc);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_scalar_shift_run_qc(uint32_t insn,
+                                                 const uint8_t *initial,
+                                                 uint32_t rm, uint32_t vpr,
+                                                 const uint8_t *expected,
+                                                 bool eci, bool expected_qc)
+{
+    test_arm_m55_mve_scalar_shift_run_qc_init(insn, initial, rm, vpr,
+                                              expected, eci, false,
+                                              expected_qc);
+}
+
+static void test_arm_m55_mve_scalar_shift_run(uint32_t insn,
+                                              const uint8_t *initial,
+                                              uint32_t rm,
+                                              const uint8_t *expected,
+                                              bool eci)
+{
+    test_arm_m55_mve_scalar_shift_run_qc(insn, initial, rm, 0, expected,
+                                         eci, false);
+}
+
+static void test_arm_m55_mve_scalar_shift(void)
+{
+    static const uint32_t vshls_base = 0xee311e60;
+    static const uint32_t vshlu_base = 0xfe311e60;
+    static const uint32_t vrshls_base = 0xee331e60;
+    static const uint32_t vrshlu_base = 0xfe331e60;
+    static const uint32_t vqshls_base = 0xee311ee0;
+    static const uint32_t vqshlu_base = 0xfe311ee0;
+    static const uint32_t vqrshls_base = 0xee331ee0;
+    static const uint32_t vqrshlu_base = 0xfe331ee0;
+    const uint8_t initial[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    const uint8_t source[16] = {
+        0x81, 0x7f, 0x12, 0xf0, 0x55, 0xaa, 0x01, 0x80,
+        0xfe, 0x10, 0x33, 0xcc, 0x09, 0x90, 0x44, 0x22,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    uint8_t shifts[16];
+    uint32_t insn;
+    bool qc;
+    size_t ncase;
+
+    for (ncase = 0; ncase < 3; ncase++) {
+        unsigned size = (unsigned)ncase;
+        unsigned esize = esizes[ncase];
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vshls_base, size, 3);
+        test_arm_mve_fill_scalar_shift(shifts, esize, (uint32_t)-1);
+        test_arm_mve_expected_shift(expected, initial, source, shifts, esize,
+                                    true, false, 0xffff);
+        test_arm_m55_mve_scalar_shift_run(insn, source, (uint32_t)-1,
+                                          expected, false);
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vshlu_base, size, 3);
+        test_arm_mve_fill_scalar_shift(shifts, esize, 2);
+        test_arm_mve_expected_shift(expected, initial, source, shifts, esize,
+                                    false, false, 0xffff);
+        test_arm_m55_mve_scalar_shift_run(insn, source, 2, expected, false);
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vrshls_base, size, 3);
+        test_arm_mve_fill_scalar_shift(shifts, esize, (uint32_t)-2);
+        test_arm_mve_expected_shift(expected, initial, source, shifts, esize,
+                                    true, true, 0xffff);
+        test_arm_m55_mve_scalar_shift_run(insn, source, (uint32_t)-2,
+                                          expected, false);
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vrshlu_base, size, 3);
+        test_arm_mve_fill_scalar_shift(shifts, esize, (uint32_t)-3);
+        test_arm_mve_expected_shift(expected, initial, source, shifts, esize,
+                                    false, true, 0xffff);
+        test_arm_m55_mve_scalar_shift_run(insn, source, (uint32_t)-3,
+                                          expected, false);
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vqshls_base, size, 3);
+        test_arm_mve_expected_qshift_scalar(expected, source, source, esize,
+                                            4, true, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_shift_run_qc(insn, source, 4, 0, expected,
+                                             false, qc);
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vqshlu_base, size, 3);
+        test_arm_mve_expected_qshift_scalar(expected, source, source, esize,
+                                            5, false, false, 0xffff, &qc);
+        test_arm_m55_mve_scalar_shift_run_qc(insn, source, 5, 0, expected,
+                                             false, qc);
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vqrshls_base, size, 3);
+        test_arm_mve_expected_qshift_scalar(expected, source, source, esize,
+                                            (uint32_t)-2, true, true, 0xffff,
+                                            &qc);
+        test_arm_m55_mve_scalar_shift_run_qc(insn, source, (uint32_t)-2, 0,
+                                             expected, false, qc);
+
+        insn = test_arm_mve_vshift_scalar_insn(0, vqrshlu_base, size, 3);
+        test_arm_mve_expected_qshift_scalar(expected, source, source, esize,
+                                            (uint32_t)-3, false, true,
+                                            0xffff, &qc);
+        test_arm_m55_mve_scalar_shift_run_qc(insn, source, (uint32_t)-3, 0,
+                                             expected, false, qc);
+    }
+
+    insn = test_arm_mve_vshift_scalar_insn(0, vqrshlu_base, 2, 3);
+    test_arm_mve_expected_qshift_scalar(expected, source, source, 4, 8,
+                                        false, true, 0x00f0, &qc);
+    test_arm_m55_mve_scalar_shift_run_qc(insn, source, 8, 0x001100f0,
+                                         expected, false, qc);
+
+    test_arm_mve_expected_qshift_scalar(expected, source, source, 4, 8,
+                                        false, true, 0xff00, &qc);
+    test_arm_m55_mve_scalar_shift_run_qc(insn, source, 8, 0, expected, true,
+                                         qc);
+
+    test_arm_mve_expected_qshift_scalar(expected, source, source, 1, 1,
+                                        true, false, 0xffff, &qc);
+    test_arm_m55_mve_scalar_shift_run_qc_init(
+        test_arm_mve_vshift_scalar_insn(0, vqshls_base, 0, 3),
+        source, 1, 0, expected, false, true, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_scalar_insn(0, vshls_base, 3, 3),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_scalar_insn(8, vshls_base, 0, 3),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_scalar_insn(0, vshls_base, 0, 13),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_scalar_insn(0, vshls_base, 0, 15),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_scalar_insn(0, vshls_base, 0, 3),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static uint32_t test_arm_mve_gpr_shift_imm_insn(uint32_t base,
+                                                unsigned rda,
+                                                unsigned shift)
+{
+    uint32_t view = base;
+
+    view |= (rda & 15) << 16;
+    view |= ((shift >> 2) & 7) << 12;
+    view |= (shift & 3) << 6;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_gpr_shift_long_imm_insn(uint32_t base,
+                                                     unsigned rdalo,
+                                                     unsigned rdahi,
+                                                     unsigned shift)
+{
+    uint32_t view = base;
+
+    view |= ((rdalo >> 1) & 7) << 17;
+    view |= ((rdahi >> 1) & 7) << 9;
+    view |= ((shift >> 2) & 7) << 12;
+    view |= (shift & 3) << 6;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_gpr_shift_reg_insn(uint32_t base,
+                                                unsigned rda,
+                                                unsigned rm)
+{
+    uint32_t view = base;
+
+    view |= (rda & 15) << 16;
+    view |= (rm & 15) << 12;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_gpr_shift_long_reg_insn(uint32_t base,
+                                                     unsigned rdalo,
+                                                     unsigned rdahi,
+                                                     unsigned rm)
+{
+    uint32_t view = base;
+
+    view |= ((rdalo >> 1) & 7) << 17;
+    view |= ((rdahi >> 1) & 7) << 9;
+    view |= (rm & 15) << 12;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static void test_arm_m55_mve_gpr_shift32_run_qc(uint32_t insn,
+                                                unsigned rda_reg,
+                                                uint32_t initial_rda,
+                                                unsigned rm_reg,
+                                                uint32_t rm,
+                                                uint32_t expected_rda,
+                                                bool expected_qc)
+{
+    const uint32_t fpscr_qc = 1U << 27;
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t fpscr = 0;
+    uint32_t got;
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rda_reg, &initial_rda));
+    if (rm_reg < 13) {
+        OK(uc_reg_write(uc, UC_ARM_REG_R0 + rm_reg, &rm));
+    }
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rda_reg, &got));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(got == expected_rda,
+                "insn=0x%08x r%u=0x%08x expected=0x%08x",
+                insn, rda_reg, got, expected_rda);
+    TEST_CHECK_(((fpscr & fpscr_qc) != 0) == expected_qc,
+                "insn=0x%08x fpscr=0x%08x expected_qc=%d",
+                insn, fpscr, expected_qc);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_gpr_shift64_run_qc(uint32_t insn,
+                                                unsigned rdalo_reg,
+                                                unsigned rdahi_reg,
+                                                uint64_t initial_rda,
+                                                unsigned rm_reg,
+                                                uint32_t rm,
+                                                uint64_t expected_rda,
+                                                bool expected_qc)
+{
+    const uint32_t fpscr_qc = 1U << 27;
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t initlo = (uint32_t)initial_rda;
+    uint32_t inithi = (uint32_t)(initial_rda >> 32);
+    uint32_t gotlo;
+    uint32_t gothi;
+    uint32_t fpscr = 0;
+    uint64_t got;
+
+    test_arm_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rdalo_reg, &initlo));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rdahi_reg, &inithi));
+    if (rm_reg < 13) {
+        OK(uc_reg_write(uc, UC_ARM_REG_R0 + rm_reg, &rm));
+    }
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rdalo_reg, &gotlo));
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rdahi_reg, &gothi));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    got = ((uint64_t)gothi << 32) | gotlo;
+    TEST_CHECK_(got == expected_rda,
+                "insn=0x%08x got=0x%016llx expected=0x%016llx",
+                insn, (unsigned long long)got,
+                (unsigned long long)expected_rda);
+    TEST_CHECK_(((fpscr & fpscr_qc) != 0) == expected_qc,
+                "insn=0x%08x fpscr=0x%08x expected_qc=%d",
+                insn, fpscr, expected_qc);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_gpr_shift(void)
+{
+    uint32_t insn;
+
+    insn = test_arm_mve_gpr_shift_imm_insn(0xea500f0f, 2, 2);
+    test_arm_m55_mve_gpr_shift32_run_qc(insn, 2, 0x40000000,
+                                        16, 0, UINT32_MAX, true);
+
+    insn = test_arm_mve_gpr_shift_imm_insn(0xea500f3f, 2, 3);
+    test_arm_m55_mve_gpr_shift32_run_qc(insn, 2, 0x20000000,
+                                        16, 0, 0x7fffffff, true);
+
+    insn = test_arm_mve_gpr_shift_imm_insn(0xea500f1f, 2, 1);
+    test_arm_m55_mve_gpr_shift32_run_qc(insn, 2, 0x80000001,
+                                        16, 0, 0x40000001, false);
+
+    insn = test_arm_mve_gpr_shift_imm_insn(0xea500f2f, 2, 1);
+    test_arm_m55_mve_gpr_shift32_run_qc(insn, 2, 0x80000001,
+                                        16, 0, 0xc0000001, false);
+
+    insn = test_arm_mve_gpr_shift_imm_insn(0xea500f1f, 2, 0);
+    test_arm_m55_mve_gpr_shift32_run_qc(insn, 2, 0x80000000,
+                                        16, 0, 1, false);
+
+    insn = test_arm_mve_gpr_shift_reg_insn(0xea500f0d, 2, 4);
+    test_arm_m55_mve_gpr_shift32_run_qc(insn, 2, 3, 4,
+                                        (uint32_t)-1, 2, false);
+
+    insn = test_arm_mve_gpr_shift_reg_insn(0xea500f2d, 2, 4);
+    test_arm_m55_mve_gpr_shift32_run_qc(insn, 2, 0x40000000, 4,
+                                        (uint32_t)-2, 0x7fffffff, true);
+
+    insn = test_arm_mve_gpr_shift_long_imm_insn(0xea50010f, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x0000000100000001ULL, 16, 0,
+        0x0000001000000010ULL, false);
+
+    insn = test_arm_mve_gpr_shift_long_imm_insn(0xea50011f, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x8000000000000000ULL, 16, 0,
+        0x0800000000000000ULL, false);
+
+    insn = test_arm_mve_gpr_shift_long_imm_insn(0xea50012f, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x8000000000000000ULL, 16, 0,
+        0xf800000000000000ULL, false);
+
+    insn = test_arm_mve_gpr_shift_long_imm_insn(0xea51010f, 2, 3, 2);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x4000000000000000ULL, 16, 0,
+        UINT64_MAX, true);
+
+    insn = test_arm_mve_gpr_shift_long_imm_insn(0xea51013f, 2, 3, 3);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x2000000000000000ULL, 16, 0,
+        INT64_MAX, true);
+
+    insn = test_arm_mve_gpr_shift_long_imm_insn(0xea51011f, 2, 3, 1);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x8000000000000001ULL, 16, 0,
+        0x4000000000000001ULL, false);
+
+    insn = test_arm_mve_gpr_shift_long_imm_insn(0xea51012f, 2, 3, 1);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x8000000000000001ULL, 16, 0,
+        0xc000000000000001ULL, false);
+
+    insn = test_arm_mve_gpr_shift_long_reg_insn(0xea50010d, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 8, 4, (uint32_t)-1, 4, false);
+
+    insn = test_arm_mve_gpr_shift_long_reg_insn(0xea50012d, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x8000000000000000ULL, 4, 4,
+        0xf800000000000000ULL, false);
+
+    insn = test_arm_mve_gpr_shift_long_reg_insn(0xea51010d, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x4000000000000000ULL, 4, 2,
+        UINT64_MAX, true);
+
+    insn = test_arm_mve_gpr_shift_long_reg_insn(0xea51012d, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x4000000000000000ULL, 4, (uint32_t)-2,
+        INT64_MAX, true);
+
+    insn = test_arm_mve_gpr_shift_long_reg_insn(0xea51018d, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x0000800000000000ULL, 4, 1,
+        0x0000ffffffffffffULL, true);
+
+    insn = test_arm_mve_gpr_shift_long_reg_insn(0xea5101ad, 2, 3, 4);
+    test_arm_m55_mve_gpr_shift64_run_qc(
+        insn, 2, 3, 0x0000400000000000ULL, 4, (uint32_t)-2,
+        0x00007fffffffffffULL, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_gpr_shift_imm_insn(0xea500f0f, 13, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_gpr_shift_reg_insn(0xea500f0d, 2, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_gpr_shift_long_imm_insn(0xea50010f, 2, 13, 4),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_gpr_shift_long_reg_insn(0xea50010d, 2, 3, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_gpr_shift_imm_insn(0xea500f0f, 2, 2),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_FETCH_UNMAPPED);
+}
+
+static uint32_t test_arm_mve_vshll_insn(unsigned qd, unsigned qm,
+                                        uint32_t base, unsigned size,
+                                        unsigned shift, bool esize_shift)
+{
+    uint32_t view = base;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    if (esize_shift) {
+        if (size == 1) {
+            view |= 1U << 18;
+        } else {
+            TEST_CHECK(size == 0);
+        }
+    } else if (size == 0) {
+        view |= 1U << 19;
+        view |= (shift & 7) << 16;
+    } else {
+        TEST_CHECK(size == 1);
+        view |= 1U << 20;
+        view |= (shift & 15) << 16;
+    }
+    return test_arm_mve_view_to_t32(view);
+}
+
+static void test_arm_mve_expected_vshll(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        const uint8_t *source,
+                                        unsigned esize, unsigned shift,
+                                        bool top, bool is_signed,
+                                        uint16_t pred)
+{
+    unsigned bits = esize * 8;
+    unsigned lesize = esize * 2;
+    size_t le;
+
+    memcpy(expected, initial, 16);
+    for (le = 0; le < 16 / lesize; le++) {
+        size_t src_off = (le * 2 + top) * esize;
+        size_t dst_off = le * lesize;
+        uint32_t src = test_arm_load_le(source + src_off, esize);
+        uint64_t result;
+
+        if (is_signed) {
+            result = (uint64_t)(test_arm_sign_extend(src, bits) *
+                                (1LL << shift));
+        } else {
+            result = (uint64_t)src << shift;
+        }
+        test_arm_mve_store_masked(expected, dst_off, lesize, result, pred);
+    }
+}
+
+static void test_arm_m55_mve_vshll(void)
+{
+    const uint8_t initial[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    const uint8_t source[16] = {
+        0x81, 0x7f, 0x12, 0xf0, 0x55, 0xaa, 0x01, 0x80,
+        0xfe, 0x10, 0x33, 0xcc, 0x09, 0x90, 0x44, 0x22,
+    };
+    uint8_t expected[16];
+    uint32_t insn;
+
+    insn = test_arm_mve_vshll_insn(0, 1, 0xeea00f40, 0, 3, false);
+    test_arm_mve_expected_vshll(expected, initial, source, 1, 3, false,
+                                true, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshll_insn(0, 1, 0xfea01f40, 1, 4, false);
+    test_arm_mve_expected_vshll(expected, initial, source, 2, 4, true,
+                                false, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshll_insn(0, 1, 0xfea00f40, 0, 0, false);
+    test_arm_mve_expected_vshll(expected, initial, source, 1, 0, false,
+                                false, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshll_insn(0, 1, 0xee310e01, 0, 8, true);
+    test_arm_mve_expected_vshll(expected, initial, source, 1, 8, false,
+                                true, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshll_insn(0, 1, 0xfe311e01, 1, 16, true);
+    test_arm_mve_expected_vshll(expected, initial, source, 2, 16, true,
+                                false, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshll_insn(0, 1, 0xeea01f40, 0, 2, false);
+    test_arm_mve_expected_vshll(expected, initial, source, 1, 2, true,
+                                true, 0xff00);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshll_insn(8, 1, 0xeea00f40, 0, 3, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshll_insn(0, 1, 0xeea00f40, 0, 3, false),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static uint64_t test_arm_mve_urshr(uint64_t value, unsigned shift)
+{
+    return (value >> shift) + ((value >> (shift - 1)) & 1);
+}
+
+static int64_t test_arm_mve_srshr(int64_t value, unsigned shift)
+{
+    return (value >> shift) + ((value >> (shift - 1)) & 1);
+}
+
+typedef enum test_arm_mve_qshrn_kind {
+    TEST_ARM_MVE_QSHRN_S,
+    TEST_ARM_MVE_QSHRN_U,
+    TEST_ARM_MVE_QSHRUN,
+    TEST_ARM_MVE_QRSHRN_S,
+    TEST_ARM_MVE_QRSHRN_U,
+    TEST_ARM_MVE_QRSHRUN,
+} test_arm_mve_qshrn_kind;
+
+static uint32_t test_arm_mve_expected_qshrn_lane(
+    uint64_t src, unsigned src_bits, unsigned dst_bits, unsigned shift,
+    test_arm_mve_qshrn_kind kind, bool *saturated)
+{
+    uint32_t dst_mask = test_arm_mve_element_mask(dst_bits);
+    int64_t signed_src;
+    int64_t signed_value;
+    uint64_t unsigned_value;
+
+    *saturated = false;
+    switch (kind) {
+    case TEST_ARM_MVE_QSHRN_S:
+        signed_src = test_arm_sign_extend((uint32_t)src, src_bits);
+        signed_value = signed_src >> shift;
+        return test_arm_mve_sat_s(signed_value, dst_bits, saturated) &
+               dst_mask;
+    case TEST_ARM_MVE_QSHRN_U:
+        unsigned_value = src >> shift;
+        return test_arm_mve_sat_u(unsigned_value, dst_bits, saturated) &
+               dst_mask;
+    case TEST_ARM_MVE_QSHRUN:
+        signed_src = test_arm_sign_extend((uint32_t)src, src_bits);
+        signed_value = signed_src >> shift;
+        break;
+    case TEST_ARM_MVE_QRSHRN_S:
+        signed_src = test_arm_sign_extend((uint32_t)src, src_bits);
+        signed_value = test_arm_mve_srshr(signed_src, shift);
+        return test_arm_mve_sat_s(signed_value, dst_bits, saturated) &
+               dst_mask;
+    case TEST_ARM_MVE_QRSHRN_U:
+        unsigned_value = test_arm_mve_urshr(src, shift);
+        return test_arm_mve_sat_u(unsigned_value, dst_bits, saturated) &
+               dst_mask;
+    case TEST_ARM_MVE_QRSHRUN:
+        signed_src = test_arm_sign_extend((uint32_t)src, src_bits);
+        signed_value = test_arm_mve_srshr(signed_src, shift);
+        break;
+    default:
+        TEST_CHECK(false);
+        return 0;
+    }
+
+    if (signed_value < 0) {
+        *saturated = true;
+        return 0;
+    }
+    return test_arm_mve_sat_u((uint64_t)signed_value, dst_bits,
+                              saturated) & dst_mask;
+}
+
+static void test_arm_mve_expected_shrn(uint8_t *expected,
+                                       const uint8_t *initial,
+                                       const uint8_t *source,
+                                       unsigned esize, unsigned shift,
+                                       bool top, bool rounded,
+                                       uint16_t pred)
+{
+    unsigned lesize = esize * 2;
+    size_t le;
+
+    memcpy(expected, initial, 16);
+    for (le = 0; le < 16 / lesize; le++) {
+        size_t src_off = le * lesize;
+        size_t dst_off = (le * 2 + top) * esize;
+        uint64_t src = test_arm_load_le(source + src_off, lesize);
+        uint64_t result = rounded ? test_arm_mve_urshr(src, shift) :
+            src >> shift;
+
+        test_arm_mve_store_masked(expected, dst_off, esize, result, pred);
+    }
+}
+
+static void test_arm_mve_expected_qshrn(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        const uint8_t *source,
+                                        unsigned esize, unsigned shift,
+                                        bool top,
+                                        test_arm_mve_qshrn_kind kind,
+                                        uint16_t pred, bool *qc)
+{
+    unsigned dst_bits = esize * 8;
+    unsigned lesize = esize * 2;
+    unsigned src_bits = lesize * 8;
+    size_t le;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (le = 0; le < 16 / lesize; le++) {
+        size_t src_off = le * lesize;
+        size_t dst_off = (le * 2 + top) * esize;
+        uint64_t src = test_arm_load_le(source + src_off, lesize);
+        bool saturated;
+        uint32_t result;
+
+        result = test_arm_mve_expected_qshrn_lane(src, src_bits, dst_bits,
+                                                  shift, kind, &saturated);
+        test_arm_mve_store_masked(expected, dst_off, esize, result, pred);
+        if (saturated && (pred & (1U << dst_off))) {
+            *qc = true;
+        }
+    }
+}
+
+static void test_arm_m55_mve_shrn_imm(void)
+{
+    const uint8_t initial[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    const uint8_t source[16] = {
+        0xff, 0x00, 0x80, 0x01, 0x7f, 0x80, 0x01, 0xff,
+        0x34, 0x12, 0x78, 0x56, 0x88, 0x77, 0x66, 0x55,
+    };
+    uint8_t expected[16];
+    uint32_t insn;
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee800fc1, 0, 4, true);
+    test_arm_mve_expected_shrn(expected, initial, source, 1, 4, false,
+                               false, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee801fc1, 1, 8, true);
+    test_arm_mve_expected_shrn(expected, initial, source, 2, 8, true,
+                               false, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xfe800fc1, 0, 8, true);
+    test_arm_mve_expected_shrn(expected, initial, source, 1, 8, false,
+                               true, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xfe801fc1, 1, 16, true);
+    test_arm_mve_expected_shrn(expected, initial, source, 2, 16, true,
+                               true, 0xffff);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected,
+                                   false);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee801fc1, 0, 4, true);
+    test_arm_mve_expected_shrn(expected, initial, source, 1, 4, true,
+                               false, 0xff00);
+    test_arm_m55_mve_shift_imm_run(insn, initial, source, 0, expected, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(8, 1, 0xee800fc1, 0, 4, true),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(0, 1, 0xee800fc1, 0, 4, true),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_qshrn_imm(void)
+{
+    const uint8_t initial[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    const uint8_t source[16] = {
+        0xff, 0x00, 0x80, 0x01, 0x7f, 0x80, 0x01, 0xff,
+        0xf8, 0x0f, 0x80, 0xff, 0x00, 0x80, 0xff, 0x7f,
+    };
+    const uint8_t eci_source[16] = {
+        0x00, 0x80, 0xff, 0x7f, 0x00, 0x80, 0xff, 0x7f,
+        0x10, 0x00, 0x20, 0x00, 0x30, 0x00, 0x40, 0x00,
+    };
+    const uint8_t pred_source[16] = {
+        0x00, 0x80, 0xff, 0x7f, 0x10, 0x00, 0x20, 0x00,
+        0x30, 0x00, 0x40, 0x00, 0x50, 0x00, 0x60, 0x00,
+    };
+    const uint8_t nonsat_source[16] = {
+        0x10, 0x00, 0x20, 0x00, 0x30, 0x00, 0x40, 0x00,
+        0x50, 0x00, 0x60, 0x00, 0x70, 0x00, 0x7f, 0x00,
+    };
+    uint8_t expected[16];
+    uint32_t insn;
+    bool qc;
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee800f40, 0, 1, true);
+    test_arm_mve_expected_qshrn(expected, initial, source, 1, 1, false,
+                                TEST_ARM_MVE_QSHRN_S, 0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xfe801f40, 1, 8, true);
+    test_arm_mve_expected_qshrn(expected, initial, source, 2, 8, true,
+                                TEST_ARM_MVE_QSHRN_U, 0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee800fc0, 0, 4, true);
+    test_arm_mve_expected_qshrn(expected, initial, source, 1, 4, false,
+                                TEST_ARM_MVE_QSHRUN, 0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee801f41, 0, 4, true);
+    test_arm_mve_expected_qshrn(expected, initial, source, 1, 4, true,
+                                TEST_ARM_MVE_QRSHRN_S, 0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xfe800f41, 0, 8, true);
+    test_arm_mve_expected_qshrn(expected, initial, source, 1, 8, false,
+                                TEST_ARM_MVE_QRSHRN_U, 0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xfe800fc0, 0, 4, true);
+    test_arm_mve_expected_qshrn(expected, initial, eci_source, 1, 4, false,
+                                TEST_ARM_MVE_QRSHRUN, 0xff00, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, eci_source, 0,
+                                      expected, true, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee801fc0, 1, 8, true);
+    test_arm_mve_expected_qshrn(expected, initial, source, 2, 8, true,
+                                TEST_ARM_MVE_QSHRUN, 0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xfe801fc0, 0, 4, true);
+    test_arm_mve_expected_qshrn(expected, initial, source, 1, 4, true,
+                                TEST_ARM_MVE_QRSHRUN, 0xffff, &qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, source, 0, expected,
+                                      false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee800f40, 0, 1, true);
+    test_arm_mve_expected_qshrn(expected, initial, pred_source, 1, 1, false,
+                                TEST_ARM_MVE_QSHRN_S, 0x00f0, &qc);
+    TEST_CHECK(!qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, pred_source,
+                                      0x001100f0, expected, false, qc);
+
+    insn = test_arm_mve_vshift_imm_insn(0, 1, 0xee800f40, 0, 1, true);
+    test_arm_mve_expected_qshrn(expected, initial, nonsat_source, 1, 1,
+                                false, TEST_ARM_MVE_QSHRN_S, 0xffff, &qc);
+    TEST_CHECK(!qc);
+    test_arm_m55_mve_shift_imm_run_qc_init(insn, initial, nonsat_source, 0,
+                                           expected, false, true, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(8, 1, 0xee800f40, 0, 1, true),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshift_imm_insn(0, 1, 0xee800f40, 0, 1, true),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+typedef enum test_arm_mve_movn_kind {
+    TEST_ARM_MVE_MOVN,
+    TEST_ARM_MVE_QMOVN_S,
+    TEST_ARM_MVE_QMOVN_U,
+    TEST_ARM_MVE_QMOVUN,
+} test_arm_mve_movn_kind;
+
+static uint32_t test_arm_mve_1op_insn(unsigned qd, unsigned qm,
+                                      uint32_t base, unsigned size)
+{
+    uint32_t view = base;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (size & 3) << 18;
+    view |= (qd & 7) << 13;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_vcvt_fixed_insn(unsigned qd, unsigned qm,
+                                             uint32_t base, unsigned size,
+                                             uint32_t shift)
+{
+    uint32_t view = base;
+    uint32_t encoded;
+
+    encoded = (size == 1) ? (16 - shift) : (32 - shift);
+    view |= (encoded & ((1U << (size == 1 ? 4 : 5)) - 1)) << 16;
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_movn_insn(unsigned qd, unsigned qm,
+                                       uint32_t base, unsigned size)
+{
+    return test_arm_mve_1op_insn(qd, qm, base, size);
+}
+
+static uint32_t test_arm_mve_expected_movn_lane(
+    uint64_t src, unsigned src_bits, unsigned dst_bits,
+    test_arm_mve_movn_kind kind, bool *saturated)
+{
+    uint32_t dst_mask = test_arm_mve_element_mask(dst_bits);
+    int64_t signed_src;
+
+    *saturated = false;
+    switch (kind) {
+    case TEST_ARM_MVE_MOVN:
+        return (uint32_t)src & dst_mask;
+    case TEST_ARM_MVE_QMOVN_S:
+        signed_src = test_arm_sign_extend((uint32_t)src, src_bits);
+        return test_arm_mve_sat_s(signed_src, dst_bits, saturated) &
+               dst_mask;
+    case TEST_ARM_MVE_QMOVN_U:
+        return test_arm_mve_sat_u(src, dst_bits, saturated) & dst_mask;
+    case TEST_ARM_MVE_QMOVUN:
+        signed_src = test_arm_sign_extend((uint32_t)src, src_bits);
+        if (signed_src < 0) {
+            *saturated = true;
+            return 0;
+        }
+        return test_arm_mve_sat_u((uint64_t)signed_src, dst_bits,
+                                  saturated) & dst_mask;
+    default:
+        TEST_CHECK(false);
+        return 0;
+    }
+}
+
+static void test_arm_mve_expected_movn(uint8_t *expected,
+                                       const uint8_t *initial,
+                                       const uint8_t *source,
+                                       unsigned esize, bool top,
+                                       test_arm_mve_movn_kind kind,
+                                       uint16_t pred, bool *qc)
+{
+    unsigned dst_bits = esize * 8;
+    unsigned lesize = esize * 2;
+    unsigned src_bits = lesize * 8;
+    size_t le;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (le = 0; le < 16 / lesize; le++) {
+        size_t src_off = le * lesize;
+        size_t dst_off = (le * 2 + top) * esize;
+        uint64_t src = test_arm_load_le(source + src_off, lesize);
+        bool saturated;
+        uint32_t result;
+
+        result = test_arm_mve_expected_movn_lane(src, src_bits, dst_bits,
+                                                 kind, &saturated);
+        test_arm_mve_store_masked(expected, dst_off, esize, result, pred);
+        if (saturated && (pred & (1U << dst_off))) {
+            *qc = true;
+        }
+    }
+}
+
+static void test_arm_m55_mve_movn(void)
+{
+    const uint8_t initial[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    const uint8_t half_source[16] = {
+        0x34, 0x12, 0x80, 0xff, 0xff, 0x00, 0x7f, 0x7f,
+        0x00, 0x80, 0xff, 0x7f, 0x01, 0x00, 0x00, 0xff,
+    };
+    const uint8_t word_source[16] = {
+        0x34, 0x12, 0x00, 0x00, 0x00, 0x80, 0xff, 0xff,
+        0x00, 0x80, 0x01, 0x00, 0x00, 0x00, 0x00, 0x80,
+    };
+    const uint8_t pred_source[16] = {
+        0x00, 0x80, 0xff, 0x7f, 0x10, 0x00, 0x20, 0x00,
+        0x00, 0x80, 0xff, 0x7f, 0x00, 0x80, 0xff, 0x7f,
+    };
+    const uint8_t eci_source[16] = {
+        0x00, 0x80, 0xff, 0x7f, 0x00, 0x80, 0xff, 0x7f,
+        0x10, 0x00, 0x20, 0x00, 0x30, 0x00, 0x40, 0x00,
+    };
+    uint8_t expected[16];
+    uint32_t insn;
+    bool qc;
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xfe310e81, 0);
+    test_arm_mve_expected_movn(expected, initial, half_source, 1, false,
+                               TEST_ARM_MVE_MOVN, 0xffff, &qc);
+    TEST_CHECK(!qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, half_source, 0,
+                                      expected, false, qc);
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xfe311e81, 1);
+    test_arm_mve_expected_movn(expected, initial, word_source, 2, true,
+                               TEST_ARM_MVE_MOVN, 0xffff, &qc);
+    TEST_CHECK(!qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, word_source, 0,
+                                      expected, false, qc);
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xee330e01, 0);
+    test_arm_mve_expected_movn(expected, initial, half_source, 1, false,
+                               TEST_ARM_MVE_QMOVN_S, 0xffff, &qc);
+    TEST_CHECK(qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, half_source, 0,
+                                      expected, false, qc);
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xfe331e01, 1);
+    test_arm_mve_expected_movn(expected, initial, word_source, 2, true,
+                               TEST_ARM_MVE_QMOVN_U, 0xffff, &qc);
+    TEST_CHECK(qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, word_source, 0,
+                                      expected, false, qc);
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xee310e81, 0);
+    test_arm_mve_expected_movn(expected, initial, half_source, 1, false,
+                               TEST_ARM_MVE_QMOVUN, 0xffff, &qc);
+    TEST_CHECK(qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, half_source, 0,
+                                      expected, false, qc);
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xee311e81, 1);
+    test_arm_mve_expected_movn(expected, initial, word_source, 2, true,
+                               TEST_ARM_MVE_QMOVUN, 0xffff, &qc);
+    TEST_CHECK(qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, word_source, 0,
+                                      expected, false, qc);
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xee330e01, 0);
+    test_arm_mve_expected_movn(expected, initial, pred_source, 1, false,
+                               TEST_ARM_MVE_QMOVN_S, 0x00f0, &qc);
+    TEST_CHECK(!qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, pred_source,
+                                      0x001100f0, expected, false, qc);
+
+    insn = test_arm_mve_movn_insn(0, 1, 0xee310e81, 0);
+    test_arm_mve_expected_movn(expected, initial, eci_source, 1, false,
+                               TEST_ARM_MVE_QMOVUN, 0xff00, &qc);
+    TEST_CHECK(!qc);
+    test_arm_m55_mve_shift_imm_run_qc(insn, initial, eci_source, 0,
+                                      expected, true, qc);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_movn_insn(0, 1, 0xfe310e81, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_movn_insn(8, 1, 0xfe310e81, 0),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_movn_insn(0, 1, 0xfe310e81, 0),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static uint32_t test_arm_mve_vaddv_insn(unsigned qm, unsigned rda,
+                                        bool is_unsigned, unsigned size,
+                                        bool accum)
+{
+    uint32_t view = 0xeef10f00;
+
+    view |= (is_unsigned ? 1U : 0U) << 28;
+    view |= (size & 3) << 18;
+    view |= ((rda >> 1) & 7) << 13;
+    view |= (accum ? 1U : 0U) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+typedef enum test_arm_mve_reduce_minmax_kind {
+    TEST_ARM_MVE_REDUCE_MAX_S,
+    TEST_ARM_MVE_REDUCE_MAX_U,
+    TEST_ARM_MVE_REDUCE_MAXA,
+    TEST_ARM_MVE_REDUCE_MIN_S,
+    TEST_ARM_MVE_REDUCE_MIN_U,
+    TEST_ARM_MVE_REDUCE_MINA,
+} test_arm_mve_reduce_minmax_kind;
+
+typedef enum test_arm_mve_reduce_fp_kind {
+    TEST_ARM_MVE_REDUCE_FMAXNMV_F32,
+    TEST_ARM_MVE_REDUCE_FMAXNMV_F16,
+    TEST_ARM_MVE_REDUCE_FMINNMV_F32,
+    TEST_ARM_MVE_REDUCE_FMINNMV_F16,
+    TEST_ARM_MVE_REDUCE_FMAXNMAV_F32,
+    TEST_ARM_MVE_REDUCE_FMAXNMAV_F16,
+    TEST_ARM_MVE_REDUCE_FMINNMAV_F32,
+    TEST_ARM_MVE_REDUCE_FMINNMAV_F16,
+} test_arm_mve_reduce_fp_kind;
+
+static uint32_t
+test_arm_mve_vmaxv_insn(unsigned qm, unsigned rda,
+                        test_arm_mve_reduce_minmax_kind kind,
+                        unsigned size)
+{
+    uint32_t view;
+
+    switch (kind) {
+    case TEST_ARM_MVE_REDUCE_MAX_S:
+        view = 0xeee20f00;
+        break;
+    case TEST_ARM_MVE_REDUCE_MAX_U:
+        view = 0xfee20f00;
+        break;
+    case TEST_ARM_MVE_REDUCE_MAXA:
+        view = 0xeee00f00;
+        break;
+    case TEST_ARM_MVE_REDUCE_MIN_S:
+        view = 0xeee20f80;
+        break;
+    case TEST_ARM_MVE_REDUCE_MIN_U:
+        view = 0xfee20f80;
+        break;
+    case TEST_ARM_MVE_REDUCE_MINA:
+        view = 0xeee00f80;
+        break;
+    default:
+        TEST_ASSERT(false);
+        view = 0;
+        break;
+    }
+
+    view |= (size & 3) << 18;
+    view |= (rda & 15) << 12;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_vmaxnmv_insn(unsigned qm, unsigned rda,
+                                          test_arm_mve_reduce_fp_kind kind)
+{
+    uint32_t view;
+
+    switch (kind) {
+    case TEST_ARM_MVE_REDUCE_FMAXNMV_F32:
+        view = 0xeeee0f00;
+        break;
+    case TEST_ARM_MVE_REDUCE_FMAXNMV_F16:
+        view = 0xfeee0f00;
+        break;
+    case TEST_ARM_MVE_REDUCE_FMINNMV_F32:
+        view = 0xeeee0f80;
+        break;
+    case TEST_ARM_MVE_REDUCE_FMINNMV_F16:
+        view = 0xfeee0f80;
+        break;
+    case TEST_ARM_MVE_REDUCE_FMAXNMAV_F32:
+        view = 0xeeec0f00;
+        break;
+    case TEST_ARM_MVE_REDUCE_FMAXNMAV_F16:
+        view = 0xfeec0f00;
+        break;
+    case TEST_ARM_MVE_REDUCE_FMINNMAV_F32:
+        view = 0xeeec0f80;
+        break;
+    case TEST_ARM_MVE_REDUCE_FMINNMAV_F16:
+        view = 0xfeec0f80;
+        break;
+    default:
+        TEST_ASSERT(false);
+        view = 0;
+        break;
+    }
+
+    view |= (rda & 15) << 12;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_vaddlv_insn(unsigned qm, unsigned rdalo,
+                                         unsigned rdahi, bool is_unsigned,
+                                         bool accum)
+{
+    uint32_t view = 0xee890f00;
+
+    view |= (is_unsigned ? 1U : 0U) << 28;
+    view |= ((rdahi >> 1) & 7) << 20;
+    view |= ((rdalo >> 1) & 7) << 13;
+    view |= (accum ? 1U : 0U) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_vabav_insn(unsigned qn, unsigned qm,
+                                        unsigned rda, bool is_unsigned,
+                                        unsigned size)
+{
+    uint32_t view = 0xee800f01;
+
+    view |= (is_unsigned ? 1U : 0U) << 28;
+    view |= (size & 3) << 20;
+    view |= (qn & 7) << 17;
+    view |= (rda & 15) << 12;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_expected_vaddv(const uint8_t *source,
+                                            unsigned esize,
+                                            bool is_unsigned,
+                                            uint32_t accum,
+                                            uint16_t pred)
+{
+    size_t i;
+
+    for (i = 0; i < 16; i += esize) {
+        uint32_t value = test_arm_load_le(source + i, esize);
+
+        if (!(pred & (1U << i))) {
+            continue;
+        }
+        if (is_unsigned) {
+            accum += value;
+        } else {
+            accum += (uint32_t)test_arm_sign_extend(value, esize * 8);
+        }
+    }
+    return accum;
+}
+
+static uint32_t
+test_arm_mve_expected_vmaxv(const uint8_t *source, unsigned esize,
+                            test_arm_mve_reduce_minmax_kind kind,
+                            uint32_t accum, uint16_t pred)
+{
+    unsigned bits = esize * 8;
+    uint64_t mask = bits == 32 ? UINT32_MAX : (1ULL << bits) - 1;
+    int64_t saccum = test_arm_sign_extend(accum & (uint32_t)mask, bits);
+    uint64_t uaccum = accum & (uint32_t)mask;
+    size_t i;
+
+    for (i = 0; i < 16; i += esize) {
+        uint32_t value = test_arm_load_le(source + i, esize);
+        int64_t svalue;
+        uint64_t uvalue;
+
+        if (!(pred & (1U << i))) {
+            continue;
+        }
+
+        switch (kind) {
+        case TEST_ARM_MVE_REDUCE_MAX_S:
+            svalue = test_arm_sign_extend(value, bits);
+            saccum = saccum >= svalue ? saccum : svalue;
+            break;
+        case TEST_ARM_MVE_REDUCE_MIN_S:
+            svalue = test_arm_sign_extend(value, bits);
+            saccum = saccum >= svalue ? svalue : saccum;
+            break;
+        case TEST_ARM_MVE_REDUCE_MAX_U:
+            uvalue = value & mask;
+            uaccum = uaccum >= uvalue ? uaccum : uvalue;
+            break;
+        case TEST_ARM_MVE_REDUCE_MIN_U:
+            uvalue = value & mask;
+            uaccum = uaccum >= uvalue ? uvalue : uaccum;
+            break;
+        case TEST_ARM_MVE_REDUCE_MAXA:
+            svalue = test_arm_sign_extend(value, bits);
+            uvalue = (uint64_t)(svalue < 0 ? -svalue : svalue);
+            uaccum = uaccum >= uvalue ? uaccum : uvalue;
+            break;
+        case TEST_ARM_MVE_REDUCE_MINA:
+            svalue = test_arm_sign_extend(value, bits);
+            uvalue = (uint64_t)(svalue < 0 ? -svalue : svalue);
+            uaccum = uaccum >= uvalue ? uvalue : uaccum;
+            break;
+        }
+    }
+
+    switch (kind) {
+    case TEST_ARM_MVE_REDUCE_MAX_S:
+    case TEST_ARM_MVE_REDUCE_MIN_S:
+        return (uint32_t)saccum;
+    default:
+        return (uint32_t)uaccum;
+    }
+}
+
+static uint64_t test_arm_mve_expected_vaddlv(const uint8_t *source,
+                                             bool is_unsigned,
+                                             uint64_t accum,
+                                             uint16_t pred)
+{
+    size_t i;
+
+    for (i = 0; i < 16; i += 4) {
+        uint32_t value = test_arm_load_le(source + i, 4);
+
+        if (!(pred & (1U << i))) {
+            continue;
+        }
+        if (is_unsigned) {
+            accum += value;
+        } else {
+            accum += (uint64_t)test_arm_sign_extend(value, 32);
+        }
+    }
+    return accum;
+}
+
+static uint32_t test_arm_mve_expected_vabav(const uint8_t *n,
+                                            const uint8_t *m,
+                                            unsigned esize,
+                                            bool is_unsigned,
+                                            uint32_t accum,
+                                            uint16_t pred)
+{
+    size_t i;
+
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = test_arm_load_le(m + i, esize);
+        int64_t slhs;
+        int64_t srhs;
+        uint64_t diff;
+
+        if (!(pred & (1U << i))) {
+            continue;
+        }
+        if (is_unsigned) {
+            diff = lhs >= rhs ? lhs - rhs : rhs - lhs;
+        } else {
+            slhs = test_arm_sign_extend(lhs, esize * 8);
+            srhs = test_arm_sign_extend(rhs, esize * 8);
+            diff = slhs >= srhs ? slhs - srhs : srhs - slhs;
+        }
+        accum += (uint32_t)diff;
+    }
+    return accum;
+}
+
+static void test_arm_m55_mve_reduce32_run(uint32_t insn,
+                                          const uint8_t *q1_data,
+                                          const uint8_t *q2_data,
+                                          unsigned rda_reg,
+                                          uint32_t initial_rda,
+                                          uint32_t vpr,
+                                          uint32_t expected_rda,
+                                          bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    uint32_t epsr;
+    uint32_t got;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q1, q1_data, 16);
+    memcpy(q2, q2_data, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rda_reg, &initial_rda));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rda_reg, &got));
+    TEST_CHECK_(got == expected_rda,
+                "insn=0x%08x r%u=0x%08x expected=0x%08x",
+                insn, rda_reg, got, expected_rda);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_reduce_fp_run(uint32_t insn,
+                                           const uint8_t *q1_data,
+                                           unsigned rda_reg,
+                                           uint32_t initial_rda,
+                                           uint32_t vpr,
+                                           uint32_t expected_rda,
+                                           bool eci, bool expected_ioc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_ioc = 1U;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q1[2];
+    uint32_t epsr;
+    uint32_t fpscr;
+    uint32_t got;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q1, q1_data, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rda_reg, &initial_rda));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rda_reg, &got));
+    TEST_CHECK_(got == expected_rda,
+                "insn=0x%08x r%u=0x%08x expected=0x%08x",
+                insn, rda_reg, got, expected_rda);
+
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(((fpscr & fpscr_ioc) != 0) == expected_ioc,
+                "insn=0x%08x fpscr=0x%08x expected_ioc=%u",
+                insn, fpscr, expected_ioc ? 1 : 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_reduce64_run(uint32_t insn,
+                                          const uint8_t *source,
+                                          unsigned rdalo_reg,
+                                          unsigned rdahi_reg,
+                                          uint64_t initial_rda,
+                                          uint32_t vpr,
+                                          uint64_t expected_rda,
+                                          bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q1[2];
+    uint32_t epsr;
+    uint32_t gotlo;
+    uint32_t gothi;
+    uint32_t initlo = (uint32_t)initial_rda;
+    uint32_t inithi = (uint32_t)(initial_rda >> 32);
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q1, source, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rdalo_reg, &initlo));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rdahi_reg, &inithi));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rdalo_reg, &gotlo));
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rdahi_reg, &gothi));
+    TEST_CHECK_((((uint64_t)gothi << 32) | gotlo) == expected_rda,
+                "insn=0x%08x got=0x%08x%08x expected=0x%016llx",
+                insn, gothi, gotlo, (unsigned long long)expected_rda);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_reduce(void)
+{
+    const uint8_t source[16] = {
+        0x01, 0xff, 0x7f, 0x80, 0x10, 0x00, 0xf0, 0xff,
+        0x78, 0x56, 0x34, 0x12, 0x88, 0xa9, 0xcb, 0xed,
+    };
+    const uint8_t other[16] = {
+        0x7f, 0x00, 0x80, 0xff, 0x20, 0x00, 0x10, 0x00,
+        0x00, 0x80, 0xff, 0x7f, 0x78, 0x56, 0x34, 0x12,
+    };
+    const uint8_t fp32_source[16] = {
+        0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0xa0, 0x40,
+        0x00, 0x00, 0xe0, 0xc0, 0x00, 0x00, 0x40, 0x40,
+    };
+    const uint8_t fp32_snan_source[16] = {
+        0x01, 0x00, 0x80, 0x7f, 0x00, 0x00, 0x00, 0xc0,
+        0x00, 0x00, 0x80, 0xbf, 0x00, 0x00, 0x00, 0x40,
+    };
+    const uint8_t fp16_source[16] = {
+        0x00, 0xc0, 0x00, 0x42, 0x00, 0x38, 0x00, 0xc4,
+        0x00, 0x3e, 0x00, 0xb4, 0x00, 0x41, 0x00, 0xc6,
+    };
+    const uint8_t fp16_snan_source[16] = {
+        0x00, 0x7d, 0x00, 0xc0, 0x00, 0xbc, 0x00, 0x40,
+        0x00, 0x38, 0x00, 0xb8, 0x00, 0x3c, 0x00, 0xc2,
+    };
+    uint32_t insn;
+    uint32_t expected32;
+    uint64_t expected64;
+
+    insn = test_arm_mve_vaddv_insn(1, 2, false, 0, false);
+    expected32 = test_arm_mve_expected_vaddv(source, 1, false, 0, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x11111111, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vaddv_insn(1, 2, true, 1, true);
+    expected32 = test_arm_mve_expected_vaddv(source, 2, true, 0x1000,
+                                             0x00f0);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x1000,
+                                  0x001100f0, expected32, false);
+
+    insn = test_arm_mve_vaddv_insn(1, 2, false, 2, false);
+    expected32 = test_arm_mve_expected_vaddv(source, 4, false, 0x12345678,
+                                             0xff00);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x12345678, 0,
+                                  expected32, true);
+
+    insn = test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MAX_S, 0);
+    expected32 = test_arm_mve_expected_vmaxv(
+        source, 1, TEST_ARM_MVE_REDUCE_MAX_S, 0x00000081, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x00000081, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MIN_S, 1);
+    expected32 = test_arm_mve_expected_vmaxv(
+        source, 2, TEST_ARM_MVE_REDUCE_MIN_S, 0x00007fff, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x00007fff, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MAX_U, 2);
+    expected32 = test_arm_mve_expected_vmaxv(
+        source, 4, TEST_ARM_MVE_REDUCE_MAX_U, 0x00001000, 0x00f0);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x00001000,
+                                  0x001100f0, expected32, false);
+
+    insn = test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MIN_U, 0);
+    expected32 = test_arm_mve_expected_vmaxv(
+        source, 1, TEST_ARM_MVE_REDUCE_MIN_U, 0x00000080, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x00000080, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MAXA, 0);
+    expected32 = test_arm_mve_expected_vmaxv(
+        source, 1, TEST_ARM_MVE_REDUCE_MAXA, 0x000000c8, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x000000c8, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MAXA, 0);
+    expected32 = test_arm_mve_expected_vmaxv(
+        source, 1, TEST_ARM_MVE_REDUCE_MAXA, 1, 0xff00);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 1, 0,
+                                  expected32, true);
+
+    insn = test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MINA, 1);
+    expected32 = test_arm_mve_expected_vmaxv(
+        source, 2, TEST_ARM_MVE_REDUCE_MINA, 0x0000ffff, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 2, 0x0000ffff, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F32);
+    test_arm_m55_mve_reduce_fp_run(insn, fp32_source, 2, 0x3f800000, 0,
+                                   0x40a00000, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMINNMV_F32);
+    test_arm_m55_mve_reduce_fp_run(insn, fp32_source, 2, 0x3f800000, 0,
+                                   0xc0e00000, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F32);
+    test_arm_m55_mve_reduce_fp_run(insn, fp32_source, 2, 0x00000000,
+                                   0, 0x40400000, true, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F32);
+    test_arm_m55_mve_reduce_fp_run(insn, fp32_source, 2, 0x3f800000,
+                                   0x001100f0, 0x40a00000, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMAV_F32);
+    test_arm_m55_mve_reduce_fp_run(insn, fp32_source, 2, 0xbf800000, 0,
+                                   0x40e00000, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMINNMAV_F32);
+    test_arm_m55_mve_reduce_fp_run(insn, fp32_source, 2, 0xc1200000, 0,
+                                   0xc1200000, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F32);
+    test_arm_m55_mve_reduce_fp_run(insn, fp32_snan_source, 2, 0x3f800000,
+                                   0, 0x40000000, false, true);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F16);
+    test_arm_m55_mve_reduce_fp_run(insn, fp16_source, 2, 0x00003c00, 0,
+                                   0x00004200, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMINNMV_F16);
+    test_arm_m55_mve_reduce_fp_run(insn, fp16_source, 2, 0x00003c00, 0,
+                                   0x0000c600, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMAV_F16);
+    test_arm_m55_mve_reduce_fp_run(insn, fp16_source, 2, 0x0000bc00, 0,
+                                   0x00004600, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMINNMAV_F16);
+    test_arm_m55_mve_reduce_fp_run(insn, fp16_source, 2, 0x0000bc00, 0,
+                                   0x0000bc00, false, false);
+
+    insn = test_arm_mve_vmaxnmv_insn(
+        1, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F16);
+    test_arm_m55_mve_reduce_fp_run(insn, fp16_snan_source, 2, 0x00003c00,
+                                   0, 0x00004000, false, true);
+
+    insn = test_arm_mve_vaddlv_insn(1, 2, 3, false, false);
+    expected64 = test_arm_mve_expected_vaddlv(source, false, 0, 0xffff);
+    test_arm_m55_mve_reduce64_run(insn, source, 2, 3,
+                                  0x2222222211111111ULL, 0, expected64,
+                                  false);
+
+    insn = test_arm_mve_vaddlv_insn(1, 2, 3, true, true);
+    expected64 = test_arm_mve_expected_vaddlv(source, true,
+                                             0x0000000100000010ULL,
+                                             0x00f0);
+    test_arm_m55_mve_reduce64_run(insn, source, 2, 3,
+                                  0x0000000100000010ULL, 0x001100f0,
+                                  expected64, false);
+
+    insn = test_arm_mve_vaddlv_insn(1, 2, 3, false, false);
+    expected64 = test_arm_mve_expected_vaddlv(source, false,
+                                             0x1234567887654321ULL,
+                                             0xff00);
+    test_arm_m55_mve_reduce64_run(insn, source, 2, 3,
+                                  0x1234567887654321ULL, 0, expected64,
+                                  true);
+
+    insn = test_arm_mve_vabav_insn(1, 2, 4, false, 0);
+    expected32 = test_arm_mve_expected_vabav(source, other, 1, false, 5,
+                                             0xffff);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 4, 5, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vabav_insn(1, 2, 4, true, 1);
+    expected32 = test_arm_mve_expected_vabav(source, other, 2, true, 0x100,
+                                             0x00f0);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 4, 0x100,
+                                  0x001100f0, expected32, false);
+
+    insn = test_arm_mve_vabav_insn(1, 2, 4, false, 2);
+    expected32 = test_arm_mve_expected_vabav(source, other, 4, false,
+                                             0x76543210, 0xff00);
+    test_arm_m55_mve_reduce32_run(insn, source, other, 4, 0x76543210, 0,
+                                  expected32, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vaddv_insn(1, 2, false, 3, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vaddlv_insn(1, 2, 13, false, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vabav_insn(8, 2, 4, false, 0),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vabav_insn(1, 2, 13, false, 0),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxv_insn(8, 2, TEST_ARM_MVE_REDUCE_MAX_S, 0),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxv_insn(1, 13, TEST_ARM_MVE_REDUCE_MAX_U, 1),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxv_insn(1, 15, TEST_ARM_MVE_REDUCE_MINA, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxnmv_insn(8, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F32),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxnmv_insn(1, 13, TEST_ARM_MVE_REDUCE_FMAXNMV_F16),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxnmv_insn(1, 15, TEST_ARM_MVE_REDUCE_FMINNMAV_F32),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vaddv_insn(1, 2, false, 0, false),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxv_insn(1, 2, TEST_ARM_MVE_REDUCE_MIN_U, 0),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmaxnmv_insn(1, 2, TEST_ARM_MVE_REDUCE_FMAXNMV_F32),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+typedef enum test_arm_mve_dualacc_kind {
+    TEST_ARM_MVE_DUALACC_ADD_S,
+    TEST_ARM_MVE_DUALACC_ADD_U,
+    TEST_ARM_MVE_DUALACC_SUB_S,
+} test_arm_mve_dualacc_kind;
+
+static uint32_t test_arm_mve_vmladav_insn(unsigned qn, unsigned qm,
+                                          unsigned rda,
+                                          test_arm_mve_dualacc_kind kind,
+                                          unsigned size, bool xchg,
+                                          bool accum)
+{
+    uint32_t view;
+
+    switch (kind) {
+    case TEST_ARM_MVE_DUALACC_ADD_S:
+        view = size == 0 ? 0xeef00f00 : 0xeef00e00;
+        break;
+    case TEST_ARM_MVE_DUALACC_ADD_U:
+        view = size == 0 ? 0xfef00f00 : 0xfef00e00;
+        break;
+    case TEST_ARM_MVE_DUALACC_SUB_S:
+    default:
+        view = size == 0 ? 0xfef00e01 : 0xeef00e01;
+        break;
+    }
+    if (size != 0) {
+        view |= ((size - 1) & 1) << 16;
+    }
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= ((rda >> 1) & 7) << 13;
+    view |= (xchg ? 1U : 0U) << 12;
+    view |= (accum ? 1U : 0U) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_set_view_bit(uint32_t insn, unsigned bit)
+{
+    uint32_t view = test_arm_mve_view_to_t32(insn);
+
+    view |= 1U << bit;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_vmlaldav_insn(unsigned qn, unsigned qm,
+                                           unsigned rdalo, unsigned rdahi,
+                                           test_arm_mve_dualacc_kind kind,
+                                           unsigned size, bool xchg,
+                                           bool accum)
+{
+    uint32_t view;
+
+    switch (kind) {
+    case TEST_ARM_MVE_DUALACC_ADD_S:
+        view = 0xee800e00;
+        break;
+    case TEST_ARM_MVE_DUALACC_ADD_U:
+        view = 0xfe800e00;
+        break;
+    case TEST_ARM_MVE_DUALACC_SUB_S:
+    default:
+        view = 0xee800e01;
+        break;
+    }
+    view |= ((size - 1) & 1) << 16;
+    view |= ((rdahi >> 1) & 7) << 20;
+    view |= ((rdalo >> 1) & 7) << 13;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= (xchg ? 1U : 0U) << 12;
+    view |= (accum ? 1U : 0U) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_vrmlaldavh_insn(unsigned qn, unsigned qm,
+                                             unsigned rdalo, unsigned rdahi,
+                                             test_arm_mve_dualacc_kind kind,
+                                             bool xchg, bool accum)
+{
+    uint32_t view;
+
+    switch (kind) {
+    case TEST_ARM_MVE_DUALACC_ADD_S:
+        view = 0xee800f00;
+        break;
+    case TEST_ARM_MVE_DUALACC_ADD_U:
+        view = 0xfe800f00;
+        break;
+    case TEST_ARM_MVE_DUALACC_SUB_S:
+    default:
+        view = 0xfe800e01;
+        break;
+    }
+    view |= ((rdahi >> 1) & 7) << 20;
+    view |= ((rdalo >> 1) & 7) << 13;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= (xchg ? 1U : 0U) << 12;
+    view |= (accum ? 1U : 0U) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_expected_dualacc32(const uint8_t *n,
+                                                const uint8_t *m,
+                                                unsigned esize,
+                                                test_arm_mve_dualacc_kind kind,
+                                                bool xchg, uint32_t accum,
+                                                uint16_t pred)
+{
+    size_t e;
+
+    for (e = 0; e < 16 / esize; e++) {
+        size_t ne = xchg ? (e ^ 1) : e;
+        uint32_t nval = test_arm_load_le(n + ne * esize, esize);
+        uint32_t mval = test_arm_load_le(m + e * esize, esize);
+        uint32_t product;
+
+        if (!(pred & (1U << (e * esize)))) {
+            continue;
+        }
+        if (kind == TEST_ARM_MVE_DUALACC_ADD_U) {
+            product = nval * mval;
+        } else {
+            int64_t sn = test_arm_sign_extend(nval, esize * 8);
+            int64_t sm = test_arm_sign_extend(mval, esize * 8);
+
+            product = (uint32_t)(sn * sm);
+        }
+        if (kind == TEST_ARM_MVE_DUALACC_SUB_S && (e & 1)) {
+            accum -= product;
+        } else {
+            accum += product;
+        }
+    }
+    return accum;
+}
+
+static uint64_t test_arm_mve_expected_dualacc64(const uint8_t *n,
+                                                const uint8_t *m,
+                                                unsigned esize,
+                                                test_arm_mve_dualacc_kind kind,
+                                                bool xchg, uint64_t accum,
+                                                uint16_t pred)
+{
+    size_t e;
+
+    for (e = 0; e < 16 / esize; e++) {
+        size_t ne = xchg ? (e ^ 1) : e;
+        uint32_t nval = test_arm_load_le(n + ne * esize, esize);
+        uint32_t mval = test_arm_load_le(m + e * esize, esize);
+        uint64_t product;
+
+        if (!(pred & (1U << (e * esize)))) {
+            continue;
+        }
+        if (kind == TEST_ARM_MVE_DUALACC_ADD_U) {
+            product = (uint64_t)nval * mval;
+        } else {
+            int64_t sn = test_arm_sign_extend(nval, esize * 8);
+            int64_t sm = test_arm_sign_extend(mval, esize * 8);
+
+            product = (uint64_t)(sn * sm);
+        }
+        if (kind == TEST_ARM_MVE_DUALACC_SUB_S && (e & 1)) {
+            accum -= product;
+        } else {
+            accum += product;
+        }
+    }
+    return accum;
+}
+
+static uint64_t
+test_arm_mve_expected_dualacc64_high(const uint8_t *n, const uint8_t *m,
+                                     test_arm_mve_dualacc_kind kind,
+                                     bool xchg, uint64_t accum,
+                                     uint16_t pred)
+{
+    size_t e;
+
+    for (e = 0; e < 4; e++) {
+        size_t ne = xchg ? (e ^ 1) : e;
+        uint32_t nval = test_arm_load_le(n + ne * 4, 4);
+        uint32_t mval = test_arm_load_le(m + e * 4, 4);
+
+        if (!(pred & (1U << (e * 4)))) {
+            continue;
+        }
+        if (kind == TEST_ARM_MVE_DUALACC_ADD_U) {
+            uint64_t product = (uint64_t)nval * mval;
+
+            accum += (product >> 8) + ((product >> 7) & 1);
+        } else {
+            int64_t sn = test_arm_sign_extend(nval, 32);
+            int64_t sm = test_arm_sign_extend(mval, 32);
+            int64_t product = sn * sm;
+
+            if (kind == TEST_ARM_MVE_DUALACC_SUB_S && (e & 1)) {
+                product = -product;
+            }
+            accum += (uint64_t)((product >> 8) + ((product >> 7) & 1));
+        }
+    }
+    return accum;
+}
+
+static void test_arm_m55_mve_dualacc64_run(uint32_t insn,
+                                           const uint8_t *q1_data,
+                                           const uint8_t *q2_data,
+                                           unsigned rdalo_reg,
+                                           unsigned rdahi_reg,
+                                           uint64_t initial_rda,
+                                           uint32_t vpr,
+                                           uint64_t expected_rda,
+                                           bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    uint32_t epsr;
+    uint32_t gotlo;
+    uint32_t gothi;
+    uint32_t initlo = (uint32_t)initial_rda;
+    uint32_t inithi = (uint32_t)(initial_rda >> 32);
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q1, q1_data, 16);
+    memcpy(q2, q2_data, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rdalo_reg, &initlo));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0 + rdahi_reg, &inithi));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rdalo_reg, &gotlo));
+    OK(uc_reg_read(uc, UC_ARM_REG_R0 + rdahi_reg, &gothi));
+    TEST_CHECK_((((uint64_t)gothi << 32) | gotlo) == expected_rda,
+                "insn=0x%08x got=0x%08x%08x expected=0x%016llx",
+                insn, gothi, gotlo, (unsigned long long)expected_rda);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_dualacc(void)
+{
+    const uint8_t n[16] = {
+        0x02, 0xfe, 0x03, 0xfd, 0x04, 0xfc, 0x05, 0xfb,
+        0x06, 0xfa, 0x07, 0xf9, 0x08, 0xf8, 0x09, 0xf7,
+    };
+    const uint8_t m[16] = {
+        0x01, 0x04, 0xff, 0x03, 0x02, 0xfe, 0x05, 0xfd,
+        0x06, 0xfc, 0x07, 0xfb, 0x08, 0xfa, 0x09, 0xf9,
+    };
+    uint32_t insn;
+    uint32_t expected32;
+    uint64_t expected64;
+
+    insn = test_arm_mve_vmladav_insn(1, 2, 4, TEST_ARM_MVE_DUALACC_ADD_S,
+                                     0, false, false);
+    expected32 = test_arm_mve_expected_dualacc32(n, m, 1,
+                                                 TEST_ARM_MVE_DUALACC_ADD_S,
+                                                 false, 0, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, n, m, 4, 0x11111111, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_set_view_bit(insn, 0);
+    test_arm_m55_mve_reduce32_run(insn, n, m, 4, 0x11111111, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmladav_insn(1, 2, 4, TEST_ARM_MVE_DUALACC_ADD_U,
+                                     1, false, true);
+    expected32 = test_arm_mve_expected_dualacc32(n, m, 2,
+                                                 TEST_ARM_MVE_DUALACC_ADD_U,
+                                                 false, 0x1000, 0x00f0);
+    test_arm_m55_mve_reduce32_run(insn, n, m, 4, 0x1000, 0x001100f0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmladav_insn(1, 2, 4, TEST_ARM_MVE_DUALACC_SUB_S,
+                                     2, true, false);
+    expected32 = test_arm_mve_expected_dualacc32(n, m, 4,
+                                                 TEST_ARM_MVE_DUALACC_SUB_S,
+                                                 true, 0x76543210, 0xff00);
+    test_arm_m55_mve_reduce32_run(insn, n, m, 4, 0x76543210, 0,
+                                  expected32, true);
+
+    insn = test_arm_mve_vmladav_insn(1, 2, 4, TEST_ARM_MVE_DUALACC_SUB_S,
+                                     0, false, false);
+    expected32 = test_arm_mve_expected_dualacc32(n, m, 1,
+                                                 TEST_ARM_MVE_DUALACC_SUB_S,
+                                                 false, 0, 0xffff);
+    test_arm_m55_mve_reduce32_run(insn, n, m, 4, 0x11111111, 0,
+                                  expected32, false);
+
+    insn = test_arm_mve_vmlaldav_insn(1, 2, 2, 3,
+                                      TEST_ARM_MVE_DUALACC_ADD_S, 1,
+                                      false, false);
+    expected64 = test_arm_mve_expected_dualacc64(n, m, 2,
+                                                 TEST_ARM_MVE_DUALACC_ADD_S,
+                                                 false, 0, 0xffff);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3,
+                                   0x2222222211111111ULL, 0,
+                                   expected64, false);
+
+    insn = test_arm_mve_vmlaldav_insn(1, 2, 2, 3,
+                                      TEST_ARM_MVE_DUALACC_ADD_S, 1,
+                                      true, true);
+    expected64 = test_arm_mve_expected_dualacc64(n, m, 2,
+                                                 TEST_ARM_MVE_DUALACC_ADD_S,
+                                                 true,
+                                                 0x0000000000000100ULL,
+                                                 0xffff);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3, 0x100, 0,
+                                   expected64, false);
+
+    insn = test_arm_mve_vmlaldav_insn(1, 2, 2, 3,
+                                      TEST_ARM_MVE_DUALACC_ADD_U, 2,
+                                      false, true);
+    expected64 = test_arm_mve_expected_dualacc64(n, m, 4,
+                                                 TEST_ARM_MVE_DUALACC_ADD_U,
+                                                 false,
+                                                 0x0000000100000010ULL,
+                                                 0x00f0);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3,
+                                   0x0000000100000010ULL, 0x001100f0,
+                                   expected64, false);
+
+    insn = test_arm_mve_vmlaldav_insn(1, 2, 2, 3,
+                                      TEST_ARM_MVE_DUALACC_SUB_S, 1,
+                                      true, false);
+    expected64 = test_arm_mve_expected_dualacc64(n, m, 2,
+                                                 TEST_ARM_MVE_DUALACC_SUB_S,
+                                                 true,
+                                                 0x1234567887654321ULL,
+                                                 0xff00);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3,
+                                   0x1234567887654321ULL, 0,
+                                   expected64, true);
+
+    insn = test_arm_mve_vrmlaldavh_insn(1, 2, 2, 3,
+                                        TEST_ARM_MVE_DUALACC_ADD_S,
+                                        false, false);
+    expected64 = test_arm_mve_expected_dualacc64_high(
+        n, m, TEST_ARM_MVE_DUALACC_ADD_S, false, 0, 0xffff);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3,
+                                   0x2222222211111111ULL, 0,
+                                   expected64, false);
+
+    insn = test_arm_mve_vrmlaldavh_insn(1, 2, 2, 3,
+                                        TEST_ARM_MVE_DUALACC_ADD_S,
+                                        true, true);
+    expected64 = test_arm_mve_expected_dualacc64_high(
+        n, m, TEST_ARM_MVE_DUALACC_ADD_S, true, 0x100, 0xffff);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3, 0x100, 0,
+                                   expected64, false);
+
+    insn = test_arm_mve_vrmlaldavh_insn(1, 2, 2, 3,
+                                        TEST_ARM_MVE_DUALACC_ADD_U,
+                                        false, true);
+    expected64 = test_arm_mve_expected_dualacc64_high(
+        n, m, TEST_ARM_MVE_DUALACC_ADD_U, false,
+        0x0000000100000010ULL, 0x00f0);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3,
+                                   0x0000000100000010ULL, 0x001100f0,
+                                   expected64, false);
+
+    insn = test_arm_mve_vrmlaldavh_insn(1, 2, 2, 3,
+                                        TEST_ARM_MVE_DUALACC_SUB_S,
+                                        false, true);
+    expected64 = test_arm_mve_expected_dualacc64_high(
+        n, m, TEST_ARM_MVE_DUALACC_SUB_S, false,
+        0x0000000000000200ULL, 0xffff);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3, 0x200, 0,
+                                   expected64, false);
+
+    insn = test_arm_mve_vrmlaldavh_insn(1, 2, 2, 3,
+                                        TEST_ARM_MVE_DUALACC_SUB_S,
+                                        true, false);
+    expected64 = test_arm_mve_expected_dualacc64_high(
+        n, m, TEST_ARM_MVE_DUALACC_SUB_S, true,
+        0x1234567887654321ULL, 0xff00);
+    test_arm_m55_mve_dualacc64_run(insn, n, m, 2, 3,
+                                   0x1234567887654321ULL, 0,
+                                   expected64, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmladav_insn(1, 2, 4, TEST_ARM_MVE_DUALACC_ADD_U, 0,
+                                  true, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vrmlaldavh_insn(1, 2, 2, 3,
+                                     TEST_ARM_MVE_DUALACC_ADD_U, true,
+                                     false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vrmlaldavh_insn(8, 2, 2, 3,
+                                     TEST_ARM_MVE_DUALACC_ADD_S, false,
+                                     false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vrmlaldavh_insn(1, 2, 2, 3,
+                                     TEST_ARM_MVE_DUALACC_ADD_S, false,
+                                     false),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmladav_insn(8, 2, 4, TEST_ARM_MVE_DUALACC_ADD_S, 0,
+                                  false, false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmlaldav_insn(1, 2, 2, 13,
+                                   TEST_ARM_MVE_DUALACC_ADD_S, 1, false,
+                                   false),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vmladav_insn(1, 2, 4, TEST_ARM_MVE_DUALACC_ADD_S, 0,
+                                  false, false),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static uint32_t test_arm_mve_vshlc_insn(unsigned qd, unsigned rdm,
+                                        unsigned shift)
+{
+    uint32_t view = 0xeea00fc0;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (shift & 31) << 16;
+    view |= (qd & 7) << 13;
+    view |= rdm & 15;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static void test_arm_mve_expected_vshlc(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        uint32_t rdm, unsigned shift,
+                                        uint16_t pred,
+                                        uint32_t *expected_rdm)
+{
+    size_t e;
+
+    memcpy(expected, initial, 16);
+    for (e = 0; e < 16 / 4; e++) {
+        size_t off = e * 4;
+        uint32_t lane = test_arm_load_le(initial + off, 4);
+        uint32_t result;
+
+        if (shift == 0) {
+            result = rdm;
+            if (pred & (1U << off)) {
+                rdm = lane;
+            }
+        } else {
+            uint32_t shiftmask = (uint32_t)((1ULL << shift) - 1);
+
+            result = (lane << shift) | (rdm & shiftmask);
+            if (pred & (1U << off)) {
+                rdm = lane >> (32 - shift);
+            }
+        }
+        test_arm_mve_store_masked(expected, off, 4, result, pred);
+    }
+    *expected_rdm = rdm;
+}
+
+static void test_arm_m55_mve_vshlc_run(uint32_t insn,
+                                       const uint8_t *initial,
+                                       uint32_t rdm,
+                                       uint32_t vpr,
+                                       const uint8_t *expected,
+                                       uint32_t expected_rdm,
+                                       bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &rdm));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &rdm));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(rdm == expected_rdm, "rdm=0x%08x expected=0x%08x",
+                rdm, expected_rdm);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_vshlc(void)
+{
+    const uint8_t initial[16] = {
+        0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a,
+        0xa9, 0xcb, 0xed, 0x0f, 0x21, 0x43, 0x65, 0x87,
+    };
+    uint8_t expected[16];
+    uint32_t insn;
+    uint32_t expected_rdm;
+    uint32_t rdm;
+
+    rdm = 0xf000000a;
+    insn = test_arm_mve_vshlc_insn(0, 1, 4);
+    test_arm_mve_expected_vshlc(expected, initial, rdm, 4, 0xffff,
+                                &expected_rdm);
+    test_arm_m55_mve_vshlc_run(insn, initial, rdm, 0, expected,
+                               expected_rdm, false);
+
+    rdm = 0x89abcdef;
+    insn = test_arm_mve_vshlc_insn(0, 1, 0);
+    test_arm_mve_expected_vshlc(expected, initial, rdm, 0, 0xffff,
+                                &expected_rdm);
+    test_arm_m55_mve_vshlc_run(insn, initial, rdm, 0, expected,
+                               expected_rdm, false);
+
+    rdm = 0x0000000b;
+    insn = test_arm_mve_vshlc_insn(0, 1, 8);
+    test_arm_mve_expected_vshlc(expected, initial, rdm, 8, 0x00f0,
+                                &expected_rdm);
+    test_arm_m55_mve_vshlc_run(insn, initial, rdm, 0x001100f0, expected,
+                               expected_rdm, false);
+
+    rdm = 0x0000000c;
+    insn = test_arm_mve_vshlc_insn(0, 1, 4);
+    test_arm_mve_expected_vshlc(expected, initial, rdm, 4, 0xff00,
+                                &expected_rdm);
+    test_arm_m55_mve_vshlc_run(insn, initial, rdm, 0, expected,
+                               expected_rdm, true);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshlc_insn(8, 1, 4),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshlc_insn(0, 13, 4),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vshlc_insn(0, 1, 4),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_vdup_run(uint32_t insn, int source_reg,
+                                      const uint8_t *initial,
+                                      uint32_t value, uint32_t vpr,
+                                      const uint8_t *expected, bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, source_reg, &value));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_vdup(void)
+{
+    const uint8_t initial[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    uint8_t expected[16];
+    uint32_t value = 0x89abcdef;
+    uint32_t vpr = 0;
+
+    test_arm_mve_expected_vdup(expected, initial, value, 1, 0xffff);
+    test_arm_m55_mve_vdup_run(0x1b10eee0, UC_ARM_REG_R1, initial, value,
+                              vpr, expected, false);
+
+    test_arm_mve_expected_vdup(expected, initial, value, 2, 0xffff);
+    test_arm_m55_mve_vdup_run(0x1b30eea0, UC_ARM_REG_R1, initial, value,
+                              vpr, expected, false);
+
+    test_arm_mve_expected_vdup(expected, initial, value, 4, 0xffff);
+    test_arm_m55_mve_vdup_run(0x1b10eea0, UC_ARM_REG_R1, initial, value,
+                              vpr, expected, false);
+
+    vpr = 0x001100f0;
+    test_arm_mve_expected_vdup(expected, initial, value, 1, 0x00f0);
+    test_arm_m55_mve_vdup_run(0x1b10eee0, UC_ARM_REG_R1, initial, value,
+                              vpr, expected, false);
+
+    vpr = 0;
+    test_arm_mve_expected_vdup(expected, initial, value, 4, 0xff00);
+    test_arm_m55_mve_vdup_run(0x1b10eea0, UC_ARM_REG_R1, initial, value,
+                              vpr, expected, true);
+
+    test_arm_mve_expected_vdup(expected, initial, value, 4, 0xffff);
+    test_arm_m55_mve_vdup_run(0xeb10eea0, UC_ARM_REG_LR, initial, value,
+                              vpr, expected, false);
+
+    test_arm_m55_mve_2op_expect_error(0x1b90eee0, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0xdb10eee0, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0xfb10eee0, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x1b30eee0, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x1b10eee0, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_vmov_2gp(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint8_t initial[16] = {
+        0x44, 0x33, 0x22, 0x11, 0x88, 0x77, 0x66, 0x55,
+        0xcc, 0xbb, 0xaa, 0x99, 0x00, 0xff, 0xee, 0xdd,
+    };
+    uint8_t expected[16];
+    uint8_t code[4];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t r1;
+    uint32_t r2;
+    uint32_t epsr;
+    uc_engine *uc;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0f01ec02);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    r1 = 0x11111111;
+    r2 = 0x22222222;
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
+    TEST_CHECK_(r1 == 0x11223344, "r1=0x%08x", r1);
+    TEST_CHECK_(r2 == 0x99aabbcc, "r2=0x%08x", r2);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f11ec02);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    r1 = 0x11111111;
+    r2 = 0x22222222;
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
+    TEST_CHECK_(r1 == 0x55667788, "r1=0x%08x", r1);
+    TEST_CHECK_(r2 == 0xddeeff00, "r2=0x%08x", r2);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f01ec02);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    r1 = 0x11111111;
+    r2 = 0x22222222;
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    TEST_CHECK_(r1 == 0x11111111, "eci r1=0x%08x", r1);
+    TEST_CHECK_(r2 == 0x99aabbcc, "eci r2=0x%08x", r2);
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f01ec12);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(expected, initial, 16);
+    r1 = 0x01020304;
+    r2 = 0xa1a2a3a4;
+    test_arm_store_le(expected, 4, r1);
+    test_arm_store_le(expected + 8, 4, r2);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "from idx0 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f11ec12);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(expected, initial, 16);
+    r1 = 0x01020304;
+    r2 = 0xa1a2a3a4;
+    test_arm_store_le(expected + 4, 4, r1);
+    test_arm_store_le(expected + 12, 4, r2);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "from idx1 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f11ec12);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(expected, initial, 16);
+    r1 = 0x01020304;
+    r2 = 0xa1a2a3a4;
+    epsr = xpsr_t | eci_a0a1;
+    test_arm_store_le(expected + 12, 4, r2);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "from eci i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f01ec11);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(expected, initial, 16);
+    r1 = 0x01020304;
+    test_arm_store_le(expected, 4, r1);
+    test_arm_store_le(expected + 8, 4, r1);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "from dup i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_m55_mve_2op_expect_error(0x0f01ec42, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f0dec02, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f01ec0f, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f01ec01, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f0dec12, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f01ec1f, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f01ec02, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static void test_arm_mve_expected_vidup(uint8_t *expected,
+                                        const uint8_t *initial,
+                                        uint32_t offset, uint32_t wrap,
+                                        uint32_t imm, unsigned esize,
+                                        bool wrapped, bool decrement,
+                                        uint16_t mask,
+                                        uint32_t *final_offset)
+{
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    for (i = 0; i < 16; i += esize) {
+        test_arm_mve_store_masked(expected, i, esize, offset, mask);
+        if (wrapped) {
+            if (decrement) {
+                if (offset == 0) {
+                    offset = wrap;
+                }
+                offset -= imm;
+            } else {
+                offset += imm;
+                if (offset == wrap) {
+                    offset = 0;
+                }
+            }
+        } else if (decrement) {
+            offset -= imm;
+        } else {
+            offset += imm;
+        }
+    }
+    *final_offset = offset;
+}
+
+static void test_arm_m55_mve_vidup_run(uint32_t insn,
+                                       const uint8_t *initial,
+                                       uint32_t rn_value, uint32_t rm_value,
+                                       uint32_t vpr,
+                                       const uint8_t *expected,
+                                       uint32_t expected_rn, bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R2, &rn_value));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &rm_value));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R2, &rn_value));
+    TEST_CHECK_(rn_value == expected_rn,
+                "insn=0x%08x rn=0x%08x expected=0x%08x",
+                insn, rn_value, expected_rn);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_vidup(void)
+{
+    const uint8_t initial[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint32_t vidup_insns[] = {
+        0x0f6eee03, 0x0f6eee13, 0x0f6eee23,
+    };
+    const uint32_t vddup_insns[] = {
+        0x1f6eee03, 0x1f6eee13, 0x1f6eee23,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    uint32_t final_offset;
+    uint32_t vpr = 0;
+    size_t n;
+
+    for (n = 0; n < 3; n++) {
+        test_arm_mve_expected_vidup(expected, initial, 0x10, 0, 1,
+                                    esizes[n], false, false, 0xffff,
+                                    &final_offset);
+        test_arm_m55_mve_vidup_run(vidup_insns[n], initial, 0x10, 0,
+                                   vpr, expected, final_offset, false);
+
+        test_arm_mve_expected_vidup(expected, initial, 0x20, 0, 1,
+                                    esizes[n], false, true, 0xffff,
+                                    &final_offset);
+        test_arm_m55_mve_vidup_run(vddup_insns[n], initial, 0x20, 0,
+                                   vpr, expected, final_offset, false);
+    }
+
+    test_arm_mve_expected_vidup(expected, initial, 2, 5, 1, 1, true,
+                                false, 0xffff, &final_offset);
+    test_arm_m55_mve_vidup_run(0x0f62ee03, initial, 2, 5, vpr, expected,
+                               final_offset, false);
+
+    test_arm_mve_expected_vidup(expected, initial, 0, 5, 1, 1, true,
+                                true, 0xffff, &final_offset);
+    test_arm_m55_mve_vidup_run(0x1f62ee03, initial, 0, 5, vpr, expected,
+                               final_offset, false);
+
+    vpr = 0x001100f0;
+    test_arm_mve_expected_vidup(expected, initial, 0x40, 0, 1, 1, false,
+                                false, 0x00f0, &final_offset);
+    test_arm_m55_mve_vidup_run(0x0f6eee03, initial, 0x40, 0, vpr,
+                               expected, final_offset, false);
+
+    vpr = 0;
+    test_arm_mve_expected_vidup(expected, initial, 0x100, 0, 1, 4, false,
+                                false, 0xff00, &final_offset);
+    test_arm_m55_mve_vidup_run(0x0f6eee23, initial, 0x100, 0, vpr,
+                               expected, final_offset, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0f6eee33, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f6eee43, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f6cee03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f6eee03, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+enum test_arm_mve_1op_kind {
+    TEST_ARM_MVE_CLS,
+    TEST_ARM_MVE_CLZ,
+    TEST_ARM_MVE_MVN,
+    TEST_ARM_MVE_ABS,
+    TEST_ARM_MVE_NEG,
+    TEST_ARM_MVE_MAXA,
+    TEST_ARM_MVE_MINA,
+    TEST_ARM_MVE_QABS,
+    TEST_ARM_MVE_QNEG,
+};
+
+static uint32_t test_arm_clz_bits(uint32_t value, unsigned bits)
+{
+    uint32_t bit;
+    uint32_t count = 0;
+
+    for (bit = 1U << (bits - 1); bit; bit >>= 1) {
+        if (value & bit) {
+            break;
+        }
+        count++;
+    }
+
+    return count;
+}
+
+static uint32_t test_arm_cls_bits(uint32_t value, unsigned bits)
+{
+    uint32_t sign = value & (1U << (bits - 1));
+    uint32_t count = 0;
+    int bit;
+
+    for (bit = (int)bits - 2; bit >= 0; bit--) {
+        bool set = (value & (1U << bit)) != 0;
+
+        if (set != (sign != 0)) {
+            break;
+        }
+        count++;
+    }
+
+    return count;
+}
+
+static void test_arm_mve_expected_1op(uint8_t *expected,
+                                      const uint8_t *initial,
+                                      const uint8_t *source,
+                                      enum test_arm_mve_1op_kind kind,
+                                      unsigned esize, uint16_t mask,
+                                      bool *qc)
+{
+    unsigned bits = esize * 8;
+    size_t i;
+
+    memcpy(expected, initial, 16);
+    *qc = false;
+    for (i = 0; i < 16; i += esize, mask >>= esize) {
+        uint32_t value = test_arm_load_le(source + i, esize);
+        uint32_t result = 0;
+        int64_t signed_value = test_arm_sign_extend(value, bits);
+
+        switch (kind) {
+        case TEST_ARM_MVE_CLS:
+            result = test_arm_cls_bits(value, bits);
+            break;
+        case TEST_ARM_MVE_CLZ:
+            result = test_arm_clz_bits(value, bits);
+            break;
+        case TEST_ARM_MVE_MVN:
+            result = ~value;
+            break;
+        case TEST_ARM_MVE_ABS:
+            result = (uint32_t)(signed_value < 0 ?
+                                -signed_value : signed_value);
+            break;
+        case TEST_ARM_MVE_NEG:
+            result = (uint32_t)-signed_value;
+            break;
+        case TEST_ARM_MVE_MAXA:
+        case TEST_ARM_MVE_MINA: {
+            uint32_t dst = test_arm_load_le(initial + i, esize);
+            uint32_t abs_value = (uint32_t)(signed_value < 0 ?
+                                           -signed_value : signed_value);
+
+            if (kind == TEST_ARM_MVE_MAXA) {
+                result = dst >= abs_value ? dst : abs_value;
+            } else {
+                result = dst >= abs_value ? abs_value : dst;
+            }
+            break;
+        }
+        case TEST_ARM_MVE_QABS:
+        case TEST_ARM_MVE_QNEG: {
+            int64_t min = -(1LL << (bits - 1));
+            int64_t max = (1LL << (bits - 1)) - 1;
+            int64_t signed_result = kind == TEST_ARM_MVE_QABS ?
+                (signed_value < 0 ? -signed_value : signed_value) :
+                -signed_value;
+
+            if (signed_result > max) {
+                signed_result = max;
+                *qc |= (mask & 1) != 0;
+            } else if (signed_result < min) {
+                signed_result = min;
+                *qc |= (mask & 1) != 0;
+            }
+            result = (uint32_t)signed_result;
+            break;
+        }
+        default:
+            break;
+        }
+
+        if (mask & 1) {
+            test_arm_store_le(expected + i, esize, result);
+        }
+    }
+}
+
+static void test_arm_mve_expected_rev(uint8_t *expected,
+                                      const uint8_t *source,
+                                      unsigned group_size,
+                                      unsigned elem_size)
+{
+    size_t group;
+
+    for (group = 0; group < 16; group += group_size) {
+        size_t e;
+        size_t elems = group_size / elem_size;
+
+        for (e = 0; e < elems; e++) {
+            memcpy(expected + group + e * elem_size,
+                   source + group + (elems - 1 - e) * elem_size,
+                   elem_size);
+        }
+    }
+}
+
+static void test_arm_m55_mve_1op_run(uint32_t insn, const uint8_t *initial,
+                                     const uint8_t *source,
+                                     const uint8_t *expected,
+                                     bool eci, bool expected_qc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_qc = 1U << 27;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    uint32_t vpr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, source, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    }
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                    insn, (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(((fpscr & fpscr_qc) != 0) == expected_qc,
+                "fpscr=0x%08x expected_qc=%d",
+                fpscr, expected_qc);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_1op(void)
+{
+    static const uint32_t cls_insns[] = {
+        0x0442ffb0, 0x0442ffb4, 0x0442ffb8,
+    };
+    static const uint32_t clz_insns[] = {
+        0x04c2ffb0, 0x04c2ffb4, 0x04c2ffb8,
+    };
+    static const uint32_t abs_insns[] = {
+        0x0342ffb1, 0x0342ffb5, 0x0342ffb9,
+    };
+    static const uint32_t neg_insns[] = {
+        0x03c2ffb1, 0x03c2ffb5, 0x03c2ffb9,
+    };
+    static const uint32_t qabs_insns[] = {
+        0x0742ffb0, 0x0742ffb4, 0x0742ffb8,
+    };
+    static const uint32_t qneg_insns[] = {
+        0x07c2ffb0, 0x07c2ffb4, 0x07c2ffb8,
+    };
+    static const struct {
+        uint32_t insn;
+        unsigned group_size;
+        unsigned elem_size;
+    } rev_cases[] = {
+        { 0x0142ffb0, 2, 1 },
+        { 0x00c2ffb0, 4, 1 },
+        { 0x00c2ffb4, 4, 2 },
+        { 0x0042ffb0, 8, 1 },
+        { 0x0042ffb4, 8, 2 },
+        { 0x0042ffb8, 8, 4 },
+    };
+    const uint8_t initial[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint8_t source[16] = {
+        0x00, 0x01, 0x7f, 0x80, 0xff, 0x55, 0xaa, 0x7f,
+        0x80, 0x00, 0x34, 0x12, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const unsigned esizes[] = { 1, 2, 4 };
+    uint8_t expected[16];
+    bool qc;
+    size_t n;
+
+    for (n = 0; n < 3; n++) {
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_CLS, esizes[n], 0xffff, &qc);
+        test_arm_m55_mve_1op_run(cls_insns[n], initial, source, expected,
+                                 false, false);
+
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_CLZ, esizes[n], 0xffff, &qc);
+        test_arm_m55_mve_1op_run(clz_insns[n], initial, source, expected,
+                                 false, false);
+
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_ABS, esizes[n], 0xffff, &qc);
+        test_arm_m55_mve_1op_run(abs_insns[n], initial, source, expected,
+                                 false, false);
+
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_NEG, esizes[n], 0xffff, &qc);
+        test_arm_m55_mve_1op_run(neg_insns[n], initial, source, expected,
+                                 false, false);
+
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_MAXA, esizes[n], 0xffff,
+                                  &qc);
+        test_arm_m55_mve_1op_run(
+            test_arm_mve_1op_insn(0, 1, 0xee330e81, n), initial, source,
+            expected, false, false);
+
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_MINA, esizes[n], 0xffff,
+                                  &qc);
+        test_arm_m55_mve_1op_run(
+            test_arm_mve_1op_insn(0, 1, 0xee331e81, n), initial, source,
+            expected, false, false);
+
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_QABS, esizes[n], 0xffff, &qc);
+        test_arm_m55_mve_1op_run(qabs_insns[n], initial, source, expected,
+                                 false, qc);
+
+        test_arm_mve_expected_1op(expected, initial, source,
+                                  TEST_ARM_MVE_QNEG, esizes[n], 0xffff, &qc);
+        test_arm_m55_mve_1op_run(qneg_insns[n], initial, source, expected,
+                                 false, qc);
+    }
+
+    for (n = 0; n < sizeof(rev_cases) / sizeof(rev_cases[0]); n++) {
+        memcpy(expected, initial, 16);
+        test_arm_mve_expected_rev(expected, source,
+                                  rev_cases[n].group_size,
+                                  rev_cases[n].elem_size);
+        test_arm_m55_mve_1op_run(rev_cases[n].insn, initial, source,
+                                 expected, false, false);
+    }
+
+    test_arm_mve_expected_1op(expected, initial, source, TEST_ARM_MVE_MVN,
+                              1, 0xffff, &qc);
+    test_arm_m55_mve_1op_run(0x05c2ffb0, initial, source, expected,
+                             false, false);
+
+    test_arm_mve_expected_1op(expected, initial, source, TEST_ARM_MVE_MVN,
+                              1, 0xff00, &qc);
+    test_arm_m55_mve_1op_run(0x05c2ffb0, initial, source, expected,
+                             true, false);
+
+    test_arm_mve_expected_1op(expected, initial, source, TEST_ARM_MVE_MAXA,
+                              1, 0xff00, &qc);
+    test_arm_m55_mve_1op_run(test_arm_mve_1op_insn(0, 1, 0xee330e81, 0),
+                             initial, source, expected, true, false);
+
+    test_arm_m55_mve_2op_expect_error(0x04c2ffbc, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x0142ffb4, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x05c2fff0, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(0x05c2ffb0, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_INSN_INVALID);
+}
+
+static uint32_t test_arm_mve_fp_2op_insn(uint32_t base, bool fp16,
+                                         unsigned qd, unsigned qn,
+                                         unsigned qm)
+{
+    uint32_t view = base;
+
+    if (fp16) {
+        view |= 1U << 20;
+    }
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_fp_2op_qdqn_insn(uint32_t base, unsigned qd,
+                                              unsigned qm)
+{
+    uint32_t view = base;
+
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_fp_2op_rev_insn(uint32_t base, bool fp32,
+                                             unsigned qd, unsigned qn,
+                                             unsigned qm)
+{
+    uint32_t view = base;
+
+    if (fp32) {
+        view |= 1U << 20;
+    }
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_fp_vcmul_insn(uint32_t base, bool fp32,
+                                           unsigned qd, unsigned qn,
+                                           unsigned qm)
+{
+    uint32_t view = base;
+
+    if (fp32) {
+        view |= 1U << 28;
+    }
+    view |= ((qd >> 3) & 1) << 22;
+    view |= (qd & 7) << 13;
+    view |= ((qn >> 3) & 1) << 7;
+    view |= (qn & 7) << 17;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static void test_arm_m55_mve_fp_2op_run(uint32_t insn,
+                                        const uint8_t *initial,
+                                        const uint8_t *n, const uint8_t *m,
+                                        uint32_t vpr,
+                                        const uint8_t *expected,
+                                        uint16_t expected_mask,
+                                        bool eci, bool expected_ioc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_ioc = 1U;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        if (expected_mask & (1U << i)) {
+            TEST_CHECK_(got[i] == expected[i],
+                        "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                        insn, (unsigned)i, got[i], expected[i]);
+        }
+    }
+    TEST_CHECK_(((fpscr & fpscr_ioc) != 0) == expected_ioc,
+                "insn=0x%08x fpscr=0x%08x expected_ioc=%u",
+                insn, fpscr, expected_ioc ? 1 : 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_fp_1op_run(uint32_t insn,
+                                        const uint8_t *initial,
+                                        const uint8_t *source,
+                                        uint32_t vpr,
+                                        const uint8_t *expected,
+                                        uint16_t expected_mask,
+                                        bool eci, bool expected_ioc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_ioc = 1U;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, source, 16);
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    for (i = 0; i < 16; i++) {
+        if (expected_mask & (1U << i)) {
+            TEST_CHECK_(got[i] == expected[i],
+                        "insn=0x%08x i=%u got=0x%02x expected=0x%02x",
+                        insn, (unsigned)i, got[i], expected[i]);
+        }
+    }
+    TEST_CHECK_(((fpscr & fpscr_ioc) != 0) == expected_ioc,
+                "insn=0x%08x fpscr=0x%08x expected_ioc=%u",
+                insn, fpscr, expected_ioc ? 1 : 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm_store_le16_vector(uint8_t *dst, const uint16_t *values,
+                                       size_t count)
+{
+    size_t i;
+
+    for (i = 0; i < count; i++) {
+        test_arm_store_le(dst + i * 2, 2, values[i]);
+    }
+}
+
+static void test_arm_store_le32_vector(uint8_t *dst, const uint32_t *values,
+                                       size_t count)
+{
+    size_t i;
+
+    for (i = 0; i < count; i++) {
+        test_arm_store_le(dst + i * 4, 4, values[i]);
+    }
+}
+
+static void test_arm_m55_mve_fp_convert_round(void)
+{
+    const uint32_t vcvt_sf_base = 0xffb30640;
+    const uint32_t vcvt_uf_base = 0xffb306c0;
+    const uint32_t vcvt_fs_base = 0xffb30740;
+    const uint32_t vcvt_fu_base = 0xffb307c0;
+    const uint32_t vcvt_sh_fixed_base = 0xefb00c50;
+    const uint32_t vcvt_uh_fixed_base = 0xffb00c50;
+    const uint32_t vcvt_hs_fixed_base = 0xefb00d50;
+    const uint32_t vcvt_hu_fixed_base = 0xffb00d50;
+    const uint32_t vcvt_sf_fixed_base = 0xefa00e50;
+    const uint32_t vcvt_uf_fixed_base = 0xffa00e50;
+    const uint32_t vcvt_fs_fixed_base = 0xefa00f50;
+    const uint32_t vcvt_fu_fixed_base = 0xffa00f50;
+    const uint32_t vcvtb_sh_base = 0xee3f0e01;
+    const uint32_t vcvtt_sh_base = 0xee3f1e01;
+    const uint32_t vcvtb_hs_base = 0xfe3f0e01;
+    const uint32_t vcvtt_hs_base = 0xfe3f1e01;
+    const uint32_t vcvtas_base = 0xffb30040;
+    const uint32_t vcvtau_base = 0xffb300c0;
+    const uint32_t vcvtns_base = 0xffb30140;
+    const uint32_t vcvtnu_base = 0xffb301c0;
+    const uint32_t vcvtps_base = 0xffb30240;
+    const uint32_t vcvtpu_base = 0xffb302c0;
+    const uint32_t vcvtms_base = 0xffb30340;
+    const uint32_t vcvtmu_base = 0xffb303c0;
+    const uint32_t vrintn_base = 0xffb20440;
+    const uint32_t vrintx_base = 0xffb204c0;
+    const uint32_t vrinta_base = 0xffb20540;
+    const uint32_t vrintz_base = 0xffb205c0;
+    const uint32_t vrintm_base = 0xffb206c0;
+    const uint32_t vrintp_base = 0xffb207c0;
+    const uint8_t initial[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    uint8_t source[16];
+    uint8_t expected[16];
+    uint32_t insn;
+
+    const uint32_t s32_values[] = {
+        1, 0xfffffffe, 3, 0xfffffffc,
+    };
+    const uint32_t f32_from_s32[] = {
+        0x3f800000, 0xc0000000, 0x40400000, 0xc0800000,
+    };
+    const uint16_t u16_values[] = {
+        0, 1, 2, 3, 4, 8, 16, 32,
+    };
+    const uint16_t f16_from_u16[] = {
+        0x0000, 0x3c00, 0x4000, 0x4200,
+        0x4400, 0x4800, 0x4c00, 0x5000,
+    };
+    const uint32_t f32_to_s32[] = {
+        0x3fe00000, 0xc0200000, 0x7f800001, 0xbf400000,
+    };
+    const uint32_t s32_from_f32[] = {
+        1, 0xfffffffe, 0, 0,
+    };
+    const uint16_t f16_to_u16[] = {
+        0x3e00, 0x4000, 0x4300, 0x4400,
+        0x4500, 0x0000, 0x4800, 0x4c00,
+    };
+    const uint16_t u16_from_f16[] = {
+        1, 2, 3, 4, 5, 0, 8, 16,
+    };
+    const uint16_t s16_fixed_values[] = {
+        4, 0xfff8, 16, 0xffe0, 0, 2, 0xfffe, 32,
+    };
+    const uint16_t f16_from_s16_fixed[] = {
+        0x3c00, 0xc000, 0x4400, 0xc800,
+        0x0000, 0x3800, 0xb800, 0x4800,
+    };
+    const uint16_t u16_fixed_values[] = {
+        0, 4, 8, 16, 32, 64, 128, 256,
+    };
+    const uint16_t f16_from_u16_fixed[] = {
+        0x0000, 0x3800, 0x3c00, 0x4000,
+        0x4400, 0x4800, 0x4c00, 0x5000,
+    };
+    const uint16_t f16_to_s16_fixed[] = {
+        0x3d00, 0xbc00, 0x3800, 0xc100,
+        0x4200, 0xb800, 0x0000, 0x4400,
+    };
+    const uint16_t s16_from_f16_fixed[] = {
+        5, 0xfffc, 2, 0xfff6, 12, 0xfffe, 0, 16,
+    };
+    const uint16_t f16_to_u16_fixed[] = {
+        0x3d00, 0x3800, 0x4000, 0x4200,
+        0x4400, 0x0000, 0x4800, 0x4c00,
+    };
+    const uint16_t u16_from_f16_fixed[] = {
+        5, 2, 8, 12, 16, 0, 32, 64,
+    };
+    const uint32_t s32_fixed_to_f32[] = {
+        16, 0xffffffe0, 64, 0xffffff80,
+    };
+    const uint32_t f32_from_s32_fixed[] = {
+        0x3f800000, 0xc0000000, 0x40800000, 0xc1000000,
+    };
+    const uint32_t u32_fixed_to_f32[] = {
+        0, 8, 16, 32,
+    };
+    const uint32_t f32_from_u32_fixed[] = {
+        0x00000000, 0x3f000000, 0x3f800000, 0x40000000,
+    };
+    const uint32_t f32_fixed_values[] = {
+        0x3fa00000, 0xbfc00000, 0x40000000, 0xc0300000,
+    };
+    const uint32_t s32_from_f32_fixed[] = {
+        10, 0xfffffff4, 16, 0xffffffea,
+    };
+    const uint32_t f32_to_u32_fixed[] = {
+        0x3fa00000, 0xbf800000, 0x7f800000, 0x7fc00000,
+    };
+    const uint32_t u32_from_f32_fixed[] = {
+        5, 0, 0xffffffff, 0,
+    };
+    const uint16_t f16_from_f32_lanes[] = {
+        0x3c00, 0xc000, 0x4200, 0xc400,
+    };
+    const uint16_t f16_hs_values[] = {
+        0x3c00, 0x4000, 0xc000, 0x4200,
+        0x4400, 0xc400, 0x3800, 0xb800,
+    };
+    const uint32_t f32_from_f16_bottom[] = {
+        0x3f800000, 0xc0000000, 0x40800000, 0x3f000000,
+    };
+    const uint32_t f32_from_f16_top[] = {
+        0x40000000, 0x40400000, 0xc0800000, 0xbf000000,
+    };
+    const uint32_t f32_tie_away[] = {
+        0x3fc00000, 0xbfc00000, 0x40100000, 0xc0100000,
+    };
+    const uint32_t s32_tie_away[] = {
+        2, 0xfffffffe, 2, 0xfffffffe,
+    };
+    const uint16_t f16_tie_away_u[] = {
+        0x3e00, 0x4100, 0xbe00, 0xb800,
+        0x7c00, 0x7e00, 0x0000, 0x3c00,
+    };
+    const uint16_t u16_tie_away[] = {
+        2, 3, 0, 0, 0xffff, 0, 0, 1,
+    };
+    const uint32_t f32_tie_even_s[] = {
+        0x3fc00000, 0x40200000, 0xbfc00000, 0xc0200000,
+    };
+    const uint32_t s32_tie_even[] = {
+        2, 2, 0xfffffffe, 0xfffffffe,
+    };
+    const uint32_t f32_tie_even_u[] = {
+        0x40200000, 0x40600000, 0x40900000, 0x40b00000,
+    };
+    const uint32_t u32_tie_even[] = {
+        2, 4, 4, 6,
+    };
+    const uint32_t f32_ceil_s[] = {
+        0x3fa00000, 0xbfa00000, 0x4f000000, 0x7fc00000,
+    };
+    const uint32_t s32_ceil[] = {
+        2, 0xffffffff, 0x7fffffff, 0,
+    };
+    const uint32_t f32_ceil_u[] = {
+        0x3fa00000, 0xbfa00000, 0x4f800000, 0x7fc00000,
+    };
+    const uint32_t u32_ceil[] = {
+        2, 0, 0xffffffff, 0,
+    };
+    const uint16_t f16_floor[] = {
+        0x3e00, 0xbe00, 0x4100, 0xc100,
+        0x4200, 0xc200, 0x0000, 0xb800,
+    };
+    const uint16_t s16_floor[] = {
+        1, 0xfffe, 2, 0xfffd, 3, 0xfffd, 0, 0xffff,
+    };
+    const uint16_t f16_floor_u[] = {
+        0x3e00, 0x4100, 0xbe00, 0xb800,
+        0x7c00, 0x7e00, 0x0000, 0x3c00,
+    };
+    const uint16_t u16_floor[] = {
+        1, 2, 0, 0, 0xffff, 0, 0, 1,
+    };
+    const uint32_t f32_round_ties[] = {
+        0x40200000, 0x40600000, 0xc0200000, 0xc0600000,
+    };
+    const uint32_t f32_round_ties_expected[] = {
+        0x40000000, 0x40800000, 0xc0000000, 0xc0800000,
+    };
+    const uint16_t f16_round_away[] = {
+        0x3e00, 0xbe00, 0x4100, 0xc100,
+        0x4200, 0xc200, 0x3800, 0xb800,
+    };
+    const uint16_t f16_round_away_expected[] = {
+        0x4000, 0xc000, 0x4200, 0xc200,
+        0x4200, 0xc200, 0x3c00, 0xbc00,
+    };
+    const uint16_t f16_round_ties_expected[] = {
+        0x4000, 0xc000, 0x4000, 0xc000,
+        0x4200, 0xc200, 0x0000, 0x8000,
+    };
+    const uint32_t f32_round_frac[] = {
+        0x3fe00000, 0xbfe00000, 0x40000000, 0xc0000000,
+    };
+    const uint32_t f32_round_zero_expected[] = {
+        0x3f800000, 0xbf800000, 0x40000000, 0xc0000000,
+    };
+    const uint32_t f32_round_floor_expected[] = {
+        0x3f800000, 0xc0000000, 0x40000000, 0xc0000000,
+    };
+    const uint32_t f32_round_ceil_expected[] = {
+        0x40000000, 0xbf800000, 0x40000000, 0xc0000000,
+    };
+
+    test_arm_store_le32_vector(source, s32_values, 4);
+    test_arm_store_le32_vector(expected, f32_from_s32, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvt_sf_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    test_arm_store_le32_vector(expected, f32_from_s32, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0x00ff00ff,
+                                expected, 0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    test_arm_store_le32_vector(expected + 8, f32_from_s32 + 2, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, true, false);
+
+    test_arm_store_le16_vector(source, u16_values, 8);
+    test_arm_store_le16_vector(expected, f16_from_u16, 8);
+    insn = test_arm_mve_1op_insn(0, 1, vcvt_uf_base, 1);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, f32_to_s32, 4);
+    test_arm_store_le32_vector(expected, s32_from_f32, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvt_fs_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, true);
+
+    test_arm_store_le16_vector(source, f16_to_u16, 8);
+    test_arm_store_le16_vector(expected, u16_from_f16, 8);
+    insn = test_arm_mve_1op_insn(0, 1, vcvt_fu_base, 1);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, s16_fixed_values, 8);
+    test_arm_store_le16_vector(expected, f16_from_s16_fixed, 8);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_sh_fixed_base, 1, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, u16_fixed_values, 8);
+    test_arm_store_le16_vector(expected, f16_from_u16_fixed, 8);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_uh_fixed_base, 1, 3);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, f16_to_s16_fixed, 8);
+    test_arm_store_le16_vector(expected, s16_from_f16_fixed, 8);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_hs_fixed_base, 1, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, f16_to_u16_fixed, 8);
+    test_arm_store_le16_vector(expected, u16_from_f16_fixed, 8);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_hu_fixed_base, 1, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, s32_fixed_to_f32, 4);
+    test_arm_store_le32_vector(expected, f32_from_s32_fixed, 4);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_sf_fixed_base, 2, 4);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, u32_fixed_to_f32, 4);
+    test_arm_store_le32_vector(expected, f32_from_u32_fixed, 4);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_uf_fixed_base, 2, 4);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, f32_fixed_values, 4);
+    test_arm_store_le32_vector(expected, s32_from_f32_fixed, 4);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_fs_fixed_base, 2, 3);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, f32_to_u32_fixed, 4);
+    test_arm_store_le32_vector(expected, u32_from_f32_fixed, 4);
+    insn = test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_fu_fixed_base, 2, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, true);
+
+    test_arm_store_le32_vector(source, f32_from_s32, 4);
+    memcpy(expected, initial, sizeof(expected));
+    test_arm_store_le(expected + 0, 2, f16_from_f32_lanes[0]);
+    test_arm_store_le(expected + 4, 2, f16_from_f32_lanes[1]);
+    test_arm_store_le(expected + 8, 2, f16_from_f32_lanes[2]);
+    test_arm_store_le(expected + 12, 2, f16_from_f32_lanes[3]);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtb_sh_base, 0);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    test_arm_store_le(expected + 2, 2, f16_from_f32_lanes[0]);
+    test_arm_store_le(expected + 6, 2, f16_from_f32_lanes[1]);
+    test_arm_store_le(expected + 10, 2, f16_from_f32_lanes[2]);
+    test_arm_store_le(expected + 14, 2, f16_from_f32_lanes[3]);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtt_sh_base, 0);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, f16_hs_values, 8);
+    test_arm_store_le32_vector(expected, f32_from_f16_bottom, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtb_hs_base, 0);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(expected, f32_from_f16_top, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtt_hs_base, 0);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, f32_tie_away, 4);
+    test_arm_store_le32_vector(expected, s32_tie_away, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtas_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, f16_tie_away_u, 8);
+    test_arm_store_le16_vector(expected, u16_tie_away, 8);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtau_base, 1);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, true);
+
+    test_arm_store_le32_vector(source, f32_tie_even_s, 4);
+    test_arm_store_le32_vector(expected, s32_tie_even, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtns_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, f32_tie_even_u, 4);
+    test_arm_store_le32_vector(expected, u32_tie_even, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtnu_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, f32_ceil_s, 4);
+    test_arm_store_le32_vector(expected, s32_ceil, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtps_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, true);
+
+    test_arm_store_le32_vector(source, f32_ceil_u, 4);
+    test_arm_store_le32_vector(expected, u32_ceil, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtpu_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, true);
+
+    test_arm_store_le16_vector(source, f16_floor, 8);
+    test_arm_store_le16_vector(expected, s16_floor, 8);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtms_base, 1);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, f16_floor_u, 8);
+    test_arm_store_le16_vector(expected, u16_floor, 8);
+    insn = test_arm_mve_1op_insn(0, 1, vcvtmu_base, 1);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, true);
+
+    test_arm_store_le32_vector(source, f32_round_ties, 4);
+    test_arm_store_le32_vector(expected, f32_round_ties_expected, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vrintn_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, f16_round_away, 8);
+    test_arm_store_le16_vector(expected, f16_round_away_expected, 8);
+    insn = test_arm_mve_1op_insn(0, 1, vrinta_base, 1);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(source, f32_round_frac, 4);
+    test_arm_store_le32_vector(expected, f32_round_zero_expected, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vrintz_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(expected, f32_round_floor_expected, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vrintm_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le32_vector(expected, f32_round_ceil_expected, 4);
+    insn = test_arm_mve_1op_insn(0, 1, vrintp_base, 2);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_store_le16_vector(source, f16_round_away, 8);
+    test_arm_store_le16_vector(expected, f16_round_ties_expected, 8);
+    insn = test_arm_mve_1op_insn(0, 1, vrintx_base, 1);
+    test_arm_m55_mve_fp_1op_run(insn, initial, source, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_1op_insn(0, 1, vcvt_sf_base, 0),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_1op_insn(8, 1, vcvt_sf_base, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vcvt_fixed_insn(0, 1, vcvt_sh_fixed_base, 1, 2),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_1op_insn(0, 1, vrintn_base, 2),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_INSN_INVALID);
+}
+
+static void test_arm_m55_mve_fp_vector(void)
+{
+    const uint32_t vabs_base = 0xffb10740;
+    const uint32_t vneg_base = 0xffb107c0;
+    const uint32_t vadd_base = 0xef000d40;
+    const uint32_t vsub_base = 0xef200d40;
+    const uint32_t vmul_base = 0xff000d50;
+    const uint32_t vabd_base = 0xff200d40;
+    const uint32_t vmaxnm_base = 0xff000f50;
+    const uint32_t vminnm_base = 0xff200f50;
+    const uint32_t vmaxnma_f32_base = 0xee3f0e41;
+    const uint32_t vminnma_f16_base = 0xfe3f1e41;
+    const uint32_t vfma_base = 0xef000c50;
+    const uint32_t vfms_base = 0xef200c50;
+    const uint32_t vfcadd90_base = 0xfc800840;
+    const uint32_t vfcadd270_base = 0xfd800840;
+    const uint32_t vcmul0_base = 0xee300e00;
+    const uint32_t vcmul90_base = 0xee300e01;
+    const uint32_t vcmul180_base = 0xee301e00;
+    const uint32_t vcmul270_base = 0xee301e01;
+    const uint32_t vcmla0_base = 0xfc200840;
+    const uint32_t vcmla90_base = 0xfca00840;
+    const uint32_t vcmla180_base = 0xfd200840;
+    const uint32_t vcmla270_base = 0xfda00840;
+    const uint8_t initial[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    uint8_t n[16];
+    uint8_t m[16];
+    uint8_t expected[16];
+    uint8_t fma_initial[16];
+    uint8_t fms_initial[16];
+    uint32_t insn;
+    size_t i;
+
+    const uint16_t h_values[] = {
+        0xbc00, 0x4000, 0xfc00, 0x7e00,
+        0x8000, 0x3c00, 0xc000, 0x0000,
+    };
+    const uint32_t s_values[] = {
+        0xbf800000, 0x40000000, 0xff800000, 0x7fc00000,
+    };
+    const uint32_t s_add_n[] = {
+        0x3f800000, 0x40000000, 0xc0800000, 0x80000000,
+    };
+    const uint32_t s_add_m[] = {
+        0x40400000, 0xbf800000, 0x3fc00000, 0x00000000,
+    };
+    const uint32_t s_add_expected[] = {
+        0x40800000, 0x3f800000, 0xc0200000, 0x00000000,
+    };
+    const uint16_t h_sub_n[] = {
+        0x4200, 0xc000, 0x3c00, 0x3800,
+        0x4000, 0xbc00, 0x0000, 0x4400,
+    };
+    const uint16_t h_sub_m[] = {
+        0x3c00, 0x4000, 0xc200, 0x3400,
+        0x3c00, 0xbc00, 0x8000, 0x4000,
+    };
+    const uint16_t h_sub_expected[] = {
+        0x4000, 0xc400, 0x4400, 0x3400,
+        0x3c00, 0x0000, 0x0000, 0x4000,
+    };
+    const uint32_t s_mul_n[] = {
+        0x3f800000, 0x7f800001, 0x40000000, 0xc0000000,
+    };
+    const uint32_t s_mul_m[] = {
+        0x40000000, 0x3f800000, 0x40400000, 0xbf800000,
+    };
+    const uint32_t s_mul_expected[] = {
+        0x40000000, 0, 0x40c00000, 0x40000000,
+    };
+    const uint32_t s_abd_n[] = {
+        0x40a00000, 0x3f800000, 0xc0000000, 0x00000000,
+    };
+    const uint32_t s_abd_m[] = {
+        0x40000000, 0x40600000, 0xc0a00000, 0x80000000,
+    };
+    const uint32_t s_abd_expected[] = {
+        0x40400000, 0x40200000, 0x40400000, 0x00000000,
+    };
+    const uint16_t h_nm_n[] = {
+        0x3c00, 0xc000, 0x4400, 0xbc00,
+        0x3800, 0xc400, 0x4200, 0xbe00,
+    };
+    const uint16_t h_nm_m[] = {
+        0x4000, 0xc200, 0x3c00, 0xc000,
+        0x3c00, 0xc000, 0xc500, 0xbc00,
+    };
+    const uint16_t h_maxnm_expected[] = {
+        0x4000, 0xc000, 0x4400, 0xbc00,
+        0x3c00, 0xc000, 0x4200, 0xbc00,
+    };
+    const uint16_t h_minnm_expected[] = {
+        0x3c00, 0xc200, 0x3c00, 0xc000,
+        0x3800, 0xc400, 0xc500, 0xbe00,
+    };
+    const uint32_t s_nma_n[] = {
+        0xbf800000, 0x40000000, 0xc0800000, 0xbf400000,
+    };
+    const uint32_t s_nma_m[] = {
+        0x40400000, 0xbfc00000, 0x40000000, 0xc0000000,
+    };
+    const uint32_t s_maxnma_expected[] = {
+        0x40400000, 0x40000000, 0x40800000, 0x40000000,
+    };
+    const uint16_t h_nma_n[] = {
+        0xc000, 0x3c00, 0xc400, 0x3800,
+        0x4200, 0x8000, 0xbe00, 0x4400,
+    };
+    const uint16_t h_nma_m[] = {
+        0x3c00, 0xc200, 0x4000, 0xb400,
+        0x4400, 0x0000, 0xc000, 0xb800,
+    };
+    const uint16_t h_minnma_expected[] = {
+        0x3c00, 0x3c00, 0x4000, 0x3400,
+        0x4200, 0x0000, 0x3e00, 0x3800,
+    };
+    const uint32_t s_fma_initial[] = {
+        0x3f800000, 0xc0000000, 0x3f000000, 0xbf800000,
+    };
+    const uint32_t s_fma_n[] = {
+        0x40000000, 0xc0400000, 0x40800000, 0xc0000000,
+    };
+    const uint32_t s_fma_nan_n[] = {
+        0x40000000, 0x7f800001, 0x40800000, 0xc0000000,
+    };
+    const uint32_t s_fma_m[] = {
+        0x40400000, 0x3f000000, 0xc0000000, 0xbf800000,
+    };
+    const uint32_t s_fma_expected[] = {
+        0x40e00000, 0xc0600000, 0xc0f00000, 0x3f800000,
+    };
+    const uint16_t h_fms_initial[] = {
+        0x3c00, 0x4000, 0xc400, 0x3800,
+        0xbc00, 0x0000, 0x4400, 0xc000,
+    };
+    const uint16_t h_fms_n[] = {
+        0x4000, 0xc200, 0x3c00, 0xc000,
+        0x3800, 0xbc00, 0x4400, 0x8000,
+    };
+    const uint16_t h_fms_m[] = {
+        0x4200, 0x3c00, 0xc000, 0xbc00,
+        0x4000, 0x4400, 0xb800, 0x3c00,
+    };
+    const uint16_t h_fms_expected[] = {
+        0xc500, 0x4500, 0xc000, 0xbe00,
+        0xc000, 0x4400, 0x4600, 0xc000,
+    };
+    const uint32_t s_fcadd_n[] = {
+        0x41200000, 0x41a00000, 0xc0a00000, 0x41000000,
+    };
+    const uint32_t s_fcadd_m[] = {
+        0x3f800000, 0x40000000, 0x40400000, 0xc0800000,
+    };
+    const uint32_t s_fcadd90_expected[] = {
+        0x41000000, 0x41a80000, 0xbf800000, 0x41300000,
+    };
+    const uint16_t h_fcadd_n[] = {
+        0x3c00, 0x4000, 0xc200, 0x4400,
+        0x3800, 0xbc00, 0xc000, 0xc400,
+    };
+    const uint16_t h_fcadd_m[] = {
+        0x3800, 0x3c00, 0x4000, 0xbc00,
+        0xc000, 0x4200, 0xb800, 0xc200,
+    };
+    const uint16_t h_fcadd270_expected[] = {
+        0x4000, 0x3e00, 0xc400, 0x4000,
+        0x4300, 0x3c00, 0xc500, 0xc300,
+    };
+    const uint32_t s_cmul_n[] = {
+        0x40000000, 0x40400000, 0xc0000000, 0x3fc00000,
+    };
+    const uint32_t s_cmul_m[] = {
+        0x40800000, 0x40a00000, 0x40400000, 0xc0800000,
+    };
+    const uint32_t s_cmul0_expected[] = {
+        0x41000000, 0x41200000, 0xc0c00000, 0x41000000,
+    };
+    const uint32_t s_cmul90_expected[] = {
+        0xc1700000, 0x41400000, 0x40c00000, 0x40900000,
+    };
+    const uint16_t h_cmul_n[] = {
+        0x4000, 0x4200, 0xc000, 0x3c00,
+        0x3800, 0xbc00, 0xc400, 0xc500,
+    };
+    const uint16_t h_cmul_m[] = {
+        0x4400, 0x4500, 0x4200, 0xc400,
+        0xc000, 0x4200, 0xb800, 0xc200,
+    };
+    const uint16_t h_cmul180_expected[] = {
+        0xc800, 0xc900, 0x4600, 0xc800,
+        0x3c00, 0xbe00, 0xc000, 0xca00,
+    };
+    const uint16_t h_cmul270_expected[] = {
+        0x4b80, 0xca00, 0xc400, 0xc200,
+        0xc200, 0xc000, 0x4b80, 0xc100,
+    };
+    const uint32_t s_cmla0_expected[] = {
+        0x41100000, 0x41000000, 0xc0b00000, 0x40e00000,
+    };
+    const uint32_t s_cmla90_expected[] = {
+        0xc1600000, 0x41200000, 0x40d00000, 0x40600000,
+    };
+    const uint16_t h_cmla180_expected[] = {
+        0xc700, 0xc800, 0x4000, 0xc780,
+        0x0000, 0xbe00, 0x4000, 0xcb00,
+    };
+    const uint16_t h_cmla270_expected[] = {
+        0x4c00, 0xc900, 0xc800, 0xc100,
+        0xc400, 0xc000, 0x4cc0, 0xc480,
+    };
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_values[i]);
+        test_arm_store_le(expected + i * 2, 2, h_values[i] & 0x7fff);
+    }
+    insn = test_arm_mve_1op_insn(0, 1, vabs_base, 1);
+    test_arm_m55_mve_1op_run(insn, initial, n, expected, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_values[i]);
+        test_arm_store_le(expected + i * 4, 4, s_values[i] ^ 0x80000000);
+    }
+    insn = test_arm_mve_1op_insn(0, 1, vneg_base, 2);
+    test_arm_m55_mve_1op_run(insn, initial, n, expected, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_values[i]);
+    }
+    for (i = 4; i < 8; i++) {
+        test_arm_store_le(expected + i * 2, 2, h_values[i] ^ 0x8000);
+    }
+    insn = test_arm_mve_1op_insn(0, 1, vneg_base, 1);
+    test_arm_m55_mve_1op_run(insn, initial, n, expected, true, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_add_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_add_m[i]);
+        test_arm_store_le(expected + i * 4, 4, s_add_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vadd_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_sub_n[i]);
+        test_arm_store_le(m + i * 2, 2, h_sub_m[i]);
+        test_arm_store_le(expected + i * 2, 2, h_sub_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vsub_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_add_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_add_m[i]);
+    }
+    for (i = 2; i < 4; i++) {
+        test_arm_store_le(expected + i * 4, 4, s_add_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vadd_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0x00000055,
+                                expected, 0xffff, true, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_mul_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_mul_m[i]);
+        test_arm_store_le(expected + i * 4, 4, s_mul_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vmul_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xff0f, false, true);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_abd_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_abd_m[i]);
+        test_arm_store_le(expected + i * 4, 4, s_abd_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vabd_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_nm_n[i]);
+        test_arm_store_le(m + i * 2, 2, h_nm_m[i]);
+        test_arm_store_le(expected + i * 2, 2, h_maxnm_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vmaxnm_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(expected + i * 2, 2, h_minnm_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vminnm_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_nma_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_nma_m[i]);
+        test_arm_store_le(expected + i * 4, 4, s_maxnma_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_qdqn_insn(vmaxnma_f32_base, 0, 2);
+    test_arm_m55_mve_fp_2op_run(insn, n, initial, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, n, sizeof(expected));
+    for (i = 2; i < 4; i++) {
+        test_arm_store_le(expected + i * 4, 4, s_maxnma_expected[i]);
+    }
+    test_arm_m55_mve_fp_2op_run(insn, n, initial, m, 0x00000055,
+                                expected, 0xffff, true, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_nma_n[i]);
+        test_arm_store_le(m + i * 2, 2, h_nma_m[i]);
+        test_arm_store_le(expected + i * 2, 2, h_minnma_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_qdqn_insn(vminnma_f16_base, 0, 2);
+    test_arm_m55_mve_fp_2op_run(insn, n, initial, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(fma_initial + i * 4, 4, s_fma_initial[i]);
+        test_arm_store_le(n + i * 4, 4, s_fma_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_fma_m[i]);
+        test_arm_store_le(expected + i * 4, 4, s_fma_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vfma_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fma_initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, fma_initial, sizeof(expected));
+    for (i = 2; i < 4; i++) {
+        test_arm_store_le(expected + i * 4, 4, s_fma_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vfma_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fma_initial, n, m, 0x00000055,
+                                expected, 0xffff, true, false);
+
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_fma_nan_n[i]);
+    }
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(expected + i * 4, 4, s_fma_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vfma_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fma_initial, n, m, 0, expected,
+                                0xff0f, false, true);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(fms_initial + i * 2, 2, h_fms_initial[i]);
+        test_arm_store_le(n + i * 2, 2, h_fms_n[i]);
+        test_arm_store_le(m + i * 2, 2, h_fms_m[i]);
+        test_arm_store_le(expected + i * 2, 2, h_fms_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_insn(vfms_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fms_initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_fcadd_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_fcadd_m[i]);
+        test_arm_store_le(expected + i * 4, 4, s_fcadd90_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_rev_insn(vfcadd90_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_fcadd_n[i]);
+        test_arm_store_le(m + i * 2, 2, h_fcadd_m[i]);
+        test_arm_store_le(expected + i * 2, 2, h_fcadd270_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_rev_insn(vfcadd270_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_cmul_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_cmul_m[i]);
+        test_arm_store_le(expected + i * 4, 4, s_cmul0_expected[i]);
+    }
+    insn = test_arm_mve_fp_vcmul_insn(vcmul0_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(expected + i * 4, 4, s_cmul90_expected[i]);
+    }
+    insn = test_arm_mve_fp_vcmul_insn(vcmul90_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_cmul_n[i]);
+        test_arm_store_le(m + i * 2, 2, h_cmul_m[i]);
+        test_arm_store_le(expected + i * 2, 2, h_cmul180_expected[i]);
+    }
+    insn = test_arm_mve_fp_vcmul_insn(vcmul180_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(expected + i * 2, 2, h_cmul270_expected[i]);
+    }
+    insn = test_arm_mve_fp_vcmul_insn(vcmul270_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(n + i * 4, 4, s_cmul_n[i]);
+        test_arm_store_le(m + i * 4, 4, s_cmul_m[i]);
+    }
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(expected + i * 4, 4, s_cmla0_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_rev_insn(vcmla0_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fma_initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le(expected + i * 4, 4, s_cmla90_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_rev_insn(vcmla90_base, true, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fma_initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(n, initial, sizeof(n));
+    memcpy(m, initial, sizeof(m));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(n + i * 2, 2, h_cmul_n[i]);
+        test_arm_store_le(m + i * 2, 2, h_cmul_m[i]);
+    }
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(expected + i * 2, 2, h_cmla180_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_rev_insn(vcmla180_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fms_initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < 8; i++) {
+        test_arm_store_le(expected + i * 2, 2, h_cmla270_expected[i]);
+    }
+    insn = test_arm_mve_fp_2op_rev_insn(vcmla270_base, false, 0, 1, 2);
+    test_arm_m55_mve_fp_2op_run(insn, fms_initial, n, m, 0, expected,
+                                0xffff, false, false);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_1op_insn(0, 1, vabs_base, 0),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_1op_insn(8, 1, vabs_base, 1),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_insn(vadd_base, false, 8, 1, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_insn(vfma_base, false, 8, 1, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_qdqn_insn(vmaxnma_f32_base, 8, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_qdqn_insn(vmaxnma_f32_base, 0, 8),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_rev_insn(vfcadd90_base, true, 8, 1, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_vcmul_insn(vcmul0_base, true, 8, 1, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_rev_insn(vcmla0_base, true, 8, 1, 2),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_insn(vadd_base, false, 0, 1, 2),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_INSN_INVALID);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_fp_2op_qdqn_insn(vmaxnma_f32_base, 0, 2),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_vldrw_vstrw(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint64_t data_addr = code_start + 0x2000;
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t mem[32] = {
+        0x44, 0x33, 0x22, 0x11, 0x88, 0x77, 0x66, 0x55,
+        0xcc, 0xbb, 0xaa, 0x99, 0x00, 0xff, 0xee, 0xdd,
+        0x04, 0x03, 0x02, 0x01, 0x08, 0x07, 0x06, 0x05,
+        0x0c, 0x0b, 0x0a, 0x09, 0x10, 0x0f, 0x0e, 0x0d,
+    };
+    const uint8_t qbytes[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    uint8_t expected[32];
+    uint8_t got_mem[32];
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t base;
+    uint32_t epsr;
+    uint32_t vpr;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x1f00ed91); /* vldrw.32 q0,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == mem[i], "load i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f04ed81); /* vstrw.32 q0,[r1,#16] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memcpy(q0, qbytes, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(expected + 16, qbytes, 16);
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK_(got_mem[i] == expected[i], "store i=%u got=0x%02x",
+                    (unsigned)i, got_mem[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f00ed91); /* vldrw.32 q0,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0x001100f0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    memset(expected, 0, 16);
+    memcpy(expected + 4, mem + 4, 4);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i], "pred load i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, (code_start + 4) | 1,
+                    code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    memcpy(expected, initial, 8);
+    memcpy(expected + 8, mem + 8, 8);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i], "eci load i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f00ed81); /* vstrw.32 q0,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memcpy(q0, qbytes, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0x001100f0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(expected + 4, qbytes + 4, 4);
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK_(got_mem[i] == expected[i], "pred store i=%u got=0x%02x",
+                    (unsigned)i, got_mem[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f04ecb1); /* vldrw.32 q0,[r1],#16 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &base));
+    TEST_CHECK_(base == (uint32_t)(data_addr + 16), "base=0x%08x", base);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == mem[i], "post load i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f04ed21); /* vstrw.32 q0,[r1,#-16]! */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memcpy(q0, qbytes, 16);
+    base = (uint32_t)(data_addr + 16);
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &base));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    TEST_CHECK_(base == (uint32_t)data_addr, "base=0x%08x", base);
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(expected, qbytes, 16);
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK_(got_mem[i] == expected[i], "pre store i=%u got=0x%02x",
+                    (unsigned)i, got_mem[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_m55_mve_ldst_expect_error(0x1f00edd1, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x1f00ed91, UC_CPU_ARM_CORTEX_M33,
+                                       UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_vldrbh_vstrbh(void)
+{
+    const uint64_t data_addr = code_start + 0x2000;
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t mem[16] = {
+        0x44, 0x33, 0x22, 0x11, 0x88, 0x77, 0x66, 0x55,
+        0xcc, 0xbb, 0xaa, 0x99, 0x00, 0xff, 0xee, 0xdd,
+    };
+    const uint8_t qbytes[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    const uint32_t load_insns[] = {
+        0x1e00ed91, /* vldrb.8 q0,[r1,#0] */
+        0x1e80ed91, /* vldrh.16 q0,[r1,#0] */
+    };
+    const uint32_t store_insns[] = {
+        0x1e00ed81, /* vstrb.8 q0,[r1,#0] */
+        0x1e80ed81, /* vstrh.16 q0,[r1,#0] */
+    };
+    uint8_t expected[16];
+    uint8_t got_mem[16];
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t base;
+    uint32_t vpr;
+    size_t i;
+    size_t n;
+
+    for (n = 0; n < sizeof(load_insns) / sizeof(load_insns[0]); n++) {
+        test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+        test_arm_emit32(code, 4, load_insns[n]);
+        uc_common_setup(&uc, UC_ARCH_ARM,
+                        UC_MODE_THUMB | UC_MODE_MCLASS,
+                        (const char *)code, sizeof(code),
+                        UC_CPU_ARM_CORTEX_M55);
+        test_arm_enable_vfp(uc);
+        memcpy(q0, initial, 16);
+        base = (uint32_t)data_addr;
+        vpr = 0;
+        OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+        OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+        OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+        OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+        OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+        for (i = 0; i < 16; i++) {
+            TEST_CHECK_(got[i] == mem[i], "load n=%u i=%u got=0x%02x",
+                        (unsigned)n, (unsigned)i, got[i]);
+        }
+        OK(uc_close(uc));
+    }
+
+    for (n = 0; n < sizeof(store_insns) / sizeof(store_insns[0]); n++) {
+        test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+        test_arm_emit32(code, 4, store_insns[n]);
+        uc_common_setup(&uc, UC_ARCH_ARM,
+                        UC_MODE_THUMB | UC_MODE_MCLASS,
+                        (const char *)code, sizeof(code),
+                        UC_CPU_ARM_CORTEX_M55);
+        test_arm_enable_vfp(uc);
+        memset(got_mem, 0xee, sizeof(got_mem));
+        memcpy(q0, qbytes, 16);
+        base = (uint32_t)data_addr;
+        vpr = 0;
+        OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+        OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+        OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+        OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+        OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+        for (i = 0; i < 16; i++) {
+            TEST_CHECK_(got_mem[i] == qbytes[i],
+                        "store n=%u i=%u got=0x%02x",
+                        (unsigned)n, (unsigned)i, got_mem[i]);
+        }
+        OK(uc_close(uc));
+    }
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x1e00ed91); /* vldrb.8 q0,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0x001100f0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    memset(expected, 0, sizeof(expected));
+    memcpy(expected + 4, mem + 4, 4);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i], "pred b load i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1e80ed81); /* vstrh.16 q0,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memcpy(q0, qbytes, 16);
+    base = (uint32_t)data_addr;
+    vpr = 0x001100f0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(expected + 4, qbytes + 4, 4);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got_mem[i] == expected[i],
+                    "pred h store i=%u got=0x%02x",
+                    (unsigned)i, got_mem[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_vldst_widen_narrow(void)
+{
+    const uint64_t data_addr = code_start + 0x2000;
+    const uint8_t mem[16] = {
+        0x80, 0x7f, 0x01, 0xff, 0x34, 0x12, 0x01, 0x80,
+        0xfe, 0xff, 0x00, 0x40, 0x55, 0xaa, 0x10, 0xf0,
+    };
+    const uint8_t qbytes[16] = {
+        0x81, 0x10, 0x82, 0x20, 0x83, 0x30, 0x84, 0x40,
+        0x85, 0x50, 0x86, 0x60, 0x87, 0x70, 0x88, 0x80,
+    };
+    const struct {
+        uint32_t insn;
+        unsigned mem_size;
+        unsigned elem_size;
+        bool is_signed;
+    } load_cases[] = {
+        { 0x0e80ed91, 1, 2, true },  /* vldrb.s16 q0,[r1,#0] */
+        { 0x0e80fd91, 1, 2, false }, /* vldrb.u16 q0,[r1,#0] */
+        { 0x0f00ed91, 1, 4, true },  /* vldrb.s32 q0,[r1,#0] */
+        { 0x0f00fd91, 1, 4, false }, /* vldrb.u32 q0,[r1,#0] */
+        { 0x0f00ed99, 2, 4, true },  /* vldrh.s32 q0,[r1,#0] */
+        { 0x0f00fd99, 2, 4, false }, /* vldrh.u32 q0,[r1,#0] */
+    };
+    const struct {
+        uint32_t insn;
+        unsigned mem_size;
+        unsigned elem_size;
+    } store_cases[] = {
+        { 0x0e80ed81, 1, 2 }, /* vstrb.16 q0,[r1,#0] */
+        { 0x0f00ed81, 1, 4 }, /* vstrb.32 q0,[r1,#0] */
+        { 0x0f00ed89, 2, 4 }, /* vstrh.32 q0,[r1,#0] */
+    };
+    uint8_t expected[16];
+    uint8_t got_mem[16];
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t base;
+    uint32_t vpr;
+    size_t i;
+    size_t n;
+
+    for (n = 0; n < sizeof(load_cases) / sizeof(load_cases[0]); n++) {
+        test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+        test_arm_emit32(code, 4, load_cases[n].insn);
+        uc_common_setup(&uc, UC_ARCH_ARM,
+                        UC_MODE_THUMB | UC_MODE_MCLASS,
+                        (const char *)code, sizeof(code),
+                        UC_CPU_ARM_CORTEX_M55);
+        test_arm_enable_vfp(uc);
+        memset(q0, 0xee, sizeof(q0));
+        memset(expected, 0, sizeof(expected));
+        base = (uint32_t)data_addr;
+        vpr = 0;
+        OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+        OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+        OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+        OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+        OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+        for (i = 0; i < 16 / load_cases[n].elem_size; i++) {
+            uint32_t value = test_arm_load_le(mem + i * load_cases[n].mem_size,
+                                              load_cases[n].mem_size);
+
+            if (load_cases[n].is_signed) {
+                if (load_cases[n].mem_size == 1 && (value & 0x80)) {
+                    value |= 0xffffff00;
+                } else if (load_cases[n].mem_size == 2 &&
+                           (value & 0x8000)) {
+                    value |= 0xffff0000;
+                }
+            }
+            test_arm_store_le(expected + i * load_cases[n].elem_size,
+                              load_cases[n].elem_size, value);
+        }
+        for (i = 0; i < 16; i++) {
+            TEST_CHECK_(got[i] == expected[i],
+                        "wide load n=%u i=%u got=0x%02x expected=0x%02x",
+                        (unsigned)n, (unsigned)i, got[i], expected[i]);
+        }
+        OK(uc_close(uc));
+    }
+
+    for (n = 0; n < sizeof(store_cases) / sizeof(store_cases[0]); n++) {
+        test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+        test_arm_emit32(code, 4, store_cases[n].insn);
+        uc_common_setup(&uc, UC_ARCH_ARM,
+                        UC_MODE_THUMB | UC_MODE_MCLASS,
+                        (const char *)code, sizeof(code),
+                        UC_CPU_ARM_CORTEX_M55);
+        test_arm_enable_vfp(uc);
+        memset(got_mem, 0xee, sizeof(got_mem));
+        memcpy(q0, qbytes, 16);
+        memset(expected, 0xee, sizeof(expected));
+        base = (uint32_t)data_addr;
+        vpr = 0;
+        OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+        OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+        OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+        OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+        OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+        for (i = 0; i < 16 / store_cases[n].elem_size; i++) {
+            uint32_t value = test_arm_load_le(qbytes + i *
+                                              store_cases[n].elem_size,
+                                              store_cases[n].mem_size);
+
+            test_arm_store_le(expected + i * store_cases[n].mem_size,
+                              store_cases[n].mem_size, value);
+        }
+        for (i = 0; i < 16; i++) {
+            TEST_CHECK_(got_mem[i] == expected[i],
+                        "narrow store n=%u i=%u got=0x%02x expected=0x%02x",
+                        (unsigned)n, (unsigned)i, got_mem[i], expected[i]);
+        }
+        OK(uc_close(uc));
+    }
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x0e80ed91); /* vldrb.s16 q0,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(q0, 0xee, sizeof(q0));
+    memset(expected, 0, sizeof(expected));
+    base = (uint32_t)data_addr;
+    vpr = 0x001100f0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    test_arm_store_le(expected + 4, 2, test_arm_load_le(mem + 2, 1));
+    test_arm_store_le(expected + 6, 2, 0xffffffff);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "pred wide load i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x0f00ed89); /* vstrh.32 q0,[r1,#0] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memcpy(q0, qbytes, 16);
+    memset(expected, 0xee, sizeof(expected));
+    base = (uint32_t)data_addr;
+    vpr = 0x001100f0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    memcpy(expected + 2, qbytes + 4, 2);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got_mem[i] == expected[i],
+                    "pred narrow store i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got_mem[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x0e84ecb1); /* vldrb.s16 q0,[r1],#4 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(q0, 0xee, sizeof(q0));
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &base));
+    TEST_CHECK_(base == (uint32_t)(data_addr + 4), "base=0x%08x", base);
+    OK(uc_close(uc));
+
+    test_arm_m55_mve_ldst_expect_error(0x0e80fd81, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x0e80ed91, UC_CPU_ARM_CORTEX_M33,
+                                       UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_sg(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint64_t data_addr = code_start + 0x2000;
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t mem[32] = {
+        0x10, 0x81, 0x22, 0x7f, 0xfe, 0x35, 0xc0, 0x49,
+        0x08, 0x09, 0xaa, 0x0b, 0x0c, 0xdd, 0x0e, 0x0f,
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xf0, 0x12,
+    };
+    const uint8_t qbytes[16] = {
+        0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28,
+        0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30,
+    };
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint8_t expected[32];
+    uint8_t got_mem[32];
+    uint32_t base;
+    uint32_t epsr;
+    uint32_t vpr;
+    size_t i;
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x0e02fc91); /* vldrb.u8 q0,[r1,q1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    for (i = 0; i < 16; i++) {
+        ((uint8_t *)q1)[i] = (uint8_t)((i * 3) & 0x1f);
+        expected[i] = mem[((uint8_t *)q1)[i]];
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "u8 gather i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x0f02ec91); /* vldrb.s32 q0,[r1,q1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    memset(expected, 0, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        uint32_t offset = (uint32_t)(i * 2 + 1);
+        uint32_t value = mem[offset];
+
+        if (value & 0x80) {
+            value |= 0xffffff00;
+        }
+        test_arm_store_le((uint8_t *)q1 + i * 4, 4, offset);
+        test_arm_store_le(expected + i * 4, 4, value);
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "s32 gather i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x0f13fc91); /* vldrh.u32 q0,[r1,q1,uxtw #1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    memset(expected, 0, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        uint32_t offset = (uint32_t)(i + 4);
+        uint32_t value = test_arm_load_le(mem + offset * 2, 2);
+
+        test_arm_store_le((uint8_t *)q1 + i * 4, 4, offset);
+        test_arm_store_le(expected + i * 4, 4, value);
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "os gather i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x0f42ec81); /* vstrw.32 q0,[r1,q1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memset(q1, 0, sizeof(q1));
+    memcpy(q0, qbytes, 16);
+    memset(expected, 0xee, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        uint32_t offset = (uint32_t)(i * 4);
+
+        test_arm_store_le((uint8_t *)q1 + i * 4, 4, offset);
+        memcpy(expected + offset, qbytes + i * 4, 4);
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK_(got_mem[i] == expected[i],
+                    "scatter i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got_mem[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x0e02fc91); /* vldrb.u8 q0,[r1,q1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    memset(expected, 0, sizeof(expected));
+    for (i = 0; i < 16; i++) {
+        ((uint8_t *)q1)[i] = (uint8_t)i;
+    }
+    expected[4] = mem[4];
+    expected[5] = mem[5];
+    expected[6] = mem[6];
+    expected[7] = mem[7];
+    base = (uint32_t)data_addr;
+    vpr = 0x001100f0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "pred gather i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    memset(expected, 0, sizeof(expected));
+    for (i = 0; i < 16; i++) {
+        ((uint8_t *)q1)[i] = (uint8_t)i;
+    }
+    memcpy(expected, initial, 8);
+    memcpy(expected + 8, mem + 8, 8);
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, (code_start + 4) | 1,
+                    code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "eci gather i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_m55_mve_ldst_expect_error(0x0e00fc91, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x0e03fc91, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x0e02fc91, UC_CPU_ARM_CORTEX_M33,
+                                       UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_sg_imm(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint64_t data_addr = code_start + 0x2000;
+    const uint8_t initial[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t qbytes[16] = {
+        0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38,
+        0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40,
+    };
+    uint8_t mem[64];
+    uint8_t expected[64];
+    uint8_t got_mem[64];
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    const uint8_t *got_q1 = (const uint8_t *)q1;
+    uint32_t epsr;
+    uint32_t vpr;
+    size_t i;
+
+    for (i = 0; i < sizeof(mem); i++) {
+        mem[i] = (uint8_t)(0x40 + i);
+    }
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x1e04fd92); /* vldrw.u32 q0,[q1,#16] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le((uint8_t *)q1 + i * 4, 4,
+                          (uint32_t)(data_addr + i * 4));
+        memcpy(expected + i * 4, mem + 16 + i * 4, 4);
+    }
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "imm w load i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1e04fdb2); /* vldrw.u32 q0,[q1,#16]! */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le((uint8_t *)q1 + i * 4, 4,
+                          (uint32_t)(data_addr + i * 4));
+    }
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q1, q1));
+    for (i = 0; i < 4; i++) {
+        uint32_t updated = test_arm_load_le(got_q1 + i * 4, 4);
+
+        TEST_CHECK_(updated == (uint32_t)(data_addr + 16 + i * 4),
+                    "imm w wb lane=%u got=0x%08x",
+                    (unsigned)i, updated);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f02fdb2); /* vldrd.u64 q0,[q1,#16]! */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0xaa, sizeof(q1));
+    test_arm_store_le((uint8_t *)q1, 4, (uint32_t)data_addr);
+    test_arm_store_le((uint8_t *)q1 + 8, 4, (uint32_t)(data_addr + 16));
+    memset(expected, 0, sizeof(expected));
+    memcpy(expected, mem + 16, 8);
+    memcpy(expected + 8, mem + 32, 8);
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q1, q1));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "imm d load i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got[i], expected[i]);
+    }
+    TEST_CHECK_(test_arm_load_le(got_q1, 4) == (uint32_t)(data_addr + 16),
+                "d wb lane0=0x%08x", test_arm_load_le(got_q1, 4));
+    TEST_CHECK_(test_arm_load_le(got_q1 + 8, 4) ==
+                (uint32_t)(data_addr + 32),
+                "d wb lane2=0x%08x", test_arm_load_le(got_q1 + 8, 4));
+    TEST_CHECK_(got_q1[4] == 0xaa && got_q1[12] == 0xaa,
+                "d wb odd lanes changed");
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1e01fd22); /* vstrw.32 q0,[q1,#-4]! */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(q0, qbytes, 16);
+    memset(q1, 0, sizeof(q1));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le((uint8_t *)q1 + i * 4, 4,
+                          (uint32_t)(data_addr + 4 + i * 4));
+        memcpy(expected + i * 4, qbytes + i * 4, 4);
+    }
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q1, q1));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got_mem[i] == expected[i],
+                    "imm w store i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got_mem[i], expected[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        uint32_t updated = test_arm_load_le(got_q1 + i * 4, 4);
+
+        TEST_CHECK_(updated == (uint32_t)(data_addr + i * 4),
+                    "imm w store wb lane=%u got=0x%08x",
+                    (unsigned)i, updated);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f01fd82); /* vstrd.64 q0,[q1,#8] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(q0, qbytes, 16);
+    memset(q1, 0xaa, sizeof(q1));
+    test_arm_store_le((uint8_t *)q1, 4, (uint32_t)data_addr);
+    test_arm_store_le((uint8_t *)q1 + 8, 4, (uint32_t)(data_addr + 16));
+    memcpy(expected + 8, qbytes, 8);
+    memcpy(expected + 24, qbytes + 8, 8);
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    for (i = 0; i < 40; i++) {
+        TEST_CHECK_(got_mem[i] == expected[i],
+                    "imm d store i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got_mem[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1e04fdb2); /* vldrw.u32 q0,[q1,#16]! */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memset(q1, 0, sizeof(q1));
+    for (i = 0; i < 4; i++) {
+        test_arm_store_le((uint8_t *)q1 + i * 4, 4,
+                          (uint32_t)(data_addr + i * 4));
+    }
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, (code_start + 4) | 1,
+                    code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK_(got[i] == initial[i],
+                    "eci imm load preserved i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    for (i = 2; i < 4; i++) {
+        uint32_t updated = test_arm_load_le(got_q1 + i * 4, 4);
+
+        TEST_CHECK_(updated == (uint32_t)(data_addr + 16 + i * 4),
+                    "eci imm wb lane=%u got=0x%08x",
+                    (unsigned)i, updated);
+    }
+    TEST_CHECK_(test_arm_load_le(got_q1, 4) == (uint32_t)data_addr,
+                "eci imm lane0 updated");
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_m55_mve_ldst_expect_error(0x1e01fd90, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x1e01fdd2, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x1e04fd92, UC_CPU_ARM_CORTEX_M33,
+                                       UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_mve_interleaved(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint64_t data_addr = code_start + 0x2000;
+    const uint8_t initial_q0[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t initial_q1[16] = {
+        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+        0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+    };
+    const uint8_t qbytes0[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    const uint8_t qbytes1[16] = {
+        0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+        0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+    };
+    const uint8_t qbytes2[16] = {
+        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+        0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+    };
+    const uint8_t qbytes3[16] = {
+        0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+        0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
+    };
+    const uint8_t off_vld2b[4] = { 0, 2, 12, 14 };
+    const uint8_t off_vld4h[4] = { 0, 0, 5, 5 };
+    const uint8_t off_vst2w[4] = { 0, 4, 24, 28 };
+    const uint8_t off_vst4b[4] = { 0, 1, 10, 11 };
+    uc_engine *uc;
+    uint8_t code[8];
+    uint8_t mem[96];
+    uint8_t expected[96];
+    uint8_t got_mem[96];
+    uint8_t exp_q0[16];
+    uint8_t exp_q1[16];
+    uint8_t exp_q2[16];
+    uint8_t exp_q3[16];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    uint64_t q3[2];
+    uint8_t *dst_q[4] = { exp_q0, exp_q1, exp_q2, exp_q3 };
+    const uint8_t *src_q[4] = { qbytes0, qbytes1, qbytes2, qbytes3 };
+    uint32_t base;
+    uint32_t epsr;
+    uint32_t vpr;
+    size_t beat;
+    size_t i;
+
+    for (i = 0; i < sizeof(mem); i++) {
+        mem[i] = (uint8_t)(0x50 + i);
+    }
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x1e00fc91); /* vld2.8 {q0,q1},[r1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial_q0, 16);
+    memcpy(q1, initial_q1, 16);
+    memcpy(exp_q0, initial_q0, 16);
+    memcpy(exp_q1, initial_q1, 16);
+    for (beat = 0; beat < 4; beat++) {
+        size_t off = off_vld2b[beat];
+        size_t addr = off * 2;
+
+        exp_q0[off] = mem[addr];
+        exp_q1[off] = mem[addr + 1];
+        exp_q0[off + 1] = mem[addr + 2];
+        exp_q1[off + 1] = mem[addr + 3];
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q1, q1));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(((uint8_t *)q0)[i] == exp_q0[i],
+                    "vld2 q0 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q0)[i], exp_q0[i]);
+        TEST_CHECK_(((uint8_t *)q1)[i] == exp_q1[i],
+                    "vld2 q1 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q1)[i], exp_q1[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1e81fcb1); /* vld4.16 {q0-q3},[r1]! */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, qbytes0, 16);
+    memcpy(q1, qbytes1, 16);
+    memcpy(q2, qbytes2, 16);
+    memcpy(q3, qbytes3, 16);
+    memcpy(exp_q0, qbytes0, 16);
+    memcpy(exp_q1, qbytes1, 16);
+    memcpy(exp_q2, qbytes2, 16);
+    memcpy(exp_q3, qbytes3, 16);
+    for (beat = 0; beat < 4; beat++) {
+        size_t off = off_vld4h[beat];
+        size_t addr = off * 8 + (beat & 1) * 4;
+        unsigned y = (beat & 1) ? 2 : 0;
+
+        test_arm_store_le(dst_q[y] + off * 2, 2,
+                          test_arm_load_le(mem + addr, 2));
+        test_arm_store_le(dst_q[y + 1] + off * 2, 2,
+                          test_arm_load_le(mem + addr + 2, 2));
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q3, q3));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q3, q3));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &base));
+    TEST_CHECK_(base == (uint32_t)(data_addr + 64),
+                "vld4 wb base=0x%08x", base);
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(((uint8_t *)q0)[i] == exp_q0[i],
+                    "vld4 q0 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q0)[i], exp_q0[i]);
+        TEST_CHECK_(((uint8_t *)q1)[i] == exp_q1[i],
+                    "vld4 q1 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q1)[i], exp_q1[i]);
+        TEST_CHECK_(((uint8_t *)q2)[i] == exp_q2[i],
+                    "vld4 q2 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q2)[i], exp_q2[i]);
+        TEST_CHECK_(((uint8_t *)q3)[i] == exp_q3[i],
+                    "vld4 q3 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q3)[i], exp_q3[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1f00fca1); /* vst2.32 {q0,q1},[r1]! */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(q0, qbytes0, 16);
+    memcpy(q1, qbytes1, 16);
+    for (beat = 0; beat < 4; beat++) {
+        size_t off = off_vst2w[beat];
+        size_t lane = (off >> 3) * 4;
+
+        memcpy(expected + off, src_q[beat & 1] + lane, 4);
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &base));
+    TEST_CHECK_(base == (uint32_t)(data_addr + 32),
+                "vst2 wb base=0x%08x", base);
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK_(got_mem[i] == expected[i],
+                    "vst2 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got_mem[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1e01fc81); /* vst4.8 {q0-q3},[r1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memset(got_mem, 0xee, sizeof(got_mem));
+    memset(expected, 0xee, sizeof(expected));
+    memcpy(q0, qbytes0, 16);
+    memcpy(q1, qbytes1, 16);
+    memcpy(q2, qbytes2, 16);
+    memcpy(q3, qbytes3, 16);
+    for (beat = 0; beat < 4; beat++) {
+        size_t off = off_vst4b[beat];
+        size_t addr = off * 4;
+
+        expected[addr] = qbytes0[off];
+        expected[addr + 1] = qbytes1[off];
+        expected[addr + 2] = qbytes2[off];
+        expected[addr + 3] = qbytes3[off];
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q3, q3));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, data_addr, got_mem, sizeof(got_mem)));
+    OK(uc_reg_read(uc, UC_ARM_REG_R1, &base));
+    TEST_CHECK_(base == (uint32_t)data_addr, "vst4 base=0x%08x", base);
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK_(got_mem[i] == expected[i],
+                    "vst4 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, got_mem[i], expected[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 4, 0x1e00fc91); /* vld2.8 {q0,q1},[r1] */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial_q0, 16);
+    memcpy(q1, initial_q1, 16);
+    memcpy(exp_q0, initial_q0, 16);
+    memcpy(exp_q1, initial_q1, 16);
+    for (beat = 2; beat < 4; beat++) {
+        size_t off = off_vld2b[beat];
+        size_t addr = off * 2;
+
+        exp_q0[off] = mem[addr];
+        exp_q1[off] = mem[addr + 1];
+        exp_q0[off + 1] = mem[addr + 2];
+        exp_q1[off + 1] = mem[addr + 3];
+    }
+    base = (uint32_t)data_addr;
+    vpr = 0;
+    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R1, &base));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, (code_start + 4) | 1,
+                    code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(((uint8_t *)q0)[i] == exp_q0[i],
+                    "eci vld2 q0 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q0)[i], exp_q0[i]);
+        TEST_CHECK_(((uint8_t *)q1)[i] == exp_q1[i],
+                    "eci vld2 q1 i=%u got=0x%02x expected=0x%02x",
+                    (unsigned)i, ((uint8_t *)q1)[i], exp_q1[i]);
+    }
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_m55_mve_ldst_expect_error(0x1f80fc91, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x1e40fc91, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0xfe00fc91, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0xbe01fc91, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x1e00fc9f, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x1e00fcbd, UC_CPU_ARM_CORTEX_M55,
+                                       UC_ERR_EXCEPTION);
+    test_arm_m55_mve_ldst_expect_error(0x1e00fc91, UC_CPU_ARM_CORTEX_M33,
+                                       UC_ERR_EXCEPTION);
+}
+
+static void test_arm_m55_vpsel(void)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint8_t initial[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    const uint8_t qn_bytes[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint8_t qm_bytes[16] = {
+        0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
+        0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+    };
+    const uint8_t expected[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+    };
+    const uint8_t expected_eci[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    uint32_t epsr;
+    uint32_t vpr;
+    size_t i;
+    uc_err err;
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, 0x0f05fe33); /* vpsel q0,q1,q2 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, qn_bytes, 16);
+    memcpy(q2, qm_bytes, 16);
+    vpr = 0x000000ff;
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected[i], "i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q0, initial, 16);
+    memcpy(q1, qn_bytes, 16);
+    memcpy(q2, qm_bytes, 16);
+    vpr = 0x0000ff00;
+    OK(uc_reg_write(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &vpr));
+    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+    epsr = xpsr_t | eci_a0a1;
+    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+    OK(uc_emu_start(uc, (code_start + 4) | 1, code_start + sizeof(code), 0,
+                    0));
+    OK(uc_reg_read(uc, UC_ARM_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK_(got[i] == expected_eci[i], "i=%u got=0x%02x",
+                    (unsigned)i, got[i]);
+    }
+    TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                "epsr=0x%08x", epsr);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f05fe73); /* vpsel q8,q1,q2 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4,
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+
+    test_arm_emit32(code, 0, 0x0f05fe33); /* vpsel q0,q1,q2 */
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, 4,
+                    UC_CPU_ARM_CORTEX_M33);
+    test_arm_enable_vfp(uc);
+    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
+    OK(uc_close(uc));
+}
+
+typedef enum test_arm_mve_cmp_kind {
+    TEST_ARM_MVE_CMP_EQ,
+    TEST_ARM_MVE_CMP_NE,
+    TEST_ARM_MVE_CMP_CS,
+    TEST_ARM_MVE_CMP_HI,
+    TEST_ARM_MVE_CMP_GE,
+    TEST_ARM_MVE_CMP_LT,
+    TEST_ARM_MVE_CMP_GT,
+    TEST_ARM_MVE_CMP_LE,
+} test_arm_mve_cmp_kind;
+
+static uint16_t test_arm_mve_expected_cmp_p0(const uint8_t *n,
+                                             const uint8_t *m,
+                                             uint32_t scalar,
+                                             unsigned esize,
+                                             bool is_scalar,
+                                             bool is_signed,
+                                             test_arm_mve_cmp_kind kind,
+                                             uint16_t pred)
+{
+    uint16_t p0 = 0;
+    unsigned i;
+
+    for (i = 0; i < 16; i += esize) {
+        uint32_t lhs = test_arm_load_le(n + i, esize);
+        uint32_t rhs = is_scalar ? scalar : test_arm_load_le(m + i, esize);
+        uint16_t lane_mask = ((1U << esize) - 1) << i;
+        bool result;
+
+        if (is_scalar && esize < 4) {
+            rhs &= (1U << (esize * 8)) - 1;
+        }
+
+        if (is_signed) {
+            int64_t slhs = test_arm_sign_extend(lhs, esize * 8);
+            int64_t srhs = test_arm_sign_extend(rhs, esize * 8);
+
+            switch (kind) {
+            case TEST_ARM_MVE_CMP_GE:
+            case TEST_ARM_MVE_CMP_CS:
+                result = slhs >= srhs;
+                break;
+            case TEST_ARM_MVE_CMP_GT:
+                result = slhs > srhs;
+                break;
+            case TEST_ARM_MVE_CMP_LT:
+                result = slhs < srhs;
+                break;
+            case TEST_ARM_MVE_CMP_LE:
+                result = slhs <= srhs;
+                break;
+            case TEST_ARM_MVE_CMP_EQ:
+                result = slhs == srhs;
+                break;
+            case TEST_ARM_MVE_CMP_NE:
+                result = slhs != srhs;
+                break;
+            default:
+                result = false;
+                break;
+            }
+        } else {
+            switch (kind) {
+            case TEST_ARM_MVE_CMP_CS:
+            case TEST_ARM_MVE_CMP_GE:
+                result = lhs >= rhs;
+                break;
+            case TEST_ARM_MVE_CMP_GT:
+            case TEST_ARM_MVE_CMP_HI:
+                result = lhs > rhs;
+                break;
+            case TEST_ARM_MVE_CMP_LT:
+                result = lhs < rhs;
+                break;
+            case TEST_ARM_MVE_CMP_LE:
+                result = lhs <= rhs;
+                break;
+            case TEST_ARM_MVE_CMP_EQ:
+                result = lhs == rhs;
+                break;
+            case TEST_ARM_MVE_CMP_NE:
+                result = lhs != rhs;
+                break;
+            default:
+                result = false;
+                break;
+            }
+        }
+
+        if (result) {
+            p0 |= lane_mask;
+        }
+    }
+
+    return p0 & pred;
+}
+
+static void test_arm_m55_mve_vcmp_run(uint32_t insn, const uint8_t *n,
+                                      const uint8_t *m, uint32_t initial_vpr,
+                                      uint32_t r3, uint32_t expected_vpr,
+                                      bool eci)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    uint32_t epsr;
+    uint32_t vpr;
+
+    test_arm_emit32(code, 0, 0x0a10eeec); /* vmsr vpr,r0 */
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q1, n, 16);
+    if (m) {
+        memcpy(q2, m, 16);
+    } else {
+        memset(q2, 0, sizeof(q2));
+    }
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &initial_vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &r3));
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    TEST_CHECK_(vpr == expected_vpr,
+                "insn=0x%08x vpr=0x%08x expected=0x%08x",
+                insn, vpr, expected_vpr);
+    if (eci) {
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_vcmp(void)
+{
+    const uint8_t bytes_n[16] = {
+        0x00, 0x01, 0x80, 0xff, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t bytes_m[16] = {
+        0x00, 0x02, 0x80, 0x7f, 0x05, 0x05, 0x06, 0x08,
+        0x18, 0x09, 0x0b, 0x0b, 0x0d, 0x0d, 0x0f, 0x0f,
+    };
+    uint8_t words_n[16];
+    uint8_t words_m[16];
+    uint8_t scalar_n[16];
+    uint8_t scalar_h[16];
+    uint32_t expected_p0;
+    uint32_t expected_vpr;
+    uint32_t r3 = 0;
+
+    test_arm_store_le(words_n, 4, 0xffffffff);
+    test_arm_store_le(words_n + 4, 4, 0x80000000);
+    test_arm_store_le(words_n + 8, 4, 0x7fffffff);
+    test_arm_store_le(words_n + 12, 4, 0x00000002);
+    test_arm_store_le(words_m, 4, 0x00000001);
+    test_arm_store_le(words_m + 4, 4, 0xffffffff);
+    test_arm_store_le(words_m + 8, 4, 0x80000000);
+    test_arm_store_le(words_m + 12, 4, 0x00000002);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(bytes_n, bytes_m, 0, 1,
+                                               false, false,
+                                               TEST_ARM_MVE_CMP_EQ, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x0f04fe03, bytes_n, bytes_m, 0, 0,
+                              expected_p0, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(words_n, words_m, 0, 4,
+                                               false, true,
+                                               TEST_ARM_MVE_CMP_GE, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x1f04fe23, words_n, words_m, 0, 0,
+                              expected_p0, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(words_n, words_m, 0, 4,
+                                               false, false,
+                                               TEST_ARM_MVE_CMP_CS, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x0f05fe23, words_n, words_m, 0, 0,
+                              expected_p0, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(bytes_n, bytes_m, 0, 1,
+                                               false, false,
+                                               TEST_ARM_MVE_CMP_HI, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x0f85fe03, bytes_n, bytes_m, 0, 0,
+                              expected_p0, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(bytes_n, bytes_m, 0, 1,
+                                               false, true,
+                                               TEST_ARM_MVE_CMP_LT, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x1f84fe03, bytes_n, bytes_m, 0, 0,
+                              expected_p0, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(words_n, words_m, 0, 4,
+                                               false, true,
+                                               TEST_ARM_MVE_CMP_LE, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x1f85fe23, words_n, words_m, 0, 0,
+                              expected_p0, false);
+
+    memset(scalar_n, 0, sizeof(scalar_n));
+    test_arm_store_le(scalar_n, 4, 0);
+    test_arm_store_le(scalar_n + 4, 4, 1);
+    test_arm_store_le(scalar_n + 8, 4, 0xffffffff);
+    test_arm_store_le(scalar_n + 12, 4, 0);
+    expected_p0 = test_arm_mve_expected_cmp_p0(scalar_n, NULL, 0, 4,
+                                               true, false,
+                                               TEST_ARM_MVE_CMP_EQ, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x0f4ffe23, scalar_n, NULL, 0, 0,
+                              expected_p0, false);
+
+    r3 = 0;
+    test_arm_store_le(scalar_n + 12, 4, 0x80000000);
+    expected_p0 = test_arm_mve_expected_cmp_p0(scalar_n, NULL, r3, 4,
+                                               true, true,
+                                               TEST_ARM_MVE_CMP_GT, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x1f63fe23, scalar_n, NULL, 0, r3,
+                              expected_p0, false);
+
+    r3 = 0x102;
+    expected_p0 = test_arm_mve_expected_cmp_p0(bytes_n, NULL, r3, 1,
+                                               true, false,
+                                               TEST_ARM_MVE_CMP_HI, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x0fe3fe03, bytes_n, NULL, 0, r3,
+                              expected_p0, false);
+
+    memset(scalar_h, 0, sizeof(scalar_h));
+    test_arm_store_le(scalar_h, 2, 0x8000);
+    test_arm_store_le(scalar_h + 2, 2, 0x8001);
+    test_arm_store_le(scalar_h + 4, 2, 0x8002);
+    test_arm_store_le(scalar_h + 6, 2, 0x7fff);
+    test_arm_store_le(scalar_h + 8, 2, 0);
+    test_arm_store_le(scalar_h + 10, 2, 0xffff);
+    test_arm_store_le(scalar_h + 12, 2, 1);
+    test_arm_store_le(scalar_h + 14, 2, 0x9000);
+    r3 = 0x8001;
+    expected_p0 = test_arm_mve_expected_cmp_p0(scalar_h, NULL, r3, 2,
+                                               true, true,
+                                               TEST_ARM_MVE_CMP_LT, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x1fc3fe13, scalar_h, NULL, 0, r3,
+                              expected_p0, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(scalar_h, NULL, r3, 2,
+                                               true, true,
+                                               TEST_ARM_MVE_CMP_LE, 0xffff);
+    test_arm_m55_mve_vcmp_run(0x1fe3fe13, scalar_h, NULL, 0, r3,
+                              expected_p0, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(bytes_n, bytes_m, 0, 1,
+                                               false, false,
+                                               TEST_ARM_MVE_CMP_NE, 0xffff);
+    expected_vpr = 0x00aa0000 | expected_p0;
+    test_arm_m55_mve_vcmp_run(0x4f84fe43, bytes_n, bytes_m, 0, 0,
+                              expected_vpr, false);
+
+    expected_p0 = test_arm_mve_expected_cmp_p0(bytes_n, bytes_m, 0, 1,
+                                               false, false,
+                                               TEST_ARM_MVE_CMP_NE, 0xffff);
+    expected_vpr = 0x00000055 | (expected_p0 & 0xff00);
+    test_arm_m55_mve_vcmp_run(0x0f84fe03, bytes_n, bytes_m, 0x00000055, 0,
+                              expected_vpr, true);
+
+    test_arm_m55_mve_2op_expect_error(0x0f20fe03, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f4dfe23, UC_CPU_ARM_CORTEX_M55,
+                                      UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(0x0f04fe03, UC_CPU_ARM_CORTEX_M33,
+                                      UC_ERR_EXCEPTION);
+}
+
+static uint32_t test_arm_mve_vcmp_fp_insn(uint32_t base, bool fp16,
+                                          unsigned mask, unsigned qn,
+                                          unsigned qm)
+{
+    uint32_t view = base;
+
+    if (fp16) {
+        view |= 1U << 28;
+    }
+    view |= ((mask >> 3) & 1) << 22;
+    view |= (mask & 7) << 13;
+    view |= (qn & 7) << 17;
+    view |= ((qm >> 3) & 1) << 5;
+    view |= (qm & 7) << 1;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static uint32_t test_arm_mve_vcmp_fp_scalar_insn(uint32_t base, bool fp16,
+                                                 unsigned mask, unsigned qn,
+                                                 unsigned rm)
+{
+    uint32_t view = base;
+
+    if (fp16) {
+        view |= 1U << 28;
+    }
+    view |= ((mask >> 3) & 1) << 22;
+    view |= (mask & 7) << 13;
+    view |= (qn & 7) << 17;
+    view |= rm & 15;
+    return test_arm_mve_view_to_t32(view);
+}
+
+static void test_arm_m55_mve_vcmp_fp_run(uint32_t insn, const uint8_t *n,
+                                         const uint8_t *m,
+                                         uint32_t initial_vpr, uint32_t r3,
+                                         uint32_t expected_vpr, bool eci,
+                                         bool expected_ioc)
+{
+    const uint32_t xpsr_t = 1U << 24;
+    const uint32_t eci_a0a1 = 2U << 12;
+    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
+    const uint32_t fpscr_ioc = 1U;
+    uc_engine *uc;
+    uint8_t code[8];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    uint32_t epsr;
+    uint32_t fpscr = 0;
+    uint32_t vpr;
+
+    test_arm_emit32(code, 0, 0x0a10eeec);
+    test_arm_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM,
+                    UC_MODE_THUMB | UC_MODE_MCLASS,
+                    (const char *)code, sizeof(code),
+                    UC_CPU_ARM_CORTEX_M55);
+    test_arm_enable_vfp(uc);
+    memcpy(q1, n, 16);
+    if (m) {
+        memcpy(q2, m, 16);
+    } else {
+        memset(q2, 0, sizeof(q2));
+    }
+    OK(uc_reg_write(uc, UC_ARM_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM_REG_Q2, q2));
+    OK(uc_reg_write(uc, UC_ARM_REG_R0, &initial_vpr));
+    OK(uc_reg_write(uc, UC_ARM_REG_R3, &r3));
+    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
+
+    if (eci) {
+        OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
+        epsr = xpsr_t | eci_a0a1;
+        OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
+        OK(uc_emu_start(uc, (code_start + 4) | 1,
+                        code_start + sizeof(code), 0, 0));
+    } else {
+        OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0,
+                        0));
+    }
+
+    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
+    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
+    TEST_CHECK_(vpr == expected_vpr,
+                "insn=0x%08x vpr=0x%08x expected=0x%08x",
+                insn, vpr, expected_vpr);
+    TEST_CHECK_(((fpscr & fpscr_ioc) != 0) == expected_ioc,
+                "insn=0x%08x fpscr=0x%08x expected_ioc=%u",
+                insn, fpscr, expected_ioc ? 1 : 0);
+    if (eci) {
+        OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
+        TEST_CHECK_((epsr & epsr_condexec_mask) == 0,
+                    "epsr=0x%08x", epsr);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm_m55_mve_vcmp_fp(void)
+{
+    const uint32_t vcmpeq_base = 0xee310f00;
+    const uint32_t vcmpne_base = 0xee310f80;
+    const uint32_t vcmplt_base = 0xee311f80;
+    const uint32_t vcmpgt_scalar_base = 0xee311f60;
+    const uint32_t vcmple_scalar_base = 0xee311fe0;
+    const uint8_t f32_eq_n[16] = {
+        0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x40,
+        0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x80, 0x40,
+    };
+    const uint8_t f32_eq_m[16] = {
+        0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0xa0, 0x40,
+    };
+    const uint8_t f32_nan_n[16] = {
+        0x00, 0x00, 0x80, 0x3f, 0x01, 0x00, 0x80, 0x7f,
+        0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0xa0, 0x40,
+    };
+    const uint8_t f32_nan_m[16] = {
+        0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x40,
+        0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0xc0, 0x40,
+    };
+    const uint8_t f16_scalar_n[16] = {
+        0x00, 0xbc, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x40,
+        0x00, 0xc0, 0x00, 0x42, 0x00, 0x38, 0x00, 0xb8,
+    };
+    const uint8_t f16_scalar_le_n[16] = {
+        0x00, 0xbc, 0x01, 0x7c, 0x00, 0x00, 0x00, 0x3c,
+        0x00, 0x80, 0x00, 0xc0, 0x00, 0x38, 0x00, 0x7c,
+    };
+    uint32_t insn;
+
+    insn = test_arm_mve_vcmp_fp_insn(vcmpeq_base, false, 0, 1, 2);
+    test_arm_m55_mve_vcmp_fp_run(insn, f32_eq_n, f32_eq_m, 0, 0,
+                                 0x00000f0f, false, false);
+
+    insn = test_arm_mve_vcmp_fp_insn(vcmpeq_base, false, 10, 1, 2);
+    test_arm_m55_mve_vcmp_fp_run(insn, f32_eq_n, f32_eq_m, 0, 0,
+                                 0x00aa0f0f, false, false);
+
+    insn = test_arm_mve_vcmp_fp_insn(vcmpne_base, false, 0, 1, 2);
+    test_arm_m55_mve_vcmp_fp_run(insn, f32_nan_n, f32_nan_m, 0, 0,
+                                 0x0000f0f0, false, true);
+
+    insn = test_arm_mve_vcmp_fp_insn(vcmplt_base, false, 0, 1, 2);
+    test_arm_m55_mve_vcmp_fp_run(insn, f32_nan_n, f32_nan_m, 0, 0,
+                                 0x0000f0f0, false, true);
+
+    insn = test_arm_mve_vcmp_fp_scalar_insn(vcmpgt_scalar_base, true, 0,
+                                            1, 15);
+    test_arm_m55_mve_vcmp_fp_run(insn, f16_scalar_n, NULL, 0, 0,
+                                 0x00003cf0, false, false);
+
+    insn = test_arm_mve_vcmp_fp_scalar_insn(vcmple_scalar_base, true, 0,
+                                            1, 3);
+    test_arm_m55_mve_vcmp_fp_run(insn, f16_scalar_le_n, NULL, 0, 0,
+                                 0x00000f3f, false, true);
+
+    insn = test_arm_mve_vcmp_fp_insn(vcmpeq_base, false, 0, 1, 2);
+    test_arm_m55_mve_vcmp_fp_run(insn, f32_eq_n, f32_eq_m, 0x00000055,
+                                 0, 0x00000f55, true, false);
+
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vcmp_fp_scalar_insn(vcmpgt_scalar_base, false, 0,
+                                         1, 13),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vcmp_fp_insn(vcmpeq_base, false, 0, 1, 8),
+        UC_CPU_ARM_CORTEX_M55, UC_ERR_EXCEPTION);
+    test_arm_m55_mve_2op_expect_error(
+        test_arm_mve_vcmp_fp_insn(vcmpeq_base, false, 0, 1, 2),
+        UC_CPU_ARM_CORTEX_M33, UC_ERR_EXCEPTION);
+}
+
 //
 // Some notes:
 //   Qemu raise a special exception EXCP_EXCEPTION_EXIT to handle the
@@ -1172,7 +12467,81 @@ TEST_LIST = {{"test_arm_nop", test_arm_nop},
              {"test_arm_thumbeb_sub", test_arm_thumbeb_sub},
              {"test_arm_thumb_ite", test_arm_thumb_ite},
              {"test_arm_m_thumb_mrs", test_arm_m_thumb_mrs},
+             {"test_arm_i8mm", test_arm_i8mm},
+             {"test_arm_bf16", test_arm_bf16},
              {"test_arm_m_control", test_arm_m_control},
+             {"test_arm_m33_sau_tt", test_arm_m33_sau_tt},
+             {"test_arm_m55_pmsav8_pxn", test_arm_m55_pmsav8_pxn},
+             {"test_arm_m55_mve_id", test_arm_m55_mve_id},
+             {"test_arm_m55_vpr_public_reg", test_arm_m55_vpr_public_reg},
+             {"test_arm_m55_vpr_sysreg", test_arm_m55_vpr_sysreg},
+             {"test_arm_m55_fpscr_ltpsize", test_arm_m55_fpscr_ltpsize},
+             {"test_arm_m55_fpscr_nzcvqc_sysreg",
+              test_arm_m55_fpscr_nzcvqc_sysreg},
+             {"test_arm_m55_sysreg_mem", test_arm_m55_sysreg_mem},
+             {"test_arm_m55_vlldm_vlstm", test_arm_m55_vlldm_vlstm},
+             {"test_arm_m55_fpcxt_sysreg", test_arm_m55_fpcxt_sysreg},
+             {"test_arm_m55_vscclrm", test_arm_m55_vscclrm},
+             {"test_arm_m55_vctp", test_arm_m55_vctp},
+             {"test_arm_m55_mve_eci", test_arm_m55_mve_eci},
+             {"test_arm_m55_vpst_vpnot", test_arm_m55_vpst_vpnot},
+             {"test_arm_m55_mve_logic", test_arm_m55_mve_logic},
+             {"test_arm_m55_mve_add_sub", test_arm_m55_mve_add_sub},
+             {"test_arm_m55_mve_scalar_2op", test_arm_m55_mve_scalar_2op},
+             {"test_arm_m55_mve_vbrsr", test_arm_m55_mve_vbrsr},
+             {"test_arm_m55_mve_scalar_halving_sat",
+              test_arm_m55_mve_scalar_halving_sat},
+             {"test_arm_m55_mve_mulh", test_arm_m55_mve_mulh},
+             {"test_arm_m55_mve_vmull", test_arm_m55_mve_vmull},
+             {"test_arm_m55_mve_minmax", test_arm_m55_mve_minmax},
+             {"test_arm_m55_mve_vabd", test_arm_m55_mve_vabd},
+             {"test_arm_m55_mve_halving", test_arm_m55_mve_halving},
+             {"test_arm_m55_mve_qaddsub", test_arm_m55_mve_qaddsub},
+             {"test_arm_m55_mve_shift", test_arm_m55_mve_shift},
+             {"test_arm_m55_mve_qdmulh", test_arm_m55_mve_qdmulh},
+             {"test_arm_m55_mve_qdmladh", test_arm_m55_mve_qdmladh},
+             {"test_arm_m55_mve_qdmull", test_arm_m55_mve_qdmull},
+             {"test_arm_m55_mve_scalar_qdmull",
+              test_arm_m55_mve_scalar_qdmull},
+             {"test_arm_m55_mve_scalar_acc",
+              test_arm_m55_mve_scalar_acc},
+             {"test_arm_m55_mve_carry", test_arm_m55_mve_carry},
+             {"test_arm_m55_mve_fp_scalar",
+              test_arm_m55_mve_fp_scalar},
+             {"test_arm_m55_mve_cadd", test_arm_m55_mve_cadd},
+             {"test_arm_m55_mve_vimm", test_arm_m55_mve_vimm},
+             {"test_arm_m55_mve_shift_imm", test_arm_m55_mve_shift_imm},
+             {"test_arm_m55_mve_scalar_shift",
+              test_arm_m55_mve_scalar_shift},
+             {"test_arm_m55_mve_gpr_shift",
+              test_arm_m55_mve_gpr_shift},
+             {"test_arm_m55_mve_qshift_imm", test_arm_m55_mve_qshift_imm},
+             {"test_arm_m55_mve_vshll", test_arm_m55_mve_vshll},
+             {"test_arm_m55_mve_shrn_imm", test_arm_m55_mve_shrn_imm},
+             {"test_arm_m55_mve_qshrn_imm", test_arm_m55_mve_qshrn_imm},
+             {"test_arm_m55_mve_movn", test_arm_m55_mve_movn},
+             {"test_arm_m55_mve_reduce", test_arm_m55_mve_reduce},
+             {"test_arm_m55_mve_dualacc", test_arm_m55_mve_dualacc},
+             {"test_arm_m55_mve_vshlc", test_arm_m55_mve_vshlc},
+             {"test_arm_m55_mve_vdup", test_arm_m55_mve_vdup},
+             {"test_arm_m55_mve_vmov_2gp", test_arm_m55_mve_vmov_2gp},
+             {"test_arm_m55_mve_vidup", test_arm_m55_mve_vidup},
+             {"test_arm_m55_mve_1op", test_arm_m55_mve_1op},
+             {"test_arm_m55_mve_fp_convert_round",
+              test_arm_m55_mve_fp_convert_round},
+             {"test_arm_m55_mve_fp_vector", test_arm_m55_mve_fp_vector},
+             {"test_arm_m55_mve_vldrw_vstrw", test_arm_m55_mve_vldrw_vstrw},
+             {"test_arm_m55_mve_vldrbh_vstrbh",
+              test_arm_m55_mve_vldrbh_vstrbh},
+             {"test_arm_m55_mve_vldst_widen_narrow",
+              test_arm_m55_mve_vldst_widen_narrow},
+             {"test_arm_m55_mve_sg", test_arm_m55_mve_sg},
+             {"test_arm_m55_mve_sg_imm", test_arm_m55_mve_sg_imm},
+             {"test_arm_m55_mve_interleaved",
+              test_arm_m55_mve_interleaved},
+             {"test_arm_m55_vpsel", test_arm_m55_vpsel},
+             {"test_arm_m55_mve_vcmp", test_arm_m55_mve_vcmp},
+             {"test_arm_m55_mve_vcmp_fp", test_arm_m55_mve_vcmp_fp},
              {"test_arm_m_exc_return", test_arm_m_exc_return},
              {"test_arm_und32_to_svc32", test_arm_und32_to_svc32},
              {"test_arm_usr32_to_svc32", test_arm_usr32_to_svc32},
diff --git a/tests/unit/test_arm64.c b/tests/unit/test_arm64.c
index 68db50ac21..f80f69a445 100644
--- a/tests/unit/test_arm64.c
+++ b/tests/unit/test_arm64.c
@@ -4,6 +4,7 @@
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <string.h>
 
 const uint64_t code_start = 0x1000;
 const uint64_t code_len = 0x4000;
@@ -147,6 +148,120 @@ static void test_arm64_v8_cas(void)
     OK(uc_close(uc));
 }
 
+static void test_arm64_lse_unaligned_exception_code(const char *code,
+                                                    size_t size)
+{
+    uc_engine *uc;
+    uint64_t x0 = 0x40001;
+    uint64_t x2 = 0x1122334455667788ull;
+    uint64_t x5 = 0x8877665544332211ull;
+    uint64_t data = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, size,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + size, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_lse_rcpc_unaligned(void)
+{
+    const char ldar[] = "\x01\xfc\xdf\xc8";
+    const char stlr[] = "\x02\xfc\x9f\xc8";
+    const char ldapr[] =
+        "\x03\xc0\xbf\xf8" /* ldapr  x3,[x0] */
+        "\x04\xc0\xbf\x78" /* ldaprh w4,[x0] */
+        "\x05\xc0\xbf\x38"; /* ldaprb w5,[x0] */
+    const char ldapur[] = "\x04\x00\x40\xd9";
+    const char stlur[] = "\x05\x00\x00\xd9";
+    uc_engine *uc;
+    uint64_t x0 = 0x40001;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0;
+    uint64_t x5 = 0;
+    uint64_t data = 0x1122334455667788ull;
+
+    test_arm64_lse_unaligned_exception_code(ldar, sizeof(ldar) - 1);
+    test_arm64_lse_unaligned_exception_code(stlr, sizeof(stlr) - 1);
+    test_arm64_lse_unaligned_exception_code(ldapur, sizeof(ldapur) - 1);
+    test_arm64_lse_unaligned_exception_code(stlur, sizeof(stlur) - 1);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldapr,
+                    sizeof(ldapr) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(ldapr) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    TEST_CHECK(x3 == 0x0011223344556677ull);
+    TEST_CHECK(x4 == 0x6677);
+    TEST_CHECK(x5 == 0x77);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_lse_signed_minmax_byte(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x02\x40\x21\x38" /* ldsmaxb w1,w2,[x0] */
+        "\x04\x50\x23\x38"; /* ldsminb w3,w4,[x0] */
+    uint64_t x0 = 0x40000;
+    uint64_t x1 = 0x7f;
+    uint64_t x2 = 0xffffffffffffffffull;
+    uint64_t x3 = 0x80;
+    uint64_t x4 = 0xffffffffffffffffull;
+    uint8_t data = 0x80;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_mem_read(uc, 0x40000, &data, sizeof(data)));
+    TEST_CHECK_(x2 == 0x80, "x2=0x%llx", x2);
+    TEST_CHECK_(x4 == 0x7f, "x4=0x%llx", x4);
+    TEST_CHECK_(data == 0x80, "data=0x%x", data);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_dgh_hint(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\xdf\x20\x03\xd5" /* dgh */
+        "\x40\x05\x80\xd2"; /* mov x0,#42 */
+    uint64_t x0 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    TEST_CHECK(x0 == 42);
+    OK(uc_close(uc));
+
+    x0 = 0;
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_A72);
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    TEST_CHECK(x0 == 42);
+    OK(uc_close(uc));
+}
+
 static void test_arm64_read_sctlr(void)
 {
     uc_engine *uc;
@@ -740,6 +855,22 @@ static bool test_arm64_pauth_cp_reg_update(uc_engine *uc, const uint32_t cpregid
     return (((reg.val & setmask) == setmask) && ((reg.val & clearmask) == 0));
 }
 
+static uint32_t test_arm64_mrs_sysreg(uint32_t rt,
+                                      const uint32_t cpregid[5])
+{
+    return 0xd5200000 | (cpregid[0] << 19) | (cpregid[1] << 16) |
+           (cpregid[2] << 12) | (cpregid[3] << 8) |
+           (cpregid[4] << 5) | rt;
+}
+
+static uint32_t test_arm64_msr_sysreg(uint32_t rt,
+                                      const uint32_t cpregid[5])
+{
+    return 0xd5000000 | (cpregid[0] << 19) | (cpregid[1] << 16) |
+           (cpregid[2] << 12) | (cpregid[3] << 8) |
+           (cpregid[4] << 5) | rt;
+}
+
 static void test_arm64_pauth_check_cpu_feat(uc_engine *uc)
 {
     // Check the CPU actually supports any form of PAuth, i.e. any APA or API
@@ -754,6 +885,166 @@ static void test_arm64_pauth_check_cpu_feat(uc_engine *uc)
     TEST_CHECK((ID_AA64ISAR1_EL1_bits & ID_AA64ISAR1_EL1_APA_API_MASK) != 0);
 }
 
+static void test_arm64_lse_rcpc_id_registers(void)
+{
+    uc_engine *uc;
+    const char code[] = "\x1f\x20\x03\xd5"; /* nop */
+    const uint32_t ID_AA64ISAR0_EL1[5] = { 3, 0, 0, 6, 0 };
+    const uint32_t ID_AA64ISAR1_EL1[5] = { 3, 0, 0, 6, 1 };
+    const uint32_t ID_AA64ISAR2_EL1[5] = { 3, 0, 0, 6, 2 };
+    const uint32_t ID_AA64MMFR2_EL1[5] = { 3, 0, 0, 7, 2 };
+    uint64_t isar0;
+    uint64_t isar1;
+    uint64_t isar2;
+    uint64_t mmfr2;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    isar0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR0_EL1);
+    isar1 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR1_EL1);
+    isar2 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR2_EL1);
+    mmfr2 = test_arm64_pauth_cp_reg_read(uc, ID_AA64MMFR2_EL1);
+    TEST_CHECK(((isar0 >> 20) & 0xf) == 2);
+    TEST_CHECK(((isar1 >> 20) & 0xf) == 2);
+    TEST_CHECK(((isar1 >> 48) & 0xf) == 1);
+    TEST_CHECK(((isar1 >> 56) & 0xf) == 0);
+    TEST_CHECK(((isar1 >> 60) & 0xf) == 0);
+    TEST_CHECK(isar2 == 0);
+    TEST_CHECK(((mmfr2 >> 32) & 0xf) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_A72);
+    isar0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR0_EL1);
+    isar1 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR1_EL1);
+    isar2 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR2_EL1);
+    mmfr2 = test_arm64_pauth_cp_reg_read(uc, ID_AA64MMFR2_EL1);
+    TEST_CHECK(((isar0 >> 20) & 0xf) == 0);
+    TEST_CHECK(((isar1 >> 20) & 0xf) == 0);
+    TEST_CHECK(((isar1 >> 48) & 0xf) == 0);
+    TEST_CHECK(((isar1 >> 56) & 0xf) == 0);
+    TEST_CHECK(((isar1 >> 60) & 0xf) == 0);
+    TEST_CHECK(isar2 == 0);
+    TEST_CHECK(((mmfr2 >> 32) & 0xf) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_a72_rejects_code(const char *code, size_t size)
+{
+    uc_engine *uc;
+    uint64_t x0 = 0x40000;
+    uint64_t x8 = 0;
+    uint64_t x9 = 0x40000;
+    uint64_t x10 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, size,
+                    UC_CPU_ARM64_A72);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X10, &x10));
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + size, 0, 0));
+    OK(uc_close(uc));
+}
+
+static void test_arm64_lse_rcpc_a72_rejects(void)
+{
+    const char casal[] = "\x28\xfd\xea\xc8";
+    const char ldapr[] = "\x03\xc0\xbf\xf8";
+    const char ldapur[] = "\x04\x00\x40\xd9";
+
+    test_arm64_a72_rejects_code(casal, sizeof(casal) - 1);
+    test_arm64_a72_rejects_code(ldapr, sizeof(ldapr) - 1);
+    test_arm64_a72_rejects_code(ldapur, sizeof(ldapur) - 1);
+}
+
+static void test_arm64_sve_id_registers(void)
+{
+    uc_engine *uc;
+    const char code[] = "\x1f\x20\x03\xd5"; /* nop */
+    const uint32_t ID_AA64ISAR1_EL1[5] = { 3, 0, 0, 6, 1 };
+    const uint32_t ID_AA64PFR0_EL1[5] = { 3, 0, 0, 4, 0 };
+    const uint32_t ID_AA64PFR1_EL1[5] = { 3, 0, 0, 4, 1 };
+    const uint32_t ID_AA64ZFR0_EL1[5] = { 3, 0, 0, 4, 4 };
+    const uint32_t ID_AA64SMFR0_EL1[5] = { 3, 0, 0, 4, 5 };
+    const uint64_t isar1_bf16 = 0xfULL << 44;
+    const uint64_t isar1_i8mm = 0xfULL << 52;
+    const uint64_t pfr0_sve = 0xfULL << 32;
+    const uint64_t pfr1_mte = 0xfULL << 8;
+    const uint64_t pfr1_sme = 0xfULL << 24;
+    const uint64_t zfr0_svever = 0xfULL;
+    const uint64_t zfr0_aes = 0xfULL << 4;
+    const uint64_t zfr0_bitperm = 0xfULL << 16;
+    const uint64_t zfr0_bf16 = 0xfULL << 20;
+    const uint64_t zfr0_sha3 = 0xfULL << 32;
+    const uint64_t zfr0_sm4 = 0xfULL << 40;
+    const uint64_t zfr0_i8mm = 0xfULL << 44;
+    const uint64_t zfr0_f32mm = 0xfULL << 52;
+    const uint64_t zfr0_f64mm = 0xfULL << 56;
+    const uint64_t smfr0_f32f32 = 1ULL << 32;
+    const uint64_t smfr0_b16f32 = 1ULL << 34;
+    const uint64_t smfr0_f16f32 = 1ULL << 35;
+    const uint64_t smfr0_i8i32 = 0xfULL << 36;
+    const uint64_t smfr0_f64f64 = 1ULL << 48;
+    const uint64_t smfr0_i16i64 = 0xfULL << 52;
+    const uint64_t smfr0_smever = 0xfULL << 56;
+    const uint64_t smfr0_fa64 = 1ULL << 63;
+    uint64_t isar1;
+    uint64_t pfr0;
+    uint64_t pfr1;
+    uint64_t zfr0;
+    uint64_t smfr0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    isar1 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR1_EL1);
+    pfr0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64PFR0_EL1);
+    pfr1 = test_arm64_pauth_cp_reg_read(uc, ID_AA64PFR1_EL1);
+    zfr0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ZFR0_EL1);
+    smfr0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64SMFR0_EL1);
+    TEST_CHECK(((isar1 & isar1_bf16) >> 44) == 1);
+    TEST_CHECK(((isar1 & isar1_i8mm) >> 52) == 1);
+    TEST_CHECK(((pfr0 & pfr0_sve) >> 32) == 1);
+    TEST_CHECK(((pfr1 & pfr1_mte) >> 8) == 2);
+    TEST_CHECK(((pfr1 & pfr1_sme) >> 24) == 1);
+    TEST_CHECK((zfr0 & zfr0_svever) == 1);
+    TEST_CHECK(((zfr0 & zfr0_aes) >> 4) == 2);
+    TEST_CHECK(((zfr0 & zfr0_bitperm) >> 16) == 1);
+    TEST_CHECK(((zfr0 & zfr0_bf16) >> 20) == 1);
+    TEST_CHECK(((zfr0 & zfr0_sha3) >> 32) == 1);
+    TEST_CHECK(((zfr0 & zfr0_sm4) >> 40) == 1);
+    TEST_CHECK(((zfr0 & zfr0_i8mm) >> 44) == 1);
+    TEST_CHECK(((zfr0 & zfr0_f32mm) >> 52) == 1);
+    TEST_CHECK(((zfr0 & zfr0_f64mm) >> 56) == 1);
+    TEST_CHECK(((smfr0 & smfr0_f32f32) >> 32) == 1);
+    TEST_CHECK(((smfr0 & smfr0_b16f32) >> 34) == 1);
+    TEST_CHECK(((smfr0 & smfr0_f16f32) >> 35) == 1);
+    TEST_CHECK(((smfr0 & smfr0_i8i32) >> 36) == 0xf);
+    TEST_CHECK(((smfr0 & smfr0_f64f64) >> 48) == 1);
+    TEST_CHECK(((smfr0 & smfr0_i16i64) >> 52) == 0xf);
+    TEST_CHECK(((smfr0 & smfr0_smever) >> 56) == 1);
+    TEST_CHECK(((smfr0 & smfr0_fa64) >> 63) == 1);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_A72);
+    isar1 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ISAR1_EL1);
+    pfr0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64PFR0_EL1);
+    pfr1 = test_arm64_pauth_cp_reg_read(uc, ID_AA64PFR1_EL1);
+    zfr0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64ZFR0_EL1);
+    smfr0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64SMFR0_EL1);
+    TEST_CHECK((isar1 & isar1_bf16) == 0);
+    TEST_CHECK((isar1 & isar1_i8mm) == 0);
+    TEST_CHECK((pfr0 & pfr0_sve) == 0);
+    TEST_CHECK((pfr1 & pfr1_mte) == 0);
+    TEST_CHECK((pfr1 & pfr1_sme) == 0);
+    TEST_CHECK(zfr0 == 0);
+    TEST_CHECK(smfr0 == 0);
+    OK(uc_close(uc));
+}
+
 #define SCTLR_EL1_EnIA (1ULL << 31)
 #define SCTLR_EL1_EnIB (1ULL << 30)
 #define SCTLR_EL1_EnDA (1ULL << 27)
@@ -944,10 +1235,11142 @@ static void test_arm64_pauth_ctl(void)
     OK(uc_close(uc));
 }
 
+static void test_arm64_mte_register_only(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x20\x0c\x82\x91" /* addg  x0,x1,#0x20,#3 */
+        "\x22\x14\x81\xd1" /* subg  x2,x1,#0x10,#5 */
+        "\x23\x10\xc4\x9a" /* irg   x3,x1,x4 */
+        "\xc5\x14\xc7\x9a" /* gmi   x5,x6,x7 */
+        "\x28\x01\xca\x9a" /* subp  x8,x9,x10 */
+        "\x8b\x01\xcd\xba"; /* subps x11,x12,x13 */
+    const uint32_t TCO[5] = { 3, 3, 4, 2, 7 };
+    const uint32_t TFSRE0_EL1[5] = { 3, 0, 5, 6, 1 };
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint32_t TFSR_EL2[5] = { 3, 4, 5, 6, 0 };
+    const uint32_t TFSR_EL3[5] = { 3, 6, 5, 6, 0 };
+    const uint64_t pstate_tco = 1ULL << 25;
+    uint64_t x1 = 0x0a00000000001000ull;
+    uint64_t x4 = 0;
+    uint64_t x6 = 0x0b00000000000000ull;
+    uint64_t x7 = 0x100;
+    uint64_t x9 = 0xaa00000000002000ull;
+    uint64_t x10 = 0xbb00000000000100ull;
+    uint64_t x12 = 0xcc00000000000100ull;
+    uint64_t x13 = 0xdd00000000000200ull;
+    uint64_t x0, x2, x3, x5, x8, x11;
+    uint32_t nzcv;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X10, &x10));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X13, &x13));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X11, &x11));
+    OK(uc_reg_read(uc, UC_ARM64_REG_NZCV, &nzcv));
+
+    test_arm64_pauth_cp_reg_write(uc, TCO, pstate_tco);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TCO) == pstate_tco);
+    test_arm64_pauth_cp_reg_write(uc, TCO, 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TCO) == 0);
+
+    test_arm64_pauth_cp_reg_write(uc, TFSRE0_EL1, 0x11);
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL1, 0x22);
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL2, 0x44);
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL3, 0x88);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSRE0_EL1) == 0x11);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0x22);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL2) == 0x44);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL3) == 0x88);
+
+    TEST_CHECK(x0 == 0x1020);
+    TEST_CHECK(x2 == 0xff0);
+    TEST_CHECK(x3 == 0x1000);
+    TEST_CHECK(x5 == 0x900);
+    TEST_CHECK(x8 == 0x1f00);
+    TEST_CHECK(x11 == 0xffffffffffffff00ull);
+    TEST_CHECK((nzcv & 0xf0000000u) == 0x80000000u);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_enable_checks(uc_engine *uc, uint64_t tcf);
+
+static void test_arm64_mte_ata_tag_generation(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x20\x0c\x82\x91" /* addg x0,x1,#0x20,#3 */
+        "\x22\x14\x81\xd1" /* subg x2,x1,#0x10,#5 */
+        "\x23\x10\xc4\x9a"; /* irg  x3,x1,x4 */
+    const uint32_t RGSR_EL1[5] = { 3, 0, 1, 0, 5 };
+    const uint32_t GCR_EL1[5] = { 3, 0, 1, 0, 6 };
+    uint64_t x1 = 0x0200000000001000ull;
+    uint64_t x4 = 0;
+    uint64_t x0, x2, x3, rgsr;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_checks(uc, 0);
+    test_arm64_pauth_cp_reg_write(uc, GCR_EL1, 0);
+    test_arm64_pauth_cp_reg_write(uc, RGSR_EL1, 9);
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    rgsr = test_arm64_pauth_cp_reg_read(uc, RGSR_EL1);
+
+    TEST_CHECK(x0 == 0x0500000000001020ull);
+    TEST_CHECK(x2 == 0x0700000000000ff0ull);
+    TEST_CHECK(x3 == 0x0900000000001000ull);
+    TEST_CHECK(rgsr == 9);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_tag_load_store(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg  x2,[x1] */
+        "\x23\x00\x60\xd9" /* ldg  x3,[x1] */
+        "\x22\x18\x60\xd9" /* stzg x2,[x1,#0x10] */
+        "\x24\x10\x60\xd9" /* ldg  x4,[x1,#0x10] */
+        "\x22\x28\xa0\xd9" /* st2g x2,[x1,#0x20] */
+        "\x25\x20\x60\xd9" /* ldg  x5,[x1,#0x20] */
+        "\x26\x30\x60\xd9" /* ldg  x6,[x1,#0x30] */
+        "\x21\xc0\x3f\x91" /* add  x1,x1,#0xff0 */
+        "\x22\x08\xa0\xd9" /* st2g x2,[x1] */
+        "\x27\x00\x60\xd9" /* ldg  x7,[x1] */
+        "\x28\x10\x60\xd9"; /* ldg  x8,[x1,#0x10] */
+    uint8_t fill[0x40];
+    uint8_t zeroed[0x10];
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0x5000;
+    uint64_t x4 = 0x6000;
+    uint64_t x5 = 0x7000;
+    uint64_t x6 = 0x8000;
+    uint64_t x7 = 0x9000;
+    uint64_t x8 = 0xa000;
+
+    memset(fill, 0xaa, sizeof(fill));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x2000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, fill, sizeof(fill)));
+    test_arm64_mte_enable_checks(uc, 0);
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_mem_read(uc, 0x40010, zeroed, sizeof(zeroed)));
+
+    TEST_CHECK(x3 == 0x0c00000000005000ull);
+    TEST_CHECK(x4 == 0x0c00000000006000ull);
+    TEST_CHECK(x5 == 0x0c00000000007000ull);
+    TEST_CHECK(x6 == 0x0c00000000008000ull);
+    TEST_CHECK(x7 == 0x0c00000000009000ull);
+    TEST_CHECK(x8 == 0x0c0000000000a000ull);
+    TEST_CHECK(memcmp(zeroed, "\0\0\0\0\0\0\0\0"
+                              "\0\0\0\0\0\0\0\0",
+                      sizeof(zeroed)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_tag_snapshot(void)
+{
+    uc_engine *uc;
+    uc_context *ctx;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x23\x00\x60\xd9"; /* ldg x3,[x1] */
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0x5000;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_context_alloc(uc, &ctx));
+    OK(uc_ctl_context_mode(uc, UC_CTL_CONTEXT_MEMORY | UC_CTL_CONTEXT_CPU));
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    test_arm64_mte_enable_checks(uc, 0);
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_context_save(uc, ctx));
+
+    x2 = 0x0d00000000000000ull;
+    x3 = 0x6000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x0d00000000006000ull);
+
+    OK(uc_context_restore(uc, ctx));
+    x3 = 0x7000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x0c00000000007000ull);
+
+    OK(uc_context_free(ctx));
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_tag_multiple(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x00\xa0\xd9" /* stgm  x2,[x1] */
+        "\x23\x00\xe0\xd9" /* ldgm  x3,[x1] */
+        "\x24\x00\x60\xd9" /* ldg   x4,[x1] */
+        "\x25\x10\x60\xd9" /* ldg   x5,[x1,#0x10] */
+        "\x26\x00\x20\xd9" /* stzgm x6,[x1] */
+        "\x27\x00\xe0\xd9"; /* ldgm  x7,[x1] */
+    const uint32_t GMID_EL1[5] = { 3, 1, 0, 0, 4 };
+    uint8_t fill[0x40];
+    uint8_t zeroed[0x40];
+    uint8_t expected_zero[0x40] = { 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0fedcba987654321ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0x5000;
+    uint64_t x5 = 0x6000;
+    uint64_t x6 = 0xa;
+    uint64_t x7 = 0;
+
+    memset(fill, 0xaa, sizeof(fill));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, fill, sizeof(fill)));
+    test_arm64_mte_enable_checks(uc, 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, GMID_EL1) == 6);
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_mem_read(uc, 0x40000, zeroed, sizeof(zeroed)));
+
+    TEST_CHECK(x3 == 0x0fedcba987654321ull);
+    TEST_CHECK(x4 == 0x0100000000005000ull);
+    TEST_CHECK(x5 == 0x0200000000006000ull);
+    TEST_CHECK(x7 == 0x0fedcba98765aaaaull);
+    TEST_CHECK(memcmp(zeroed, expected_zero, sizeof(zeroed)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static uint64_t test_arm64_mte_page_desc(uint64_t pa, unsigned attridx)
+{
+    return (pa & 0x0000fffffffff000ull) | 0x743 | (attridx << 2);
+}
+
+static void test_arm64_mte_set_page_desc(uc_engine *uc, uint64_t va,
+                                         uint64_t desc)
+{
+    const uint64_t l3_base = 0x102000;
+    uint64_t l3_addr = l3_base + ((va >> 12) & 0x1ff) * sizeof(desc);
+
+    OK(uc_mem_write(uc, l3_addr, &desc, sizeof(desc)));
+}
+
+static void test_arm64_mte_enable_identity_map(uc_engine *uc,
+                                               uint64_t normal_va)
+{
+    const uint32_t MAIR_EL1[5] = { 3, 0, 10, 2, 0 };
+    const uint32_t TCR_EL1[5] = { 3, 0, 2, 0, 2 };
+    const uint32_t TTBR0_EL1[5] = { 3, 0, 2, 0, 0 };
+    const uint64_t l1_base = 0x100000;
+    const uint64_t l2_base = 0x101000;
+    const uint64_t l3_base = 0x102000;
+    const uint64_t mair = 0xf0 | (0xff << 8);
+    const uint64_t tcr = 0x180803f20ull | (1ULL << 37);
+    uint64_t l1[512] = { 0 };
+    uint64_t l2[512] = { 0 };
+    uint64_t l3[512];
+    size_t i;
+
+    for (i = 0; i < 512; i++) {
+        unsigned attridx = (normal_va != UINT64_MAX &&
+                            i == ((normal_va >> 12) & 0x1ff)) ? 1 : 0;
+
+        l3[i] = test_arm64_mte_page_desc(i << 12, attridx);
+    }
+
+    l1[0] = l2_base | 3;
+    l2[0] = l3_base | 3;
+
+    OK(uc_mem_map(uc, l1_base, 0x3000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, l1_base, l1, sizeof(l1)));
+    OK(uc_mem_write(uc, l2_base, l2, sizeof(l2)));
+    OK(uc_mem_write(uc, l3_base, l3, sizeof(l3)));
+
+    test_arm64_pauth_cp_reg_write(uc, MAIR_EL1, mair);
+    test_arm64_pauth_cp_reg_write(uc, TCR_EL1, tcr);
+    test_arm64_pauth_cp_reg_write(uc, TTBR0_EL1, l1_base);
+}
+
+static void test_arm64_mte_enable_checks_with_normal_page(uc_engine *uc,
+                                                          uint64_t tcf,
+                                                          uint64_t normal_va)
+{
+    const uint32_t SCTLR_EL1[5] = { 3, 0, 1, 0, 0 };
+    const uint32_t HCR_EL2[5] = { 3, 4, 1, 1, 0 };
+    const uint32_t SCR_EL3[5] = { 3, 6, 1, 1, 0 };
+
+    test_arm64_mte_enable_identity_map(uc, normal_va);
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, SCR_EL3, 0,
+                                              1ULL | (1ULL << 10) |
+                                              (1ULL << 26)));
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, HCR_EL2, 0, 1ULL << 56));
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, SCTLR_EL1, 0,
+                                              1ULL | (1ULL << 2) |
+                                              (1ULL << 12) |
+                                              (1ULL << 43) | tcf));
+}
+
+static void test_arm64_mte_enable_checks(uc_engine *uc, uint64_t tcf)
+{
+    test_arm64_mte_enable_checks_with_normal_page(uc, tcf, UINT64_MAX);
+}
+
+static void test_arm64_mte_store_tag_at(uc_engine *uc, uint64_t ptr,
+                                        uint64_t tag)
+{
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &ptr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &tag));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+}
+
+static uint64_t test_arm64_mte_load_tag_at(uc_engine *uc, uint64_t ptr,
+                                           uint64_t value)
+{
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &ptr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &value));
+    OK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &value));
+    return value;
+}
+
+static void test_arm64_mte_checked_scalar_access(void)
+{
+    uc_engine *uc;
+    const char sync_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x83\x00\x40\xf9" /* ldr x3,[x4] */
+        "\xc5\x00\x40\xf9"; /* ldr x5,[x6] */
+    const char async_store_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xc3\x00\x00\xf9"; /* str x3,[x6] */
+    const char tco_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xc3\x00\x40\xf9"; /* ldr x3,[x6] */
+    const uint32_t TCO[5] = { 3, 3, 4, 2, 7 };
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint64_t pstate_tco = 1ULL << 25;
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0x0c00000000040000ull;
+    uint64_t x5 = 0;
+    uint64_t x6 = 0x0d00000000040000ull;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t stored = 0xaabbccddeeff0011ull;
+    uint64_t mem;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, sync_code,
+                    sizeof(sync_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(sync_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, async_store_code,
+                    sizeof(async_store_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    x3 = stored;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start,
+                    code_start + sizeof(async_store_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    TEST_CHECK(mem == stored);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, tco_code,
+                    sizeof(tco_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_pauth_cp_reg_write(uc, TCO, pstate_tco);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    x3 = 0;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(tco_code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_tco_msr_imm(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9"  /* stg x2,[x1] */
+        "\x9f\x41\x03\xd5"  /* msr tco,#1 */
+        "\xc3\x00\x40\xf9"  /* ldr x3,[x6] */
+        "\x9f\x40\x03\xd5"  /* msr tco,#0 */
+        "\xc5\x00\x40\xf9"; /* ldr x5,[x6] */
+    const uint32_t TCO[5] = { 3, 3, 4, 2, 7 };
+    const uint64_t pstate_tco = 1ULL << 25;
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x5 = 0;
+    uint64_t x6 = 0x0d00000000040000ull;
+    uint64_t data = 0x1122334455667788ull;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(x5 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TCO) == 0);
+
+    test_arm64_pauth_cp_reg_write(uc, TCO, pstate_tco);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TCO) == pstate_tco);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code + 4, 4,
+                    UC_CPU_ARM64_A72);
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + 4, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_lse_atomic_asym_sync_no_side_effect(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x23\x00\x22\xf8"; /* ldadd x2,x3,[x1] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t mem = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x1 = 0x0d00000000040000ull;
+    x2 = 0x0102030405060708ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(mem == data);
+    TEST_CHECK(x3 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_ldapr_sync_tag_check(void)
+{
+    uc_engine *uc;
+    const char non_sp_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x23\xc0\xbf\xf8"; /* ldapr x3,[x1] */
+    const char sp_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe3\xc3\xbf\xf8"; /* ldapr x3,[sp] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0xaabbccddeeff0011ull;
+    uint64_t sp = 0x0d00000000040000ull;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t mem = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, non_sp_code,
+                    sizeof(non_sp_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x1 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(non_sp_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(mem == data);
+    TEST_CHECK(x3 == 0xaabbccddeeff0011ull);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, sp_code,
+                    sizeof(sp_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    x3 = 0;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start + 4,
+                    code_start + sizeof(sp_code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_lse_cas_asym_async_side_effect(void)
+{
+    uc_engine *uc;
+    const char cas_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x28\xfd\xea\xc8"; /* casal x10,x8,[x9] */
+    const char casp_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x82\xfc\x60\x48"; /* caspal x0,x1,x2,x3,[x4] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x0;
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3;
+    uint64_t x4;
+    uint64_t x8;
+    uint64_t x9;
+    uint64_t x10;
+    uint64_t mem[2];
+    uint64_t old_pair[2] = {
+        0x1122334455667788ull,
+        0x8877665544332211ull,
+    };
+    uint64_t new_pair[2] = {
+        0x0102030405060708ull,
+        0x9080706050403020ull,
+    };
+
+    mem[0] = 0;
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, cas_code,
+                    sizeof(cas_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &mem[0], sizeof(mem[0])));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x8 = 0xaabbccddeeff0011ull;
+    x9 = 0x0d00000000040000ull;
+    x10 = 0;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X10, &x10));
+    OK(uc_emu_start(uc, code_start + 4,
+                    code_start + sizeof(cas_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40000, &mem[0], sizeof(mem[0])));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X10, &x10));
+    TEST_CHECK(mem[0] == x8);
+    TEST_CHECK(x10 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, casp_code,
+                    sizeof(casp_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, old_pair, sizeof(old_pair)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x0 = old_pair[0];
+    x1 = old_pair[1];
+    x2 = new_pair[0];
+    x3 = new_pair[1];
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start + 4,
+                    code_start + sizeof(casp_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X1, &x1));
+    TEST_CHECK(mem[0] == new_pair[0]);
+    TEST_CHECK(mem[1] == new_pair[1]);
+    TEST_CHECK(x0 == old_pair[0]);
+    TEST_CHECK(x1 == old_pair[1]);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_exclusive_asym_access(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xc5\x7c\x5f\xc8" /* ldxr x5,[x6] */
+        "\xc8\x7c\x07\xc8"; /* stxr w7,x8,[x6] */
+    const char pair_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x25\x19\x7f\xc8" /* ldxp x5,x6,[x9] */
+        "\x28\x29\x27\xc8"; /* stxp w7,x8,x10,[x9] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x5 = 0;
+    uint64_t x6;
+    uint64_t x7 = 0xff;
+    uint64_t x8 = 0xaabbccddeeff0011ull;
+    uint64_t x9;
+    uint64_t x10;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t mem = 0;
+    uint64_t pair_mem[2];
+    uint64_t pair_old[2] = {
+        0x1122334455667788ull,
+        0x8877665544332211ull,
+    };
+    uint64_t pair_new[2] = {
+        0x0102030405060708ull,
+        0x9080706050403020ull,
+    };
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x6 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    TEST_CHECK(mem == data);
+    TEST_CHECK(x5 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x6 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    TEST_CHECK(x5 == data);
+
+    x6 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start + 8,
+                    code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(mem == x8);
+    TEST_CHECK(x7 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, pair_code,
+                    sizeof(pair_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, pair_old, sizeof(pair_old)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x5 = 0;
+    x6 = 0;
+    x9 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, pair_mem, sizeof(pair_mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(pair_mem[0] == pair_old[0]);
+    TEST_CHECK(pair_mem[1] == pair_old[1]);
+    TEST_CHECK(x5 == 0);
+    TEST_CHECK(x6 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, pair_code,
+                    sizeof(pair_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, pair_old, sizeof(pair_old)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x9 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(x5 == pair_old[0]);
+    TEST_CHECK(x6 == pair_old[1]);
+
+    x7 = 0xff;
+    x8 = pair_new[0];
+    x9 = 0x0d00000000040000ull;
+    x10 = pair_new[1];
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X10, &x10));
+    OK(uc_emu_start(uc, code_start + 8,
+                    code_start + sizeof(pair_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40000, pair_mem, sizeof(pair_mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(pair_mem[0] == pair_new[0]);
+    TEST_CHECK(pair_mem[1] == pair_new[1]);
+    TEST_CHECK(x7 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_sp_addressing_tagchecked(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe3\x03\x40\xf9" /* ldr x3,[sp] */
+        "\xe4\x6b\x60\xf8" /* ldr x4,[sp,x0] */
+        "\xe5\x03\x00\xf9" /* str x5,[sp] */
+        "\xe5\x6b\x20\xf8"; /* str x5,[sp,x0] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x0 = 0;
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0;
+    uint64_t x5;
+    uint64_t sp = 0x0d00000000040000ull;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t stored = 0xaabbccddeeff0011ull;
+    uint64_t mem = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    TEST_CHECK(uc_emu_start(uc, code_start + 8, code_start + 12, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(x4 == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    x5 = stored;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start + 12, code_start + 16, 0, 1));
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    TEST_CHECK(mem == stored);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    x5 = stored ^ 0xffffffffffffffffull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_emu_start(uc, code_start + 16,
+                    code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    TEST_CHECK(mem == x5);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_sp_writeback_tagchecked(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe3\x0f\x41\xf8" /* ldr x3,[sp,#0x10]! */
+        "\xe5\x07\x01\xf8"; /* str x5,[sp],#0x10 */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x5 = 0xaabbccddeeff0011ull;
+    uint64_t sp;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t mem = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40010, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    sp = 0x0d0000000003fff0ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(x3 == 0);
+    TEST_CHECK(sp == 0x0d0000000003fff0ull);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    sp = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start + 8,
+                    code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(mem == x5);
+    TEST_CHECK(sp == 0x0d00000000040010ull);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_pair_sp_tagchecked(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe3\x13\x40\xa9" /* ldp x3,x4,[sp] */
+        "\xe5\x1b\x81\xa8"; /* stp x5,x6,[sp],#0x10 */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0;
+    uint64_t x5 = 0xaabbccddeeff0011ull;
+    uint64_t x6 = 0x8877665544332211ull;
+    uint64_t sp = 0x0d00000000040000ull;
+    uint64_t pair[2] = {
+        0x1122334455667788ull,
+        0x0102030405060708ull,
+    };
+    uint64_t mem[2];
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, pair, sizeof(pair)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(x3 == pair[0]);
+    TEST_CHECK(x4 == pair[1]);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start + 8,
+                    code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_reg_read(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(mem[0] == x5);
+    TEST_CHECK(mem[1] == x6);
+    TEST_CHECK(sp == 0x0d00000000040010ull);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_pac_load_sp_tagchecked(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe3\x07\x20\xf8" /* ldraa x3,[sp] */
+        "\xe4\x1f\x20\xf8" /* ldraa x4,[sp,#8]! */
+        "\xe3\x07\xa0\xf8" /* ldrab x3,[sp] */
+        "\xe4\x1f\xa0\xf8"; /* ldrab x4,[sp,#8]! */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0;
+    uint64_t sp;
+    uint64_t data = 0x1122334455667788ull;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    sp = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start + 4, code_start + 8, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    x3 = 0;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    sp = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start + 12, code_start + 16, 0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40008, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    sp = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(uc_emu_start(uc, code_start + 8,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_read(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(x4 == 0);
+    TEST_CHECK(sp == 0x0d00000000040000ull);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40008, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    x4 = 0;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+
+    sp = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(uc_emu_start(uc, code_start + 16,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_read(uc, UC_ARM64_REG_SP, &sp));
+    TEST_CHECK(x4 == 0);
+    TEST_CHECK(sp == 0x0d00000000040000ull);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_tcma0_tag_zero_unchecked(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x83\x00\x40\xf9"; /* ldr x3,[x4] */
+    const uint32_t TCR_EL1[5] = { 3, 0, 2, 0, 2 };
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint64_t tcma0 = 1ULL << 57;
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0x40000;
+    uint64_t data = 0x1122334455667788ull;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, TCR_EL1, 0, tcma0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    x3 = 0;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_ldapur_stlur_unchecked(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg    x2,[x1] */
+        "\x83\x00\x40\xd9" /* ldapur x3,[x4] */
+        "\x85\x00\x00\xd9"; /* stlur  x5,[x4] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0x0d00000000040000ull;
+    uint64_t x5 = 0x8877665544332211ull;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t written = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_mem_read(uc, 0x40000, &written, sizeof(written)));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(written == x5);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_ldapur_stlur_variants_unchecked(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg      x2,[x1] */
+        "\x86\x00\x40\x19" /* ldapurb  w6,[x4] */
+        "\x87\x20\x40\x59" /* ldapurh  w7,[x4,#2] */
+        "\x88\x40\x80\x19" /* ldapursb x8,[x4,#4] */
+        "\x89\x50\xc0\x19" /* ldapursb w9,[x4,#5] */
+        "\x8a\x60\x80\x59" /* ldapursh x10,[x4,#6] */
+        "\x8b\x80\xc0\x59" /* ldapursh w11,[x4,#8] */
+        "\x8c\xc0\x80\x99" /* ldapursw x12,[x4,#12] */
+        "\x8d\xa0\x00\x19" /* stlurb   w13,[x4,#10] */
+        "\x8e\xc0\x00\x59"; /* stlurh   w14,[x4,#12] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint8_t data[16] = {
+        0x7e, 0x11, 0x34, 0x12, 0x80, 0x81, 0x00, 0x80,
+        0x34, 0x80, 0xaa, 0xbb, 0x98, 0xba, 0xdc, 0xfe,
+    };
+    uint8_t written[16];
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x4 = 0x0d00000000040000ull;
+    uint64_t x6 = 0;
+    uint64_t x7 = 0;
+    uint64_t x8 = 0;
+    uint64_t x9 = 0;
+    uint64_t x10 = 0;
+    uint64_t x11 = 0;
+    uint64_t x12 = 0;
+    uint64_t x13 = 0x5a;
+    uint64_t x14 = 0xbeef;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X13, &x13));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X14, &x14));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X10, &x10));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X11, &x11));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_mem_read(uc, 0x40000, written, sizeof(written)));
+    TEST_CHECK(x6 == 0x7e);
+    TEST_CHECK(x7 == 0x1234);
+    TEST_CHECK(x8 == 0xffffffffffffff80ull);
+    TEST_CHECK(x9 == 0x00000000ffffff81ull);
+    TEST_CHECK(x10 == 0xffffffffffff8000ull);
+    TEST_CHECK(x11 == 0x00000000ffff8034ull);
+    TEST_CHECK(x12 == 0xfffffffffedcba98ull);
+    TEST_CHECK(written[10] == 0x5a);
+    TEST_CHECK(written[12] == 0xef);
+    TEST_CHECK(written[13] == 0xbe);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_unpriv_sp_no_tag_check(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg  x2,[x1] */
+        "\xe3\x0b\x40\xf8"; /* ldtr x3,[sp] */
+    const uint32_t TFSRE0_EL1[5] = { 3, 0, 5, 6, 1 };
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t sp = 0x0d00000000040000ull;
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t desc;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_checks(uc, (1ULL << 38) | (1ULL << 42));
+    desc = test_arm64_mte_page_desc(0x40000, 0) | (1ULL << 4);
+    test_arm64_mte_set_page_desc(uc, 0x40000, desc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_SP, &sp));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == data);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSRE0_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_unpriv_async_tag_check(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg  x2,[x1] */
+        "\x83\x08\x40\xf8"; /* ldtr x3,[x4] */
+    const uint32_t TFSRE0_EL1[5] = { 3, 0, 5, 6, 1 };
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint32_t pstate_el1h = 5;
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0x0d00000000040000ull;
+    uint64_t tfsre0;
+    uint64_t tfsr;
+    uc_err err;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    test_arm64_mte_enable_checks(uc, (2ULL << 38) | (1ULL << 42));
+    OK(uc_reg_write(uc, UC_ARM64_REG_PSTATE, &pstate_el1h));
+    OK(uc_ctl_flush_tb(uc));
+    OK(uc_ctl_flush_tlb(uc));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+    err = uc_emu_start(uc, code_start + 4, code_start + sizeof(code) - 1,
+                       0, 1);
+    tfsre0 = test_arm64_pauth_cp_reg_read(uc, TFSRE0_EL1);
+    tfsr = test_arm64_pauth_cp_reg_read(uc, TFSR_EL1);
+    TEST_CHECK_(err == UC_ERR_OK && tfsre0 == 1 && tfsr == 0,
+                "err=%u tfsre0=0x%llx tfsr=0x%llx", err,
+                (unsigned long long)tfsre0, (unsigned long long)tfsr);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_page_attrs(void)
+{
+    uc_engine *uc;
+    const char tag_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x23\x00\x60\xd9"; /* ldg x3,[x1] */
+    const char check_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x85\x00\x40\xf9"; /* ldr x5,[x4] */
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint64_t tagged_page = 0x40000;
+    uint64_t normal_page = 0x41000;
+    uint64_t tag = 0x0c00000000000000ull;
+    uint64_t tagged_value = 0x1122334455667788ull;
+    uint64_t normal_value = 0x8877665544332211ull;
+    uint64_t x1, x2, x3, x4, x5;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, tag_code,
+                    sizeof(tag_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, tagged_page, 0x2000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, tagged_page, &tagged_value, sizeof(tagged_value)));
+    OK(uc_mem_write(uc, normal_page, &normal_value, sizeof(normal_value)));
+    test_arm64_mte_enable_checks_with_normal_page(uc, 1ULL << 40,
+                                                  normal_page);
+
+    x1 = tagged_page;
+    x2 = tag;
+    x3 = 0x5000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(tag_code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x0c00000000005000ull);
+
+    x1 = normal_page;
+    x2 = 0x0d00000000000000ull;
+    x3 = 0x6000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(tag_code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x6000);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, check_code,
+                    sizeof(check_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, tagged_page, 0x2000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, tagged_page, &tagged_value, sizeof(tagged_value)));
+    test_arm64_mte_enable_checks_with_normal_page(uc, 1ULL << 40,
+                                                  normal_page);
+    x1 = tagged_page;
+    x2 = tag;
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(check_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, check_code,
+                    sizeof(check_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, tagged_page, 0x2000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, normal_page, &normal_value, sizeof(normal_value)));
+    test_arm64_mte_enable_checks_with_normal_page(uc, 1ULL << 40,
+                                                  normal_page);
+    x1 = normal_page;
+    x2 = tag;
+    x4 = 0x0d00000000041000ull;
+    x5 = 0;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(check_code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    TEST_CHECK(x5 == normal_value);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_enable_stage2_identity(uc_engine *uc)
+{
+    const uint32_t VTCR_EL2[5] = { 3, 4, 2, 1, 2 };
+    const uint32_t VTTBR_EL2[5] = { 3, 4, 2, 1, 0 };
+    const uint64_t table_base = 0x110000;
+    const uint64_t vtcr_t0sz_1gb = 34;
+    uint64_t l2[512] = { 0 };
+
+    l2[0] = 0x4fd;
+
+    OK(uc_mem_map(uc, table_base, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, table_base, l2, sizeof(l2)));
+    test_arm64_pauth_cp_reg_write(uc, VTCR_EL2, vtcr_t0sz_1gb);
+    test_arm64_pauth_cp_reg_write(uc, VTTBR_EL2, table_base);
+}
+
+static void test_arm64_mte_enable_dct(uc_engine *uc, bool dct)
+{
+    const uint32_t SCTLR_EL1[5] = { 3, 0, 1, 0, 0 };
+    const uint32_t HCR_EL2[5] = { 3, 4, 1, 1, 0 };
+    const uint32_t SCR_EL3[5] = { 3, 6, 1, 1, 0 };
+    uint64_t hcr = (1ULL << 56) | (1ULL << 31) | (1ULL << 12);
+
+    if (dct) {
+        hcr |= 1ULL << 57;
+    }
+
+    test_arm64_mte_enable_stage2_identity(uc);
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, SCR_EL3, 0,
+                                              1ULL | (1ULL << 10) |
+                                              (1ULL << 26)));
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, HCR_EL2, 0, hcr));
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, SCTLR_EL1, 0,
+                                              (1ULL << 2) |
+                                              (1ULL << 12) |
+                                              (1ULL << 43)));
+}
+
+static void test_arm64_mte_hcr_dct(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x23\x00\x60\xd9"; /* ldg x3,[x1] */
+    uint64_t data = 0x1122334455667788ull;
+    uint64_t x1;
+    uint64_t x2;
+    uint64_t x3;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_dct(uc, true);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    x3 = 0x5000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x0c00000000005000ull);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, &data, sizeof(data)));
+    test_arm64_mte_enable_dct(uc, false);
+    x1 = 0x40000;
+    x2 = 0x0c00000000000000ull;
+    x3 = 0x5000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x5000);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_cross_page_fault_priority(void)
+{
+    uc_engine *uc;
+    const char st2g_code[] =
+        "\x22\x08\xa0\xd9" /* st2g x2,[x1] */
+        "\x23\x00\x60\xd9"; /* ldg  x3,[x1] */
+    const char checked_load_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x83\x00\x40\xf9"; /* ldr x3,[x4] */
+    uint64_t x1;
+    uint64_t x2;
+    uint64_t x3;
+    uint64_t x4;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, st2g_code,
+                    sizeof(st2g_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    test_arm64_mte_enable_checks(uc, 0);
+    x1 = 0x40ff0;
+    x2 = 0x0c00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + 4, 0, 1) ==
+               UC_ERR_WRITE_UNMAPPED);
+    OK(uc_mem_map(uc, 0x41000, 0x1000, UC_PROT_ALL));
+    x3 = 0x5000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(st2g_code) - 1,
+                    0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x5000);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, st2g_code,
+                    sizeof(st2g_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x2000, UC_PROT_ALL));
+    OK(uc_mem_protect(uc, 0x41000, 0x1000, UC_PROT_READ));
+    test_arm64_mte_enable_checks(uc, 0);
+    x1 = 0x40ff0;
+    x2 = 0x0d00000000000000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + 4, 0, 1) ==
+               UC_ERR_WRITE_PROT);
+    OK(uc_mem_protect(uc, 0x41000, 0x1000, UC_PROT_ALL));
+    x3 = 0x6000;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(st2g_code) - 1,
+                    0, 1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(x3 == 0x6000);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, checked_load_code,
+                    sizeof(checked_load_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    x1 = 0x40ff0;
+    x2 = 0x0c00000000000000ull;
+    x3 = 0;
+    x4 = 0x0d00000000040ffcull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 1));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(checked_load_code) - 1,
+                            0, 1) == UC_ERR_READ_UNMAPPED);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_ata_disabled_tag_op_probe(void)
+{
+    uc_engine *uc;
+    const char ldg_code[] = "\x23\x00\x60\xd9"; /* ldg x3,[x1] */
+    const char ldgm_code[] = "\x23\x00\xe0\xd9"; /* ldgm x3,[x1] */
+    const char stgm_code[] = "\x22\x00\xa0\xd9"; /* stgm x2,[x1] */
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0123456789abcdefull;
+    uint64_t x3 = 0x0c00000000005000ull;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldg_code,
+                    sizeof(ldg_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(ldg_code) - 1,
+                            0, 0) == UC_ERR_READ_UNMAPPED);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldgm_code,
+                    sizeof(ldgm_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(ldgm_code) - 1,
+                            0, 0) == UC_ERR_READ_UNMAPPED);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, stgm_code,
+                    sizeof(stgm_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(stgm_code) - 1,
+                            0, 0) == UC_ERR_WRITE_UNMAPPED);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, stgm_code,
+                    sizeof(stgm_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_READ));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(stgm_code) - 1,
+                            0, 0) == UC_ERR_WRITE_PROT);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_enable_sve_vq(uc_engine *uc, uint64_t zcr_len)
+{
+    const uint32_t CPACR_EL1[5] = { 3, 0, 1, 0, 2 };
+    const uint32_t CPTR_EL3[5] = { 3, 6, 1, 1, 2 };
+    const uint32_t ZCR_EL1[5] = { 3, 0, 1, 2, 0 };
+    const uint32_t ZCR_EL2[5] = { 3, 4, 1, 2, 0 };
+    const uint32_t ZCR_EL3[5] = { 3, 6, 1, 2, 0 };
+
+    test_arm64_pauth_cp_reg_write(uc, CPACR_EL1,
+                                  (3ULL << 16) | (3ULL << 20));
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(uc, CPTR_EL3, 0, 1ULL << 8));
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL1, zcr_len);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL2, zcr_len);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL3, zcr_len);
+}
+
+static void test_arm64_mte_enable_sve(uc_engine *uc)
+{
+    test_arm64_mte_enable_sve_vq(uc, 0);
+}
+
+static void test_arm64_sve2_non_temporal_gather_scatter(void)
+{
+    uc_engine *uc;
+    const char load_code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xe2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x7] */
+        "\x41\xa0\x04\x84" /* ldnt1b { z1.s },p0/z,[z2.s,x4] */
+        "\xc1\xe0\x40\xe5"; /* st1w { z1.s },p0,[x6] */
+    const char store_code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc1\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x6] */
+        "\xe2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x7] */
+        "\x41\x20\x44\xe4"; /* stnt1b { z1.s },p0,[z2.s,x4] */
+    uint64_t x4 = 0x40000;
+    uint64_t x6 = 0x40100;
+    uint64_t x7 = 0x40200;
+    uint8_t mem[16];
+    uint8_t expected[16];
+    uint32_t offsets[4];
+    uint32_t words[4];
+    int i;
+
+    for (i = 0; i < (int)sizeof(expected); i++) {
+        expected[i] = (uint8_t)(0x20 + i);
+    }
+    for (i = 0; i < 4; i++) {
+        offsets[i] = (uint32_t)i;
+        words[i] = 0;
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, load_code,
+                    sizeof(load_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(load_code) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(words[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+
+    memset(expected, 0xa5, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        offsets[i] = (uint32_t)i;
+        words[i] = (uint32_t)(0x90 + i);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, store_code,
+                    sizeof(store_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(store_code) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(mem[i] == (uint8_t)words[i]);
+    }
+    for (i = 4; i < (int)sizeof(mem); i++) {
+        TEST_CHECK(mem[i] == 0xa5);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_bitwise_ternary(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xe1\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x7] */
+        "\x02\xa1\x40\xa5" /* ld1w { z2.s },p0/z,[x8] */
+        "\xca\xa0\x40\xa5" /* ld1w { z10.s },p0/z,[x6] */
+        "\xcb\xa0\x40\xa5" /* ld1w { z11.s },p0/z,[x6] */
+        "\xcc\xa0\x40\xa5" /* ld1w { z12.s },p0/z,[x6] */
+        "\xcd\xa0\x40\xa5" /* ld1w { z13.s },p0/z,[x6] */
+        "\xce\xa0\x40\xa5" /* ld1w { z14.s },p0/z,[x6] */
+        "\xcf\xa0\x40\xa5" /* ld1w { z15.s },p0/z,[x6] */
+        "\x4a\x38\x21\x04" /* eor3 z10.d,z10.d,z1.d,z2.d */
+        "\x4b\x3c\x21\x04" /* bsl z11.d,z11.d,z1.d,z2.d */
+        "\x4c\x38\x61\x04" /* bcax z12.d,z12.d,z1.d,z2.d */
+        "\x4d\x3c\x61\x04" /* bsl1n z13.d,z13.d,z1.d,z2.d */
+        "\x4e\x3c\xa1\x04" /* bsl2n z14.d,z14.d,z1.d,z2.d */
+        "\x4f\x3c\xe1\x04" /* nbsl z15.d,z15.d,z1.d,z2.d */
+        "\x2a\xe1\x40\xe5" /* st1w { z10.s },p0,[x9] */
+        "\x4b\xe1\x40\xe5" /* st1w { z11.s },p0,[x10] */
+        "\x6c\xe1\x40\xe5" /* st1w { z12.s },p0,[x11] */
+        "\x8d\xe1\x40\xe5" /* st1w { z13.s },p0,[x12] */
+        "\xae\xe1\x40\xe5" /* st1w { z14.s },p0,[x13] */
+        "\xcf\xe1\x40\xe5"; /* st1w { z15.s },p0,[x14] */
+    uint32_t n[4] = {
+        0x01234567, 0x89abcdef, 0x10203040, 0xfedcba98
+    };
+    uint32_t m[4] = {
+        0xf0f0aa55, 0x00ff00ff, 0x87654321, 0x13579bdf
+    };
+    uint32_t k[4] = {
+        0xff00ff00, 0x0f0f0f0f, 0xaaaaaaaa, 0x55555555
+    };
+    uint32_t expected[6][4];
+    uint32_t got[4];
+    uint64_t x6 = 0x40000;
+    uint64_t x7 = 0x40100;
+    uint64_t x8 = 0x40200;
+    uint64_t out[6] = {
+        0x40300, 0x40320, 0x40340, 0x40360, 0x40380, 0x403a0
+    };
+    int i, j;
+
+    for (i = 0; i < 4; i++) {
+        expected[0][i] = n[i] ^ m[i] ^ k[i];
+        expected[1][i] = (n[i] & k[i]) | (m[i] & ~k[i]);
+        expected[2][i] = n[i] ^ (m[i] & ~k[i]);
+        expected[3][i] = (~n[i] & k[i]) | (m[i] & ~k[i]);
+        expected[4][i] = (n[i] & k[i]) | (~m[i] & ~k[i]);
+        expected[5][i] = ~((n[i] & k[i]) | (m[i] & ~k[i]));
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, n, sizeof(n)));
+    OK(uc_mem_write(uc, 0x40100, m, sizeof(m)));
+    OK(uc_mem_write(uc, 0x40200, k, sizeof(k)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &out[0]));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X10, &out[1]));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X11, &out[2]));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &out[3]));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X13, &out[4]));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X14, &out[5]));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    for (i = 0; i < 6; i++) {
+        OK(uc_mem_read(uc, out[i], got, sizeof(got)));
+        for (j = 0; j < 4; j++) {
+            TEST_CHECK(got[j] == expected[i][j]);
+        }
+    }
+
+    OK(uc_close(uc));
+}
+
+static uint8_t test_arm64_ror8(uint8_t value, unsigned int shift)
+{
+    return (uint8_t)((value >> shift) | (value << (8 - shift)));
+}
+
+static uint16_t test_arm64_ror16(uint16_t value, unsigned int shift)
+{
+    return (uint16_t)((value >> shift) | (value << (16 - shift)));
+}
+
+static uint32_t test_arm64_ror32(uint32_t value, unsigned int shift)
+{
+    return (value >> shift) | (value << (32 - shift));
+}
+
+static uint64_t test_arm64_ror64(uint64_t value, unsigned int shift)
+{
+    return (value >> shift) | (value << (64 - shift));
+}
+
+static uint16_t test_arm64_pmull8(uint8_t op1, uint8_t op2)
+{
+    uint16_t result = 0;
+    int i;
+
+    for (i = 0; i < 8; i++) {
+        if ((op1 >> i) & 1) {
+            result ^= (uint16_t)op2 << i;
+        }
+    }
+    return result;
+}
+
+static uint64_t test_arm64_pmull32(uint32_t op1, uint32_t op2)
+{
+    uint64_t result = 0;
+    int i;
+
+    for (i = 0; i < 32; i++) {
+        if ((op1 >> i) & 1) {
+            result ^= (uint64_t)op2 << i;
+        }
+    }
+    return result;
+}
+
+static void test_arm64_pmull64(uint64_t op1, uint64_t op2, uint64_t *lo,
+                               uint64_t *hi)
+{
+    uint64_t result_lo = 0;
+    uint64_t result_hi = 0;
+    int i;
+
+    for (i = 0; i < 64; i++) {
+        if ((op1 >> i) & 1) {
+            result_lo ^= op2 << i;
+            if (i != 0) {
+                result_hi ^= op2 >> (64 - i);
+            }
+        }
+    }
+    *lo = result_lo;
+    *hi = result_hi;
+}
+
+static uint64_t test_arm64_bitextract(uint64_t data, uint64_t mask, int n)
+{
+    uint64_t result = 0;
+    int db, rb = 0;
+
+    for (db = 0; db < n; db++) {
+        if ((mask >> db) & 1) {
+            result |= ((data >> db) & 1) << rb;
+            rb++;
+        }
+    }
+    return result;
+}
+
+static uint64_t test_arm64_bitdeposit(uint64_t data, uint64_t mask, int n)
+{
+    uint64_t result = 0;
+    int rb, db = 0;
+
+    for (rb = 0; rb < n; rb++) {
+        if ((mask >> rb) & 1) {
+            result |= ((data >> db) & 1) << rb;
+            db++;
+        }
+    }
+    return result;
+}
+
+static uint64_t test_arm64_bitgroup(uint64_t data, uint64_t mask, int n)
+{
+    uint64_t masked = 0, unmasked = 0;
+    int db, rbm = 0, rbu = 0;
+
+    for (db = 0; db < n; db++) {
+        uint64_t bit = (data >> db) & 1;
+
+        if ((mask >> db) & 1) {
+            masked |= bit << rbm++;
+        } else {
+            unmasked |= bit << rbu++;
+        }
+    }
+    return rbm == 64 ? masked : masked | (unmasked << rbm);
+}
+
+static bool test_arm64_has_u8(const uint8_t *values, int count, uint8_t needle)
+{
+    int i;
+
+    for (i = 0; i < count; i++) {
+        if (values[i] == needle) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool test_arm64_has_u16(const uint16_t *values, int count,
+                               uint16_t needle)
+{
+    int i;
+
+    for (i = 0; i < count; i++) {
+        if (values[i] == needle) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static void test_arm64_sve2_xar(void)
+{
+    uc_engine *uc;
+    const char code_b[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x4] */
+        "\xa1\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x5] */
+        "\x20\x34\x2d\x04" /* xar z0.b,z0.b,z1.b,#3 */
+        "\xc0\xe0\x00\xe4" /* st1b { z0.b },p0,[x6] */
+        "\x82\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x4] */
+        "\xa3\xa0\x00\xa4" /* ld1b { z3.b },p0/z,[x5] */
+        "\x62\x34\x28\x04" /* xar z2.b,z2.b,z3.b,#8 */
+        "\xe2\xe0\x00\xe4"; /* st1b { z2.b },p0,[x7] */
+    const char code_h[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\xa0\xa4" /* ld1h { z0.h },p0/z,[x4] */
+        "\xa1\xa0\xa0\xa4" /* ld1h { z1.h },p0/z,[x5] */
+        "\x20\x34\x3b\x04" /* xar z0.h,z0.h,z1.h,#5 */
+        "\xc0\xe0\xa0\xe4"; /* st1h { z0.h },p0,[x6] */
+    const char code_s[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\x40\xa5" /* ld1w { z0.s },p0/z,[x4] */
+        "\xa1\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x5] */
+        "\x20\x34\x73\x04" /* xar z0.s,z0.s,z1.s,#13 */
+        "\xc0\xe0\x40\xe5"; /* st1w { z0.s },p0,[x6] */
+    const char code_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\xe0\xa5" /* ld1d { z0.d },p0/z,[x4] */
+        "\xa1\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x5] */
+        "\x20\x34\xe3\x04" /* xar z0.d,z0.d,z1.d,#29 */
+        "\xc0\xe0\xe0\xe5"; /* st1d { z0.d },p0,[x6] */
+    uint8_t n_b[16], m_b[16], exp_b[16], exp_b_xor[16], got_b[16];
+    uint16_t n_h[8], m_h[8], exp_h[8], got_h[8];
+    uint32_t n_s[4], m_s[4], exp_s[4], got_s[4];
+    uint64_t n_d[2], m_d[2], exp_d[2], got_d[2];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    uint64_t x7 = 0x40300;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        n_b[i] = (uint8_t)(0x11 + i * 7);
+        m_b[i] = (uint8_t)(0xc3 - i * 5);
+        exp_b[i] = test_arm64_ror8((uint8_t)(n_b[i] ^ m_b[i]), 3);
+        exp_b_xor[i] = n_b[i] ^ m_b[i];
+    }
+    for (i = 0; i < 8; i++) {
+        n_h[i] = (uint16_t)(0x1234 + i * 0x101);
+        m_h[i] = (uint16_t)(0xf0e1 - i * 0x111);
+        exp_h[i] = test_arm64_ror16((uint16_t)(n_h[i] ^ m_h[i]), 5);
+    }
+    for (i = 0; i < 4; i++) {
+        n_s[i] = 0x10203040u + (uint32_t)i * 0x11111111u;
+        m_s[i] = 0xfedcba98u - (uint32_t)i * 0x01020304u;
+        exp_s[i] = test_arm64_ror32(n_s[i] ^ m_s[i], 13);
+    }
+    n_d[0] = 0x0123456789abcdefull;
+    n_d[1] = 0xfedcba9876543210ull;
+    m_d[0] = 0x0f1e2d3c4b5a6978ull;
+    m_d[1] = 0x8877665544332211ull;
+    for (i = 0; i < 2; i++) {
+        exp_d[i] = test_arm64_ror64(n_d[i] ^ m_d[i], 29);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_b,
+                    sizeof(code_b) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_b, sizeof(n_b)));
+    OK(uc_mem_write(uc, x5, m_b, sizeof(m_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_b) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_b[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_b_xor[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_h,
+                    sizeof(code_h) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_h, sizeof(n_h)));
+    OK(uc_mem_write(uc, x5, m_h, sizeof(m_h)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_h) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_h[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_s,
+                    sizeof(code_s) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_s, sizeof(n_s)));
+    OK(uc_mem_write(uc, x5, m_s, sizeof(m_s)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_s) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == exp_s[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_d,
+                    sizeof(code_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_d, sizeof(n_d)));
+    OK(uc_mem_write(uc, x5, m_d, sizeof(m_d)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_d) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_d[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_pmull(void)
+{
+    uc_engine *uc;
+    const char code_h[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x20\x68\x42\x45" /* pmullb z0.h,z1.b,z2.b */
+        "\x23\x6c\x42\x45" /* pmullt z3.h,z1.b,z2.b */
+        "\xc0\xe0\xa0\xe4" /* st1h { z0.h },p0,[x6] */
+        "\xe3\xe0\xa0\xe4"; /* st1h { z3.h },p0,[x7] */
+    const char code_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x4] */
+        "\xa2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x5] */
+        "\x20\x68\xc2\x45" /* pmullb z0.d,z1.s,z2.s */
+        "\x23\x6c\xc2\x45" /* pmullt z3.d,z1.s,z2.s */
+        "\xc0\xe0\xe0\xe5" /* st1d { z0.d },p0,[x6] */
+        "\xe3\xe0\xe0\xe5"; /* st1d { z3.d },p0,[x7] */
+    const char code_q[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x4] */
+        "\xa2\xa0\xe0\xa5" /* ld1d { z2.d },p0/z,[x5] */
+        "\x20\x68\x02\x45" /* pmullb z0.q,z1.d,z2.d */
+        "\x23\x6c\x02\x45" /* pmullt z3.q,z1.d,z2.d */
+        "\xc0\xe0\xe0\xe5" /* st1d { z0.d },p0,[x6] */
+        "\xe3\xe0\xe0\xe5"; /* st1d { z3.d },p0,[x7] */
+    uint8_t n_b[16], m_b[16];
+    uint16_t exp_h_b[8], exp_h_t[8], got_h[8];
+    uint32_t n_s[4], m_s[4];
+    uint64_t exp_d_b[2], exp_d_t[2], got_d[2];
+    uint64_t n_d[2], m_d[2];
+    uint64_t exp_q_b[2], exp_q_t[2], got_q[2];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    uint64_t x7 = 0x40300;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        n_b[i] = (uint8_t)(0x21 + i * 9);
+        m_b[i] = (uint8_t)(0xf3 - i * 7);
+    }
+    for (i = 0; i < 8; i++) {
+        exp_h_b[i] = test_arm64_pmull8(n_b[2 * i], m_b[2 * i]);
+        exp_h_t[i] = test_arm64_pmull8(n_b[2 * i + 1], m_b[2 * i + 1]);
+    }
+    for (i = 0; i < 4; i++) {
+        n_s[i] = 0x13579bdfu + (uint32_t)i * 0x01010101u;
+        m_s[i] = 0xfdb97531u - (uint32_t)i * 0x02020202u;
+    }
+    for (i = 0; i < 2; i++) {
+        exp_d_b[i] = test_arm64_pmull32(n_s[2 * i], m_s[2 * i]);
+        exp_d_t[i] = test_arm64_pmull32(n_s[2 * i + 1],
+                                        m_s[2 * i + 1]);
+    }
+    n_d[0] = 0x0123456789abcdefull;
+    n_d[1] = 0xfedcba9876543210ull;
+    m_d[0] = 0x0f1e2d3c4b5a6978ull;
+    m_d[1] = 0x8877665544332211ull;
+    test_arm64_pmull64(n_d[0], m_d[0], &exp_q_b[0], &exp_q_b[1]);
+    test_arm64_pmull64(n_d[1], m_d[1], &exp_q_t[0], &exp_q_t[1]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_h,
+                    sizeof(code_h) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_b, sizeof(n_b)));
+    OK(uc_mem_write(uc, x5, m_b, sizeof(m_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_h) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_h_b[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_h_t[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_d,
+                    sizeof(code_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_s, sizeof(n_s)));
+    OK(uc_mem_write(uc, x5, m_s, sizeof(m_s)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_d) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_d_b[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_d_t[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_q,
+                    sizeof(code_q) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_d, sizeof(n_d)));
+    OK(uc_mem_write(uc, x5, m_d, sizeof(m_d)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_q) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_q, sizeof(got_q)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_q[i] == exp_q_b[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_q, sizeof(got_q)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_q[i] == exp_q_t[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_emit32(uint8_t *code, int offset, uint32_t insn)
+{
+    code[offset] = (uint8_t)insn;
+    code[offset + 1] = (uint8_t)(insn >> 8);
+    code[offset + 2] = (uint8_t)(insn >> 16);
+    code[offset + 3] = (uint8_t)(insn >> 24);
+}
+
+static void test_arm64_mte_simd_fp_single_access(void)
+{
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint64_t tag0 = 0x0c00000000000000ull;
+    const uint64_t tag1 = 0x0d00000000000000ull;
+    uint8_t load_code[8];
+    uint8_t store_code[8];
+    uint8_t mem[64];
+    uint8_t actual[64];
+    uint64_t q0_initial[2] = {
+        0x1111222233334444ull, 0x5555666677778888ull
+    };
+    uint64_t q0_store[2] = {
+        0x1021324354657687ull, 0x98a9bacbdcedfe0full
+    };
+    uint64_t q0[2];
+    uint64_t x4 = 0x0c00000000040008ull;
+    uc_engine *uc;
+    size_t i;
+
+    for (i = 0; i < sizeof(mem); i++) {
+        mem[i] = (uint8_t)(0x40 + i);
+    }
+    test_arm64_emit32(load_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(load_code, 4, 0x3dc00080); /* ldr q0,[x4] */
+    test_arm64_emit32(store_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(store_code, 4, 0x3d800080); /* str q0,[x4] */
+
+    memcpy(q0, q0_initial, sizeof(q0));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)load_code, sizeof(load_code),
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(load_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_Q0, q0));
+    TEST_CHECK(memcmp(q0, q0_initial, sizeof(q0)) == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)store_code, sizeof(store_code),
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q0, q0_store));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(store_code),
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40000, actual, sizeof(actual)));
+    TEST_CHECK(memcmp(actual, mem, 8) == 0);
+    TEST_CHECK(memcmp(actual + 8, q0_store, sizeof(q0_store)) == 0);
+    TEST_CHECK(memcmp(actual + 24, mem + 24, sizeof(actual) - 24) == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_advsimd_struct_range(void)
+{
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint64_t tag0 = 0x0c00000000000000ull;
+    const uint64_t tag1 = 0x0d00000000000000ull;
+    uint8_t load_code[8];
+    uint8_t store_code[8];
+    uint8_t mem[64];
+    uint8_t actual[64];
+    uint8_t expected[64];
+    uint64_t q0_initial[2] = {
+        0x0102030405060708ull, 0x1112131415161718ull
+    };
+    uint64_t q1_initial[2] = {
+        0x2122232425262728ull, 0x3132333435363738ull
+    };
+    uint64_t q0_store[2] = {
+        0x405162738495a6b7ull, 0xc8d9eafb0c1d2e3full
+    };
+    uint64_t q1_store[2] = {
+        0x1020304050607080ull, 0x90a0b0c0d0e0f000ull
+    };
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t x4 = 0x0c00000000040000ull;
+    uc_engine *uc;
+    size_t i;
+
+    for (i = 0; i < sizeof(mem); i++) {
+        mem[i] = (uint8_t)(0x80 + i);
+    }
+    test_arm64_emit32(load_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(load_code, 4, 0x4c40a080);
+    test_arm64_emit32(store_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(store_code, 4, 0x4c00a080);
+
+    memcpy(q0, q0_initial, sizeof(q0));
+    memcpy(q1, q1_initial, sizeof(q1));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)load_code, sizeof(load_code),
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(load_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_ARM64_REG_Q0, q0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_Q1, q1));
+    TEST_CHECK(memcmp(q0, q0_initial, sizeof(q0)) == 0);
+    TEST_CHECK(memcmp(q1, q1_initial, sizeof(q1)) == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+
+    memcpy(expected, mem, sizeof(expected));
+    memcpy(expected, q0_store, sizeof(q0_store));
+    memcpy(expected + sizeof(q0_store), q1_store, sizeof(q1_store));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)store_code, sizeof(store_code),
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q0, q0_store));
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q1, q1_store));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(store_code),
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40000, actual, sizeof(actual)));
+    TEST_CHECK(memcmp(actual, expected, sizeof(actual)) == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_i8mm_q_run(uint32_t insn, const uint8_t *initial,
+                                  const uint8_t *n, const uint8_t *m,
+                                  const uint8_t *expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint8_t *got = (const uint8_t *)q0;
+    size_t i;
+
+    test_arm64_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q2, q2));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_Q0, q0));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_i8mm_expect_exception(uint32_t insn,
+                                             uc_cpu_arm64 cpu)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+
+    test_arm64_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), cpu);
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_i8mm_advsimd(void)
+{
+    const uint8_t initial[16] = {
+        0x10, 0x00, 0x00, 0x00, 0xf0, 0xff, 0xff, 0xff,
+        0x80, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff,
+    };
+    const uint8_t n[16] = {
+        0x01, 0x7f, 0x80, 0xff, 0x10, 0xf0, 0x22, 0xdd,
+        0x40, 0xc0, 0x55, 0xaa, 0x7e, 0x82, 0x01, 0xff,
+    };
+    const uint8_t m[16] = {
+        0x02, 0xfe, 0x03, 0xfd, 0x80, 0x7f, 0x04, 0xfc,
+        0x11, 0xef, 0x66, 0x99, 0x08, 0xf8, 0x01, 0xff,
+    };
+    const uint8_t exp_usdot[16] = {
+        0x97, 0xfd, 0xff, 0xff, 0x14, 0x6c, 0x00, 0x00,
+        0x78, 0xd5, 0xff, 0xff, 0x61, 0xfe, 0xff, 0xff,
+    };
+    const uint8_t exp_sudot_idx[16] = {
+        0x19, 0x43, 0x00, 0x00, 0xb1, 0xea, 0xff, 0xff,
+        0x78, 0xb7, 0xff, 0xff, 0x08, 0x92, 0xff, 0xff,
+    };
+    const uint8_t exp_usdot_idx[16] = {
+        0x19, 0xc4, 0xff, 0xff, 0xb1, 0xa5, 0xff, 0xff,
+        0x78, 0xd5, 0xff, 0xff, 0x08, 0x99, 0xff, 0xff,
+    };
+    const uint8_t exp_smmla[16] = {
+        0xbb, 0xee, 0xff, 0xff, 0x3e, 0xc6, 0xff, 0xff,
+        0x07, 0x86, 0xff, 0xff, 0x59, 0x54, 0x00, 0x00,
+    };
+    const uint8_t exp_ummla[16] = {
+        0xbb, 0xd4, 0x02, 0x00, 0x3e, 0x07, 0x03, 0x00,
+        0x07, 0xe3, 0x02, 0x00, 0x59, 0xbe, 0x02, 0x00,
+    };
+    const uint8_t exp_usmmla[16] = {
+        0xbb, 0x69, 0x00, 0x00, 0x3e, 0xbc, 0xff, 0xff,
+        0x07, 0xfc, 0xff, 0xff, 0x59, 0xd3, 0xff, 0xff,
+    };
+
+    test_arm64_i8mm_q_run(0x4e829c20, initial, n, m, exp_usdot);
+    test_arm64_i8mm_q_run(0x4f02f820, initial, n, m, exp_sudot_idx);
+    test_arm64_i8mm_q_run(0x4f82f820, initial, n, m, exp_usdot_idx);
+    test_arm64_i8mm_q_run(0x4e82a420, initial, n, m, exp_smmla);
+    test_arm64_i8mm_q_run(0x6e82a420, initial, n, m, exp_ummla);
+    test_arm64_i8mm_q_run(0x4e82ac20, initial, n, m, exp_usmmla);
+
+    test_arm64_i8mm_expect_exception(0x4e829c20, UC_CPU_ARM64_A72);
+    test_arm64_i8mm_expect_exception(0x0e82a420, UC_CPU_ARM64_MAX);
+}
+
+static void test_arm64_bf16_scalar_convert(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t s0;
+    uint32_t s1 = 0x3fc00000u;
+
+    test_arm64_emit32(code, 0, 0x1e634020);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_S1, &s1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_S0, &s0));
+    TEST_CHECK(s0 == 0x00003fc0u);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_bf16_vector_convert_run(uint32_t insn,
+                                               const uint16_t *initial,
+                                               const uint32_t *source,
+                                               const uint16_t *expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    const uint16_t *got = (const uint16_t *)q0;
+    size_t i;
+
+    test_arm64_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    memcpy(q0, initial, 16);
+    memcpy(q1, source, 16);
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q1, q1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_Q0, q0));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn %08x lane %zu got %04x expected %04x",
+                    insn, i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_bf16_q_run(uint32_t insn, const uint32_t *initial,
+                                  const uint16_t *n, const uint16_t *m,
+                                  const uint32_t *expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t q0[2];
+    uint64_t q1[2];
+    uint64_t q2[2];
+    const uint32_t *got = (const uint32_t *)q0;
+    size_t i;
+
+    test_arm64_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    memcpy(q0, initial, 16);
+    memcpy(q1, n, 16);
+    memcpy(q2, m, 16);
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q0, q0));
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q1, q1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_Q2, q2));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_Q0, q0));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn %08x lane %zu got %08x expected %08x",
+                    insn, i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_bf16_advsimd(void)
+{
+    const uint16_t init_h[8] = {
+        0x1111, 0x2222, 0x3333, 0x4444,
+        0xaaaa, 0xbbbb, 0xcccc, 0xdddd,
+    };
+    const uint32_t fp32_source[4] = {
+        0x3f800000u, 0xc0000000u, 0x40400000u, 0x40800000u,
+    };
+    const uint16_t exp_bfcvtn[8] = {
+        0x3f80, 0xc000, 0x4040, 0x4080,
+        0x0000, 0x0000, 0x0000, 0x0000,
+    };
+    const uint16_t exp_bfcvtn2[8] = {
+        0x1111, 0x2222, 0x3333, 0x4444,
+        0x3f80, 0xc000, 0x4040, 0x4080,
+    };
+    const uint32_t init_s[4] = {
+        0x41200000u, 0x41a00000u, 0x41f00000u, 0x42200000u,
+    };
+    const uint16_t n_pair[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x3f80, 0x3f80, 0x4000, 0x4000,
+    };
+    const uint16_t m_pair[8] = {
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+        0x4000, 0x4040, 0x4080, 0x40a0,
+    };
+    const uint32_t exp_bfdot[4] = {
+        0x41d80000u, 0x42920000u, 0x420c0000u, 0x42680000u,
+    };
+    const uint32_t exp_bfdot_idx[4] = {
+        0x41d80000u, 0x426c0000u, 0x42240000u, 0x42780000u,
+    };
+    const uint16_t n_mmla[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+    };
+    const uint16_t m_mmla[8] = {
+        0x3f80, 0x3f80, 0x4000, 0x4000,
+        0x4040, 0x4040, 0x4080, 0x4080,
+    };
+    const uint32_t exp_bfmmla[4] = {
+        0x41d80000u, 0x42640000u, 0x428e0000u, 0x43050000u,
+    };
+    const uint16_t n_long[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+    };
+    const uint16_t m_long[8] = {
+        0x4000, 0x4040, 0x4080, 0x40a0,
+        0x40c0, 0x40e0, 0x4100, 0x4110,
+    };
+    const uint32_t exp_bfmlalb[4] = {
+        0x41400000u, 0x42000000u, 0x42700000u, 0x42c00000u,
+    };
+    const uint32_t exp_bfmlalt[4] = {
+        0x41800000u, 0x42200000u, 0x42900000u, 0x42e00000u,
+    };
+    const uint32_t exp_bfmlalb_idx[4] = {
+        0x41400000u, 0x41d00000u, 0x42200000u, 0x42580000u,
+    };
+    const uint32_t exp_bfmlalt_idx[4] = {
+        0x41600000u, 0x41e00000u, 0x42280000u, 0x42600000u,
+    };
+
+    test_arm64_bf16_scalar_convert();
+    test_arm64_bf16_vector_convert_run(0x0ea16820, init_h, fp32_source,
+                                       exp_bfcvtn);
+    test_arm64_bf16_vector_convert_run(0x4ea16820, init_h, fp32_source,
+                                       exp_bfcvtn2);
+    test_arm64_bf16_q_run(0x6e42fc20, init_s, n_pair, m_pair, exp_bfdot);
+    test_arm64_bf16_q_run(0x4f42f020, init_s, n_pair, m_pair,
+                          exp_bfdot_idx);
+    test_arm64_bf16_q_run(0x6e42ec20, init_s, n_mmla, m_mmla, exp_bfmmla);
+    test_arm64_bf16_q_run(0x2ec2fc20, init_s, n_long, m_long, exp_bfmlalb);
+    test_arm64_bf16_q_run(0x6ec2fc20, init_s, n_long, m_long, exp_bfmlalt);
+    test_arm64_bf16_q_run(0x0fc2f020, init_s, n_long, m_long,
+                          exp_bfmlalb_idx);
+    test_arm64_bf16_q_run(0x4fc2f020, init_s, n_long, m_long,
+                          exp_bfmlalt_idx);
+
+    test_arm64_i8mm_expect_exception(0x1e634020, UC_CPU_ARM64_A72);
+    test_arm64_i8mm_expect_exception(0x6e42fc20, UC_CPU_ARM64_A72);
+    test_arm64_i8mm_expect_exception(0x2e42ec20, UC_CPU_ARM64_MAX);
+}
+
+static void test_arm64_sve2_mul_run(uint32_t insn, int esz, const void *n,
+                                    const void *m, const void *expected,
+                                    size_t size)
+{
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t ld1_z2[4] = {
+        0xa400a0a2, 0xa4a0a0a2, 0xa540a0a2, 0xa5e0a0a2,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    uc_engine *uc;
+    uint8_t code[20];
+    uint8_t got[32];
+    const uint8_t *exp = expected;
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    size_t i;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, ld1_z1[esz]);
+    test_arm64_emit32(code, 8, ld1_z2[esz]);
+    test_arm64_emit32(code, 12, insn);
+    test_arm64_emit32(code, 16, st1_z0[esz]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n, size));
+    OK(uc_mem_write(uc, x5, m, size));
+    if (size > 16) {
+        test_arm64_mte_enable_sve_vq(uc, 1);
+    } else {
+        test_arm64_mte_enable_sve(uc);
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, size));
+    for (i = 0; i < size; i++) {
+        TEST_CHECK(got[i] == exp[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_widen_run(uint32_t insn, int dst_esz,
+                                      int src1_esz, int src2_esz,
+                                      const void *n, size_t n_size,
+                                      const void *m, size_t m_size,
+                                      const void *expected, size_t out_size)
+{
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t ld1_z2[4] = {
+        0xa400a0a2, 0xa4a0a0a2, 0xa540a0a2, 0xa5e0a0a2,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    uc_engine *uc;
+    uint8_t code[28];
+    uint8_t got[32];
+    const uint8_t *exp = expected;
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    size_t code_size = 0;
+    size_t i;
+
+    test_arm64_emit32(code, code_size, 0x2518e3e0);
+    code_size += 4;
+    test_arm64_emit32(code, code_size, ld1_z1[src1_esz]);
+    code_size += 4;
+    if (m != NULL) {
+        test_arm64_emit32(code, code_size, ld1_z2[src2_esz]);
+        code_size += 4;
+    }
+    test_arm64_emit32(code, code_size, insn);
+    code_size += 4;
+    test_arm64_emit32(code, code_size, st1_z0[dst_esz]);
+    code_size += 4;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    code_size, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n, n_size));
+    if (m != NULL) {
+        OK(uc_mem_write(uc, x5, m, m_size));
+    }
+    if (out_size > 16 || n_size > 16 || m_size > 16) {
+        test_arm64_mte_enable_sve_vq(uc, 1);
+    } else {
+        test_arm64_mte_enable_sve(uc);
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + code_size, 0, 0));
+    OK(uc_mem_read(uc, x6, got, out_size));
+    for (i = 0; i < out_size; i++) {
+        TEST_CHECK(got[i] == exp[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_narrow_run(uint32_t insn, int dst_esz,
+                                       int src_esz, const void *initial,
+                                       const void *n, const void *m,
+                                       const void *expected, size_t size)
+{
+    static const uint32_t ld1_z0[4] = {
+        0xa400a060, 0xa4a0a060, 0xa540a060, 0xa5e0a060,
+    };
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t ld1_z2[4] = {
+        0xa400a0a2, 0xa4a0a0a2, 0xa540a0a2, 0xa5e0a0a2,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    uc_engine *uc;
+    uint8_t code[24];
+    uint8_t got[32];
+    const uint8_t *exp = expected;
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x5 = 0x40200;
+    uint64_t x6 = 0x40300;
+    size_t i;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, ld1_z0[dst_esz]);
+    test_arm64_emit32(code, 8, ld1_z1[src_esz]);
+    test_arm64_emit32(code, 12, ld1_z2[src_esz]);
+    test_arm64_emit32(code, 16, insn);
+    test_arm64_emit32(code, 20, st1_z0[dst_esz]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, initial, size));
+    OK(uc_mem_write(uc, x4, n, size));
+    OK(uc_mem_write(uc, x5, m, size));
+    if (size > 16) {
+        test_arm64_mte_enable_sve_vq(uc, 1);
+    } else {
+        test_arm64_mte_enable_sve(uc);
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, size));
+    for (i = 0; i < size; i++) {
+        TEST_CHECK(got[i] == exp[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_fp_convert_run(uint32_t ptrue, uint32_t insn,
+                                           int dst_esz, int src_esz,
+                                           const void *initial,
+                                           const void *source,
+                                           const void *expected,
+                                           size_t size)
+{
+    static const uint32_t ld1_z0[4] = {
+        0xa400a060, 0xa4a0a060, 0xa540a060, 0xa5e0a060,
+    };
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    uc_engine *uc;
+    uint8_t code[28];
+    uint8_t got[32];
+    const uint8_t *exp = expected;
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x6 = 0x40200;
+    size_t i;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, ld1_z0[dst_esz]);
+    test_arm64_emit32(code, 8, ld1_z1[src_esz]);
+    test_arm64_emit32(code, 12, ptrue);
+    test_arm64_emit32(code, 16, insn);
+    test_arm64_emit32(code, 20, 0x2518e3e0);
+    test_arm64_emit32(code, 24, st1_z0[dst_esz]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, initial, size));
+    OK(uc_mem_write(uc, x4, source, size));
+    if (size > 16) {
+        test_arm64_mte_enable_sve_vq(uc, 1);
+    } else {
+        test_arm64_mte_enable_sve(uc);
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, size));
+    for (i = 0; i < size; i++) {
+        TEST_CHECK_(got[i] == exp[i],
+                    "insn %08x byte %zu got %02x expected %02x",
+                    insn, i, got[i], exp[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_fp_convert(void)
+{
+    const uint32_t ptrue_b = 0x2518e3e0;
+    const uint32_t ptrue_s_vl2 = 0x2598e040;
+    const uint16_t init_h[8] = {
+        0x1110, 0xaaaa, 0x1111, 0xbbbb,
+        0x1112, 0xcccc, 0x1113, 0xdddd,
+    };
+    const uint16_t exp_fcvtnt_sh[8] = {
+        0x1110, 0x3c00, 0x1111, 0xc000,
+        0x1112, 0xcccc, 0x1113, 0xdddd,
+    };
+    const uint16_t src_h[8] = {
+        0xaaaa, 0x3c00, 0xbbbb, 0xc000,
+        0xcccc, 0x3800, 0xdddd, 0x4400,
+    };
+    const uint32_t src_s[4] = {
+        0x3f800000u, 0xc0000000u, 0x3f000000u, 0x40800000u,
+    };
+    const uint32_t src_s_top[4] = {
+        0xdeadbeefu, 0x3f800000u, 0xcafebabeu, 0xc0000000u,
+    };
+    const uint32_t init_s[4] = {
+        0x11111111u, 0xaaaaaaaau, 0x22222222u, 0xbbbbbbbbu,
+    };
+    const uint32_t exp_fcvtlt_hs[4] = {
+        0x3f800000u, 0xc0000000u, 0x3f000000u, 0x40800000u,
+    };
+    const uint32_t exp_fcvtnt_ds[4] = {
+        0x11111111u, 0x3fc00000u, 0x22222222u, 0xc0000000u,
+    };
+    const uint32_t exp_fcvtx_ds[4] = {
+        0x3f800001u, 0x00000000u, 0xc0000000u, 0x00000000u,
+    };
+    const uint32_t exp_fcvtxnt_ds[4] = {
+        0x11111111u, 0x3f800001u, 0x22222222u, 0xc0000000u,
+    };
+    const uint64_t src_d[2] = {
+        0x3ff8000000000000ull, 0xc000000000000000ull,
+    };
+    const uint64_t src_d_odd[2] = {
+        0x3ff0000010000000ull, 0xc000000000000000ull,
+    };
+    const uint64_t exp_fcvtlt_sd[2] = {
+        0x3ff0000000000000ull, 0xc000000000000000ull,
+    };
+    const uint64_t init_d[2] = {
+        0xaaaaaaaaaaaaaaaaull, 0xbbbbbbbbbbbbbbbbull,
+    };
+
+    test_arm64_sve2_fp_convert_run(ptrue_s_vl2, 0x6488a020, 1, 2,
+                                   init_h, src_s, exp_fcvtnt_sh,
+                                   sizeof(exp_fcvtnt_sh));
+    test_arm64_sve2_fp_convert_run(ptrue_b, 0x6489a020, 2, 1,
+                                   init_s, src_h, exp_fcvtlt_hs,
+                                   sizeof(exp_fcvtlt_hs));
+    test_arm64_sve2_fp_convert_run(ptrue_b, 0x64caa020, 2, 3,
+                                   init_s, src_d, exp_fcvtnt_ds,
+                                   sizeof(exp_fcvtnt_ds));
+    test_arm64_sve2_fp_convert_run(ptrue_b, 0x64cba020, 3, 2,
+                                   init_d, src_s_top, exp_fcvtlt_sd,
+                                   sizeof(exp_fcvtlt_sd));
+    test_arm64_sve2_fp_convert_run(ptrue_b, 0x650aa020, 2, 3,
+                                   init_s, src_d_odd, exp_fcvtx_ds,
+                                   sizeof(exp_fcvtx_ds));
+    test_arm64_sve2_fp_convert_run(ptrue_b, 0x640aa020, 2, 3,
+                                   init_s, src_d_odd, exp_fcvtxnt_ds,
+                                   sizeof(exp_fcvtxnt_ds));
+}
+
+static void test_arm64_sve2_fp_pairwise_run(uint32_t ptrue, uint32_t insn,
+                                            int esz, const void *n,
+                                            const void *m,
+                                            const void *expected,
+                                            size_t size)
+{
+    static const uint32_t ld1_z0[4] = {
+        0xa400a060, 0xa4a0a060, 0xa540a060, 0xa5e0a060,
+    };
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    uc_engine *uc;
+    uint8_t code[28];
+    uint8_t got[32];
+    const uint8_t *exp = expected;
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x6 = 0x40200;
+    size_t i;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, ld1_z0[esz]);
+    test_arm64_emit32(code, 8, ld1_z1[esz]);
+    test_arm64_emit32(code, 12, ptrue);
+    test_arm64_emit32(code, 16, insn);
+    test_arm64_emit32(code, 20, 0x2518e3e0);
+    test_arm64_emit32(code, 24, st1_z0[esz]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, n, size));
+    OK(uc_mem_write(uc, x4, m, size));
+    if (size > 16) {
+        test_arm64_mte_enable_sve_vq(uc, 1);
+    } else {
+        test_arm64_mte_enable_sve(uc);
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, size));
+    for (i = 0; i < size; i++) {
+        TEST_CHECK_(got[i] == exp[i],
+                    "insn %08x byte %zu got %02x expected %02x",
+                    insn, i, got[i], exp[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_flogb_run(uint32_t ptrue, uint32_t insn, int esz,
+                                      const void *initial, const void *source,
+                                      const void *expected, size_t size,
+                                      uint32_t fpcr)
+{
+    static const uint32_t ld1_z0[4] = {
+        0xa400a060, 0xa4a0a060, 0xa540a060, 0xa5e0a060,
+    };
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    uc_engine *uc;
+    uint8_t code[28];
+    uint8_t got[32];
+    const uint8_t *exp = expected;
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x6 = 0x40200;
+    size_t i;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, ld1_z0[esz]);
+    test_arm64_emit32(code, 8, ld1_z1[esz]);
+    test_arm64_emit32(code, 12, ptrue);
+    test_arm64_emit32(code, 16, insn);
+    test_arm64_emit32(code, 20, 0x2518e3e0);
+    test_arm64_emit32(code, 24, st1_z0[esz]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, initial, size));
+    OK(uc_mem_write(uc, x4, source, size));
+    if (size > 16) {
+        test_arm64_mte_enable_sve_vq(uc, 1);
+    } else {
+        test_arm64_mte_enable_sve(uc);
+    }
+    if (fpcr != 0) {
+        OK(uc_reg_write(uc, UC_ARM64_REG_FPCR, &fpcr));
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, size));
+    for (i = 0; i < size; i++) {
+        TEST_CHECK_(got[i] == exp[i],
+                    "insn %08x byte %zu got %02x expected %02x",
+                    insn, i, got[i], exp[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_fp_pairwise_flogb(void)
+{
+    const uint32_t ptrue_b = 0x2518e3e0;
+    const uint32_t ptrue_s_vl2 = 0x2598e040;
+    const uint16_t n_h[8] = {
+        0x3c00, 0x4000, 0xbc00, 0xc000,
+        0x3800, 0x4400, 0x4200, 0xb800,
+    };
+    const uint16_t m_h[8] = {
+        0x4000, 0x4200, 0x4400, 0xbc00,
+        0xc000, 0x3800, 0xbc00, 0x4400,
+    };
+    const uint16_t exp_faddp_h[8] = {
+        0x4200, 0x4500, 0xc200, 0x4200,
+        0x4480, 0xbe00, 0x4100, 0x4200,
+    };
+    const uint16_t exp_fmaxnmp_h[8] = {
+        0x4000, 0x4200, 0xbc00, 0x4400,
+        0x4400, 0x3800, 0x4200, 0x4400,
+    };
+    const uint16_t exp_fminnmp_h[8] = {
+        0x3c00, 0x4000, 0xc000, 0xbc00,
+        0x3800, 0xc000, 0xb800, 0xbc00,
+    };
+    const uint16_t exp_fmaxp_h[8] = {
+        0x4000, 0x4200, 0xbc00, 0x4400,
+        0x4400, 0x3800, 0x4200, 0x4400,
+    };
+    const uint16_t exp_fminp_h[8] = {
+        0x3c00, 0x4000, 0xc000, 0xbc00,
+        0x3800, 0xc000, 0xb800, 0xbc00,
+    };
+    const uint32_t n_s[4] = {
+        0x3f800000u, 0x40000000u, 0xc0800000u, 0x40a00000u,
+    };
+    const uint32_t m_s[4] = {
+        0x41200000u, 0xc0400000u, 0x40e00000u, 0x41000000u,
+    };
+    const uint32_t exp_faddp_s[4] = {
+        0x40400000u, 0x40e00000u, 0x3f800000u, 0x41700000u,
+    };
+    const uint32_t exp_fmaxnmp_s[4] = {
+        0x40000000u, 0x41200000u, 0x40a00000u, 0x41000000u,
+    };
+    const uint32_t exp_fminnmp_s[4] = {
+        0x3f800000u, 0xc0400000u, 0xc0800000u, 0x40e00000u,
+    };
+    const uint64_t n_d[2] = {
+        0x3ff8000000000000ull, 0xc000000000000000ull,
+    };
+    const uint64_t m_d[2] = {
+        0x4008000000000000ull, 0x3ff0000000000000ull,
+    };
+    const uint64_t exp_faddp_d[2] = {
+        0xbfe0000000000000ull, 0x4010000000000000ull,
+    };
+    const uint64_t exp_fmaxnmp_d[2] = {
+        0x3ff8000000000000ull, 0x4008000000000000ull,
+    };
+    const uint64_t exp_fminnmp_d[2] = {
+        0xc000000000000000ull, 0x3ff0000000000000ull,
+    };
+    const uint32_t n_nan_s[4] = {
+        0x7fc00000u, 0x3f800000u, 0x40000000u, 0x40400000u,
+    };
+    const uint32_t m_nan_s[4] = {
+        0x7fc00000u, 0x40000000u, 0x40800000u, 0x40a00000u,
+    };
+    const uint32_t exp_fmaxnmp_nan_s[4] = {
+        0x3f800000u, 0x40000000u, 0x40400000u, 0x40a00000u,
+    };
+    const uint32_t exp_fmaxp_nan_s[4] = {
+        0x7fc00000u, 0x7fc00000u, 0x40400000u, 0x40a00000u,
+    };
+    const uint32_t init_s[4] = {
+        0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+    };
+    const uint32_t exp_faddp_s_vl2[4] = {
+        0x40400000u, 0x40e00000u, 0xc0800000u, 0x40a00000u,
+    };
+    const uint16_t init_h[8] = {
+        0x1111, 0x2222, 0x3333, 0x4444,
+        0x5555, 0x6666, 0x7777, 0x8888,
+    };
+    const uint16_t flogb_h[8] = {
+        0x3c00, 0x4000, 0x3800, 0x0001,
+        0x0000, 0x7c00, 0x7e00, 0xc000,
+    };
+    const uint16_t exp_flogb_h[8] = {
+        0x0000, 0x0001, 0xffff, 0xffe8,
+        0x8000, 0x7fff, 0x8000, 0x0001,
+    };
+    const uint16_t exp_flogb_h_fz[8] = {
+        0x0000, 0x0001, 0xffff, 0x8000,
+        0x8000, 0x7fff, 0x8000, 0x0001,
+    };
+    const uint32_t flogb_s[4] = {
+        0x3f800000u, 0x40000000u, 0x3f400000u, 0x00000001u,
+    };
+    const uint32_t exp_flogb_s[4] = {
+        0x00000000u, 0x00000001u, 0xffffffffu, 0xffffff6bu,
+    };
+    const uint64_t flogb_d[2] = {
+        0x3ff0000000000000ull, 0x7ff0000000000000ull,
+    };
+    const uint64_t exp_flogb_d[2] = {
+        0x0000000000000000ull, 0x7fffffffffffffffull,
+    };
+    const uint32_t fpcr_fz16 = 1u << 19;
+
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64508020, 1, n_h, m_h,
+                                    exp_faddp_h, sizeof(exp_faddp_h));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64548020, 1, n_h, m_h,
+                                    exp_fmaxnmp_h, sizeof(exp_fmaxnmp_h));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64558020, 1, n_h, m_h,
+                                    exp_fminnmp_h, sizeof(exp_fminnmp_h));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64568020, 1, n_h, m_h,
+                                    exp_fmaxp_h, sizeof(exp_fmaxp_h));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64578020, 1, n_h, m_h,
+                                    exp_fminp_h, sizeof(exp_fminp_h));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64908020, 2, n_s, m_s,
+                                    exp_faddp_s, sizeof(exp_faddp_s));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64948020, 2, n_s, m_s,
+                                    exp_fmaxnmp_s, sizeof(exp_fmaxnmp_s));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64958020, 2, n_s, m_s,
+                                    exp_fminnmp_s, sizeof(exp_fminnmp_s));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64968020, 2, n_s, m_s,
+                                    exp_fmaxnmp_s, sizeof(exp_fmaxnmp_s));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64978020, 2, n_s, m_s,
+                                    exp_fminnmp_s, sizeof(exp_fminnmp_s));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64d08020, 3, n_d, m_d,
+                                    exp_faddp_d, sizeof(exp_faddp_d));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64d48020, 3, n_d, m_d,
+                                    exp_fmaxnmp_d, sizeof(exp_fmaxnmp_d));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64d58020, 3, n_d, m_d,
+                                    exp_fminnmp_d, sizeof(exp_fminnmp_d));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64d68020, 3, n_d, m_d,
+                                    exp_fmaxnmp_d, sizeof(exp_fmaxnmp_d));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64d78020, 3, n_d, m_d,
+                                    exp_fminnmp_d, sizeof(exp_fminnmp_d));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64948020, 2, n_nan_s,
+                                    m_nan_s, exp_fmaxnmp_nan_s,
+                                    sizeof(exp_fmaxnmp_nan_s));
+    test_arm64_sve2_fp_pairwise_run(ptrue_b, 0x64968020, 2, n_nan_s,
+                                    m_nan_s, exp_fmaxp_nan_s,
+                                    sizeof(exp_fmaxp_nan_s));
+    test_arm64_sve2_fp_pairwise_run(ptrue_s_vl2, 0x64908020, 2, n_s, m_s,
+                                    exp_faddp_s_vl2,
+                                    sizeof(exp_faddp_s_vl2));
+
+    test_arm64_sve2_flogb_run(ptrue_b, 0x651aa020, 1, init_h, flogb_h,
+                              exp_flogb_h, sizeof(exp_flogb_h), 0);
+    test_arm64_sve2_flogb_run(ptrue_b, 0x651aa020, 1, init_h, flogb_h,
+                              exp_flogb_h_fz, sizeof(exp_flogb_h_fz),
+                              fpcr_fz16);
+    test_arm64_sve2_flogb_run(ptrue_b, 0x651ca020, 2, init_s, flogb_s,
+                              exp_flogb_s, sizeof(exp_flogb_s), 0);
+    test_arm64_sve2_flogb_run(ptrue_b, 0x651ea020, 3, n_d, flogb_d,
+                              exp_flogb_d, sizeof(exp_flogb_d), 0);
+
+    test_arm64_i8mm_expect_exception(0x64908020, UC_CPU_ARM64_A72);
+    test_arm64_i8mm_expect_exception(0x651ca020, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sve2_fmlal_run(uint32_t insn, const uint32_t *initial,
+                                      const uint16_t *n, const uint16_t *m,
+                                      const uint32_t *expected, size_t size,
+                                      uint32_t fpcr)
+{
+    uc_engine *uc;
+    uint8_t code[24];
+    uint8_t got[32];
+    const uint8_t *exp = (const uint8_t *)expected;
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x5 = 0x40200;
+    uint64_t x6 = 0x40300;
+    size_t i;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, 0xa540a060);
+    test_arm64_emit32(code, 8, 0xa4a0a081);
+    test_arm64_emit32(code, 12, 0xa4a0a0a2);
+    test_arm64_emit32(code, 16, insn);
+    test_arm64_emit32(code, 20, 0xe540e0c0);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, initial, size));
+    OK(uc_mem_write(uc, x4, n, size));
+    OK(uc_mem_write(uc, x5, m, size));
+    if (size > 16) {
+        test_arm64_mte_enable_sve_vq(uc, 1);
+    } else {
+        test_arm64_mte_enable_sve(uc);
+    }
+    if (fpcr != 0) {
+        OK(uc_reg_write(uc, UC_ARM64_REG_FPCR, &fpcr));
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, size));
+    for (i = 0; i < size; i++) {
+        TEST_CHECK_(got[i] == exp[i],
+                    "insn %08x byte %zu got %02x expected %02x",
+                    insn, i, got[i], exp[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_fmlal(void)
+{
+    const uint32_t init_s[4] = {
+        0x3f800000u, 0x41200000u, 0xc0000000u, 0x3f000000u,
+    };
+    const uint16_t n_h[8] = {
+        0x3c00, 0x4000, 0xbc00, 0xc000,
+        0x3800, 0x4400, 0x4200, 0xb800,
+    };
+    const uint16_t m_h[8] = {
+        0x4000, 0x4200, 0x4400, 0xbc00,
+        0xc000, 0x3800, 0xbc00, 0x4400,
+    };
+    const uint32_t exp_fmlalb[4] = {
+        0x40400000u, 0x40c00000u, 0xc0400000u, 0xc0200000u,
+    };
+    const uint32_t exp_fmlalt[4] = {
+        0x40e00000u, 0x41400000u, 0x00000000u, 0xbfc00000u,
+    };
+    const uint32_t exp_fmlslb[4] = {
+        0xbf800000u, 0x41600000u, 0xbf800000u, 0x40600000u,
+    };
+    const uint32_t exp_fmlslt[4] = {
+        0xc0a00000u, 0x41000000u, 0xc0800000u, 0x40200000u,
+    };
+    const uint32_t init_idx_s[8] = {
+        0x3f800000u, 0x41200000u, 0xc0000000u, 0x3f000000u,
+        0xbf800000u, 0x40400000u, 0x40800000u, 0xbf000000u,
+    };
+    const uint16_t n_idx_h[16] = {
+        0x3c00, 0x4000, 0xbc00, 0xc000,
+        0x3800, 0x4400, 0x4200, 0xb800,
+        0x4000, 0x3c00, 0x4200, 0x4400,
+        0xbc00, 0x3800, 0xc000, 0xb800,
+    };
+    const uint16_t m_idx_h[16] = {
+        0x0000, 0x0000, 0x0000, 0x4000,
+        0x0000, 0x0000, 0x0000, 0x0000,
+        0x0000, 0x0000, 0x0000, 0xc000,
+        0x0000, 0x0000, 0x0000, 0x0000,
+    };
+    const uint32_t exp_fmlalb_idx[8] = {
+        0x40400000u, 0x41000000u, 0xbf800000u, 0x40d00000u,
+        0xc0a00000u, 0xc0400000u, 0x40c00000u, 0x40600000u,
+    };
+    const uint32_t exp_fmlalt_idx[8] = {
+        0x40a00000u, 0x40c00000u, 0x40c00000u, 0xbf000000u,
+        0xc0400000u, 0xc0a00000u, 0x40400000u, 0x3f000000u,
+    };
+    const uint32_t exp_fmlslb_idx[8] = {
+        0xbf800000u, 0x41400000u, 0xc0400000u, 0xc0b00000u,
+        0x40400000u, 0x41100000u, 0x40000000u, 0xc0900000u,
+    };
+    const uint32_t exp_fmlslt_idx[8] = {
+        0xc0400000u, 0x41600000u, 0xc1200000u, 0x3fc00000u,
+        0x3f800000u, 0x41300000u, 0x40a00000u, 0xbfc00000u,
+    };
+    const uint32_t zero_s[4] = { 0, 0, 0, 0 };
+    const uint16_t subnormal_h[8] = {
+        0x0001, 0x3c00, 0x0002, 0x3c00,
+        0x0003, 0x3c00, 0x0004, 0x3c00,
+    };
+    const uint16_t one_bottom_h[8] = {
+        0x3c00, 0x0000, 0x3c00, 0x0000,
+        0x3c00, 0x0000, 0x3c00, 0x0000,
+    };
+    const uint32_t fpcr_fz16 = 1u << 19;
+
+    test_arm64_sve2_fmlal_run(0x64a28020, init_s, n_h, m_h,
+                              exp_fmlalb, sizeof(exp_fmlalb), 0);
+    test_arm64_sve2_fmlal_run(0x64a28420, init_s, n_h, m_h,
+                              exp_fmlalt, sizeof(exp_fmlalt), 0);
+    test_arm64_sve2_fmlal_run(0x64a2a020, init_s, n_h, m_h,
+                              exp_fmlslb, sizeof(exp_fmlslb), 0);
+    test_arm64_sve2_fmlal_run(0x64a2a420, init_s, n_h, m_h,
+                              exp_fmlslt, sizeof(exp_fmlslt), 0);
+    test_arm64_sve2_fmlal_run(0x64aa4820, init_idx_s, n_idx_h, m_idx_h,
+                              exp_fmlalb_idx, sizeof(exp_fmlalb_idx), 0);
+    test_arm64_sve2_fmlal_run(0x64aa4c20, init_idx_s, n_idx_h, m_idx_h,
+                              exp_fmlalt_idx, sizeof(exp_fmlalt_idx), 0);
+    test_arm64_sve2_fmlal_run(0x64aa6820, init_idx_s, n_idx_h, m_idx_h,
+                              exp_fmlslb_idx, sizeof(exp_fmlslb_idx), 0);
+    test_arm64_sve2_fmlal_run(0x64aa6c20, init_idx_s, n_idx_h, m_idx_h,
+                              exp_fmlslt_idx, sizeof(exp_fmlslt_idx), 0);
+    test_arm64_sve2_fmlal_run(0x64a28020, zero_s, subnormal_h,
+                              one_bottom_h, zero_s, sizeof(zero_s),
+                              fpcr_fz16);
+
+    test_arm64_i8mm_expect_exception(0x64a28020, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sve2_mul_base(void)
+{
+    uc_engine *uc;
+    uint8_t invalid_code[8];
+    const uint8_t n_b[16] = {
+        0x80, 0x7f, 0xf0, 0x11, 0x22, 0xdd, 0x33, 0xcc,
+        0x44, 0xbb, 0x55, 0xaa, 0x66, 0x99, 0x77, 0x88,
+    };
+    const uint8_t m_b[16] = {
+        0x02, 0xfe, 0x10, 0xf1, 0x80, 0x03, 0x7f, 0x81,
+        0x55, 0xaa, 0x66, 0x99, 0x77, 0x88, 0xff, 0x01,
+    };
+    const uint8_t exp_mul_b[16] = {
+        0x00, 0x02, 0x00, 0x01, 0x00, 0x97, 0x4d, 0xcc,
+        0x94, 0x2e, 0xde, 0x9a, 0x6a, 0x48, 0x89, 0x88,
+    };
+    const uint8_t exp_smulh_b[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xef, 0xff, 0x19, 0x19,
+        0x16, 0x17, 0x21, 0x22, 0x2f, 0x30, 0xff, 0xff,
+    };
+    const uint8_t exp_umulh_b[16] = {
+        0x01, 0x7e, 0x0f, 0x10, 0x11, 0x02, 0x19, 0x66,
+        0x16, 0x7c, 0x21, 0x65, 0x2f, 0x51, 0x76, 0x00,
+    };
+    const uint8_t exp_pmul_b[16] = {
+        0x00, 0xaa, 0x00, 0xe1, 0x00, 0x67, 0x91, 0xcc,
+        0x14, 0x4e, 0x1e, 0x5a, 0x12, 0x48, 0x2d, 0x88,
+    };
+    const uint16_t n_h[8] = {
+        0x8001, 0x7fff, 0xf123, 0x1234,
+        0xaaaa, 0x5555, 0x0101, 0xfefe,
+    };
+    const uint16_t m_h[8] = {
+        0x0002, 0xfffe, 0x1357, 0x8000,
+        0x2222, 0xdddd, 0x7fff, 0x8001,
+    };
+    const uint16_t exp_mul_h[8] = {
+        0x0002, 0x0002, 0x8be5, 0x0000,
+        0x3e94, 0x0b61, 0x7eff, 0xfefe,
+    };
+    const uint16_t exp_smulh_h[8] = {
+        0xffff, 0xffff, 0xfee0, 0xf6e6,
+        0xf49f, 0xf49f, 0x0080, 0x0080,
+    };
+    const uint16_t exp_umulh_h[8] = {
+        0x0001, 0x7ffe, 0x1237, 0x091a,
+        0x16c1, 0x49f4, 0x0080, 0x7f7f,
+    };
+    const uint32_t n_s[4] = {
+        0x80000001u, 0x7fffffffu, 0xf1234567u, 0x12345678u,
+    };
+    const uint32_t m_s[4] = {
+        0x00000002u, 0xfffffffeu, 0x13579bdfu, 0x80000000u,
+    };
+    const uint32_t exp_mul_s[4] = {
+        0x00000002u, 0x00000002u, 0xa3bfd1b9u, 0x00000000u,
+    };
+    const uint32_t exp_smulh_s[4] = {
+        0xffffffffu, 0xffffffffu, 0xfee08816u, 0xf6e5d4c4u,
+    };
+    const uint32_t exp_umulh_s[4] = {
+        0x00000001u, 0x7ffffffeu, 0x123823f5u, 0x091a2b3cu,
+    };
+    const uint64_t n_d[2] = {
+        0x8000000000000001ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t m_d[2] = {
+        0x0000000000000002ull, 0xfffffffffffffffeull,
+    };
+    const uint64_t exp_mul_d[2] = {
+        0x0000000000000002ull, 0x0000000000000002ull,
+    };
+    const uint64_t exp_smulh_d[2] = {
+        0xffffffffffffffffull, 0xffffffffffffffffull,
+    };
+    const uint64_t exp_umulh_d[2] = {
+        0x0000000000000001ull, 0x7ffffffffffffffeull,
+    };
+    const uint8_t sq_n_b[16] = {
+        0x80, 0x7f, 0x40, 0xc0, 0x20, 0xe0, 0x55, 0xab,
+        0x10, 0xf0, 0x33, 0xcd, 0x01, 0xff, 0x7e, 0x82,
+    };
+    const uint8_t sq_m_b[16] = {
+        0x80, 0x7f, 0x40, 0x40, 0xe0, 0x20, 0x55, 0xab,
+        0x7f, 0x81, 0xcd, 0x33, 0x80, 0x80, 0x02, 0xfe,
+    };
+    const uint8_t exp_sqdmulh_b[16] = {
+        0x7f, 0x7e, 0x20, 0xe0, 0xf8, 0xf8, 0x38, 0x38,
+        0x0f, 0x0f, 0xeb, 0xeb, 0xff, 0x01, 0x01, 0x01,
+    };
+    const uint8_t exp_sqrdmulh_b[16] = {
+        0x7f, 0x7e, 0x20, 0xe0, 0xf8, 0xf8, 0x38, 0x38,
+        0x10, 0x10, 0xec, 0xec, 0xff, 0x01, 0x02, 0x02,
+    };
+    const uint16_t sq_n_h[8] = {
+        0x8000, 0x7fff, 0x4000, 0xc000,
+        0x2000, 0xe000, 0x5555, 0x0001,
+    };
+    const uint16_t sq_m_h[8] = {
+        0x8000, 0x7fff, 0x4000, 0x4000,
+        0xe000, 0x2000, 0x5555, 0x4000,
+    };
+    const uint16_t exp_sqdmulh_h[8] = {
+        0x7fff, 0x7ffe, 0x2000, 0xe000,
+        0xf800, 0xf800, 0x38e3, 0x0000,
+    };
+    const uint16_t exp_sqrdmulh_h[8] = {
+        0x7fff, 0x7ffe, 0x2000, 0xe000,
+        0xf800, 0xf800, 0x38e3, 0x0001,
+    };
+    const uint32_t sq_n_s[4] = {
+        0x80000000u, 0x7fffffffu, 0x40000000u, 0x00000001u,
+    };
+    const uint32_t sq_m_s[4] = {
+        0x80000000u, 0x7fffffffu, 0x40000000u, 0x40000000u,
+    };
+    const uint32_t exp_sqdmulh_s[4] = {
+        0x7fffffffu, 0x7ffffffeu, 0x20000000u, 0x00000000u,
+    };
+    const uint32_t exp_sqrdmulh_s[4] = {
+        0x7fffffffu, 0x7ffffffeu, 0x20000000u, 0x00000001u,
+    };
+    const uint64_t sq_n_d[2] = {
+        0x8000000000000000ull, 0x0000000000000001ull,
+    };
+    const uint64_t sq_m_d[2] = {
+        0x8000000000000000ull, 0x4000000000000000ull,
+    };
+    const uint64_t exp_sqdmulh_d[2] = {
+        0x7fffffffffffffffull, 0x0000000000000000ull,
+    };
+    const uint64_t exp_sqrdmulh_d[2] = {
+        0x7fffffffffffffffull, 0x0000000000000001ull,
+    };
+
+    test_arm64_sve2_mul_run(0x04226020, 0, n_b, m_b, exp_mul_b,
+                            sizeof(n_b));
+    test_arm64_sve2_mul_run(0x04226820, 0, n_b, m_b, exp_smulh_b,
+                            sizeof(n_b));
+    test_arm64_sve2_mul_run(0x04226c20, 0, n_b, m_b, exp_umulh_b,
+                            sizeof(n_b));
+    test_arm64_sve2_mul_run(0x04226420, 0, n_b, m_b, exp_pmul_b,
+                            sizeof(n_b));
+    test_arm64_sve2_mul_run(0x04626020, 1, n_h, m_h, exp_mul_h,
+                            sizeof(n_h));
+    test_arm64_sve2_mul_run(0x04626820, 1, n_h, m_h, exp_smulh_h,
+                            sizeof(n_h));
+    test_arm64_sve2_mul_run(0x04626c20, 1, n_h, m_h, exp_umulh_h,
+                            sizeof(n_h));
+    test_arm64_sve2_mul_run(0x04a26020, 2, n_s, m_s, exp_mul_s,
+                            sizeof(n_s));
+    test_arm64_sve2_mul_run(0x04a26820, 2, n_s, m_s, exp_smulh_s,
+                            sizeof(n_s));
+    test_arm64_sve2_mul_run(0x04a26c20, 2, n_s, m_s, exp_umulh_s,
+                            sizeof(n_s));
+    test_arm64_sve2_mul_run(0x04e26020, 3, n_d, m_d, exp_mul_d,
+                            sizeof(n_d));
+    test_arm64_sve2_mul_run(0x04e26820, 3, n_d, m_d, exp_smulh_d,
+                            sizeof(n_d));
+    test_arm64_sve2_mul_run(0x04e26c20, 3, n_d, m_d, exp_umulh_d,
+                            sizeof(n_d));
+    test_arm64_sve2_mul_run(0x04227020, 0, sq_n_b, sq_m_b,
+                            exp_sqdmulh_b, sizeof(sq_n_b));
+    test_arm64_sve2_mul_run(0x04227420, 0, sq_n_b, sq_m_b,
+                            exp_sqrdmulh_b, sizeof(sq_n_b));
+    test_arm64_sve2_mul_run(0x04627020, 1, sq_n_h, sq_m_h,
+                            exp_sqdmulh_h, sizeof(sq_n_h));
+    test_arm64_sve2_mul_run(0x04627420, 1, sq_n_h, sq_m_h,
+                            exp_sqrdmulh_h, sizeof(sq_n_h));
+    test_arm64_sve2_mul_run(0x04a27020, 2, sq_n_s, sq_m_s,
+                            exp_sqdmulh_s, sizeof(sq_n_s));
+    test_arm64_sve2_mul_run(0x04a27420, 2, sq_n_s, sq_m_s,
+                            exp_sqrdmulh_s, sizeof(sq_n_s));
+    test_arm64_sve2_mul_run(0x04e27020, 3, sq_n_d, sq_m_d,
+                            exp_sqdmulh_d, sizeof(sq_n_d));
+    test_arm64_sve2_mul_run(0x04e27420, 3, sq_n_d, sq_m_d,
+                            exp_sqrdmulh_d, sizeof(sq_n_d));
+
+    test_arm64_emit32(invalid_code, 0, 0x2518e3e0);
+    test_arm64_emit32(invalid_code, 4, 0x04626420);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)invalid_code, sizeof(invalid_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_sve(uc);
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(invalid_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_mul_indexed(void)
+{
+    const uint16_t n_h[16] = {
+        0x8001, 0x7fff, 0x4000, 0xc000, 0x1111, 0xeeee, 0x5555, 0xaaaa,
+        0x1234, 0xedcc, 0x0101, 0xfefe, 0x7000, 0x9000, 0x2222, 0xdddd,
+    };
+    const uint16_t m_h[16] = {
+        0x0002, 0x7fff, 0x4000, 0xc000, 0x3333, 0x2000, 0x5555, 0xaaaa,
+        0x1111, 0xeeee, 0x0100, 0xff00, 0x6000, 0xe000, 0x4444, 0xbbbb,
+    };
+    const uint16_t exp_mul_h[16] = {
+        0x2000, 0xe000, 0x0000, 0x0000, 0x2000, 0xc000, 0xa000, 0x4000,
+        0x8000, 0x8000, 0xe000, 0x4000, 0x0000, 0x0000, 0xc000, 0x6000,
+    };
+    const uint16_t exp_sqdmulh_h[16] = {
+        0xe000, 0x1fff, 0x1000, 0xf000, 0x0444, 0xfbbb, 0x1555, 0xeaaa,
+        0xfb73, 0x048d, 0xffbf, 0x0040, 0xe400, 0x1c00, 0xf777, 0x0888,
+    };
+    const uint16_t exp_sqrdmulh_h[16] = {
+        0xe000, 0x2000, 0x1000, 0xf000, 0x0444, 0xfbbc, 0x1555, 0xeaab,
+        0xfb73, 0x048d, 0xffc0, 0x0041, 0xe400, 0x1c00, 0xf778, 0x0889,
+    };
+    const uint32_t n_s[8] = {
+        0x80000001u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x11111111u, 0xeeeeeeeeu, 0x00000001u, 0xffffffffu,
+    };
+    const uint32_t m_s[8] = {
+        0x00000002u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x22222222u, 0xddddddddu, 0x40000000u, 0x80000000u,
+    };
+    const uint32_t exp_mul_s[8] = {
+        0x40000000u, 0xc0000000u, 0x00000000u, 0x00000000u,
+        0x40000000u, 0x80000000u, 0x40000000u, 0xc0000000u,
+    };
+    const uint32_t exp_sqdmulh_s[8] = {
+        0xc0000000u, 0x3fffffffu, 0x20000000u, 0xe0000000u,
+        0x08888888u, 0xf7777777u, 0x00000000u, 0xffffffffu,
+    };
+    const uint32_t exp_sqrdmulh_s[8] = {
+        0xc0000001u, 0x40000000u, 0x20000000u, 0xe0000000u,
+        0x08888889u, 0xf7777777u, 0x00000001u, 0x00000000u,
+    };
+    const uint64_t n_d[4] = {
+        0x8000000000000001ull, 0x7fffffffffffffffull,
+        0x4000000000000000ull, 0xc000000000000000ull,
+    };
+    const uint64_t m_d[4] = {
+        0x0000000000000002ull, 0x4000000000000000ull,
+        0x2222222222222222ull, 0x8000000000000000ull,
+    };
+    const uint64_t exp_mul_d[4] = {
+        0x4000000000000000ull, 0xc000000000000000ull,
+        0x0000000000000000ull, 0x0000000000000000ull,
+    };
+    const uint64_t exp_sqdmulh_d[4] = {
+        0xc000000000000000ull, 0x3fffffffffffffffull,
+        0xc000000000000000ull, 0x4000000000000000ull,
+    };
+    const uint64_t exp_sqrdmulh_d[4] = {
+        0xc000000000000001ull, 0x4000000000000000ull,
+        0xc000000000000000ull, 0x4000000000000000ull,
+    };
+
+    test_arm64_sve2_mul_run(0x446af820, 1, n_h, m_h, exp_mul_h,
+                            sizeof(n_h));
+    test_arm64_sve2_mul_run(0x446af020, 1, n_h, m_h, exp_sqdmulh_h,
+                            sizeof(n_h));
+    test_arm64_sve2_mul_run(0x446af420, 1, n_h, m_h, exp_sqrdmulh_h,
+                            sizeof(n_h));
+    test_arm64_sve2_mul_run(0x44b2f820, 2, n_s, m_s, exp_mul_s,
+                            sizeof(n_s));
+    test_arm64_sve2_mul_run(0x44b2f020, 2, n_s, m_s, exp_sqdmulh_s,
+                            sizeof(n_s));
+    test_arm64_sve2_mul_run(0x44b2f420, 2, n_s, m_s, exp_sqrdmulh_s,
+                            sizeof(n_s));
+    test_arm64_sve2_mul_run(0x44f2f820, 3, n_d, m_d, exp_mul_d,
+                            sizeof(n_d));
+    test_arm64_sve2_mul_run(0x44f2f020, 3, n_d, m_d, exp_sqdmulh_d,
+                            sizeof(n_d));
+    test_arm64_sve2_mul_run(0x44f2f420, 3, n_d, m_d, exp_sqrdmulh_d,
+                            sizeof(n_d));
+}
+
+static void test_arm64_sve2_widen_indexed(void)
+{
+    const uint16_t n_h[16] = {
+        0x8001, 0x7fff, 0x4000, 0xc000, 0x1111, 0xeeee, 0x5555, 0xaaaa,
+        0x1234, 0xedcc, 0x0101, 0xfefe, 0x7000, 0x9000, 0x2222, 0xdddd,
+    };
+    const uint16_t m_h[16] = {
+        0x0002, 0x7fff, 0x4000, 0xc000, 0x3333, 0x2000, 0x5555, 0xaaaa,
+        0x1111, 0xeeee, 0x0100, 0xff00, 0x6000, 0xe000, 0x4444, 0xbbbb,
+    };
+    const uint32_t n_s[8] = {
+        0x80000001u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x11111111u, 0xeeeeeeeeu, 0x00000001u, 0xffffffffu,
+    };
+    const uint32_t m_s[8] = {
+        0x00000002u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x22222222u, 0xddddddddu, 0x40000000u, 0x80000000u,
+    };
+    const uint32_t exp_smullb_s[8] = {
+        0xf0002000u, 0x08000000u, 0x02222000u, 0x0aaaa000u,
+        0xfdb98000u, 0xffdfe000u, 0xf2000000u, 0xfbbbc000u,
+    };
+    const uint32_t exp_sqdmullt_s[8] = {
+        0x1fffc000u, 0xf0000000u, 0xfbbb8000u, 0xeaaa8000u,
+        0x048d0000u, 0x00408000u, 0x1c000000u, 0x0888c000u,
+    };
+    const uint64_t exp_umullt_d[4] = {
+        0x5fffffff40000000ull, 0x9000000000000000ull,
+        0x7777777700000000ull, 0x7fffffff80000000ull,
+    };
+    const uint32_t init_s[8] = {
+        0x00000010u, 0x7ffffff0u, 0x80000020u, 0x12345678u,
+        0xfffffff0u, 0x80000010u, 0x7fffff00u, 0x01020304u,
+    };
+    const uint64_t init_d[4] = {
+        0x0000000000000010ull, 0x7ffffffffffffff0ull,
+        0x8000000000000020ull, 0x0123456789abcdefull,
+    };
+    const uint32_t exp_smlalb_s[8] = {
+        0xf0002010u, 0x87fffff0u, 0x82222020u, 0x1cdef678u,
+        0xfdb97ff0u, 0x7fdfe010u, 0x71ffff00u, 0xfcbdc304u,
+    };
+    const uint32_t exp_sqdmlalb_s[8] = {
+        0xe0004010u, 0x7fffffffu, 0x84444020u, 0x27899678u,
+        0xfb72fff0u, 0x80000000u, 0x63ffff00u, 0xf8798304u,
+    };
+    const uint64_t exp_umlslt_d[4] = {
+        0xa0000000c0000010ull, 0xeffffffffffffff0ull,
+        0x0888888900000020ull, 0x8123456809abcdefull,
+    };
+    const uint64_t exp_sqdmlslt_d[4] = {
+        0x3fffffff80000010ull, 0x5ffffffffffffff0ull,
+        0x8000000000000000ull, 0x0123456689abcdefull,
+    };
+
+    test_arm64_sve2_widen_run(0x44b2c820, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_smullb_s,
+                              sizeof(exp_smullb_s));
+    test_arm64_sve2_widen_run(0x44b2ec20, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_sqdmullt_s,
+                              sizeof(exp_sqdmullt_s));
+    test_arm64_sve2_widen_run(0x44f2dc20, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_umullt_d,
+                              sizeof(exp_umullt_d));
+    test_arm64_sve2_narrow_run(0x44b28820, 2, 1, init_s, n_h, m_h,
+                               exp_smlalb_s, sizeof(exp_smlalb_s));
+    test_arm64_sve2_narrow_run(0x44b22820, 2, 1, init_s, n_h, m_h,
+                               exp_sqdmlalb_s, sizeof(exp_sqdmlalb_s));
+    test_arm64_sve2_narrow_run(0x44f2bc20, 3, 2, init_d, n_s, m_s,
+                               exp_umlslt_d, sizeof(exp_umlslt_d));
+    test_arm64_sve2_narrow_run(0x44f23c20, 3, 2, init_d, n_s, m_s,
+                               exp_sqdmlslt_d, sizeof(exp_sqdmlslt_d));
+}
+
+static void test_arm64_sve2_widen_accumulate(void)
+{
+    const uint8_t n_b[16] = {
+        0x80, 0x7f, 0x40, 0xc0, 0x11, 0xee, 0x55, 0xaa,
+        0x10, 0xf0, 0x33, 0xcd, 0x01, 0xff, 0x7e, 0x82,
+    };
+    const uint8_t m_b[16] = {
+        0x02, 0x7f, 0x40, 0xc0, 0x33, 0x20, 0x55, 0xaa,
+        0x7f, 0x81, 0xcd, 0x33, 0x80, 0x80, 0x02, 0xfe,
+    };
+    const uint16_t n_h[16] = {
+        0x8001, 0x7fff, 0x4000, 0xc000, 0x1111, 0xeeee, 0x5555, 0xaaaa,
+        0x1234, 0xedcc, 0x0101, 0xfefe, 0x7000, 0x9000, 0x2222, 0xdddd,
+    };
+    const uint16_t m_h[16] = {
+        0x0002, 0x7fff, 0x4000, 0xc000, 0x3333, 0x2000, 0x5555, 0xaaaa,
+        0x1111, 0xeeee, 0x0100, 0xff00, 0x6000, 0xe000, 0x4444, 0xbbbb,
+    };
+    const uint32_t n_s[8] = {
+        0x80000001u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x11111111u, 0xeeeeeeeeu, 0x00000001u, 0xffffffffu,
+    };
+    const uint32_t m_s[8] = {
+        0x00000002u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x22222222u, 0xddddddddu, 0x40000000u, 0x80000000u,
+    };
+    const uint16_t init_h[8] = {
+        0x0010, 0x7ff0, 0x8020, 0x5678,
+        0xfff0, 0x8010, 0x7f00, 0x0304,
+    };
+    const uint32_t init_s[8] = {
+        0x00000010u, 0x7ffffff0u, 0x80000020u, 0x12345678u,
+        0xfffffff0u, 0x80000010u, 0x7fffff00u, 0x01020304u,
+    };
+    const uint64_t init_d[4] = {
+        0x0000000000000010ull, 0x7ffffffffffffff0ull,
+        0x8000000000000020ull, 0x0123456789abcdefull,
+    };
+    const uint16_t exp_smlalb_h[8] = {
+        0xff10, 0x8ff0, 0x8383, 0x72b1,
+        0x07e0, 0x75e7, 0x7e80, 0x0400,
+    };
+    const uint32_t exp_umlalt_s[8] = {
+        0x3fff0011u, 0x0ffffff0u, 0x9dddc020u, 0x83fa8f5cu,
+        0xddf0bb98u, 0x7dff0210u, 0xfdffff00u, 0xa3b48273u,
+    };
+    const uint64_t exp_smlslt_d[4] = {
+        0xc00000010000000full, 0x6ffffffffffffff0ull,
+        0x7db97530be0246aaull, 0x0123456709abcdefull,
+    };
+    const uint32_t exp_sqdmlalb_s[8] = {
+        0xfffe0014u, 0x7fffffffu, 0x86d392e6u, 0x4b1772eau,
+        0x026d52d8u, 0x80020210u, 0x7fffffffu, 0x13363514u,
+    };
+    const uint32_t exp_sqdmlalbt_s[8] = {
+        0x8002000eu, 0x5ffffff0u, 0x84444020u, 0xd9508f5cu,
+        0xfd9288a0u, 0x80000000u, 0x63ffff00u, 0xeecd8cb0u,
+    };
+    const uint64_t exp_sqdmlslt_d[4] = {
+        0x800000020000000eull, 0x5ffffffffffffff0ull,
+        0x8000000000000000ull, 0x0123456689abcdefull,
+    };
+
+    test_arm64_sve2_narrow_run(0x44424020, 1, 0, init_h, n_b, m_b,
+                               exp_smlalb_h, sizeof(exp_smlalb_h));
+    test_arm64_sve2_narrow_run(0x44824c20, 2, 1, init_s, n_h, m_h,
+                               exp_umlalt_s, sizeof(exp_umlalt_s));
+    test_arm64_sve2_narrow_run(0x44c25420, 3, 2, init_d, n_s, m_s,
+                               exp_smlslt_d, sizeof(exp_smlslt_d));
+    test_arm64_sve2_narrow_run(0x44826020, 2, 1, init_s, n_h, m_h,
+                               exp_sqdmlalb_s, sizeof(exp_sqdmlalb_s));
+    test_arm64_sve2_narrow_run(0x44820820, 2, 1, init_s, n_h, m_h,
+                               exp_sqdmlalbt_s, sizeof(exp_sqdmlalbt_s));
+    test_arm64_sve2_narrow_run(0x44c26c20, 3, 2, init_d, n_s, m_s,
+                               exp_sqdmlslt_d, sizeof(exp_sqdmlslt_d));
+}
+
+static void test_arm64_sve2_abs_accumulate(void)
+{
+    const uint8_t aba_n_b[16] = {
+        0x80, 0x7f, 0x40, 0xc0, 0x11, 0xee, 0x55, 0xaa,
+        0x10, 0xf0, 0x33, 0xcd, 0x01, 0xff, 0x7e, 0x82,
+    };
+    const uint8_t aba_m_b[16] = {
+        0x02, 0x7f, 0x40, 0xc0, 0x33, 0x20, 0x55, 0xaa,
+        0x7f, 0x81, 0xcd, 0x33, 0x80, 0x80, 0x02, 0xfe,
+    };
+    const uint16_t aba_n_h[8] = {
+        0x8001, 0x7fff, 0x4000, 0xc000,
+        0x1111, 0xeeee, 0x5555, 0xaaaa,
+    };
+    const uint16_t aba_m_h[8] = {
+        0x0002, 0x7fff, 0x4000, 0xc000,
+        0x3333, 0x2000, 0x5555, 0xaaaa,
+    };
+    const uint32_t aba_n_s[4] = {
+        0x80000001u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+    };
+    const uint32_t aba_m_s[4] = {
+        0x00000002u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+    };
+    const uint64_t aba_n_d[2] = {
+        0x8000000000000000ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t aba_m_d[2] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint8_t aba_init_b[16] = {
+        0x10, 0x80, 0xff, 0x01, 0x7f, 0x00, 0xaa, 0x55,
+        0xfe, 0x02, 0x40, 0xc0, 0x11, 0xee, 0x33, 0xcd,
+    };
+    const uint16_t aba_init_h[8] = {
+        0x0010, 0x7ff0, 0x8020, 0x5678,
+        0xfff0, 0x8010, 0x7f00, 0x0304,
+    };
+    const uint32_t aba_init_s[4] = {
+        0x00000010u, 0x7ffffff0u, 0x80000020u, 0x12345678u,
+    };
+    const uint64_t aba_init_d[2] = {
+        0x0000000000000010ull, 0x7ffffffffffffff0ull,
+    };
+    const uint16_t exp_sabalb_h[8] = {
+        0x0092, 0x7ff0, 0x8042, 0x5678,
+        0x005f, 0x8076, 0x7f81, 0x0380,
+    };
+    const uint16_t exp_sabalt_h[8] = {
+        0x0010, 0x7ff0, 0x8052, 0x5678,
+        0x005f, 0x8076, 0x7f7f, 0x0380,
+    };
+    const uint16_t exp_uabalb_h[8] = {
+        0x008e, 0x7ff0, 0x8042, 0x5678,
+        0x005f, 0x80aa, 0x7f7f, 0x0380,
+    };
+    const uint16_t exp_uabalt_h[8] = {
+        0x0010, 0x7ff0, 0x80ee, 0x5678,
+        0x005f, 0x80aa, 0x7f7f, 0x0380,
+    };
+    const uint32_t exp_sabalb_s[4] = {
+        0x00008011u, 0x7ffffff0u, 0x80002242u, 0x12345678u,
+    };
+    const uint32_t exp_sabalt_s[4] = {
+        0x00000010u, 0x7ffffff0u, 0x80003132u, 0x12345678u,
+    };
+    const uint32_t exp_uabalb_s[4] = {
+        0x0000800fu, 0x7ffffff0u, 0x80002242u, 0x12345678u,
+    };
+    const uint32_t exp_uabalt_s[4] = {
+        0x00000010u, 0x7ffffff0u, 0x8000cf0eu, 0x12345678u,
+    };
+    const uint64_t exp_sabalb_d[2] = {
+        0x0000000080000011ull, 0x7ffffffffffffff0ull,
+    };
+    const uint64_t exp_sabalt_d[2] = {
+        0x0000000000000010ull, 0x7ffffffffffffff0ull,
+    };
+    const uint64_t exp_uabalb_d[2] = {
+        0x000000008000000full, 0x7ffffffffffffff0ull,
+    };
+    const uint64_t exp_uabalt_d[2] = {
+        0x0000000000000010ull, 0x7ffffffffffffff0ull,
+    };
+    const uint8_t exp_saba_b[16] = {
+        0x92, 0x80, 0xff, 0x01, 0xa1, 0x32, 0xaa, 0x55,
+        0x6d, 0x71, 0xa6, 0x26, 0x92, 0x6d, 0xaf, 0x49,
+    };
+    const uint8_t exp_uaba_b[16] = {
+        0x8e, 0x80, 0xff, 0x01, 0xa1, 0xce, 0xaa, 0x55,
+        0x6d, 0x71, 0xda, 0x5a, 0x90, 0x6d, 0xaf, 0x49,
+    };
+    const uint16_t exp_saba_h[8] = {
+        0x8011, 0x7ff0, 0x8020, 0x5678,
+        0x2212, 0xb122, 0x7f00, 0x0304,
+    };
+    const uint16_t exp_uaba_h[8] = {
+        0x800f, 0x7ff0, 0x8020, 0x5678,
+        0x2212, 0x4efe, 0x7f00, 0x0304,
+    };
+    const uint32_t exp_saba_s[4] = {
+        0x80000011u, 0x7ffffff0u, 0x80000020u, 0x12345678u,
+    };
+    const uint32_t exp_uaba_s[4] = {
+        0x8000000fu, 0x7ffffff0u, 0x80000020u, 0x12345678u,
+    };
+    const uint64_t exp_saba_d[2] = {
+        0x000000000000000full, 0x7fffffffffffffefull,
+    };
+    const uint64_t exp_uaba_d[2] = {
+        0x0000000000000011ull, 0x7ffffffffffffff1ull,
+    };
+
+    test_arm64_sve2_narrow_run(0x4542c020, 1, 0, aba_init_h, aba_n_b,
+                               aba_m_b, exp_sabalb_h,
+                               sizeof(exp_sabalb_h));
+    test_arm64_sve2_narrow_run(0x4542c420, 1, 0, aba_init_h, aba_n_b,
+                               aba_m_b, exp_sabalt_h,
+                               sizeof(exp_sabalt_h));
+    test_arm64_sve2_narrow_run(0x4542c820, 1, 0, aba_init_h, aba_n_b,
+                               aba_m_b, exp_uabalb_h,
+                               sizeof(exp_uabalb_h));
+    test_arm64_sve2_narrow_run(0x4542cc20, 1, 0, aba_init_h, aba_n_b,
+                               aba_m_b, exp_uabalt_h,
+                               sizeof(exp_uabalt_h));
+    test_arm64_sve2_narrow_run(0x4582c020, 2, 1, aba_init_s, aba_n_h,
+                               aba_m_h, exp_sabalb_s,
+                               sizeof(exp_sabalb_s));
+    test_arm64_sve2_narrow_run(0x4582c420, 2, 1, aba_init_s, aba_n_h,
+                               aba_m_h, exp_sabalt_s,
+                               sizeof(exp_sabalt_s));
+    test_arm64_sve2_narrow_run(0x4582c820, 2, 1, aba_init_s, aba_n_h,
+                               aba_m_h, exp_uabalb_s,
+                               sizeof(exp_uabalb_s));
+    test_arm64_sve2_narrow_run(0x4582cc20, 2, 1, aba_init_s, aba_n_h,
+                               aba_m_h, exp_uabalt_s,
+                               sizeof(exp_uabalt_s));
+    test_arm64_sve2_narrow_run(0x45c2c020, 3, 2, aba_init_d, aba_n_s,
+                               aba_m_s, exp_sabalb_d,
+                               sizeof(exp_sabalb_d));
+    test_arm64_sve2_narrow_run(0x45c2c420, 3, 2, aba_init_d, aba_n_s,
+                               aba_m_s, exp_sabalt_d,
+                               sizeof(exp_sabalt_d));
+    test_arm64_sve2_narrow_run(0x45c2c820, 3, 2, aba_init_d, aba_n_s,
+                               aba_m_s, exp_uabalb_d,
+                               sizeof(exp_uabalb_d));
+    test_arm64_sve2_narrow_run(0x45c2cc20, 3, 2, aba_init_d, aba_n_s,
+                               aba_m_s, exp_uabalt_d,
+                               sizeof(exp_uabalt_d));
+
+    test_arm64_sve2_narrow_run(0x4502f820, 0, 0, aba_init_b, aba_n_b,
+                               aba_m_b, exp_saba_b, sizeof(exp_saba_b));
+    test_arm64_sve2_narrow_run(0x4542f820, 1, 1, aba_init_h, aba_n_h,
+                               aba_m_h, exp_saba_h, sizeof(exp_saba_h));
+    test_arm64_sve2_narrow_run(0x4582f820, 2, 2, aba_init_s, aba_n_s,
+                               aba_m_s, exp_saba_s, sizeof(exp_saba_s));
+    test_arm64_sve2_narrow_run(0x45c2f820, 3, 3, aba_init_d, aba_n_d,
+                               aba_m_d, exp_saba_d, sizeof(exp_saba_d));
+    test_arm64_sve2_narrow_run(0x4502fc20, 0, 0, aba_init_b, aba_n_b,
+                               aba_m_b, exp_uaba_b, sizeof(exp_uaba_b));
+    test_arm64_sve2_narrow_run(0x4542fc20, 1, 1, aba_init_h, aba_n_h,
+                               aba_m_h, exp_uaba_h, sizeof(exp_uaba_h));
+    test_arm64_sve2_narrow_run(0x4582fc20, 2, 2, aba_init_s, aba_n_s,
+                               aba_m_s, exp_uaba_s, sizeof(exp_uaba_s));
+    test_arm64_sve2_narrow_run(0x45c2fc20, 3, 3, aba_init_d, aba_n_d,
+                               aba_m_d, exp_uaba_d, sizeof(exp_uaba_d));
+}
+
+static void test_arm64_sve2_cadd_sqcadd(void)
+{
+    const uint8_t cadd_n_b[16] = {
+        0x10, 0x20, 0x7f, 0x7e, 0x80, 0x81, 0xff, 0x01,
+        0x40, 0xc0, 0x7f, 0x80, 0x01, 0xfe, 0x55, 0xaa,
+    };
+    const uint8_t cadd_m_b[16] = {
+        0x01, 0x02, 0x7f, 0x01, 0x80, 0x80, 0x7f, 0xff,
+        0xc0, 0x40, 0x01, 0x7f, 0xff, 0x02, 0xaa, 0x55,
+    };
+    const uint16_t cadd_n_h[8] = {
+        0x0010, 0x0020, 0x7fff, 0x7ffe,
+        0x8000, 0x8001, 0xffff, 0x0001,
+    };
+    const uint16_t cadd_m_h[8] = {
+        0x0001, 0x0002, 0x7fff, 0x0001,
+        0x8000, 0x8000, 0x7fff, 0xffff,
+    };
+    const uint32_t cadd_n_s[4] = {
+        0x00000010u, 0x00000020u, 0x7fffffffu, 0x80000000u,
+    };
+    const uint32_t cadd_m_s[4] = {
+        0x00000001u, 0x00000002u, 0x7fffffffu, 0x80000000u,
+    };
+    const uint64_t cadd_n_d[2] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint64_t cadd_m_d[2] = {
+        0x0000000000000001ull, 0xffffffffffffffffull,
+    };
+    const uint8_t exp_cadd90_b[16] = {
+        0x0e, 0x21, 0x7e, 0xfd, 0x00, 0x01, 0x00, 0x80,
+        0x00, 0x80, 0x00, 0x81, 0xff, 0xfd, 0x00, 0x54,
+    };
+    const uint8_t exp_cadd270_b[16] = {
+        0x12, 0x1f, 0x80, 0xff, 0x00, 0x01, 0xfe, 0x82,
+        0x80, 0x00, 0xfe, 0x7f, 0x03, 0xff, 0xaa, 0x00,
+    };
+    const uint8_t exp_sqcadd90_b[16] = {
+        0x0e, 0x21, 0x7e, 0x7f, 0x00, 0x80, 0x00, 0x7f,
+        0x00, 0x80, 0x00, 0x81, 0xff, 0xfd, 0x00, 0x80,
+    };
+    const uint8_t exp_sqcadd270_b[16] = {
+        0x12, 0x1f, 0x7f, 0xff, 0x80, 0x01, 0xfe, 0x82,
+        0x7f, 0x00, 0x7f, 0x80, 0x03, 0xff, 0x7f, 0x00,
+    };
+    const uint16_t exp_cadd90_h[8] = {
+        0x000e, 0x0021, 0x7ffe, 0xfffd,
+        0x0000, 0x0001, 0x0000, 0x8000,
+    };
+    const uint16_t exp_cadd270_h[8] = {
+        0x0012, 0x001f, 0x8000, 0xffff,
+        0x0000, 0x0001, 0xfffe, 0x8002,
+    };
+    const uint16_t exp_sqcadd90_h[8] = {
+        0x000e, 0x0021, 0x7ffe, 0x7fff,
+        0x0000, 0x8000, 0x0000, 0x7fff,
+    };
+    const uint16_t exp_sqcadd270_h[8] = {
+        0x0012, 0x001f, 0x7fff, 0xffff,
+        0x8000, 0x0001, 0xfffe, 0x8002,
+    };
+    const uint32_t exp_cadd90_s[4] = {
+        0x0000000eu, 0x00000021u, 0xffffffffu, 0xffffffffu,
+    };
+    const uint32_t exp_cadd270_s[4] = {
+        0x00000012u, 0x0000001fu, 0xffffffffu, 0x00000001u,
+    };
+    const uint32_t exp_sqcadd90_s[4] = {
+        0x0000000eu, 0x00000021u, 0x7fffffffu, 0xffffffffu,
+    };
+    const uint32_t exp_sqcadd270_s[4] = {
+        0x00000012u, 0x0000001fu, 0xffffffffu, 0x80000000u,
+    };
+    const uint64_t exp_cadd90_d[2] = {
+        0x8000000000000000ull, 0x8000000000000001ull,
+    };
+    const uint64_t exp_cadd270_d[2] = {
+        0x7ffffffffffffffeull, 0x7fffffffffffffffull,
+    };
+    const uint64_t exp_sqcadd90_d[2] = {
+        0x7fffffffffffffffull, 0x8000000000000001ull,
+    };
+    const uint64_t exp_sqcadd270_d[2] = {
+        0x7ffffffffffffffeull, 0x8000000000000000ull,
+    };
+
+    test_arm64_sve2_narrow_run(0x4500d840, 0, 0, cadd_n_b, cadd_n_b,
+                               cadd_m_b, exp_cadd90_b,
+                               sizeof(exp_cadd90_b));
+    test_arm64_sve2_narrow_run(0x4500dc40, 0, 0, cadd_n_b, cadd_n_b,
+                               cadd_m_b, exp_cadd270_b,
+                               sizeof(exp_cadd270_b));
+    test_arm64_sve2_narrow_run(0x4501d840, 0, 0, cadd_n_b, cadd_n_b,
+                               cadd_m_b, exp_sqcadd90_b,
+                               sizeof(exp_sqcadd90_b));
+    test_arm64_sve2_narrow_run(0x4501dc40, 0, 0, cadd_n_b, cadd_n_b,
+                               cadd_m_b, exp_sqcadd270_b,
+                               sizeof(exp_sqcadd270_b));
+
+    test_arm64_sve2_narrow_run(0x4540d840, 1, 1, cadd_n_h, cadd_n_h,
+                               cadd_m_h, exp_cadd90_h,
+                               sizeof(exp_cadd90_h));
+    test_arm64_sve2_narrow_run(0x4540dc40, 1, 1, cadd_n_h, cadd_n_h,
+                               cadd_m_h, exp_cadd270_h,
+                               sizeof(exp_cadd270_h));
+    test_arm64_sve2_narrow_run(0x4541d840, 1, 1, cadd_n_h, cadd_n_h,
+                               cadd_m_h, exp_sqcadd90_h,
+                               sizeof(exp_sqcadd90_h));
+    test_arm64_sve2_narrow_run(0x4541dc40, 1, 1, cadd_n_h, cadd_n_h,
+                               cadd_m_h, exp_sqcadd270_h,
+                               sizeof(exp_sqcadd270_h));
+
+    test_arm64_sve2_narrow_run(0x4580d840, 2, 2, cadd_n_s, cadd_n_s,
+                               cadd_m_s, exp_cadd90_s,
+                               sizeof(exp_cadd90_s));
+    test_arm64_sve2_narrow_run(0x4580dc40, 2, 2, cadd_n_s, cadd_n_s,
+                               cadd_m_s, exp_cadd270_s,
+                               sizeof(exp_cadd270_s));
+    test_arm64_sve2_narrow_run(0x4581d840, 2, 2, cadd_n_s, cadd_n_s,
+                               cadd_m_s, exp_sqcadd90_s,
+                               sizeof(exp_sqcadd90_s));
+    test_arm64_sve2_narrow_run(0x4581dc40, 2, 2, cadd_n_s, cadd_n_s,
+                               cadd_m_s, exp_sqcadd270_s,
+                               sizeof(exp_sqcadd270_s));
+
+    test_arm64_sve2_narrow_run(0x45c0d840, 3, 3, cadd_n_d, cadd_n_d,
+                               cadd_m_d, exp_cadd90_d,
+                               sizeof(exp_cadd90_d));
+    test_arm64_sve2_narrow_run(0x45c0dc40, 3, 3, cadd_n_d, cadd_n_d,
+                               cadd_m_d, exp_cadd270_d,
+                               sizeof(exp_cadd270_d));
+    test_arm64_sve2_narrow_run(0x45c1d840, 3, 3, cadd_n_d, cadd_n_d,
+                               cadd_m_d, exp_sqcadd90_d,
+                               sizeof(exp_sqcadd90_d));
+    test_arm64_sve2_narrow_run(0x45c1dc40, 3, 3, cadd_n_d, cadd_n_d,
+                               cadd_m_d, exp_sqcadd270_d,
+                               sizeof(exp_sqcadd270_d));
+}
+
+static void test_arm64_sve2_sqrdmla(void)
+{
+    const uint8_t sqrd_n_b[16] = {
+        0x80, 0x7f, 0x40, 0xc0, 0x11, 0xee, 0x55, 0xaa,
+        0x10, 0xf0, 0x33, 0xcd, 0x01, 0xff, 0x7e, 0x82,
+    };
+    const uint8_t sqrd_m_b[16] = {
+        0x80, 0x7f, 0x40, 0xc0, 0x33, 0x20, 0x55, 0xaa,
+        0x7f, 0x81, 0xcd, 0x33, 0x80, 0x80, 0x02, 0xfe,
+    };
+    const uint8_t sqrd_a_b[16] = {
+        0x7f, 0x80, 0x10, 0xf0, 0x01, 0xff, 0x55, 0xaa,
+        0x40, 0xc0, 0x02, 0xfe, 0x7e, 0x82, 0x00, 0xff,
+    };
+    const uint16_t sqrd_n_h[16] = {
+        0x8001, 0x7fff, 0x4000, 0xc000,
+        0x1111, 0xeeee, 0x5555, 0xaaaa,
+        0x1234, 0xedcc, 0x0101, 0xfefe,
+        0x7000, 0x9000, 0x2222, 0xdddd,
+    };
+    const uint16_t sqrd_m_h[16] = {
+        0x0002, 0x7fff, 0x4000, 0xc000,
+        0x3333, 0x2000, 0x5555, 0xaaaa,
+        0x1111, 0xeeee, 0x0100, 0xff00,
+        0x6000, 0xe000, 0x4444, 0xbbbb,
+    };
+    const uint16_t sqrd_a_h[16] = {
+        0x7fff, 0x8000, 0x0010, 0xfff0,
+        0x4000, 0xc000, 0x0001, 0xffff,
+        0x1357, 0x2468, 0x9753, 0xeca9,
+        0x4000, 0xc000, 0x7fff, 0x8000,
+    };
+    const uint32_t sqrd_n_s[8] = {
+        0x80000001u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x11111111u, 0xeeeeeeeeu, 0x00000001u, 0xffffffffu,
+    };
+    const uint32_t sqrd_m_s[8] = {
+        0x00000002u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+        0x22222222u, 0xddddddddu, 0x40000000u, 0x80000000u,
+    };
+    const uint32_t sqrd_a_s[8] = {
+        0x7fffffffu, 0x80000000u, 0x00000010u, 0xfffffff0u,
+        0xfffffff0u, 0x80000010u, 0x7fffff00u, 0x01020304u,
+    };
+    const uint64_t sqrd_n_d[4] = {
+        0x8000000000000001ull, 0x7fffffffffffffffull,
+        0x4000000000000000ull, 0xc000000000000000ull,
+    };
+    const uint64_t sqrd_m_d[4] = {
+        0x0000000000000002ull, 0x7fffffffffffffffull,
+        0x2222222222222222ull, 0x8000000000000000ull,
+    };
+    const uint64_t sqrd_a_d[4] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+        0x8000000000000020ull, 0x0123456789abcdefull,
+    };
+    const uint8_t exp_sqrdmlah_b[16] = {
+        0x7f, 0xfe, 0x30, 0x10, 0x08, 0xfb, 0x7f, 0xe4,
+        0x50, 0xd0, 0xee, 0xea, 0x7d, 0x83, 0x02, 0x01,
+    };
+    const uint8_t exp_sqrdmlsh_b[16] = {
+        0xff, 0x80, 0xf0, 0xd0, 0xfa, 0x04, 0x1d, 0x80,
+        0x30, 0xb0, 0x16, 0x12, 0x7f, 0x81, 0xfe, 0xfd,
+    };
+    const uint16_t exp_sqrdmlah_h[8] = {
+        0x7ffd, 0xfffe, 0x2010, 0x1ff0,
+        0x46d4, 0xbbbc, 0x38e4, 0x38e3,
+    };
+    const uint16_t exp_sqrdmlsh_h[8] = {
+        0x7fff, 0x8000, 0xe010, 0xdff0,
+        0x392c, 0xc445, 0xc71e, 0xc71b,
+    };
+    const uint32_t exp_sqrdmlah_s[4] = {
+        0x7ffffffdu, 0xfffffffeu, 0x20000010u, 0x1ffffff0u,
+    };
+    const uint32_t exp_sqrdmlsh_s[4] = {
+        0x7fffffffu, 0x80000000u, 0xe0000010u, 0xdffffff0u,
+    };
+    const uint64_t exp_sqrdmlah_d[2] = {
+        0x7ffffffffffffffdull, 0xfffffffffffffffeull,
+    };
+    const uint64_t exp_sqrdmlsh_d[2] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint16_t exp_sqrdmlah_idx_h[16] = {
+        0x5fff, 0xa000, 0x1010, 0xeff0,
+        0x4444, 0xbbbc, 0x1556, 0xeaaa,
+        0x0eca, 0x28f5, 0x9713, 0xecea,
+        0x2400, 0xdc00, 0x7777, 0x8889,
+    };
+    const uint16_t exp_sqrdmlsh_idx_h[16] = {
+        0x7fff, 0x8000, 0xf010, 0x0ff0,
+        0x3bbc, 0xc445, 0xeaac, 0x1555,
+        0x17e4, 0x1fdb, 0x9793, 0xec69,
+        0x5c00, 0xa400, 0x7fff, 0x8000,
+    };
+    const uint32_t exp_sqrdmlah_idx_s[8] = {
+        0x40000000u, 0xc0000000u, 0x20000010u, 0xdffffff0u,
+        0x08888879u, 0x80000000u, 0x7fffff01u, 0x01020304u,
+    };
+    const uint32_t exp_sqrdmlsh_idx_s[8] = {
+        0x7fffffffu, 0x80000000u, 0xe0000010u, 0x1ffffff0u,
+        0xf7777768u, 0x88888899u, 0x7fffff00u, 0x01020305u,
+    };
+    const uint64_t exp_sqrdmlah_idx_d[4] = {
+        0x0000000000000001ull, 0xfffffffffffffffeull,
+        0x8000000000000000ull, 0x4123456789abcdefull,
+    };
+    const uint64_t exp_sqrdmlsh_idx_d[4] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+        0xc000000000000020ull, 0xc123456789abcdefull,
+    };
+
+    test_arm64_sve2_narrow_run(0x44027020, 0, 0, sqrd_a_b, sqrd_n_b,
+                               sqrd_m_b, exp_sqrdmlah_b,
+                               sizeof(exp_sqrdmlah_b));
+    test_arm64_sve2_narrow_run(0x44027420, 0, 0, sqrd_a_b, sqrd_n_b,
+                               sqrd_m_b, exp_sqrdmlsh_b,
+                               sizeof(exp_sqrdmlsh_b));
+    test_arm64_sve2_narrow_run(0x44427020, 1, 1, sqrd_a_h, sqrd_n_h,
+                               sqrd_m_h, exp_sqrdmlah_h,
+                               sizeof(exp_sqrdmlah_h));
+    test_arm64_sve2_narrow_run(0x44427420, 1, 1, sqrd_a_h, sqrd_n_h,
+                               sqrd_m_h, exp_sqrdmlsh_h,
+                               sizeof(exp_sqrdmlsh_h));
+    test_arm64_sve2_narrow_run(0x44827020, 2, 2, sqrd_a_s, sqrd_n_s,
+                               sqrd_m_s, exp_sqrdmlah_s,
+                               sizeof(exp_sqrdmlah_s));
+    test_arm64_sve2_narrow_run(0x44827420, 2, 2, sqrd_a_s, sqrd_n_s,
+                               sqrd_m_s, exp_sqrdmlsh_s,
+                               sizeof(exp_sqrdmlsh_s));
+    test_arm64_sve2_narrow_run(0x44c27020, 3, 3, sqrd_a_d, sqrd_n_d,
+                               sqrd_m_d, exp_sqrdmlah_d,
+                               sizeof(exp_sqrdmlah_d));
+    test_arm64_sve2_narrow_run(0x44c27420, 3, 3, sqrd_a_d, sqrd_n_d,
+                               sqrd_m_d, exp_sqrdmlsh_d,
+                               sizeof(exp_sqrdmlsh_d));
+
+    test_arm64_sve2_narrow_run(0x446a1020, 1, 1, sqrd_a_h, sqrd_n_h,
+                               sqrd_m_h, exp_sqrdmlah_idx_h,
+                               sizeof(exp_sqrdmlah_idx_h));
+    test_arm64_sve2_narrow_run(0x446a1420, 1, 1, sqrd_a_h, sqrd_n_h,
+                               sqrd_m_h, exp_sqrdmlsh_idx_h,
+                               sizeof(exp_sqrdmlsh_idx_h));
+    test_arm64_sve2_narrow_run(0x44b21020, 2, 2, sqrd_a_s, sqrd_n_s,
+                               sqrd_m_s, exp_sqrdmlah_idx_s,
+                               sizeof(exp_sqrdmlah_idx_s));
+    test_arm64_sve2_narrow_run(0x44b21420, 2, 2, sqrd_a_s, sqrd_n_s,
+                               sqrd_m_s, exp_sqrdmlsh_idx_s,
+                               sizeof(exp_sqrdmlsh_idx_s));
+    test_arm64_sve2_narrow_run(0x44f21020, 3, 3, sqrd_a_d, sqrd_n_d,
+                               sqrd_m_d, exp_sqrdmlah_idx_d,
+                               sizeof(exp_sqrdmlah_idx_d));
+    test_arm64_sve2_narrow_run(0x44f21420, 3, 3, sqrd_a_d, sqrd_n_d,
+                               sqrd_m_d, exp_sqrdmlsh_idx_d,
+                               sizeof(exp_sqrdmlsh_idx_d));
+}
+
+static void test_arm64_sve2_complex_dot(void)
+{
+    const uint8_t n_b[16] = {
+        0x02, 0x03, 0xfe, 0x05, 0x7f, 0x80, 0x10, 0xf0,
+        0x11, 0xee, 0x40, 0xc0, 0x55, 0xaa, 0x01, 0xff,
+    };
+    const uint8_t m_b[16] = {
+        0x04, 0xfd, 0x06, 0xfa, 0x80, 0x7f, 0xf0, 0x10,
+        0x22, 0xdd, 0xc0, 0x40, 0xaa, 0x55, 0xff, 0x02,
+    };
+    const uint8_t a_b[16] = {
+        0x10, 0x20, 0x30, 0x40, 0x7f, 0x80, 0x01, 0xff,
+        0x55, 0xaa, 0x00, 0x7f, 0x80, 0x01, 0xfe, 0x02,
+    };
+    const uint16_t n_h[16] = {
+        0x0002, 0xfffd, 0x1234, 0xedcc,
+        0x7fff, 0x8000, 0x1111, 0xeeee,
+        0x4000, 0xc000, 0x0101, 0xfefe,
+        0x5555, 0xaaaa, 0x0001, 0xffff,
+    };
+    const uint16_t m_h[16] = {
+        0x0004, 0xfffb, 0x0100, 0xff00,
+        0x8000, 0x7fff, 0x2222, 0xdddd,
+        0x2000, 0xe000, 0x3333, 0xcccc,
+        0xaaaa, 0x5555, 0xffff, 0x0002,
+    };
+    const uint16_t a_h[16] = {
+        0x0010, 0xfff0, 0x1234, 0xedcc,
+        0x7fff, 0x8000, 0x0101, 0xfefe,
+        0x4000, 0xc000, 0x1357, 0xeca9,
+        0x8000, 0x7fff, 0x00ff, 0xff00,
+    };
+    const uint32_t n_s[8] = {
+        0x00000002u, 0xfffffffdu, 0x12345678u, 0xedcba988u,
+        0x7fffffffu, 0x80000000u, 0x11111111u, 0xeeeeeeeeu,
+    };
+    const uint32_t m_s[8] = {
+        0x00000004u, 0xfffffffbu, 0x01010101u, 0xfefefeffu,
+        0x80000000u, 0x7fffffffu, 0x22222222u, 0xddddddddu,
+    };
+    const uint32_t a_s[8] = {
+        0x00000010u, 0xfffffff0u, 0x12345678u, 0xedcba988u,
+        0x7fffffffu, 0x80000000u, 0x01020304u, 0xfefdfcfcu,
+    };
+    const uint64_t n_d[4] = {
+        0x0000000000000002ull, 0xfffffffffffffffdull,
+        0x123456789abcdef0ull, 0xedcba98765432110ull,
+    };
+    const uint64_t m_d[4] = {
+        0x0000000000000004ull, 0xfffffffffffffffbull,
+        0x0102030405060708ull, 0xfefdfcfbfaf9f8f7ull,
+    };
+    const uint64_t a_d[4] = {
+        0x0000000000000010ull, 0xfffffffffffffff0ull,
+        0x123456789abcdef0ull, 0xedcba98765432110ull,
+    };
+    const uint8_t exp_cmla_b_rot0[16] = {
+        0x18, 0x1a, 0x24, 0x4c, 0xff, 0x81, 0x01, 0xff,
+        0x97, 0x57, 0x00, 0x7f, 0xf2, 0x3a, 0xfd, 0x04,
+    };
+    const uint16_t exp_cmla_h_rot90[16] = {
+        0x0001, 0xffe4, 0xde34, 0xb9cc,
+        0xffff, 0x8000, 0x478b, 0x569a,
+        0x4000, 0xc000, 0x78ef, 0x5343,
+        0x638e, 0xb8e3, 0x0101, 0xff01,
+    };
+    const uint32_t exp_cmla_s_rot180[8] = {
+        0x00000008u, 0xfffffffau, 0xfd318800u, 0x02ce7800u,
+        0xffffffffu, 0x7fffffffu, 0xf2377cc2u, 0x1ed9944fu,
+    };
+    const uint64_t exp_cmla_d_rot270[4] = {
+        0x000000000000001full, 0xfffffffffffffffcull,
+        0xd5b249791f194560ull, 0x9f15460f4ee2a890ull,
+    };
+    const uint8_t exp_sqrdcmlah_b_rot0[16] = {
+        0x10, 0x20, 0x30, 0x40, 0x00, 0xfe, 0xff, 0x01,
+        0x5a, 0xa5, 0xe0, 0x7f, 0x80, 0x39, 0xfe, 0x02,
+    };
+    const uint16_t exp_sqrdcmlah_h_rot90[16] = {
+        0x0010, 0xfff0, 0x1210, 0xeda8,
+        0x7fff, 0x0000, 0xfc74, 0xfa71,
+        0x3000, 0xb000, 0x12f0, 0xec42,
+        0xb8e4, 0x7fff, 0x00ff, 0xff00,
+    };
+    const uint32_t exp_sqrdcmlah_s_rot180[8] = {
+        0x00000010u, 0xfffffff0u, 0x120fc93eu, 0xedf036c2u,
+        0x7fffffffu, 0x80000000u, 0xfc74ed66u, 0x038b129au,
+    };
+    const uint64_t exp_sqrdcmlah_d_rot270[4] = {
+        0x0000000000000010ull, 0xfffffffffffffff0ull,
+        0x12590864b23531dcull, 0xedf05b737cbb73fbull,
+    };
+    const uint32_t exp_cdot_s_rot0[8] = {
+        0x00000018u, 0xfffffffeu, 0x123456bcu, 0xedcba996u,
+        0x80003f7fu, 0x80003f80u, 0x01020304u, 0xfefdfcfcu,
+    };
+    const uint64_t exp_cdot_d_rot90[4] = {
+        0x0000000000000010ull, 0xfffffffffffffffaull,
+        0x1234567897dccc30ull, 0xedcba9876262fbb9ull,
+    };
+    const uint16_t exp_cmla_idx_h[16] = {
+        0x99a7, 0x998a, 0xad18, 0x9ae4,
+        0xffff, 0x8000, 0x478b, 0x569a,
+        0xc000, 0x0000, 0x155b, 0xedab,
+        0x2aac, 0xd555, 0x0101, 0xff01,
+    };
+    const uint32_t exp_cmla_idx_s[8] = {
+        0x03030313u, 0x030302f3u, 0x273724f0u, 0x02ce7800u,
+        0xffffffffu, 0x80000000u, 0x42ffbc7au, 0x2feaa560u,
+    };
+    const uint16_t exp_sqrdcmlah_idx_h[16] = {
+        0x0012, 0xffee, 0x2468, 0xdb98,
+        0x7fff, 0x8000, 0x1212, 0xeded,
+        0x6aab, 0x9556, 0x1402, 0xebfe,
+        0xb8e4, 0x471c, 0x0100, 0xfeff,
+    };
+    const uint32_t exp_sqrdcmlah_idx_s[8] = {
+        0x00000010u, 0xfffffff0u, 0x120fc93eu, 0xeda71c4eu,
+        0x5ddddddcu, 0x80000000u, 0xfc74ed65u, 0xfa70e75eu,
+    };
+    const uint32_t exp_cdot_idx_s[8] = {
+        0x0000000eu, 0xfffffff1u, 0x12345668u, 0xedcba996u,
+        0x8000117fu, 0x7fffee80u, 0x01020304u, 0xfefdfcfcu,
+    };
+    const uint64_t exp_cdot_idx_d[4] = {
+        0x000000000000000eull, 0xffffffffffffffeeull,
+        0x12345678987a2698ull, 0xedcba9876785d05dull,
+    };
+
+    test_arm64_sve2_narrow_run(0x44022020, 0, 0, a_b, n_b, m_b,
+                               exp_cmla_b_rot0, sizeof(exp_cmla_b_rot0));
+    test_arm64_sve2_narrow_run(0x44422420, 1, 1, a_h, n_h, m_h,
+                               exp_cmla_h_rot90,
+                               sizeof(exp_cmla_h_rot90));
+    test_arm64_sve2_narrow_run(0x44822820, 2, 2, a_s, n_s, m_s,
+                               exp_cmla_s_rot180,
+                               sizeof(exp_cmla_s_rot180));
+    test_arm64_sve2_narrow_run(0x44c22c20, 3, 3, a_d, n_d, m_d,
+                               exp_cmla_d_rot270,
+                               sizeof(exp_cmla_d_rot270));
+
+    test_arm64_sve2_narrow_run(0x44023020, 0, 0, a_b, n_b, m_b,
+                               exp_sqrdcmlah_b_rot0,
+                               sizeof(exp_sqrdcmlah_b_rot0));
+    test_arm64_sve2_narrow_run(0x44423420, 1, 1, a_h, n_h, m_h,
+                               exp_sqrdcmlah_h_rot90,
+                               sizeof(exp_sqrdcmlah_h_rot90));
+    test_arm64_sve2_narrow_run(0x44823820, 2, 2, a_s, n_s, m_s,
+                               exp_sqrdcmlah_s_rot180,
+                               sizeof(exp_sqrdcmlah_s_rot180));
+    test_arm64_sve2_narrow_run(0x44c23c20, 3, 3, a_d, n_d, m_d,
+                               exp_sqrdcmlah_d_rot270,
+                               sizeof(exp_sqrdcmlah_d_rot270));
+
+    test_arm64_sve2_narrow_run(0x44821020, 2, 2, a_s, n_s, m_s,
+                               exp_cdot_s_rot0, sizeof(exp_cdot_s_rot0));
+    test_arm64_sve2_narrow_run(0x44c21420, 3, 3, a_d, n_d, m_d,
+                               exp_cdot_d_rot90, sizeof(exp_cdot_d_rot90));
+
+    test_arm64_sve2_narrow_run(0x44ba6420, 1, 1, a_h, n_h, m_h,
+                               exp_cmla_idx_h, sizeof(exp_cmla_idx_h));
+    test_arm64_sve2_narrow_run(0x44f26c20, 2, 2, a_s, n_s, m_s,
+                               exp_cmla_idx_s, sizeof(exp_cmla_idx_s));
+    test_arm64_sve2_narrow_run(0x44b27820, 1, 1, a_h, n_h, m_h,
+                               exp_sqrdcmlah_idx_h,
+                               sizeof(exp_sqrdcmlah_idx_h));
+    test_arm64_sve2_narrow_run(0x44f27420, 2, 2, a_s, n_s, m_s,
+                               exp_sqrdcmlah_idx_s,
+                               sizeof(exp_sqrdcmlah_idx_s));
+    test_arm64_sve2_narrow_run(0x44ba4020, 2, 2, a_s, n_s, m_s,
+                               exp_cdot_idx_s, sizeof(exp_cdot_idx_s));
+    test_arm64_sve2_narrow_run(0x44f24c20, 3, 3, a_d, n_d, m_d,
+                               exp_cdot_idx_d, sizeof(exp_cdot_idx_d));
+}
+
+static void test_arm64_sve_i8mm(void)
+{
+    uc_engine *uc;
+    uint8_t invalid_code[8];
+    const uint8_t initial[32] = {
+        0x10, 0x00, 0x00, 0x00, 0xf0, 0xff, 0xff, 0xff,
+        0x80, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff,
+        0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+        0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+    };
+    const uint8_t n[32] = {
+        0x01, 0x7f, 0x80, 0xff, 0x10, 0xf0, 0x22, 0xdd,
+        0x40, 0xc0, 0x55, 0xaa, 0x7e, 0x82, 0x01, 0xff,
+        0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
+        0x7f, 0x80, 0x01, 0xff, 0xaa, 0x55, 0x10, 0xf0,
+    };
+    const uint8_t m[32] = {
+        0x02, 0xfe, 0x03, 0xfd, 0x80, 0x7f, 0x04, 0xfc,
+        0x11, 0xef, 0x66, 0x99, 0x08, 0xf8, 0x01, 0xff,
+        0x01, 0x02, 0x03, 0x04, 0x80, 0x81, 0x7e, 0x7f,
+        0x11, 0x22, 0xdd, 0xee, 0xff, 0x00, 0x55, 0xaa,
+    };
+    const uint8_t exp_usdot[32] = {
+        0x97, 0xfd, 0xff, 0xff, 0x14, 0x6c, 0x00, 0x00,
+        0x78, 0xd5, 0xff, 0xff, 0x61, 0xfe, 0xff, 0xff,
+        0x5d, 0x03, 0x00, 0x00, 0x12, 0x3a, 0x00, 0x00,
+        0x61, 0x07, 0x00, 0x00, 0x0a, 0xb4, 0xff, 0xff,
+    };
+    const uint8_t exp_sudot_idx[32] = {
+        0x19, 0x43, 0x00, 0x00, 0xb1, 0xea, 0xff, 0xff,
+        0x78, 0xb7, 0xff, 0xff, 0x08, 0x92, 0xff, 0xff,
+        0xe9, 0xc1, 0x00, 0x00, 0xfa, 0xc3, 0xff, 0xff,
+        0x61, 0xf7, 0xff, 0xff, 0x88, 0x04, 0x00, 0x00,
+    };
+    const uint8_t exp_usdot_idx[32] = {
+        0x19, 0xc4, 0xff, 0xff, 0xb1, 0xa5, 0xff, 0xff,
+        0x78, 0xd5, 0xff, 0xff, 0x08, 0x99, 0xff, 0xff,
+        0xe9, 0xf3, 0xff, 0xff, 0xfa, 0xf3, 0xff, 0xff,
+        0x61, 0x07, 0x00, 0x00, 0x88, 0x03, 0x00, 0x00,
+    };
+    const uint8_t exp_smmla[32] = {
+        0xbb, 0xee, 0xff, 0xff, 0x3e, 0xc6, 0xff, 0xff,
+        0x07, 0x86, 0xff, 0xff, 0x59, 0x54, 0x00, 0x00,
+        0x6d, 0x3f, 0x00, 0x00, 0x66, 0xee, 0xff, 0xff,
+        0x46, 0x00, 0x00, 0x00, 0x68, 0x02, 0x00, 0x00,
+    };
+    const uint8_t exp_ummla[32] = {
+        0xbb, 0xd4, 0x02, 0x00, 0x3e, 0x07, 0x03, 0x00,
+        0x07, 0xe3, 0x02, 0x00, 0x59, 0xbe, 0x02, 0x00,
+        0x6d, 0x93, 0x01, 0x00, 0x66, 0x44, 0x02, 0x00,
+        0x46, 0x04, 0x01, 0x00, 0x68, 0x55, 0x02, 0x00,
+    };
+    const uint8_t exp_usmmla[32] = {
+        0xbb, 0x69, 0x00, 0x00, 0x3e, 0xbc, 0xff, 0xff,
+        0x07, 0xfc, 0xff, 0xff, 0x59, 0xd3, 0xff, 0xff,
+        0x6d, 0x3d, 0x00, 0x00, 0x66, 0xec, 0xff, 0xff,
+        0x46, 0x05, 0x00, 0x00, 0x68, 0xbb, 0xff, 0xff,
+    };
+
+    test_arm64_sve2_narrow_run(0x44827820, 2, 2, initial, n, m,
+                               exp_usdot, sizeof(exp_usdot));
+    test_arm64_sve2_narrow_run(0x44b21c20, 2, 2, initial, n, m,
+                               exp_sudot_idx, sizeof(exp_sudot_idx));
+    test_arm64_sve2_narrow_run(0x44b21820, 2, 2, initial, n, m,
+                               exp_usdot_idx, sizeof(exp_usdot_idx));
+    test_arm64_sve2_narrow_run(0x45029820, 2, 2, initial, n, m,
+                               exp_smmla, sizeof(exp_smmla));
+    test_arm64_sve2_narrow_run(0x45c29820, 2, 2, initial, n, m,
+                               exp_ummla, sizeof(exp_ummla));
+    test_arm64_sve2_narrow_run(0x45829820, 2, 2, initial, n, m,
+                               exp_usmmla, sizeof(exp_usmmla));
+
+    test_arm64_emit32(invalid_code, 0, 0x2518e3e0);
+    test_arm64_emit32(invalid_code, 4, 0x44427820);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)invalid_code, sizeof(invalid_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_sve(uc);
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(invalid_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve_bf16(void)
+{
+    const uint32_t ptrue_b = 0x2518e3e0;
+    const uint16_t init_h[8] = {
+        0x1110, 0xaaaa, 0x1111, 0xbbbb,
+        0x1112, 0xcccc, 0x1113, 0xdddd,
+    };
+    const uint32_t source_s[4] = {
+        0x3f800000u, 0xc0000000u, 0x40400000u, 0x40800000u,
+    };
+    const uint32_t exp_bfcvt[4] = {
+        0x00003f80u, 0x0000c000u, 0x00004040u, 0x00004080u,
+    };
+    const uint16_t exp_bfcvtnt[8] = {
+        0x1110, 0x3f80, 0x1111, 0xc000,
+        0x1112, 0x4040, 0x1113, 0x4080,
+    };
+    const uint32_t init_s[4] = {
+        0x41200000u, 0x41a00000u, 0x41f00000u, 0x42200000u,
+    };
+    const uint16_t n_pair[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x3f80, 0x3f80, 0x4000, 0x4000,
+    };
+    const uint16_t m_pair[8] = {
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+        0x4000, 0x4040, 0x4080, 0x40a0,
+    };
+    const uint32_t exp_bfdot[4] = {
+        0x41d80000u, 0x42920000u, 0x420c0000u, 0x42680000u,
+    };
+    const uint32_t exp_bfdot_idx[4] = {
+        0x41d80000u, 0x426c0000u, 0x42240000u, 0x42780000u,
+    };
+    const uint16_t n_mmla[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+    };
+    const uint16_t m_mmla[8] = {
+        0x3f80, 0x3f80, 0x4000, 0x4000,
+        0x4040, 0x4040, 0x4080, 0x4080,
+    };
+    const uint32_t exp_bfmmla[4] = {
+        0x41d80000u, 0x42640000u, 0x428e0000u, 0x43050000u,
+    };
+    const uint16_t n_long[8] = {
+        0x3f80, 0x4000, 0x4040, 0x4080,
+        0x40a0, 0x40c0, 0x40e0, 0x4100,
+    };
+    const uint16_t m_long[8] = {
+        0x4000, 0x4040, 0x4080, 0x40a0,
+        0x40c0, 0x40e0, 0x4100, 0x4110,
+    };
+    const uint32_t exp_bfmlalb[4] = {
+        0x41400000u, 0x42000000u, 0x42700000u, 0x42c00000u,
+    };
+    const uint32_t exp_bfmlalt[4] = {
+        0x41800000u, 0x42200000u, 0x42900000u, 0x42e00000u,
+    };
+    const uint32_t exp_bfmlalb_idx[4] = {
+        0x41400000u, 0x41d00000u, 0x42200000u, 0x42580000u,
+    };
+    const uint32_t exp_bfmlalt_idx[4] = {
+        0x41600000u, 0x41e00000u, 0x42280000u, 0x42600000u,
+    };
+
+    test_arm64_sve2_fp_convert_run(ptrue_b, 0x658aa020, 2, 2,
+                                   init_s, source_s, exp_bfcvt,
+                                   sizeof(exp_bfcvt));
+    test_arm64_sve2_fp_convert_run(ptrue_b, 0x648aa020, 1, 2,
+                                   init_h, source_s, exp_bfcvtnt,
+                                   sizeof(exp_bfcvtnt));
+    test_arm64_sve2_narrow_run(0x64628020, 2, 2, init_s, n_pair,
+                               m_pair, exp_bfdot, sizeof(exp_bfdot));
+    test_arm64_sve2_narrow_run(0x64624020, 2, 2, init_s, n_pair,
+                               m_pair, exp_bfdot_idx,
+                               sizeof(exp_bfdot_idx));
+    test_arm64_sve2_narrow_run(0x6462e420, 2, 2, init_s, n_mmla,
+                               m_mmla, exp_bfmmla, sizeof(exp_bfmmla));
+    test_arm64_sve2_narrow_run(0x64e28020, 2, 1, init_s, n_long,
+                               m_long, exp_bfmlalb, sizeof(exp_bfmlalb));
+    test_arm64_sve2_narrow_run(0x64e28420, 2, 1, init_s, n_long,
+                               m_long, exp_bfmlalt, sizeof(exp_bfmlalt));
+    test_arm64_sve2_narrow_run(0x64e24020, 2, 1, init_s, n_long,
+                               m_long, exp_bfmlalb_idx,
+                               sizeof(exp_bfmlalb_idx));
+    test_arm64_sve2_narrow_run(0x64e24420, 2, 1, init_s, n_long,
+                               m_long, exp_bfmlalt_idx,
+                               sizeof(exp_bfmlalt_idx));
+
+    test_arm64_i8mm_expect_exception(0x658aa020, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sve_qperm_run(uint32_t insn, uint64_t zcr_len,
+                                     const uint8_t *n, const uint8_t *m,
+                                     const uint8_t *expected, size_t size)
+{
+    uc_engine *uc;
+    uint8_t code[20];
+    uint8_t got[64];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    size_t i;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, 0xa400a081);
+    test_arm64_emit32(code, 8, 0xa400a0a2);
+    test_arm64_emit32(code, 12, insn);
+    test_arm64_emit32(code, 16, 0xe400e0c0);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n, size));
+    OK(uc_mem_write(uc, x5, m, size));
+    test_arm64_mte_enable_sve_vq(uc, zcr_len);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, size));
+    for (i = 0; i < size; i++) {
+        TEST_CHECK(got[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve_qperm_expect_exception(uint32_t insn,
+                                                  uint64_t zcr_len)
+{
+    uc_engine *uc;
+    uint8_t code[8];
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_sve_vq(uc, zcr_len);
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code),
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve_f32mm_f64mm(void)
+{
+    const uint32_t init_s[8] = {
+        0x41200000u, 0x41a00000u, 0x41f00000u, 0x42200000u,
+        0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+    };
+    const uint32_t n_s[8] = {
+        0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+        0xbf800000u, 0x3f000000u, 0x40000000u, 0xbf000000u,
+    };
+    const uint32_t m_s[8] = {
+        0x40a00000u, 0x40c00000u, 0x40e00000u, 0x41000000u,
+        0x40800000u, 0xc0000000u, 0x3f800000u, 0x40400000u,
+    };
+    const uint32_t exp_s[8] = {
+        0x41d80000u, 0x422c0000u, 0x428a0000u, 0x42ba0000u,
+        0xc0800000u, 0x40200000u, 0x41400000u, 0x40900000u,
+    };
+    const uint64_t init_d[4] = {
+        0x4024000000000000ull, 0x4034000000000000ull,
+        0x403e000000000000ull, 0x4044000000000000ull,
+    };
+    const uint64_t n_d[4] = {
+        0x3ff0000000000000ull, 0x4000000000000000ull,
+        0x4008000000000000ull, 0x4010000000000000ull,
+    };
+    const uint64_t m_d[4] = {
+        0x4014000000000000ull, 0x4018000000000000ull,
+        0x401c000000000000ull, 0x4020000000000000ull,
+    };
+    const uint64_t exp_d[4] = {
+        0x403b000000000000ull, 0x4045800000000000ull,
+        0x4051400000000000ull, 0x4057400000000000ull,
+    };
+    uint8_t n_q[64];
+    uint8_t m_q[64];
+    uint8_t exp_q[64];
+    int q, i;
+
+    test_arm64_sve2_narrow_run(0x64a2e420, 2, 2, init_s, n_s, m_s,
+                               exp_s, sizeof(exp_s));
+    test_arm64_sve2_narrow_run(0x64e2e420, 3, 3, init_d, n_d, m_d,
+                               exp_d, sizeof(exp_d));
+
+    for (q = 0; q < 4; q++) {
+        for (i = 0; i < 16; i++) {
+            n_q[q * 16 + i] = 0x10 + q * 0x10 + i;
+            m_q[q * 16 + i] = 0x80 + q * 0x10 + i;
+        }
+    }
+
+    memcpy(exp_q, n_q + 0x00, 16);
+    memcpy(exp_q + 0x10, m_q + 0x00, 16);
+    memcpy(exp_q + 0x20, n_q + 0x10, 16);
+    memcpy(exp_q + 0x30, m_q + 0x10, 16);
+    test_arm64_sve_qperm_run(0x05a20020, 3, n_q, m_q, exp_q, 64);
+
+    memcpy(exp_q, n_q + 0x20, 16);
+    memcpy(exp_q + 0x10, m_q + 0x20, 16);
+    memcpy(exp_q + 0x20, n_q + 0x30, 16);
+    memcpy(exp_q + 0x30, m_q + 0x30, 16);
+    test_arm64_sve_qperm_run(0x05a20420, 3, n_q, m_q, exp_q, 64);
+
+    memcpy(exp_q, n_q + 0x00, 16);
+    memcpy(exp_q + 0x10, n_q + 0x20, 16);
+    memcpy(exp_q + 0x20, m_q + 0x00, 16);
+    memcpy(exp_q + 0x30, m_q + 0x20, 16);
+    test_arm64_sve_qperm_run(0x05a20820, 3, n_q, m_q, exp_q, 64);
+
+    memcpy(exp_q, n_q + 0x10, 16);
+    memcpy(exp_q + 0x10, n_q + 0x30, 16);
+    memcpy(exp_q + 0x20, m_q + 0x10, 16);
+    memcpy(exp_q + 0x30, m_q + 0x30, 16);
+    test_arm64_sve_qperm_run(0x05a20c20, 3, n_q, m_q, exp_q, 64);
+
+    memcpy(exp_q, n_q + 0x00, 16);
+    memcpy(exp_q + 0x10, m_q + 0x00, 16);
+    memcpy(exp_q + 0x20, n_q + 0x20, 16);
+    memcpy(exp_q + 0x30, m_q + 0x20, 16);
+    test_arm64_sve_qperm_run(0x05a21820, 3, n_q, m_q, exp_q, 64);
+
+    memcpy(exp_q, n_q + 0x10, 16);
+    memcpy(exp_q + 0x10, m_q + 0x10, 16);
+    memcpy(exp_q + 0x20, n_q + 0x30, 16);
+    memcpy(exp_q + 0x30, m_q + 0x30, 16);
+    test_arm64_sve_qperm_run(0x05a21c20, 3, n_q, m_q, exp_q, 64);
+
+    memset(exp_q, 0, sizeof(exp_q));
+    memcpy(exp_q, n_q + 0x10, 16);
+    memcpy(exp_q + 0x10, m_q + 0x10, 16);
+    test_arm64_sve_qperm_run(0x05a20420, 2, n_q, m_q, exp_q, 48);
+
+    memset(exp_q, 0, sizeof(exp_q));
+    memcpy(exp_q, n_q + 0x00, 16);
+    memcpy(exp_q + 0x10, n_q + 0x20, 16);
+    memcpy(exp_q + 0x20, m_q + 0x10, 16);
+    test_arm64_sve_qperm_run(0x05a20820, 2, n_q, m_q, exp_q, 48);
+
+    memset(exp_q, 0, sizeof(exp_q));
+    memcpy(exp_q, n_q + 0x10, 16);
+    memcpy(exp_q + 0x10, m_q + 0x10, 16);
+    test_arm64_sve_qperm_run(0x05a21c20, 2, n_q, m_q, exp_q, 48);
+
+    test_arm64_sve_qperm_expect_exception(0x05a20020, 0);
+    test_arm64_i8mm_expect_exception(0x64a2e420, UC_CPU_ARM64_A72);
+    test_arm64_i8mm_expect_exception(0x64e2e420, UC_CPU_ARM64_A72);
+    test_arm64_i8mm_expect_exception(0x05a20020, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_foundation(void)
+{
+    const uint32_t SVCR[5] = { 3, 3, 4, 2, 2 };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t TPIDR2_EL0[5] = { 3, 3, 13, 0, 5 };
+    uint8_t code[52];
+    uc_engine *uc;
+    uint64_t x0;
+    uint64_t x1;
+    uint64_t x2;
+    uint64_t x3;
+    uint64_t x4;
+    uint64_t x5;
+    uint64_t x6;
+    uint64_t value;
+
+    test_arm64_emit32(code, 0, 0xd51812c4);  /* msr smcr_el1, x4 */
+    test_arm64_emit32(code, 4, 0xd53812c0);  /* mrs x0, smcr_el1 */
+    test_arm64_emit32(code, 8, 0xd51b4244);  /* msr svcr, x4 */
+    test_arm64_emit32(code, 12, 0xd53b4241); /* mrs x1, svcr */
+    test_arm64_emit32(code, 16, 0xd503467f); /* smstop smza */
+    test_arm64_emit32(code, 20, 0xd503437f); /* smstart sm */
+    test_arm64_emit32(code, 24, 0xd53b4242); /* mrs x2, svcr */
+    test_arm64_emit32(code, 28, 0xd503457f); /* smstart za */
+    test_arm64_emit32(code, 32, 0xd53b4243); /* mrs x3, svcr */
+    test_arm64_emit32(code, 36, 0xd503427f); /* smstop sm */
+    test_arm64_emit32(code, 40, 0xd53b4245); /* mrs x5, svcr */
+    test_arm64_emit32(code, 44, 0xd503447f); /* smstop za */
+    test_arm64_emit32(code, 48, 0xd53b4246); /* mrs x6, svcr */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SVCR) == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SMCR_EL1) == 0x80000001);
+    value = 0x123456789abcdef0ULL;
+    test_arm64_pauth_cp_reg_write(uc, TPIDR2_EL0, value);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TPIDR2_EL0) == value);
+    x4 = 0xffffffffffffffffULL;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(x0 == 0x8000000f);
+    TEST_CHECK(x1 == 3);
+    TEST_CHECK(x2 == 1);
+    TEST_CHECK(x3 == 3);
+    TEST_CHECK(x5 == 2);
+    TEST_CHECK(x6 == 0);
+    OK(uc_close(uc));
+
+    test_arm64_i8mm_expect_exception(0xd503437f, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_svlength(void)
+{
+    const uint32_t SVCR[5] = { 3, 3, 4, 2, 2 };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[28];
+    uint8_t no_fa64_code[20];
+    uc_engine *uc;
+    uint64_t x0;
+    uint64_t x1;
+    uint64_t x2;
+    uint64_t x3;
+    uint64_t x4;
+    uint64_t x5;
+    uint64_t x6;
+    uint64_t pc;
+    uint64_t smcr_el1;
+    uint64_t svcr;
+    uc_err err;
+
+    test_arm64_emit32(code, 0, 0xd51812c4);  /* msr smcr_el1, x4 */
+    test_arm64_emit32(code, 4, 0x04bf5820);  /* rdsvl x0, #1 */
+    test_arm64_emit32(code, 8, 0x04bf5fe1);  /* rdsvl x1, #-1 */
+    test_arm64_emit32(code, 12, 0x04235842); /* addsvl x2, x3, #2 */
+    test_arm64_emit32(code, 16, 0x04635845); /* addspl x5, x3, #2 */
+    test_arm64_emit32(code, 20, 0xd503437f); /* smstart sm */
+    test_arm64_emit32(code, 24, 0x04bf5026); /* rdvl x6, #1 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    x3 = 0x1000;
+    x4 = 0x80000003;
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, x4);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, x4);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0);
+    OK(uc_reg_read(uc, UC_ARM64_REG_PC, &pc));
+    smcr_el1 = test_arm64_pauth_cp_reg_read(uc, SMCR_EL1);
+    svcr = test_arm64_pauth_cp_reg_read(uc, SVCR);
+    TEST_CHECK_(err == UC_ERR_OK,
+                "err=%u pc=0x%llx smcr_el1=0x%llx svcr=0x%llx",
+                (unsigned)err, (unsigned long long)pc,
+                (unsigned long long)smcr_el1, (unsigned long long)svcr);
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(x0 == 64);
+    TEST_CHECK(x1 == (uint64_t)-64);
+    TEST_CHECK(x2 == 0x1080);
+    TEST_CHECK(x5 == 0x1010);
+    TEST_CHECK_(x6 == 64, "x6 = 0x%llx", (unsigned long long)x6);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(no_fa64_code, 0, 0xd51812c4);  /* msr smcr_el1, x4 */
+    test_arm64_emit32(no_fa64_code, 4, 0xd503437f);  /* smstart sm */
+    test_arm64_emit32(no_fa64_code, 8, 0x04bf5020);  /* rdvl x0, #1 */
+    test_arm64_emit32(no_fa64_code, 12, 0x0e013c01); /* umov w1, v0.b[0] */
+    test_arm64_emit32(no_fa64_code, 16, 0x4e010c00); /* dup v0.16b, w0 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)no_fa64_code, sizeof(no_fa64_code),
+                    UC_CPU_ARM64_MAX);
+    x4 = 3;
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, x4);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, x4);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    err = uc_emu_start(uc, code_start,
+                       code_start + sizeof(no_fa64_code), 0, 0);
+    OK(uc_reg_read(uc, UC_ARM64_REG_PC, &pc));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u pc=0x%llx",
+                (unsigned)err, (unsigned long long)pc);
+    TEST_CHECK(x0 == 64);
+    TEST_CHECK(pc == code_start + 16);
+    OK(uc_close(uc));
+
+    test_arm64_i8mm_expect_exception(0x04bf5820, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_nonstreaming_sve_ffr_one(uint32_t insn)
+{
+    const uint32_t CPACR_EL1[5] = { 3, 0, 1, 0, 2 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[4];
+    uint8_t streaming_code[12];
+    uc_engine *uc;
+    uint64_t x4;
+    uint64_t pc;
+    uc_err err;
+
+    test_arm64_emit32(code, 0, insn);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_pauth_cp_reg_write(uc, CPACR_EL1,
+                                  (3ULL << 16) | (3ULL << 20) |
+                                  (3ULL << 24));
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == UC_ERR_OK, "insn=0x%x err=%u", insn, (unsigned)err);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(streaming_code, 0, 0xd51812c4);  /* msr smcr_el1, x4 */
+    test_arm64_emit32(streaming_code, 4, 0xd503437f);  /* smstart sm */
+    test_arm64_emit32(streaming_code, 8, insn);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)streaming_code, sizeof(streaming_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_pauth_cp_reg_write(uc, CPACR_EL1,
+                                  (3ULL << 16) | (3ULL << 20) |
+                                  (3ULL << 24));
+    x4 = 3;
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, x4);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, x4);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    err = uc_emu_start(uc, code_start, code_start + sizeof(streaming_code),
+                       0, 0);
+    OK(uc_reg_read(uc, UC_ARM64_REG_PC, &pc));
+    TEST_CHECK_(err == UC_ERR_EXCEPTION, "insn=0x%x err=%u pc=0x%llx",
+                insn, (unsigned)err, (unsigned long long)pc);
+    TEST_CHECK_(pc == code_start + 8,
+                "insn=0x%x pc=0x%llx", insn, (unsigned long long)pc);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_nonstreaming_sve_ffr(void)
+{
+    test_arm64_sme_nonstreaming_sve_ffr_one(0x252c9000); /* setffr */
+    test_arm64_sme_nonstreaming_sve_ffr_one(0x2518f000); /* rdffr p0.b,p0/z */
+    test_arm64_sme_nonstreaming_sve_ffr_one(0x2519f000); /* rdffr p0.b */
+    test_arm64_sme_nonstreaming_sve_ffr_one(0x25289000); /* wrffr p0.b */
+}
+
+static void test_arm64_sme_nonstreaming_sve_setup(uc_engine *uc,
+                                                  uint64_t zcr_len,
+                                                  bool memory)
+{
+    const uint32_t CPACR_EL1[5] = { 3, 0, 1, 0, 2 };
+    uint8_t mem[0x400];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0;
+    uint64_t x6 = 0x40000;
+    uint64_t x7 = 8;
+    uint64_t x8 = 0x40100;
+    uint64_t x9 = 0x40200;
+    uint64_t x10 = 3;
+    int i;
+
+    test_arm64_mte_enable_sve_vq(uc, zcr_len);
+    test_arm64_pauth_cp_reg_write(uc, CPACR_EL1,
+                                  (3ULL << 16) | (3ULL << 20) |
+                                  (3ULL << 24));
+
+    if (memory) {
+        for (i = 0; i < (int)sizeof(mem); i++) {
+            mem[i] = (uint8_t)i;
+        }
+        OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+        OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    }
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X10, &x10));
+}
+
+static void test_arm64_sme_nonstreaming_sve_one(const char *name,
+                                                uint32_t insn,
+                                                uint64_t zcr_len,
+                                                bool ptrue, bool memory,
+                                                bool streaming_ok)
+{
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[8];
+    uint8_t streaming_code[20];
+    uint64_t smcr = 3;
+    uint64_t pc;
+    size_t off = 0;
+    size_t target_off;
+    uc_engine *uc;
+    uc_err err;
+
+    if (ptrue) {
+        test_arm64_emit32(code, off, 0x2518e3e0); /* ptrue p0.b */
+        off += 4;
+    }
+    test_arm64_emit32(code, off, insn);
+    off += 4;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    off, UC_CPU_ARM64_MAX);
+    test_arm64_sme_nonstreaming_sve_setup(uc, zcr_len, memory);
+    err = uc_emu_start(uc, code_start, code_start + off, 0, 0);
+    TEST_CHECK_(err == UC_ERR_OK, "%s insn=0x%x err=%u", name, insn,
+                (unsigned)err);
+    OK(uc_close(uc));
+
+    off = 0;
+    if (ptrue) {
+        test_arm64_emit32(streaming_code, off, 0x2518e3e0); /* ptrue p0.b */
+        off += 4;
+    }
+    test_arm64_emit32(streaming_code, off, 0xd51812ca); /* msr smcr_el1,x10 */
+    off += 4;
+    test_arm64_emit32(streaming_code, off, 0xd503437f); /* smstart sm */
+    off += 4;
+    target_off = off;
+    test_arm64_emit32(streaming_code, off, insn);
+    off += 4;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)streaming_code, off, UC_CPU_ARM64_MAX);
+    test_arm64_sme_nonstreaming_sve_setup(uc, zcr_len, memory);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    err = uc_emu_start(uc, code_start, code_start + off, 0, 0);
+    OK(uc_reg_read(uc, UC_ARM64_REG_PC, &pc));
+    if (streaming_ok) {
+        TEST_CHECK_(err == UC_ERR_OK, "%s insn=0x%x err=%u pc=0x%llx",
+                    name, insn, (unsigned)err, (unsigned long long)pc);
+    } else {
+        TEST_CHECK_(err == UC_ERR_EXCEPTION,
+                    "%s insn=0x%x err=%u pc=0x%llx", name, insn,
+                    (unsigned)err, (unsigned long long)pc);
+        TEST_CHECK_(pc == code_start + target_off,
+                    "%s insn=0x%x pc=0x%llx", name, insn,
+                    (unsigned long long)pc);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_nonstreaming_sve_misc(void)
+{
+    static const struct {
+        const char *name;
+        uint32_t insn;
+        uint64_t zcr_len;
+        bool ptrue;
+        bool memory;
+    } nonstreaming[] = {
+        { "bext", 0x4502b020, 0, true, false },
+        { "adr_s32", 0x0420a000, 0, false, false },
+        { "fexpa_s", 0x04a0b800, 0, false, false },
+        { "ftssel_s", 0x04a0b000, 0, false, false },
+        { "zip1_q", 0x05a20020, 3, false, false },
+        { "compact_s", 0x05a18000, 0, true, false },
+        { "histcnt_s", 0x45a2c020, 0, true, false },
+        { "histseg", 0x4522a020, 0, false, false },
+        { "aese", 0x4522e040, 0, false, false },
+        { "smmla", 0x45029820, 0, false, false },
+        { "bfmmla", 0x6462e420, 0, false, false },
+        { "fmmla_s", 0x64a2e420, 0, false, false },
+        { "ftmad_s", 0x65908000, 0, false, false },
+        { "fadda_s", 0x65982000, 0, true, false },
+        { "ftsmul_s", 0x65800c00, 0, false, false },
+        { "ldff1b", 0xa4056081, 0, true, true },
+        { "ldnf1b", 0xa410a081, 0, true, true },
+        { "ld1row", 0xa52120c1, 3, true, true },
+        { "ldnt1b", 0x8404a041, 0, true, true },
+        { "stnt1b", 0xe4442041, 0, true, true },
+        { "prf_ns", 0x8400e000, 0, false, false },
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(nonstreaming) / sizeof(nonstreaming[0]); i++) {
+        test_arm64_sme_nonstreaming_sve_one(nonstreaming[i].name,
+                                            nonstreaming[i].insn,
+                                            nonstreaming[i].zcr_len,
+                                            nonstreaming[i].ptrue,
+                                            nonstreaming[i].memory, false);
+    }
+
+    test_arm64_sme_nonstreaming_sve_one("prf_contiguous", 0x85c00000,
+                                        0, false, false, true);
+}
+
+enum {
+    TEST_ARM64_SME_OP_S = 2,
+    TEST_ARM64_SME_OP_D = 3,
+    TEST_ARM64_SME_OP_Q = 4,
+};
+
+static uint32_t test_arm64_sme_mova_esz_imm_insn(int esz, bool to_vec,
+                                                 bool vertical, int zr,
+                                                 int za_imm)
+{
+    uint32_t insn = 0xc0000000;
+
+    if (esz == TEST_ARM64_SME_OP_Q) {
+        insn |= 3U << 22;
+        insn |= 1U << 16;
+    } else {
+        insn |= (uint32_t)esz << 22;
+    }
+    if (to_vec) {
+        insn |= 1U << 17;
+        insn |= (uint32_t)za_imm << 5;
+        insn |= (uint32_t)zr;
+    } else {
+        insn |= (uint32_t)zr << 5;
+        insn |= (uint32_t)za_imm;
+    }
+    if (vertical) {
+        insn |= 1U << 15;
+    }
+    return insn;
+}
+
+static uint32_t test_arm64_sme_mova_esz_insn(int esz, bool to_vec,
+                                             bool vertical, int zr)
+{
+    return test_arm64_sme_mova_esz_imm_insn(esz, to_vec, vertical, zr, 0);
+}
+
+static uint32_t test_arm64_sme_mova_insn(bool to_vec, bool vertical, int zr)
+{
+    return test_arm64_sme_mova_esz_insn(0, to_vec, vertical, zr);
+}
+
+static uint32_t test_arm64_sme_adda_insn(int esz, bool vertical)
+{
+    uint32_t insn;
+
+    if (esz == 2) {
+        insn = 0xc0900000;
+    } else {
+        insn = 0xc0d00000;
+    }
+    if (vertical) {
+        insn |= 1U << 16;
+    }
+    insn |= 1U << 5;
+    return insn;
+}
+
+static uint32_t test_arm64_sme_ldstr_insn(bool store, int rv, int rn, int imm)
+{
+    uint32_t insn = 0xe1000000;
+
+    if (store) {
+        insn |= 1U << 21;
+    }
+    insn |= (uint32_t)(rv - 12) << 13;
+    insn |= (uint32_t)rn << 5;
+    insn |= (uint32_t)imm;
+    return insn;
+}
+
+enum {
+    TEST_ARM64_SME_FMOPA_S,
+    TEST_ARM64_SME_FMOPA_D,
+    TEST_ARM64_SME_BFMOPA,
+    TEST_ARM64_SME_FMOPA_H,
+};
+
+static uint32_t test_arm64_sme_imopa_insn(int esz, int kind, bool sub)
+{
+    uint32_t insn = 0xa0000000;
+
+    insn |= (uint32_t)esz << 22;
+    if (kind & 2) {
+        insn |= 1U << 24;
+    }
+    if (kind & 1) {
+        insn |= 1U << 21;
+    }
+    insn |= 2U << 16;
+    insn |= 1U << 5;
+    if (sub) {
+        insn |= 1U << 4;
+    }
+    return insn;
+}
+
+static uint32_t test_arm64_sme_fpout_insn(int kind, bool sub)
+{
+    uint32_t insn;
+
+    switch (kind) {
+    case TEST_ARM64_SME_FMOPA_S:
+        insn = 0x80800000;
+        break;
+    case TEST_ARM64_SME_FMOPA_D:
+        insn = 0x80c00000;
+        break;
+    case TEST_ARM64_SME_BFMOPA:
+        insn = 0x81800000;
+        break;
+    default:
+        insn = 0x81a00000;
+        break;
+    }
+
+    insn |= 2U << 16;
+    insn |= 1U << 5;
+    if (sub) {
+        insn |= 1U << 4;
+    }
+    return insn;
+}
+
+static uint32_t test_arm64_sme_ldst1_insn(int esz, bool store, bool vertical,
+                                          int rn, int rm, int za_imm)
+{
+    uint32_t insn;
+
+    if (esz == TEST_ARM64_SME_OP_Q) {
+        insn = 0xe1c00000;
+    } else {
+        insn = 0xe0000000 | ((uint32_t)esz << 22);
+    }
+    if (store) {
+        insn |= 1U << 21;
+    }
+    if (vertical) {
+        insn |= 1U << 15;
+    }
+    insn |= (uint32_t)rm << 16;
+    insn |= (uint32_t)rn << 5;
+    insn |= (uint32_t)za_imm;
+    return insn;
+}
+
+static uint32_t test_arm64_sve_ldr_p_insn(int pd, int rn)
+{
+    return 0x85800000 | ((uint32_t)rn << 5) | (uint32_t)pd;
+}
+
+static uint32_t test_arm64_sve_str_p_insn(int pd, int rn)
+{
+    return 0xe5800000 | ((uint32_t)rn << 5) | (uint32_t)pd;
+}
+
+static uint32_t test_arm64_sme_psel_insn(int esz, int pd, int pn, int pm,
+                                         int rv, int imm)
+{
+    uint32_t insn = 0x25000000 | (1U << 21) | (1U << 14);
+
+    insn |= (uint32_t)pd;
+    insn |= (uint32_t)pm << 5;
+    insn |= (uint32_t)pn << 10;
+    insn |= (uint32_t)(rv - 12) << 16;
+
+    switch (esz) {
+    case 0:
+        insn |= 1U << 18;
+        insn |= (uint32_t)(imm >> 2) << 22;
+        insn |= (uint32_t)(imm & 3) << 19;
+        break;
+    case 1:
+        insn |= 1U << 19;
+        insn |= (uint32_t)(imm >> 1) << 22;
+        insn |= (uint32_t)(imm & 1) << 20;
+        break;
+    case 2:
+        insn |= 1U << 20;
+        insn |= (uint32_t)imm << 22;
+        break;
+    default:
+        insn |= 1U << 22;
+        insn |= (uint32_t)imm << 23;
+        break;
+    }
+    return insn;
+}
+
+static void test_arm64_sme_za_only_expect_exception(uint32_t insn)
+{
+    const uint32_t SVCR[5] = { 3, 3, 4, 2, 2 };
+    uint8_t code[8];
+    uc_engine *uc;
+    uc_err err;
+
+    test_arm64_emit32(code, 0, 0xd503457f); /* smstart za */
+    test_arm64_emit32(code, 4, insn);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0);
+    TEST_CHECK(err == UC_ERR_EXCEPTION);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SVCR) == 2);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_zero_mova(void)
+{
+    const uint32_t SVCR[5] = { 3, 3, 4, 2, 2 };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t roundtrip_code[24];
+    uint8_t zero_code[28];
+    uint8_t partial_zero_code[24];
+    uint8_t input[32];
+    uint8_t partial_input[64];
+    uint8_t output[32];
+    uint8_t partial_output[64];
+    uint8_t expected_zero[32];
+    uint8_t expected_partial[64];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x6 = 0x40200;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(i * 7 + 3);
+    }
+    for (i = 0; i < sizeof(partial_input); i++) {
+        partial_input[i] = (uint8_t)(i * 5 + 1);
+    }
+    memset(expected_zero, 0, sizeof(expected_zero));
+    memcpy(expected_partial, partial_input, sizeof(expected_partial));
+    memset(expected_partial, 0, sizeof(input));
+
+    test_arm64_emit32(roundtrip_code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(roundtrip_code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(roundtrip_code, 8, 0xa400a081);  /* ld1b z1.b */
+    test_arm64_emit32(roundtrip_code, 12,
+                      test_arm64_sme_mova_insn(false, true, 1));
+    test_arm64_emit32(roundtrip_code, 16,
+                      test_arm64_sme_mova_insn(true, true, 0));
+    test_arm64_emit32(roundtrip_code, 20, 0xe400e0c0); /* st1b z0.b */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)roundtrip_code, sizeof(roundtrip_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(roundtrip_code),
+                    0, 0));
+    OK(uc_mem_read(uc, x6, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, input, sizeof(output)) == 0);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(zero_code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(zero_code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(zero_code, 8, 0xa400a081);  /* ld1b z1.b */
+    test_arm64_emit32(zero_code, 12,
+                      test_arm64_sme_mova_insn(false, true, 1));
+    test_arm64_emit32(zero_code, 16, 0xc00800ff); /* zero {za} */
+    test_arm64_emit32(zero_code, 20,
+                      test_arm64_sme_mova_insn(true, true, 0));
+    test_arm64_emit32(zero_code, 24, 0xe400e0c0); /* st1b z0.b */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)zero_code, sizeof(zero_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(zero_code), 0, 0));
+    OK(uc_mem_read(uc, x6, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, expected_zero, sizeof(output)) == 0);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(partial_zero_code, 0, 0xd503457f);  /* smstart za */
+    test_arm64_emit32(partial_zero_code, 4,
+                      test_arm64_sme_ldstr_insn(false, 12, 4, 0));
+    test_arm64_emit32(partial_zero_code, 8,
+                      test_arm64_sme_ldstr_insn(false, 12, 4, 1));
+    test_arm64_emit32(partial_zero_code, 12, 0xc0080001); /* zero {za0.h} */
+    test_arm64_emit32(partial_zero_code, 16,
+                      test_arm64_sme_ldstr_insn(true, 12, 6, 0));
+    test_arm64_emit32(partial_zero_code, 20,
+                      test_arm64_sme_ldstr_insn(true, 12, 6, 1));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)partial_zero_code,
+                    sizeof(partial_zero_code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, partial_input, sizeof(partial_input)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start,
+                    code_start + sizeof(partial_zero_code), 0, 0));
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SVCR) == 2);
+    OK(uc_mem_read(uc, x6, partial_output, sizeof(partial_output)));
+    TEST_CHECK(memcmp(partial_output, expected_partial,
+                      sizeof(partial_output)) == 0);
+    OK(uc_close(uc));
+
+    test_arm64_sme_za_only_expect_exception(
+        test_arm64_sme_mova_insn(false, true, 1));
+
+    test_arm64_i8mm_expect_exception(0xc00800ff, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_context_save_restore(void)
+{
+    const uint32_t SVCR[5] = { 3, 3, 4, 2, 2 };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    const uint32_t TPIDR2_EL0[5] = { 3, 3, 13, 0, 5 };
+    const uint32_t ZCR_EL1[5] = { 3, 0, 1, 2, 0 };
+    const uint32_t ZCR_EL2[5] = { 3, 4, 1, 2, 0 };
+    const uint32_t ZCR_EL3[5] = { 3, 6, 1, 2, 0 };
+    uint8_t code[40];
+    uint8_t input[32];
+    uint8_t output[32];
+    uc_engine *uc;
+    uc_context *ctx;
+    uint64_t x4 = 0x40000;
+    uint64_t x6 = 0x40200;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+    uint64_t zcr = 1;
+    uint64_t saved_tpidr2 = 0x0123456789abcdefULL;
+    uint64_t mutated_tpidr2 = 0xfedcba9876543210ULL;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(0x23 + i * 9);
+    }
+    memset(output, 0xa5, sizeof(output));
+
+    test_arm64_emit32(code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(code, 8, 0xa400a081);  /* ld1b z1.b */
+    test_arm64_emit32(code, 12,
+                      test_arm64_sme_mova_insn(false, true, 1));
+    test_arm64_emit32(code, 16, 0xd503477f); /* smstart smza */
+    test_arm64_emit32(code, 20, 0xc00800ff); /* zero {za} */
+    test_arm64_emit32(code, 24, 0xd503467f); /* smstop smza */
+    test_arm64_emit32(code, 28, 0x2518e3e0); /* ptrue p0.b */
+    test_arm64_emit32(code, 32,
+                      test_arm64_sme_mova_insn(true, true, 0));
+    test_arm64_emit32(code, 36, 0xe400e0c0); /* st1b z0.b */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_context_alloc(uc, &ctx));
+    OK(uc_ctl_context_mode(uc, UC_CTL_CONTEXT_CPU));
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    test_arm64_pauth_cp_reg_write(uc, TPIDR2_EL0, saved_tpidr2);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL1, zcr);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL2, zcr);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL3, zcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, sizeof(input)));
+    OK(uc_mem_write(uc, x6, output, sizeof(output)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+
+    OK(uc_emu_start(uc, code_start, code_start + 16, 0, 0));
+    OK(uc_context_save(uc, ctx));
+
+    OK(uc_emu_start(uc, code_start + 16, code_start + 28, 0, 0));
+
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, 0);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, 0);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, 0);
+    test_arm64_pauth_cp_reg_write(uc, TPIDR2_EL0, mutated_tpidr2);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL1, 0);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL2, 0);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL3, 0);
+    OK(uc_context_restore(uc, ctx));
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SVCR) == 3);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SMCR_EL1) == smcr);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SMCR_EL2) == smcr);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SMCR_EL3) == smcr);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TPIDR2_EL0) == saved_tpidr2);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, ZCR_EL1) == zcr);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, ZCR_EL2) == zcr);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, ZCR_EL3) == zcr);
+
+    OK(uc_emu_start(uc, code_start + 28, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, input, sizeof(output)) == 0);
+
+    OK(uc_context_free(ctx));
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_mova_q_horizontal(void)
+{
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[24];
+    uint8_t input[32];
+    uint8_t output[32];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x6 = 0x40200;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(0x5a + i * 11);
+    }
+
+    test_arm64_emit32(code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(code, 8, 0xa400a081);  /* ld1b z1.b */
+    test_arm64_emit32(code, 12,
+                      test_arm64_sme_mova_esz_insn(TEST_ARM64_SME_OP_Q,
+                                                   false, false, 1));
+    test_arm64_emit32(code, 16,
+                      test_arm64_sme_mova_esz_insn(TEST_ARM64_SME_OP_Q,
+                                                   true, false, 0));
+    test_arm64_emit32(code, 20, 0xe400e0c0); /* st1b z0.b */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, input, sizeof(output)) == 0);
+    OK(uc_close(uc));
+
+    test_arm64_i8mm_expect_exception(
+        test_arm64_sme_mova_esz_insn(TEST_ARM64_SME_OP_Q, false, false, 1),
+        UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_adda_run(int esz, bool vertical, const void *input,
+                                    const void *expected, size_t size)
+{
+    static const uint32_t ptrue[4] = {
+        0x2518e3e0, 0x2558e3e0, 0x2598e3e0, 0x25d8e3e0,
+    };
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[28];
+    uint8_t output[32];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x6 = 0x40200;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+
+    test_arm64_emit32(code, 0, 0xd503477f); /* smstart smza */
+    test_arm64_emit32(code, 4, ptrue[esz]);
+    test_arm64_emit32(code, 8, ld1_z1[esz]);
+    test_arm64_emit32(code, 12,
+                      test_arm64_sme_mova_esz_insn(esz, false, true, 1));
+    test_arm64_emit32(code, 16, test_arm64_sme_adda_insn(esz, vertical));
+    test_arm64_emit32(code, 20,
+                      test_arm64_sme_mova_esz_insn(esz, true, true, 0));
+    test_arm64_emit32(code, 24, st1_z0[esz]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, size));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, output, size));
+    TEST_CHECK(memcmp(output, expected, size) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_adda(void)
+{
+    const uint32_t input_s[8] = {
+        1, 3, 5, 7, 11, 13, 17, 19,
+    };
+    const uint32_t exp_addha_s[8] = {
+        2, 4, 6, 8, 12, 14, 18, 20,
+    };
+    const uint32_t exp_addva_s[8] = {
+        2, 6, 10, 14, 22, 26, 34, 38,
+    };
+    const uint64_t input_d[4] = {
+        0x100000000ULL, 0x100000003ULL,
+        0x100000005ULL, 0x100000007ULL,
+    };
+    const uint64_t exp_addha_d[4] = {
+        0x200000000ULL, 0x200000003ULL,
+        0x200000005ULL, 0x200000007ULL,
+    };
+    const uint64_t exp_addva_d[4] = {
+        0x200000000ULL, 0x200000006ULL,
+        0x20000000aULL, 0x20000000eULL,
+    };
+
+    test_arm64_sme_adda_run(2, false, input_s, exp_addha_s, sizeof(input_s));
+    test_arm64_sme_adda_run(2, true, input_s, exp_addva_s, sizeof(input_s));
+    test_arm64_sme_adda_run(3, false, input_d, exp_addha_d, sizeof(input_d));
+    test_arm64_sme_adda_run(3, true, input_d, exp_addva_d, sizeof(input_d));
+
+    test_arm64_sme_za_only_expect_exception(
+        test_arm64_sme_adda_insn(TEST_ARM64_SME_OP_S, false));
+
+    test_arm64_i8mm_expect_exception(test_arm64_sme_adda_insn(2, false),
+                                     UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_ldstr(void)
+{
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t ldr_code[20];
+    uint8_t str_code[20];
+    uint8_t input[32];
+    uint8_t output[32];
+    uint8_t store_mem[64];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x6 = 0x40200;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(0x80 + i * 5);
+    }
+    memset(store_mem, 0xa5, sizeof(store_mem));
+
+    test_arm64_emit32(ldr_code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(ldr_code, 4,
+                      test_arm64_sme_ldstr_insn(false, 12, 4, 0));
+    test_arm64_emit32(ldr_code, 8, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(ldr_code, 12,
+                      test_arm64_sme_mova_insn(true, false, 0));
+    test_arm64_emit32(ldr_code, 16, 0xe400e0c0); /* st1b z0.b */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)ldr_code, sizeof(ldr_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(ldr_code), 0, 0));
+    OK(uc_mem_read(uc, x6, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, input, sizeof(output)) == 0);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(str_code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(str_code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(str_code, 8, 0xa400a081);  /* ld1b z1.b */
+    test_arm64_emit32(str_code, 12,
+                      test_arm64_sme_mova_esz_imm_insn(0, false,
+                                                       false, 1, 1));
+    test_arm64_emit32(str_code, 16,
+                      test_arm64_sme_ldstr_insn(true, 12, 6, 1));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)str_code, sizeof(str_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, sizeof(input)));
+    OK(uc_mem_write(uc, x6, store_mem, sizeof(store_mem)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(str_code), 0, 0));
+    OK(uc_mem_read(uc, x6, store_mem, sizeof(store_mem)));
+    for (i = 0; i < 32; i++) {
+        TEST_CHECK(store_mem[i] == 0xa5);
+    }
+    TEST_CHECK(memcmp(store_mem + 32, input, sizeof(input)) == 0);
+    OK(uc_close(uc));
+
+    test_arm64_i8mm_expect_exception(
+        test_arm64_sme_ldstr_insn(false, 12, 4, 0), UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_ldst1_run(int esz, bool vertical)
+{
+    static const uint32_t ptrue[4] = {
+        0x2518e3e0, 0x2558e3e0, 0x2598e3e0, 0x25d8e3e0,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[24];
+    uint8_t input[80];
+    uint8_t expected[32];
+    uint8_t vec_output[32];
+    uint8_t za_output[32];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 1;
+    uint64_t x6 = 0x40200;
+    uint64_t x7 = 0x40300;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+    size_t addr_off = (size_t)1 << esz;
+    size_t code_off = 0;
+    int pred_esz = esz == TEST_ARM64_SME_OP_Q ? TEST_ARM64_SME_OP_D : esz;
+    bool can_mova_to_vec = esz != TEST_ARM64_SME_OP_Q;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(0x31 + i * 13);
+    }
+    memcpy(expected, input + addr_off, sizeof(expected));
+
+    test_arm64_emit32(code, code_off, 0xd503477f);  /* smstart smza */
+    code_off += 4;
+    test_arm64_emit32(code, code_off, ptrue[pred_esz]);
+    code_off += 4;
+    test_arm64_emit32(code, code_off,
+                      test_arm64_sme_ldst1_insn(esz, false, vertical,
+                                                4, 5, 0));
+    code_off += 4;
+    if (can_mova_to_vec) {
+        test_arm64_emit32(code, code_off,
+                          test_arm64_sme_mova_esz_insn(esz, true,
+                                                       vertical, 0));
+        code_off += 4;
+        test_arm64_emit32(code, code_off, st1_z0[esz]);
+        code_off += 4;
+    }
+    test_arm64_emit32(code, code_off,
+                      test_arm64_sme_ldst1_insn(esz, true, vertical,
+                                                7, 31, 0));
+    code_off += 4;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    code_off, UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x4000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + code_off, 0, 0));
+    OK(uc_mem_read(uc, x7, za_output, sizeof(za_output)));
+    TEST_CHECK(memcmp(za_output, expected, sizeof(expected)) == 0);
+    if (can_mova_to_vec) {
+        OK(uc_mem_read(uc, x6, vec_output, sizeof(vec_output)));
+        TEST_CHECK(memcmp(vec_output, expected, sizeof(expected)) == 0);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_ldst1(void)
+{
+    const uint32_t SVCR[5] = { 3, 3, 4, 2, 2 };
+    uint8_t za_only_code[12];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0;
+    uint64_t x12 = 0;
+    uc_err err;
+    int esz;
+
+    for (esz = 0; esz <= TEST_ARM64_SME_OP_Q; esz++) {
+        test_arm64_sme_ldst1_run(esz, false);
+        test_arm64_sme_ldst1_run(esz, true);
+    }
+
+    test_arm64_emit32(za_only_code, 0, 0xd503457f); /* smstart za */
+    test_arm64_emit32(za_only_code, 4, 0x2518e3e0); /* ptrue p0.b */
+    test_arm64_emit32(za_only_code, 8,
+                      test_arm64_sme_ldst1_insn(0, false, false,
+                                                4, 5, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)za_only_code, sizeof(za_only_code),
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, x4, 0x1000, UC_PROT_ALL));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    err = uc_emu_start(uc, code_start, code_start + sizeof(za_only_code),
+                       0, 0);
+    TEST_CHECK(err == UC_ERR_EXCEPTION);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SVCR) == 2);
+    OK(uc_close(uc));
+
+    test_arm64_i8mm_expect_exception(
+        test_arm64_sme_ldst1_insn(0, false, false, 4, 5, 0),
+        UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_ldst1_fault_no_partial(void)
+{
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[20];
+    uint8_t initial[32];
+    uint8_t fault_src[8];
+    uint8_t before[8];
+    uint8_t output[32];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40ff8;
+    uint64_t x7 = 0x40200;
+    uint64_t smcr = 0x80000001;
+    uc_err err;
+    size_t i;
+
+    for (i = 0; i < sizeof(initial); i++) {
+        initial[i] = (uint8_t)(0x60 + i);
+    }
+    for (i = 0; i < sizeof(fault_src); i++) {
+        fault_src[i] = (uint8_t)(0xb0 + i);
+        before[i] = (uint8_t)(0xc0 + i);
+    }
+
+    test_arm64_emit32(code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(code, 8,
+                      test_arm64_sme_ldst1_insn(0, false, false,
+                                                4, 31, 0));
+    test_arm64_emit32(code, 12,
+                      test_arm64_sme_ldst1_insn(0, false, false,
+                                                5, 31, 0));
+    test_arm64_emit32(code, 16,
+                      test_arm64_sme_ldst1_insn(0, true, false,
+                                                7, 31, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, initial, sizeof(initial)));
+    OK(uc_mem_write(uc, x5, fault_src, sizeof(fault_src)));
+    memset(output, 0xa5, sizeof(output));
+    OK(uc_mem_write(uc, x7, output, sizeof(output)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+
+    err = uc_emu_start(uc, code_start, code_start + 16, 0, 0);
+    TEST_CHECK_(err == UC_ERR_READ_UNMAPPED, "err=%u", err);
+    OK(uc_emu_start(uc, code_start + 16, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x7, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, initial, sizeof(output)) == 0);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(code, 8,
+                      test_arm64_sme_ldst1_insn(0, false, false,
+                                                4, 31, 0));
+    test_arm64_emit32(code, 12,
+                      test_arm64_sme_ldst1_insn(0, true, false,
+                                                5, 31, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code, 16,
+                    UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, initial, sizeof(initial)));
+    OK(uc_mem_write(uc, x5, before, sizeof(before)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+
+    err = uc_emu_start(uc, code_start, code_start + 16, 0, 0);
+    TEST_CHECK_(err == UC_ERR_WRITE_UNMAPPED, "err=%u", err);
+    OK(uc_mem_read(uc, x5, output, sizeof(before)));
+    TEST_CHECK(memcmp(output, before, sizeof(before)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_psel_run(int esz, uint64_t x12, int imm,
+                                    uint16_t pm_mask, bool copy)
+{
+    uc_engine *uc;
+    uint8_t code[16];
+    const uint8_t pn[2] = { 0xb5, 0x4a };
+    uint8_t pm[2];
+    uint8_t expected[2];
+    uint8_t out[2];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+
+    pm[0] = (uint8_t)pm_mask;
+    pm[1] = (uint8_t)(pm_mask >> 8);
+    if (copy) {
+        memcpy(expected, pn, sizeof(expected));
+    } else {
+        memset(expected, 0, sizeof(expected));
+    }
+    memset(out, 0xa5, sizeof(out));
+
+    test_arm64_emit32(code, 0, test_arm64_sve_ldr_p_insn(1, 4));
+    test_arm64_emit32(code, 4, test_arm64_sve_ldr_p_insn(2, 5));
+    test_arm64_emit32(code, 8,
+                      test_arm64_sme_psel_insn(esz, 0, 1, 2, 12, imm));
+    test_arm64_emit32(code, 12, test_arm64_sve_str_p_insn(0, 6));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, pn, sizeof(pn)));
+    OK(uc_mem_write(uc, x5, pm, sizeof(pm)));
+    OK(uc_mem_write(uc, x6, out, sizeof(out)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, out, sizeof(out)));
+    TEST_CHECK(memcmp(out, expected, sizeof(out)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_psel(void)
+{
+    test_arm64_sme_psel_run(0, 14, 5, 1U << 3, true);
+    test_arm64_sme_psel_run(0, 14, 5, 0, false);
+    test_arm64_sme_psel_run(1, 7, 3, 1U << 4, true);
+    test_arm64_sme_psel_run(1, 7, 3, 0, false);
+    test_arm64_sme_psel_run(2, 3, 3, 1U << 8, true);
+    test_arm64_sme_psel_run(2, 3, 3, 0, false);
+    test_arm64_sme_psel_run(3, 4, 1, 1U << 8, true);
+    test_arm64_sme_psel_run(3, 4, 1, 0, false);
+
+    test_arm64_i8mm_expect_exception(
+        test_arm64_sme_psel_insn(0, 0, 1, 2, 12, 0), UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_ldst1_mte_enable_tcf(uc_engine *uc, uint64_t tcf)
+{
+    const uint32_t CPACR_EL1[5] = { 3, 0, 1, 0, 2 };
+    const uint32_t CPTR_EL3[5] = { 3, 6, 1, 1, 2 };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint64_t smcr = 0x80000001;
+
+    test_arm64_mte_enable_checks(uc, tcf);
+    test_arm64_mte_enable_sve(uc);
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(
+        uc, CPACR_EL1, 0, (3ULL << 16) | (3ULL << 20) | (3ULL << 24)));
+    TEST_CHECK(test_arm64_pauth_cp_reg_update(
+        uc, CPTR_EL3, 0, (1ULL << 8) | (1ULL << 12)));
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+}
+
+static void test_arm64_sme_ldst1_mte_enable(uc_engine *uc)
+{
+    test_arm64_sme_ldst1_mte_enable_tcf(uc, 1ULL << 40);
+}
+
+static void test_arm64_sme_ldst1_mte_seed_tag_pair(uc_engine *uc,
+                                                   uint64_t addr,
+                                                   uint64_t tag0,
+                                                   uint64_t tag1)
+{
+    test_arm64_mte_store_tag_at(uc, addr, tag0);
+    test_arm64_mte_store_tag_at(uc, addr + 0x10, tag1);
+}
+
+static void test_arm64_sme_ldst1_mte_seed_tags(uc_engine *uc,
+                                               uint64_t input_addr,
+                                               uint64_t output_addr,
+                                               uint64_t tag)
+{
+    test_arm64_sme_ldst1_mte_seed_tag_pair(uc, input_addr, tag, tag);
+    test_arm64_sme_ldst1_mte_seed_tag_pair(uc, output_addr, tag, tag);
+}
+
+static void test_arm64_sme_ldst1_mte_run(uint64_t load_addr,
+                                         uint64_t store_addr, bool tco,
+                                         uc_err expected_err,
+                                         bool expect_store,
+                                         uint64_t input_tag1,
+                                         uint64_t output_tag1)
+{
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SVCR[5] = { 3, 3, 4, 2, 2 };
+    const uint32_t TCO[5] = { 3, 3, 4, 2, 7 };
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint64_t pstate_tco = 1ULL << 25;
+    uint8_t code[24];
+    uint8_t input[32];
+    uint8_t initial_output[32];
+    uint8_t output[32];
+    uc_engine *uc;
+    uc_err err;
+    uint64_t pc = 0;
+    uint64_t svcr = 0;
+    uint64_t smcr_el1 = 0;
+    uint64_t x4 = load_addr;
+    uint64_t x7 = store_addr;
+    uint64_t x12 = 0;
+    uint64_t tag = 0x0c00000000000000ull;
+    uint64_t tag_probe = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(0x70 + i);
+        initial_output[i] = 0xa5;
+    }
+
+    test_arm64_emit32(code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(code, 4, 0xd9600023);  /* ldg x3,[x1] */
+    test_arm64_emit32(code, 8, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 12, 0x2518e3e0); /* ptrue p0.b */
+    test_arm64_emit32(code, 16,
+                      test_arm64_sme_ldst1_insn(0, false, false,
+                                                4, 31, 0));
+    test_arm64_emit32(code, 20,
+                      test_arm64_sme_ldst1_insn(0, true, false,
+                                                7, 31, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, input, sizeof(input)));
+    OK(uc_mem_write(uc, 0x40100, initial_output, sizeof(initial_output)));
+    test_arm64_sme_ldst1_mte_enable(uc);
+    test_arm64_sme_ldst1_mte_seed_tag_pair(uc, 0x40000, tag, input_tag1);
+    test_arm64_sme_ldst1_mte_seed_tag_pair(uc, 0x40100, tag, output_tag1);
+    tag_probe = test_arm64_mte_load_tag_at(uc, 0x40000, 0);
+    TEST_CHECK_(tag_probe == tag, "tag=0x%llx",
+                (unsigned long long)tag_probe);
+    tag_probe = test_arm64_mte_load_tag_at(uc, 0x40010, 0);
+    TEST_CHECK_(tag_probe == input_tag1, "tag=0x%llx",
+                (unsigned long long)tag_probe);
+    tag_probe = test_arm64_mte_load_tag_at(uc, 0x40100, 0);
+    TEST_CHECK_(tag_probe == tag, "tag=0x%llx",
+                (unsigned long long)tag_probe);
+    tag_probe = test_arm64_mte_load_tag_at(uc, 0x40110, 0);
+    TEST_CHECK_(tag_probe == output_tag1, "tag=0x%llx",
+                (unsigned long long)tag_probe);
+    if (tco) {
+        test_arm64_pauth_cp_reg_write(uc, TCO, pstate_tco);
+    }
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+
+    err = uc_emu_start(uc, code_start + 8, code_start + sizeof(code), 0, 0);
+    OK(uc_reg_read(uc, UC_ARM64_REG_PC, &pc));
+    svcr = test_arm64_pauth_cp_reg_read(uc, SVCR);
+    smcr_el1 = test_arm64_pauth_cp_reg_read(uc, SMCR_EL1);
+    TEST_CHECK_(err == expected_err,
+                "err=%u expected=%u pc=0x%llx svcr=0x%llx smcr=0x%llx",
+                (unsigned)err, (unsigned)expected_err,
+                (unsigned long long)pc, (unsigned long long)svcr,
+                (unsigned long long)smcr_el1);
+    OK(uc_mem_read(uc, 0x40100, output, sizeof(output)));
+    if (expect_store) {
+        TEST_CHECK(memcmp(output, input, sizeof(output)) == 0);
+    } else {
+        TEST_CHECK(memcmp(output, initial_output, sizeof(output)) == 0);
+    }
+    if (tco) {
+        TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_ldst1_mte_load_fault_preserve(void)
+{
+    uint8_t code[28];
+    uint8_t input[32];
+    uint8_t output[32];
+    uc_engine *uc;
+    uc_err err;
+    uint64_t x4 = 0x0c00000000040000ull;
+    uint64_t x5 = 0x0d00000000040000ull;
+    uint64_t x7 = 0x0c00000000040100ull;
+    uint64_t x12 = 0;
+    uint64_t tag = 0x0c00000000000000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(0x90 + i);
+        output[i] = 0xa5;
+    }
+
+    test_arm64_emit32(code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(code, 4, 0xd9600023);  /* ldg x3,[x1] */
+    test_arm64_emit32(code, 8, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 12, 0x2518e3e0); /* ptrue p0.b */
+    test_arm64_emit32(code, 16,
+                      test_arm64_sme_ldst1_insn(0, false, false,
+                                                4, 31, 0));
+    test_arm64_emit32(code, 20,
+                      test_arm64_sme_ldst1_insn(0, false, false,
+                                                5, 31, 0));
+    test_arm64_emit32(code, 24,
+                      test_arm64_sme_ldst1_insn(0, true, false,
+                                                7, 31, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, input, sizeof(input)));
+    OK(uc_mem_write(uc, 0x40100, output, sizeof(output)));
+    test_arm64_sme_ldst1_mte_enable(uc);
+    test_arm64_sme_ldst1_mte_seed_tags(uc, 0x40000, 0x40100, tag);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+
+    err = uc_emu_start(uc, code_start + 8, code_start + 24, 0, 0);
+    TEST_CHECK(err == UC_ERR_EXCEPTION);
+    OK(uc_emu_start(uc, code_start + 24, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, 0x40100, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, input, sizeof(output)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_ldst1_mte_store_fault_priority(void)
+{
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint8_t code[20];
+    uint8_t initial[16];
+    uint8_t actual[16];
+    uc_engine *uc;
+    uc_err err;
+    uint64_t x5 = 0x0d00000000040ff8ull;
+    uint64_t x12 = 0;
+    uint64_t tag = 0x0c00000000000000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(initial); i++) {
+        initial[i] = (uint8_t)(0xe0 + i);
+    }
+
+    test_arm64_emit32(code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(code, 4, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 8, 0xc00800ff);  /* zero {za} */
+    test_arm64_emit32(code, 12, 0x2518e3e0); /* ptrue p0.b */
+    test_arm64_emit32(code, 16,
+                      test_arm64_sme_ldst1_insn(0, true, false,
+                                                5, 31, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40ff0, initial, sizeof(initial)));
+    test_arm64_sme_ldst1_mte_enable_tcf(uc, 3ULL << 40);
+    test_arm64_mte_store_tag_at(uc, 0x40ff0, tag);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+
+    err = uc_emu_start(uc, code_start + 4, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == UC_ERR_WRITE_UNMAPPED, "err=%u", (unsigned)err);
+    OK(uc_mem_read(uc, 0x40ff0, actual, sizeof(actual)));
+    TEST_CHECK(memcmp(actual, initial, sizeof(initial)) == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_ldst1_mte(void)
+{
+    uint64_t tag = 0x0c00000000000000ull;
+    uint64_t other_tag = 0x0e00000000000000ull;
+
+    test_arm64_sme_ldst1_mte_load_fault_preserve();
+    test_arm64_sme_ldst1_mte_store_fault_priority();
+
+    test_arm64_sme_ldst1_mte_run(0x0c00000000040000ull,
+                                 0x0c00000000040100ull,
+                                 false, UC_ERR_OK, true, tag, tag);
+    test_arm64_sme_ldst1_mte_run(0x0d00000000040000ull,
+                                 0x0c00000000040100ull,
+                                 false, UC_ERR_EXCEPTION, false, tag, tag);
+    test_arm64_sme_ldst1_mte_run(0x0c00000000040000ull,
+                                 0x0c00000000040100ull,
+                                 false, UC_ERR_EXCEPTION, false,
+                                 other_tag, tag);
+    test_arm64_sme_ldst1_mte_run(0x0c00000000040000ull,
+                                 0x0d00000000040100ull,
+                                 false, UC_ERR_EXCEPTION, false, tag, tag);
+    test_arm64_sme_ldst1_mte_run(0x0c00000000040000ull,
+                                 0x0c00000000040100ull,
+                                 false, UC_ERR_EXCEPTION, false,
+                                 tag, other_tag);
+    test_arm64_sme_ldst1_mte_run(0x0d00000000040000ull,
+                                 0x0d00000000040100ull,
+                                 true, UC_ERR_OK, true, tag, tag);
+}
+
+static int64_t test_arm64_sme_byte_value(uint8_t value, bool is_unsigned)
+{
+    if (is_unsigned) {
+        return value;
+    }
+    return (int8_t)value;
+}
+
+static int64_t test_arm64_sme_half_value(uint16_t value, bool is_unsigned)
+{
+    if (is_unsigned) {
+        return value;
+    }
+    return (int16_t)value;
+}
+
+static void test_arm64_sme_imopa_s_expected(uint32_t *expected,
+                                            const uint8_t *n,
+                                            const uint8_t *m,
+                                            int kind, bool sub)
+{
+    bool n_unsigned = (kind & 2) != 0;
+    bool m_unsigned = (kind & 1) != 0;
+    int col;
+
+    for (col = 0; col < 4; col++) {
+        int lane;
+        int64_t sum0 = 0;
+        int64_t sum1 = 0;
+
+        for (lane = 0; lane < 4; lane++) {
+            sum0 += test_arm64_sme_byte_value(n[lane], n_unsigned) *
+                    test_arm64_sme_byte_value(m[col * 8 + lane],
+                                              m_unsigned);
+            sum1 += test_arm64_sme_byte_value(n[lane + 4], n_unsigned) *
+                    test_arm64_sme_byte_value(m[col * 8 + lane + 4],
+                                              m_unsigned);
+        }
+        expected[col * 2] = (uint32_t)(sub ? -sum0 : sum0);
+        expected[col * 2 + 1] = (uint32_t)(sub ? -sum1 : sum1);
+    }
+}
+
+static void test_arm64_sme_imopa_d_expected(uint64_t *expected,
+                                            const uint16_t *n,
+                                            const uint16_t *m,
+                                            int kind, bool sub)
+{
+    bool n_unsigned = (kind & 2) != 0;
+    bool m_unsigned = (kind & 1) != 0;
+    int col;
+
+    for (col = 0; col < 4; col++) {
+        int lane;
+        int64_t sum = 0;
+
+        for (lane = 0; lane < 4; lane++) {
+            sum += test_arm64_sme_half_value(n[lane], n_unsigned) *
+                   test_arm64_sme_half_value(m[col * 4 + lane], m_unsigned);
+        }
+        expected[col] = (uint64_t)(sub ? -sum : sum);
+    }
+}
+
+static void test_arm64_sme_imopa_run(int esz, int kind, bool sub,
+                                     const void *input_n,
+                                     const void *input_m,
+                                     const void *expected, size_t size)
+{
+    static const uint32_t ptrue[2] = {
+        0x2518e3e0, 0x2558e3e0,
+    };
+    static const uint32_t ld1_z1[2] = {
+        0xa400a081, 0xa4a0a081,
+    };
+    static const uint32_t ld1_z2[2] = {
+        0xa400a0a2, 0xa4a0a0a2,
+    };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[28];
+    uint8_t output[32];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+    int load_esz = esz == TEST_ARM64_SME_OP_D ? 1 : 0;
+
+    test_arm64_emit32(code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 4, 0xc00800ff);  /* zero {za} */
+    test_arm64_emit32(code, 8, ptrue[load_esz]);
+    test_arm64_emit32(code, 12, ld1_z1[load_esz]);
+    test_arm64_emit32(code, 16, ld1_z2[load_esz]);
+    test_arm64_emit32(code, 20, test_arm64_sme_imopa_insn(esz, kind, sub));
+    test_arm64_emit32(code, 24,
+                      test_arm64_sme_ldstr_insn(true, 12, 6, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input_n, size));
+    OK(uc_mem_write(uc, x5, input_m, size));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, output, size));
+    TEST_CHECK(memcmp(output, expected, size) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_imopa(void)
+{
+    uint8_t n_b[32] = {
+        0x7f, 0x80, 0x02, 0xff, 0x10, 0xf0, 0x04, 0x81,
+    };
+    uint8_t m_b[32] = {
+        0x02, 0x7e, 0x80, 0xff, 0x11, 0xef, 0x40, 0xc0,
+        0x01, 0x02, 0x03, 0x04, 0x80, 0x7f, 0xfe, 0x10,
+        0xf0, 0x10, 0x08, 0xf8, 0x20, 0xe0, 0x55, 0xaa,
+        0x7f, 0x81, 0x33, 0xcd, 0x12, 0x34, 0x56, 0x78,
+    };
+    uint16_t n_h[16] = {
+        0x7fff, 0x8000, 0x0002, 0xffff,
+    };
+    uint16_t m_h[16] = {
+        0x0002, 0x7ffe, 0x8000, 0xffff,
+        0x0001, 0x0002, 0x0003, 0x0004,
+        0x8000, 0x7fff, 0xfffe, 0x0010,
+        0x1234, 0xedcc, 0x0101, 0xf0f0,
+    };
+    uint32_t expected_s[8];
+    uint64_t expected_d[4];
+    int kind;
+
+    for (kind = 0; kind < 4; kind++) {
+        test_arm64_sme_imopa_s_expected(expected_s, n_b, m_b, kind, false);
+        test_arm64_sme_imopa_run(TEST_ARM64_SME_OP_S, kind, false,
+                                 n_b, m_b, expected_s,
+                                 sizeof(expected_s));
+        test_arm64_sme_imopa_s_expected(expected_s, n_b, m_b, kind, true);
+        test_arm64_sme_imopa_run(TEST_ARM64_SME_OP_S, kind, true,
+                                 n_b, m_b, expected_s,
+                                 sizeof(expected_s));
+
+        test_arm64_sme_imopa_d_expected(expected_d, n_h, m_h, kind, false);
+        test_arm64_sme_imopa_run(TEST_ARM64_SME_OP_D, kind, false,
+                                 n_h, m_h, expected_d,
+                                 sizeof(expected_d));
+        test_arm64_sme_imopa_d_expected(expected_d, n_h, m_h, kind, true);
+        test_arm64_sme_imopa_run(TEST_ARM64_SME_OP_D, kind, true,
+                                 n_h, m_h, expected_d,
+                                 sizeof(expected_d));
+    }
+
+    test_arm64_sme_za_only_expect_exception(
+        test_arm64_sme_imopa_insn(TEST_ARM64_SME_OP_S, 0, false));
+
+    test_arm64_i8mm_expect_exception(
+        test_arm64_sme_imopa_insn(TEST_ARM64_SME_OP_S, 0, false),
+        UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sme_fpout_run(int kind, bool sub, int pred_esz,
+                                     const void *input_n,
+                                     const void *input_m,
+                                     const void *expected)
+{
+    static const uint32_t ptrue[4] = {
+        0x2518e3e0, 0x2558e3e0, 0x2598e3e0, 0x25d8e3e0,
+    };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL3[5] = { 3, 6, 1, 2, 6 };
+    uint8_t code[32];
+    uint8_t output[32];
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    uint64_t x12 = 0;
+    uint64_t smcr = 0x80000001;
+
+    test_arm64_emit32(code, 0, 0xd503477f);  /* smstart smza */
+    test_arm64_emit32(code, 4, 0xc00800ff);  /* zero {za} */
+    test_arm64_emit32(code, 8, ptrue[0]);
+    test_arm64_emit32(code, 12, 0xa400a081); /* ld1b z1.b */
+    test_arm64_emit32(code, 16, 0xa400a0a2); /* ld1b z2.b */
+    test_arm64_emit32(code, 20, ptrue[pred_esz]);
+    test_arm64_emit32(code, 24, test_arm64_sme_fpout_insn(kind, sub));
+    test_arm64_emit32(code, 28,
+                      test_arm64_sme_ldstr_insn(true, 12, 6, 0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, smcr);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL3, smcr);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input_n, sizeof(output)));
+    OK(uc_mem_write(uc, x5, input_m, sizeof(output)));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, output, sizeof(output)));
+    TEST_CHECK(memcmp(output, expected, sizeof(output)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sme_fpout(void)
+{
+    const uint32_t n_s[8] = {
+        0x40000000u, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint32_t m_s[8] = {
+        0x40400000u, 0x40800000u, 0xbf800000u, 0x3f000000u,
+        0x40000000u, 0xc0400000u, 0x41000000u, 0xbe800000u,
+    };
+    const uint32_t exp_s_add[8] = {
+        0x40c00000u, 0x41000000u, 0xc0000000u, 0x3f800000u,
+        0x40800000u, 0xc0c00000u, 0x41800000u, 0xbf000000u,
+    };
+    const uint32_t exp_s_sub[8] = {
+        0xc0c00000u, 0xc1000000u, 0x40000000u, 0xbf800000u,
+        0xc0800000u, 0x40c00000u, 0xc1800000u, 0x3f000000u,
+    };
+    const uint64_t n_d[4] = {
+        0x4000000000000000ull, 0, 0, 0,
+    };
+    const uint64_t m_d[4] = {
+        0x4008000000000000ull, 0x4010000000000000ull,
+        0xbff0000000000000ull, 0x3fe0000000000000ull,
+    };
+    const uint64_t exp_d_add[4] = {
+        0x4018000000000000ull, 0x4020000000000000ull,
+        0xc000000000000000ull, 0x3ff0000000000000ull,
+    };
+    const uint64_t exp_d_sub[4] = {
+        0xc018000000000000ull, 0xc020000000000000ull,
+        0x4000000000000000ull, 0xbff0000000000000ull,
+    };
+    const uint32_t n_h[8] = {
+        0x40003c00u, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint32_t m_h[8] = {
+        0x44004200u, 0x3c003c00u, 0x42004000u, 0x3c00bc00u,
+        0x38003800u, 0xbc004400u, 0x38004800u, 0xbc00c400u,
+    };
+    const uint32_t exp_pair_add[8] = {
+        0x41300000u, 0x40400000u, 0x41000000u, 0x3f800000u,
+        0x3fc00000u, 0x40000000u, 0x41100000u, 0xc0c00000u,
+    };
+    const uint32_t exp_pair_sub[8] = {
+        0xc1300000u, 0xc0400000u, 0xc1000000u, 0xbf800000u,
+        0xbfc00000u, 0xc0000000u, 0xc1100000u, 0x40c00000u,
+    };
+    const uint32_t n_bf[8] = {
+        0x40003f80u, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint32_t m_bf[8] = {
+        0x40804040u, 0x3f803f80u, 0x40404000u, 0x3f80bf80u,
+        0x3f003f00u, 0xbf804080u, 0x3f004100u, 0xbf80c080u,
+    };
+
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_FMOPA_S, false,
+                             TEST_ARM64_SME_OP_S, n_s, m_s, exp_s_add);
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_FMOPA_S, true,
+                             TEST_ARM64_SME_OP_S, n_s, m_s, exp_s_sub);
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_FMOPA_D, false,
+                             TEST_ARM64_SME_OP_D, n_d, m_d, exp_d_add);
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_FMOPA_D, true,
+                             TEST_ARM64_SME_OP_D, n_d, m_d, exp_d_sub);
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_FMOPA_H, false, 1,
+                             n_h, m_h, exp_pair_add);
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_FMOPA_H, true, 1,
+                             n_h, m_h, exp_pair_sub);
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_BFMOPA, false, 1,
+                             n_bf, m_bf, exp_pair_add);
+    test_arm64_sme_fpout_run(TEST_ARM64_SME_BFMOPA, true, 1,
+                             n_bf, m_bf, exp_pair_sub);
+
+    test_arm64_sme_za_only_expect_exception(
+        test_arm64_sme_fpout_insn(TEST_ARM64_SME_FMOPA_S, false));
+    test_arm64_sme_za_only_expect_exception(
+        test_arm64_sme_fpout_insn(TEST_ARM64_SME_BFMOPA, false));
+
+    test_arm64_i8mm_expect_exception(
+        test_arm64_sme_fpout_insn(TEST_ARM64_SME_FMOPA_S, false),
+        UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_sve2_widen_add_shift(void)
+{
+    const uint8_t n_b[16] = {
+        0x80, 0x7f, 0x40, 0xc0, 0x11, 0xee, 0x55, 0xaa,
+        0x10, 0xf0, 0x33, 0xcd, 0x01, 0xff, 0x7e, 0x82,
+    };
+    const uint8_t m_b[16] = {
+        0x02, 0x7f, 0x40, 0xc0, 0x33, 0x20, 0x55, 0xaa,
+        0x7f, 0x81, 0xcd, 0x33, 0x80, 0x80, 0x02, 0xfe,
+    };
+    const uint8_t sq_n_b[16] = {
+        0x80, 0x7f, 0x40, 0xc0, 0x20, 0xe0, 0x55, 0xab,
+        0x10, 0xf0, 0x33, 0xcd, 0x01, 0xff, 0x7e, 0x82,
+    };
+    const uint8_t sq_m_b[16] = {
+        0x80, 0x7f, 0x40, 0x40, 0xe0, 0x20, 0x55, 0xab,
+        0x7f, 0x81, 0xcd, 0x33, 0x80, 0x80, 0x02, 0xfe,
+    };
+    const uint16_t n_h[8] = {
+        0x8001, 0x7fff, 0x4000, 0xc000,
+        0x1111, 0xeeee, 0x5555, 0xaaaa,
+    };
+    const uint16_t m_h[8] = {
+        0x0002, 0x7fff, 0x4000, 0xc000,
+        0x3333, 0x2000, 0x5555, 0xaaaa,
+    };
+    const uint32_t n_s[4] = {
+        0x80000001u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+    };
+    const uint32_t m_s[4] = {
+        0x00000002u, 0x7fffffffu, 0x40000000u, 0xc0000000u,
+    };
+    const uint64_t n_d[2] = {
+        0x8000000000000001ull, 0x7fffffffffffffffull,
+    };
+    const uint16_t exp_saddlb[8] = {
+        0xff82, 0x0080, 0x0044, 0x00aa,
+        0x008f, 0x0000, 0xff81, 0x0080,
+    };
+    const uint16_t exp_uaddlb[8] = {
+        0x0082, 0x0080, 0x0044, 0x00aa,
+        0x008f, 0x0100, 0x0081, 0x0080,
+    };
+    const uint16_t exp_ssublb[8] = {
+        0xff7e, 0x0000, 0xffde, 0x0000,
+        0xff91, 0x0066, 0x0081, 0x007c,
+    };
+    const uint16_t exp_usublb[8] = {
+        0x007e, 0x0000, 0xffde, 0x0000,
+        0xff91, 0xff66, 0xff81, 0x007c,
+    };
+    const uint16_t exp_saddlt[8] = {
+        0x00fe, 0xff80, 0x000e, 0xff54,
+        0xff71, 0x0000, 0xff7f, 0xff80,
+    };
+    const uint16_t exp_uaddlt[8] = {
+        0x00fe, 0x0180, 0x010e, 0x0154,
+        0x0171, 0x0100, 0x017f, 0x0180,
+    };
+    const uint16_t exp_ssublt[8] = {
+        0x0000, 0x0000, 0xffce, 0x0000,
+        0x006f, 0xff9a, 0x007f, 0xff84,
+    };
+    const uint16_t exp_usublt[8] = {
+        0x0000, 0x0000, 0x00ce, 0x0000,
+        0x006f, 0x009a, 0x007f, 0xff84,
+    };
+    const uint16_t exp_saddwb[8] = {
+        0x8003, 0x803f, 0x4033, 0xc055,
+        0x1190, 0xeebb, 0x54d5, 0xaaac,
+    };
+    const uint16_t exp_uaddwb[8] = {
+        0x8003, 0x803f, 0x4033, 0xc055,
+        0x1190, 0xefbb, 0x55d5, 0xaaac,
+    };
+    const uint16_t exp_ssubwb[8] = {
+        0x7fff, 0x7fbf, 0x3fcd, 0xbfab,
+        0x1092, 0xef21, 0x55d5, 0xaaa8,
+    };
+    const uint16_t exp_usubwb[8] = {
+        0x7fff, 0x7fbf, 0x3fcd, 0xbfab,
+        0x1092, 0xee21, 0x54d5, 0xaaa8,
+    };
+    const uint16_t exp_saddwt[8] = {
+        0x8080, 0x7fbf, 0x4020, 0xbfaa,
+        0x1092, 0xef21, 0x54d5, 0xaaa8,
+    };
+    const uint16_t exp_uaddwt[8] = {
+        0x8080, 0x80bf, 0x4020, 0xc0aa,
+        0x1192, 0xef21, 0x55d5, 0xaba8,
+    };
+    const uint16_t exp_ssubwt[8] = {
+        0x7f82, 0x803f, 0x3fe0, 0xc056,
+        0x1190, 0xeebb, 0x55d5, 0xaaac,
+    };
+    const uint16_t exp_usubwt[8] = {
+        0x7f82, 0x7f3f, 0x3fe0, 0xbf56,
+        0x1090, 0xeebb, 0x54d5, 0xa9ac,
+    };
+    const uint16_t exp_sshllb[8] = {
+        0xff80, 0x0040, 0x0011, 0x0055,
+        0x0010, 0x0033, 0x0001, 0x007e,
+    };
+    const uint16_t exp_ushllb[8] = {
+        0x0080, 0x0040, 0x0011, 0x0055,
+        0x0010, 0x0033, 0x0001, 0x007e,
+    };
+    const uint16_t exp_sshllt[8] = {
+        0x007f, 0xffc0, 0xffee, 0xffaa,
+        0xfff0, 0xffcd, 0xffff, 0xff82,
+    };
+    const uint16_t exp_ushllt[8] = {
+        0x007f, 0x00c0, 0x00ee, 0x00aa,
+        0x00f0, 0x00cd, 0x00ff, 0x0082,
+    };
+    const uint32_t exp_saddlb_s[4] = {
+        0xffff8003u, 0x00008000u, 0x00004444u, 0x0000aaaau,
+    };
+    const uint32_t exp_uaddlt_s[4] = {
+        0x0000fffeu, 0x00018000u, 0x00010eeeu, 0x00015554u,
+    };
+    const uint32_t exp_ssublb_s[4] = {
+        0xffff7fffu, 0x00000000u, 0xffffdddeu, 0x00000000u,
+    };
+    const uint32_t exp_usublt_s[4] = {
+        0x00000000u, 0x00000000u, 0x0000ceeeu, 0x00000000u,
+    };
+    const uint64_t exp_saddlt_d[2] = {
+        0x00000000fffffffeull, 0xffffffff80000000ull,
+    };
+    const uint64_t exp_uaddlb_d[2] = {
+        0x0000000080000003ull, 0x0000000080000000ull,
+    };
+    const uint64_t exp_ssublt_d[2] = {
+        0x0000000000000000ull, 0x0000000000000000ull,
+    };
+    const uint64_t exp_usublb_d[2] = {
+        0x000000007fffffffull, 0x0000000000000000ull,
+    };
+    const uint32_t exp_saddwb_s[4] = {
+        0x80000003u, 0x80003fffu, 0x40003333u, 0xc0005555u,
+    };
+    const uint32_t exp_uaddwt_s[4] = {
+        0x80008000u, 0x8000bfffu, 0x40002000u, 0xc000aaaau,
+    };
+    const uint32_t exp_ssubwt_s[4] = {
+        0x7fff8002u, 0x80003fffu, 0x3fffe000u, 0xc0005556u,
+    };
+    const uint32_t exp_usubwb_s[4] = {
+        0x7fffffffu, 0x7fffbfffu, 0x3fffcccdu, 0xbfffaaabu,
+    };
+    const uint64_t exp_saddwt_d[2] = {
+        0x8000000080000000ull, 0x7fffffffbfffffffull,
+    };
+    const uint64_t exp_uaddwb_d[2] = {
+        0x8000000000000003ull, 0x800000003fffffffull,
+    };
+    const uint64_t exp_ssubwb_d[2] = {
+        0x7fffffffffffffffull, 0x7fffffffbfffffffull,
+    };
+    const uint64_t exp_usubwt_d[2] = {
+        0x7fffffff80000002ull, 0x7fffffff3fffffffull,
+    };
+    const uint32_t exp_sshllb_s4[4] = {
+        0xfff80010u, 0x00040000u, 0x00011110u, 0x00055550u,
+    };
+    const uint64_t exp_ushllt_d7[2] = {
+        0x0000003fffffff80ull, 0x0000006000000000ull,
+    };
+    const uint32_t exp_saddlbt_s[4] = {
+        0x00000000u, 0x00000000u, 0x00003111u, 0xffffffffu,
+    };
+    const uint64_t exp_ssubltb_d[2] = {
+        0x000000007ffffffdull, 0xffffffff80000000ull,
+    };
+    const uint32_t exp_sabdlb_s[4] = {
+        0x00008001u, 0x00000000u, 0x00002222u, 0x00000000u,
+    };
+    const uint64_t exp_uabdlb_d[2] = {
+        0x000000007fffffffull, 0x0000000000000000ull,
+    };
+    const uint32_t exp_smullb_s[4] = {
+        0xffff0002u, 0x10000000u, 0x0369c963u, 0x1c718e39u,
+    };
+    const uint64_t exp_umullt_d[2] = {
+        0x3fffffff00000001ull, 0x9000000000000000ull,
+    };
+    const uint16_t exp_sqdmullb_h[8] = {
+        0x7fff, 0x2000, 0xf800, 0x3872,
+        0x0fe0, 0xebae, 0xff00, 0x01f8,
+    };
+
+    test_arm64_sve2_widen_run(0x45420020, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_saddlb,
+                              sizeof(exp_saddlb));
+    test_arm64_sve2_widen_run(0x45420820, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_uaddlb,
+                              sizeof(exp_uaddlb));
+    test_arm64_sve2_widen_run(0x45421020, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_ssublb,
+                              sizeof(exp_ssublb));
+    test_arm64_sve2_widen_run(0x45421820, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_usublb,
+                              sizeof(exp_usublb));
+    test_arm64_sve2_widen_run(0x45420420, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_saddlt,
+                              sizeof(exp_saddlt));
+    test_arm64_sve2_widen_run(0x45420c20, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_uaddlt,
+                              sizeof(exp_uaddlt));
+    test_arm64_sve2_widen_run(0x45421420, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_ssublt,
+                              sizeof(exp_ssublt));
+    test_arm64_sve2_widen_run(0x45421c20, 1, 0, 0, n_b, sizeof(n_b),
+                              m_b, sizeof(m_b), exp_usublt,
+                              sizeof(exp_usublt));
+
+    test_arm64_sve2_widen_run(0x45424020, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_saddwb,
+                              sizeof(exp_saddwb));
+    test_arm64_sve2_widen_run(0x45424820, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_uaddwb,
+                              sizeof(exp_uaddwb));
+    test_arm64_sve2_widen_run(0x45425020, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_ssubwb,
+                              sizeof(exp_ssubwb));
+    test_arm64_sve2_widen_run(0x45425820, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_usubwb,
+                              sizeof(exp_usubwb));
+    test_arm64_sve2_widen_run(0x45424420, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_saddwt,
+                              sizeof(exp_saddwt));
+    test_arm64_sve2_widen_run(0x45424c20, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_uaddwt,
+                              sizeof(exp_uaddwt));
+    test_arm64_sve2_widen_run(0x45425420, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_ssubwt,
+                              sizeof(exp_ssubwt));
+    test_arm64_sve2_widen_run(0x45425c20, 1, 1, 0, n_h, sizeof(n_h),
+                              m_b, sizeof(m_b), exp_usubwt,
+                              sizeof(exp_usubwt));
+
+    test_arm64_sve2_widen_run(0x4508a020, 1, 0, 0, n_b, sizeof(n_b),
+                              NULL, 0, exp_sshllb, sizeof(exp_sshllb));
+    test_arm64_sve2_widen_run(0x4508a820, 1, 0, 0, n_b, sizeof(n_b),
+                              NULL, 0, exp_ushllb, sizeof(exp_ushllb));
+    test_arm64_sve2_widen_run(0x4508a420, 1, 0, 0, n_b, sizeof(n_b),
+                              NULL, 0, exp_sshllt, sizeof(exp_sshllt));
+    test_arm64_sve2_widen_run(0x4508ac20, 1, 0, 0, n_b, sizeof(n_b),
+                              NULL, 0, exp_ushllt, sizeof(exp_ushllt));
+
+    test_arm64_sve2_widen_run(0x45820020, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_saddlb_s,
+                              sizeof(exp_saddlb_s));
+    test_arm64_sve2_widen_run(0x45820c20, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_uaddlt_s,
+                              sizeof(exp_uaddlt_s));
+    test_arm64_sve2_widen_run(0x45821020, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_ssublb_s,
+                              sizeof(exp_ssublb_s));
+    test_arm64_sve2_widen_run(0x45821c20, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_usublt_s,
+                              sizeof(exp_usublt_s));
+    test_arm64_sve2_widen_run(0x45c20420, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_saddlt_d,
+                              sizeof(exp_saddlt_d));
+    test_arm64_sve2_widen_run(0x45c20820, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_uaddlb_d,
+                              sizeof(exp_uaddlb_d));
+    test_arm64_sve2_widen_run(0x45c21420, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_ssublt_d,
+                              sizeof(exp_ssublt_d));
+    test_arm64_sve2_widen_run(0x45c21820, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_usublb_d,
+                              sizeof(exp_usublb_d));
+
+    test_arm64_sve2_widen_run(0x45824020, 2, 2, 1, n_s, sizeof(n_s),
+                              m_h, sizeof(m_h), exp_saddwb_s,
+                              sizeof(exp_saddwb_s));
+    test_arm64_sve2_widen_run(0x45824c20, 2, 2, 1, n_s, sizeof(n_s),
+                              m_h, sizeof(m_h), exp_uaddwt_s,
+                              sizeof(exp_uaddwt_s));
+    test_arm64_sve2_widen_run(0x45825420, 2, 2, 1, n_s, sizeof(n_s),
+                              m_h, sizeof(m_h), exp_ssubwt_s,
+                              sizeof(exp_ssubwt_s));
+    test_arm64_sve2_widen_run(0x45825820, 2, 2, 1, n_s, sizeof(n_s),
+                              m_h, sizeof(m_h), exp_usubwb_s,
+                              sizeof(exp_usubwb_s));
+    test_arm64_sve2_widen_run(0x45c24420, 3, 3, 2, n_d, sizeof(n_d),
+                              m_s, sizeof(m_s), exp_saddwt_d,
+                              sizeof(exp_saddwt_d));
+    test_arm64_sve2_widen_run(0x45c24820, 3, 3, 2, n_d, sizeof(n_d),
+                              m_s, sizeof(m_s), exp_uaddwb_d,
+                              sizeof(exp_uaddwb_d));
+    test_arm64_sve2_widen_run(0x45c25020, 3, 3, 2, n_d, sizeof(n_d),
+                              m_s, sizeof(m_s), exp_ssubwb_d,
+                              sizeof(exp_ssubwb_d));
+    test_arm64_sve2_widen_run(0x45c25c20, 3, 3, 2, n_d, sizeof(n_d),
+                              m_s, sizeof(m_s), exp_usubwt_d,
+                              sizeof(exp_usubwt_d));
+
+    test_arm64_sve2_widen_run(0x4514a020, 2, 1, 0, n_h, sizeof(n_h),
+                              NULL, 0, exp_sshllb_s4,
+                              sizeof(exp_sshllb_s4));
+    test_arm64_sve2_widen_run(0x4547ac20, 3, 2, 0, n_s, sizeof(n_s),
+                              NULL, 0, exp_ushllt_d7,
+                              sizeof(exp_ushllt_d7));
+
+    test_arm64_sve2_widen_run(0x45828020, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_saddlbt_s,
+                              sizeof(exp_saddlbt_s));
+    test_arm64_sve2_widen_run(0x45c28c20, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_ssubltb_d,
+                              sizeof(exp_ssubltb_d));
+    test_arm64_sve2_widen_run(0x45823020, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_sabdlb_s,
+                              sizeof(exp_sabdlb_s));
+    test_arm64_sve2_widen_run(0x45c23820, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_uabdlb_d,
+                              sizeof(exp_uabdlb_d));
+    test_arm64_sve2_widen_run(0x45827020, 2, 1, 1, n_h, sizeof(n_h),
+                              m_h, sizeof(m_h), exp_smullb_s,
+                              sizeof(exp_smullb_s));
+    test_arm64_sve2_widen_run(0x45c27c20, 3, 2, 2, n_s, sizeof(n_s),
+                              m_s, sizeof(m_s), exp_umullt_d,
+                              sizeof(exp_umullt_d));
+    test_arm64_sve2_widen_run(0x45426020, 1, 0, 0, sq_n_b, sizeof(sq_n_b),
+                              sq_m_b, sizeof(sq_m_b), exp_sqdmullb_h,
+                              sizeof(exp_sqdmullb_h));
+}
+
+static void test_arm64_sve2_addhn(void)
+{
+    const uint8_t init_b[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint16_t init_h[8] = {
+        0xa000, 0xa001, 0xa002, 0xa003,
+        0xa004, 0xa005, 0xa006, 0xa007,
+    };
+    const uint32_t init_s[4] = {
+        0xa0000000u, 0xa0000001u, 0xa0000002u, 0xa0000003u,
+    };
+    const uint16_t n_h[8] = {
+        0x1200, 0x12ff, 0x8000, 0x7fff,
+        0xffff, 0x0001, 0x00ff, 0xff00,
+    };
+    const uint16_t m_h[8] = {
+        0x0100, 0x0002, 0x8000, 0x0001,
+        0x0002, 0xffff, 0xff00, 0x0100,
+    };
+    const uint32_t n_s[4] = {
+        0x12000000u, 0x12ff0001u, 0x80000000u, 0x7fffffffu,
+    };
+    const uint32_t m_s[4] = {
+        0x01000000u, 0x00020000u, 0x80000000u, 0x00000001u,
+    };
+    const uint64_t n_d[2] = {
+        0x1200000000000000ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t m_d[2] = {
+        0x0100000000000000ull, 0x0000000100000001ull,
+    };
+    const uint8_t exp_addhnb_h[16] = {
+        0x13, 0x00, 0x13, 0x00, 0x00, 0x00, 0x80, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00,
+    };
+    const uint8_t exp_subhnb_h[16] = {
+        0x11, 0x00, 0x12, 0x00, 0x00, 0x00, 0x7f, 0x00,
+        0xff, 0x00, 0x00, 0x00, 0x01, 0x00, 0xfe, 0x00,
+    };
+    const uint8_t exp_rsubhnt_h[16] = {
+        0xa0, 0x11, 0xa2, 0x13, 0xa4, 0x00, 0xa6, 0x80,
+        0xa8, 0x00, 0xaa, 0x00, 0xac, 0x02, 0xae, 0xfe,
+    };
+    const uint16_t exp_addhnt_s[8] = {
+        0xa000, 0x1300, 0xa002, 0x1301,
+        0xa004, 0x0000, 0xa006, 0x8000,
+    };
+    const uint16_t exp_raddhnb_s[8] = {
+        0x1300, 0x0000, 0x1301, 0x0000,
+        0x0000, 0x0000, 0x8000, 0x0000,
+    };
+    const uint16_t exp_subhnt_s[8] = {
+        0xa000, 0x1100, 0xa002, 0x12fd,
+        0xa004, 0x0000, 0xa006, 0x7fff,
+    };
+    const uint32_t exp_raddhnt_d[4] = {
+        0xa0000000u, 0x13000000u, 0xa0000002u, 0x80000001u,
+    };
+    const uint32_t exp_rsubhnb_d[4] = {
+        0x11000000u, 0x00000000u, 0x7fffffffu, 0x00000000u,
+    };
+
+    test_arm64_sve2_narrow_run(0x45626020, 0, 1, init_b, n_h, m_h,
+                               exp_addhnb_h, sizeof(exp_addhnb_h));
+    test_arm64_sve2_narrow_run(0x45627020, 0, 1, init_b, n_h, m_h,
+                               exp_subhnb_h, sizeof(exp_subhnb_h));
+    test_arm64_sve2_narrow_run(0x45627c20, 0, 1, init_b, n_h, m_h,
+                               exp_rsubhnt_h, sizeof(exp_rsubhnt_h));
+    test_arm64_sve2_narrow_run(0x45a26420, 1, 2, init_h, n_s, m_s,
+                               exp_addhnt_s, sizeof(exp_addhnt_s));
+    test_arm64_sve2_narrow_run(0x45a26820, 1, 2, init_h, n_s, m_s,
+                               exp_raddhnb_s, sizeof(exp_raddhnb_s));
+    test_arm64_sve2_narrow_run(0x45a27420, 1, 2, init_h, n_s, m_s,
+                               exp_subhnt_s, sizeof(exp_subhnt_s));
+    test_arm64_sve2_narrow_run(0x45e26c20, 2, 3, init_s, n_d, m_d,
+                               exp_raddhnt_d, sizeof(exp_raddhnt_d));
+    test_arm64_sve2_narrow_run(0x45e27820, 2, 3, init_s, n_d, m_d,
+                               exp_rsubhnb_d, sizeof(exp_rsubhnb_d));
+}
+
+static void test_arm64_sve2_xtn(void)
+{
+    const uint8_t init_b[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint16_t init_h[8] = {
+        0xa000, 0xa001, 0xa002, 0xa003,
+        0xa004, 0xa005, 0xa006, 0xa007,
+    };
+    const uint32_t init_s[4] = {
+        0xa0000000u, 0xa0000001u, 0xa0000002u, 0xa0000003u,
+    };
+    const uint16_t src_h[8] = {
+        0x007f, 0x0080, 0xff80, 0xff7f,
+        0x0000, 0xffff, 0x1234, 0x8000,
+    };
+    const uint32_t src_s[4] = {
+        0x00007fffu, 0x00008000u, 0xffff8000u, 0xffff7fffu,
+    };
+    const uint64_t src_d[2] = {
+        0x000000007fffffffull, 0x0000000080000000ull,
+    };
+    const uint8_t exp_sqxtnb_h[16] = {
+        0x7f, 0x00, 0x7f, 0x00, 0x80, 0x00, 0x80, 0x00,
+        0x00, 0x00, 0xff, 0x00, 0x7f, 0x00, 0x80, 0x00,
+    };
+    const uint8_t exp_sqxtnt_h[16] = {
+        0xa0, 0x7f, 0xa2, 0x7f, 0xa4, 0x80, 0xa6, 0x80,
+        0xa8, 0x00, 0xaa, 0xff, 0xac, 0x7f, 0xae, 0x80,
+    };
+    const uint16_t exp_uqxtnb_s[8] = {
+        0x7fff, 0x0000, 0x8000, 0x0000,
+        0xffff, 0x0000, 0xffff, 0x0000,
+    };
+    const uint8_t exp_sqxtunb_h[16] = {
+        0x7f, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00,
+    };
+    const uint16_t exp_sqxtunt_s[8] = {
+        0xa000, 0x7fff, 0xa002, 0x8000,
+        0xa004, 0x0000, 0xa006, 0x0000,
+    };
+    const uint32_t exp_uqxtnt_d[4] = {
+        0xa0000000u, 0x7fffffffu, 0xa0000002u, 0x80000000u,
+    };
+
+    test_arm64_sve2_narrow_run(0x45284020, 0, 1, init_b, src_h, src_h,
+                               exp_sqxtnb_h, sizeof(exp_sqxtnb_h));
+    test_arm64_sve2_narrow_run(0x45284420, 0, 1, init_b, src_h, src_h,
+                               exp_sqxtnt_h, sizeof(exp_sqxtnt_h));
+    test_arm64_sve2_narrow_run(0x45304820, 1, 2, init_h, src_s, src_s,
+                               exp_uqxtnb_s, sizeof(exp_uqxtnb_s));
+    test_arm64_sve2_narrow_run(0x45285020, 0, 1, init_b, src_h, src_h,
+                               exp_sqxtunb_h, sizeof(exp_sqxtunb_h));
+    test_arm64_sve2_narrow_run(0x45305420, 1, 2, init_h, src_s, src_s,
+                               exp_sqxtunt_s, sizeof(exp_sqxtunt_s));
+    test_arm64_sve2_narrow_run(0x45604c20, 2, 3, init_s, src_d, src_d,
+                               exp_uqxtnt_d, sizeof(exp_uqxtnt_d));
+}
+
+static void test_arm64_sve2_shift_narrow(void)
+{
+    const uint8_t init_b[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint16_t init_h[8] = {
+        0xa000, 0xa001, 0xa002, 0xa003,
+        0xa004, 0xa005, 0xa006, 0xa007,
+    };
+    const uint32_t init_s[4] = {
+        0xa0000000u, 0xa0000001u, 0xa0000002u, 0xa0000003u,
+    };
+    const uint16_t src_h[8] = {
+        0x0000, 0x00ff, 0x0100, 0x7fff,
+        0x8000, 0xff00, 0xffff, 0x1234,
+    };
+    const uint32_t src_s[4] = {
+        0x00000000u, 0x0000ffffu, 0x00010000u, 0xffffffffu,
+    };
+    const uint32_t src_s_signed[4] = {
+        0x00010000u, 0x7fffffffu, 0x80000000u, 0xffff0000u,
+    };
+    const uint64_t src_d[2] = {
+        0x00000000ffffffffull, 0xffffffff00000000ull,
+    };
+    const uint64_t src_d_signed[2] = {
+        0x0000000100000000ull, 0x8000000000000000ull,
+    };
+    const uint8_t exp_shrnb_h[16] = {
+        0x00, 0x00, 0x1f, 0x00, 0x20, 0x00, 0xff, 0x00,
+        0x00, 0x00, 0xe0, 0x00, 0xff, 0x00, 0x46, 0x00,
+    };
+    const uint8_t exp_shrnt_h[16] = {
+        0xa0, 0x00, 0xa2, 0x0f, 0xa4, 0x10, 0xa6, 0xff,
+        0xa8, 0x00, 0xaa, 0xf0, 0xac, 0xff, 0xae, 0x23,
+    };
+    const uint16_t exp_rshrnb_s[8] = {
+        0x0000, 0x0000, 0x0800, 0x0000,
+        0x0800, 0x0000, 0x0000, 0x0000,
+    };
+    const uint16_t exp_rshrnt_s[8] = {
+        0xa000, 0x0000, 0xa002, 0x0400,
+        0xa004, 0x0400, 0xa006, 0x0000,
+    };
+    const uint16_t exp_sqshrunb_s[8] = {
+        0x1000, 0x0000, 0xffff, 0x0000,
+        0x0000, 0x0000, 0x0000, 0x0000,
+    };
+    const uint32_t exp_sqrshrunt_d[4] = {
+        0xa0000000u, 0x00000002u,
+        0xa0000002u, 0x00000000u,
+    };
+    const uint32_t exp_sqrshrnt_d[4] = {
+        0xa0000000u, 0x00000002u,
+        0xa0000002u, 0x80000000u,
+    };
+    const uint8_t exp_sqshrnb_h[16] = {
+        0x00, 0x00, 0x3f, 0x00, 0x40, 0x00, 0x7f, 0x00,
+        0x80, 0x00, 0xc0, 0x00, 0xff, 0x00, 0x7f, 0x00,
+    };
+    const uint8_t exp_sqrshrunb_h[16] = {
+        0x00, 0x00, 0x80, 0x00, 0x80, 0x00, 0xff, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00,
+    };
+    const uint32_t exp_uqshrnb_d[4] = {
+        0x0fffffffu, 0x00000000u, 0xffffffffu, 0x00000000u,
+    };
+    const uint16_t exp_uqshrnt_s[8] = {
+        0xa000, 0x0000, 0xa002, 0x1fff,
+        0xa004, 0x2000, 0xa006, 0xffff,
+    };
+    const uint8_t exp_uqrshrnb_h[16] = {
+        0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x80, 0x00,
+        0x80, 0x00, 0xff, 0x00, 0xff, 0x00, 0x12, 0x00,
+    };
+    const uint8_t exp_uqrshrnt_h[16] = {
+        0xa0, 0x00, 0xa2, 0x01, 0xa4, 0x01, 0xa6, 0x80,
+        0xa8, 0x80, 0xaa, 0xff, 0xac, 0xff, 0xae, 0x12,
+    };
+
+    test_arm64_sve2_narrow_run(0x452d1020, 0, 1, init_b, src_h, src_h,
+                               exp_shrnb_h, sizeof(exp_shrnb_h));
+    test_arm64_sve2_narrow_run(0x452c1420, 0, 1, init_b, src_h, src_h,
+                               exp_shrnt_h, sizeof(exp_shrnt_h));
+    test_arm64_sve2_narrow_run(0x453b1820, 1, 2, init_h, src_s, src_s,
+                               exp_rshrnb_s, sizeof(exp_rshrnb_s));
+    test_arm64_sve2_narrow_run(0x453a1c20, 1, 2, init_h, src_s, src_s,
+                               exp_rshrnt_s, sizeof(exp_rshrnt_s));
+    test_arm64_sve2_narrow_run(0x453c0020, 1, 2, init_h, src_s_signed,
+                               src_s_signed, exp_sqshrunb_s,
+                               sizeof(exp_sqshrunb_s));
+    test_arm64_sve2_narrow_run(0x45610c20, 2, 3, init_s, src_d_signed,
+                               src_d_signed, exp_sqrshrunt_d,
+                               sizeof(exp_sqrshrunt_d));
+    test_arm64_sve2_narrow_run(0x45612c20, 2, 3, init_s, src_d_signed,
+                               src_d_signed, exp_sqrshrnt_d,
+                               sizeof(exp_sqrshrnt_d));
+    test_arm64_sve2_narrow_run(0x452e2020, 0, 1, init_b, src_h, src_h,
+                               exp_sqshrnb_h, sizeof(exp_sqshrnb_h));
+    test_arm64_sve2_narrow_run(0x452f0820, 0, 1, init_b, src_h, src_h,
+                               exp_sqrshrunb_h, sizeof(exp_sqrshrunb_h));
+    test_arm64_sve2_narrow_run(0x457c3020, 2, 3, init_s, src_d, src_d,
+                               exp_uqshrnb_d, sizeof(exp_uqshrnb_d));
+    test_arm64_sve2_narrow_run(0x453d3420, 1, 2, init_h, src_s, src_s,
+                               exp_uqshrnt_s, sizeof(exp_uqshrnt_s));
+    test_arm64_sve2_narrow_run(0x45283820, 0, 1, init_b, src_h, src_h,
+                               exp_uqrshrnb_h, sizeof(exp_uqrshrnb_h));
+    test_arm64_sve2_narrow_run(0x45283c20, 0, 1, init_b, src_h, src_h,
+                               exp_uqrshrnt_h, sizeof(exp_uqrshrnt_h));
+}
+
+static void test_arm64_sve2_shift_accumulate(void)
+{
+    const uint8_t init_b[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    const uint8_t src_b[16] = {
+        0x80, 0x7f, 0xff, 0x01, 0x40, 0xc0, 0x00, 0xfe,
+        0x55, 0xaa, 0x08, 0xf8, 0x81, 0x7e, 0x02, 0x00,
+    };
+    const uint8_t exp_ssra_b[16] = {
+        0x0f, 0x11, 0x11, 0x13, 0x14, 0x14, 0x16, 0x16,
+        0x18, 0x18, 0x1a, 0x1a, 0x1b, 0x1d, 0x1e, 0x1f,
+    };
+    const uint16_t init_h[8] = {
+        0x1000, 0x1001, 0x1002, 0x1003,
+        0x1004, 0x1005, 0x1006, 0x1007,
+    };
+    const uint16_t src_h[8] = {
+        0x00f0, 0xffff, 0x8000, 0x0001,
+        0x1234, 0xf000, 0x0ff0, 0x7000,
+    };
+    const uint16_t exp_usra_h[8] = {
+        0x100f, 0x2000, 0x1802, 0x1003,
+        0x1127, 0x1f05, 0x1105, 0x1707,
+    };
+    const uint32_t init_s[4] = {
+        0x10000000u, 0x10000001u, 0x10000002u, 0x10000003u,
+    };
+    const uint32_t src_s[4] = {
+        0x0000001fu, 0xfffffff1u, 0x80000000u, 0x7ffffff8u,
+    };
+    const uint32_t exp_srsra_s[4] = {
+        0x10000002u, 0x10000000u, 0x08000002u, 0x18000003u,
+    };
+    const uint64_t init_d[2] = {
+        0x1000000000000000ull, 0x1000000000000001ull,
+    };
+    const uint64_t src_d[2] = {
+        0x8000000000000000ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t exp_ursra_d[2] = {
+        0x1000000000000001ull, 0x1000000000000001ull,
+    };
+
+    test_arm64_sve2_narrow_run(0x4508e020, 0, 0, init_b, src_b, src_b,
+                               exp_ssra_b, sizeof(exp_ssra_b));
+    test_arm64_sve2_narrow_run(0x451ce420, 1, 1, init_h, src_h, src_h,
+                               exp_usra_h, sizeof(exp_usra_h));
+    test_arm64_sve2_narrow_run(0x455ce820, 2, 2, init_s, src_s, src_s,
+                               exp_srsra_s, sizeof(exp_srsra_s));
+    test_arm64_sve2_narrow_run(0x4580ec20, 3, 3, init_d, src_d, src_d,
+                               exp_ursra_d, sizeof(exp_ursra_d));
+}
+
+static uint64_t test_arm64_sve2_load_elem(const uint8_t *data, size_t offset,
+                                          int esz)
+{
+    uint64_t value = 0;
+    unsigned bytes = 1u << esz;
+    unsigned i;
+
+    for (i = 0; i < bytes; i++) {
+        value |= (uint64_t)data[offset + i] << (i * 8);
+    }
+    return value;
+}
+
+static void test_arm64_sve2_store_elem(uint8_t *data, size_t offset, int esz,
+                                       uint64_t value)
+{
+    unsigned bytes = 1u << esz;
+    unsigned i;
+
+    for (i = 0; i < bytes; i++) {
+        data[offset + i] = (uint8_t)(value >> (i * 8));
+    }
+}
+
+static uint64_t test_arm64_sve2_elem_mask(int esz)
+{
+    unsigned bits = 8u << esz;
+
+    return bits == 64 ? ~0ull : ((1ull << bits) - 1);
+}
+
+static uint64_t test_arm64_sve2_shift_insert_expected(uint64_t old,
+                                                      uint64_t src,
+                                                      int esz,
+                                                      unsigned shift,
+                                                      bool left)
+{
+    unsigned bits = 8u << esz;
+    uint64_t mask = test_arm64_sve2_elem_mask(esz);
+
+    old &= mask;
+    src &= mask;
+    if (left) {
+        uint64_t low_mask;
+
+        if (shift == 0) {
+            return src;
+        }
+        low_mask = (1ull << shift) - 1;
+        return (old & low_mask) | ((src << shift) & mask);
+    } else {
+        uint64_t low_mask;
+
+        if (shift == bits) {
+            return old;
+        }
+        low_mask = (1ull << (bits - shift)) - 1;
+        return (old & (mask & ~low_mask)) | ((src >> shift) & low_mask);
+    }
+}
+
+static void test_arm64_sve2_shift_insert_run(uint32_t insn, int esz,
+                                             unsigned shift, bool left)
+{
+    static const uint32_t ld1_z0[4] = {
+        0xa400a060, 0xa4a0a060, 0xa540a060, 0xa5e0a060,
+    };
+    static const uint32_t ld1_z1[4] = {
+        0xa400a081, 0xa4a0a081, 0xa540a081, 0xa5e0a081,
+    };
+    static const uint32_t st1_z0[4] = {
+        0xe400e0c0, 0xe4a0e0c0, 0xe540e0c0, 0xe5e0e0c0,
+    };
+    uc_engine *uc;
+    uint8_t code[20];
+    uint8_t initial[32];
+    uint8_t source[32];
+    uint8_t expected[32];
+    uint8_t got[32];
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x6 = 0x40200;
+    unsigned bytes = 1u << esz;
+    size_t i;
+
+    for (i = 0; i < sizeof(initial); i++) {
+        initial[i] = (uint8_t)(0xa5u + i * 13u);
+        source[i] = (uint8_t)(0x3cu + i * 29u);
+    }
+    memcpy(expected, initial, sizeof(expected));
+    for (i = 0; i < sizeof(expected); i += bytes) {
+        uint64_t old = test_arm64_sve2_load_elem(initial, i, esz);
+        uint64_t src = test_arm64_sve2_load_elem(source, i, esz);
+        uint64_t res;
+
+        res = test_arm64_sve2_shift_insert_expected(old, src, esz,
+                                                    shift, left);
+        test_arm64_sve2_store_elem(expected, i, esz, res);
+    }
+
+    test_arm64_emit32(code, 0, 0x2518e3e0);
+    test_arm64_emit32(code, 4, ld1_z0[esz]);
+    test_arm64_emit32(code, 8, ld1_z1[esz]);
+    test_arm64_emit32(code, 12, insn);
+    test_arm64_emit32(code, 16, st1_z0[esz]);
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, initial, sizeof(initial)));
+    OK(uc_mem_write(uc, x4, source, sizeof(source)));
+    test_arm64_mte_enable_sve_vq(uc, 1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, x6, got, sizeof(got)));
+    for (i = 0; i < sizeof(got); i++) {
+        TEST_CHECK_(got[i] == expected[i],
+                    "insn %08x byte %zu got %02x expected %02x",
+                    insn, i, got[i], expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_shift_insert(void)
+{
+    test_arm64_sve2_shift_insert_run(0x450df020, 0, 3, false);
+    test_arm64_sve2_shift_insert_run(0x451cf020, 1, 4, false);
+    test_arm64_sve2_shift_insert_run(0x455bf020, 2, 5, false);
+    test_arm64_sve2_shift_insert_run(0x45d8f020, 3, 8, false);
+    test_arm64_sve2_shift_insert_run(0x4508f020, 0, 8, false);
+
+    test_arm64_sve2_shift_insert_run(0x450bf420, 0, 3, true);
+    test_arm64_sve2_shift_insert_run(0x4514f420, 1, 4, true);
+    test_arm64_sve2_shift_insert_run(0x4545f420, 2, 5, true);
+    test_arm64_sve2_shift_insert_run(0x4588f420, 3, 8, true);
+    test_arm64_sve2_shift_insert_run(0x4508f420, 0, 0, true);
+}
+
+static void test_arm64_sve2_sat_unary(void)
+{
+    uc_engine *uc;
+    const char code_sqabs_b[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x3] */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\x80\xe0\x18\x25" /* ptrue p0.b,vl4 */
+        "\x20\xa0\x08\x44" /* sqabs z0.b,p0/m,z1.b */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_sqneg_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x4] */
+        "\x20\xa0\xc9\x44" /* sqneg z0.d,p0/m,z1.d */
+        "\xc0\xe0\xe0\xe5"; /* st1d { z0.d },p0,[x6] */
+    const uint8_t init_b[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint8_t src_b[16] = {
+        0x80, 0x81, 0xff, 0x7f, 0x01, 0xfe, 0x40, 0xc0,
+        0x00, 0x55, 0xaa, 0x02, 0xfd, 0x10, 0xf0, 0x7e,
+    };
+    const uint8_t exp_sqabs_b[16] = {
+        0x7f, 0x7f, 0x01, 0x7f, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint64_t src_d[2] = {
+        0x8000000000000000ull, 0xffffffffffffffffull,
+    };
+    const uint64_t exp_sqneg_d[2] = {
+        0x7fffffffffffffffull, 0x0000000000000001ull,
+    };
+    uint8_t got_b[16];
+    uint64_t got_d[2];
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x6 = 0x40200;
+    int i;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_sqabs_b,
+                    sizeof(code_sqabs_b) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, init_b, sizeof(init_b)));
+    OK(uc_mem_write(uc, x4, src_b, sizeof(src_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_sqabs_b) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_sqabs_b[i]);
+    }
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_sqneg_d,
+                    sizeof(code_sqneg_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, src_d, sizeof(src_d)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_sqneg_d) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_sqneg_d[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_adalp(void)
+{
+    const uint16_t init_h[8] = {
+        0x0100, 0x0101, 0x0102, 0x0103,
+        0x0104, 0x0105, 0x0106, 0x0107,
+    };
+    const uint32_t init_s[4] = {
+        0x01000000u, 0x01000001u,
+        0x01000002u, 0x01000003u,
+    };
+    const uint64_t init_d[2] = {
+        0x0100000000000000ull, 0x0100000000000001ull,
+    };
+    const uint8_t src_b[16] = {
+        0x02, 0x01, 0x80, 0xff, 0x81, 0x7f, 0x80, 0x80,
+        0x01, 0x00, 0xf0, 0x10, 0x0f, 0xf0, 0xff, 0xff,
+    };
+    const uint16_t src_h[8] = {
+        0x0001, 0xffff, 0x8000, 0x7fff,
+        0xff00, 0x0100, 0x1234, 0xedcc,
+    };
+    const uint32_t src_s[4] = {
+        0x00000001u, 0xffffffffu, 0x80000000u, 0x7fffffffu,
+    };
+    const uint32_t src_s_unsigned[4] = {
+        0x00000001u, 0x00000002u, 0x80000000u, 0xffffffffu,
+    };
+    const uint16_t exp_sadalp_h[8] = {
+        0x0103, 0x0080, 0x0102, 0x0003,
+        0x0105, 0x0105, 0x0105, 0x0105,
+    };
+    const uint32_t exp_sadalp_s[4] = {
+        0x01000000u, 0x01000000u,
+        0x01000002u, 0x01000003u,
+    };
+    const uint64_t exp_sadalp_d[2] = {
+        0x0100000000000000ull, 0x0100000000000000ull,
+    };
+    const uint16_t exp_uadalp_h[8] = {
+        0x0103, 0x0280, 0x0202, 0x0203,
+        0x0105, 0x0205, 0x0205, 0x0305,
+    };
+    const uint32_t exp_uadalp_s[4] = {
+        0x01010000u, 0x01010000u,
+        0x01010002u, 0x01010003u,
+    };
+    const uint64_t exp_uadalp_d[2] = {
+        0x0100000000000003ull, 0x0100000180000000ull,
+    };
+
+    test_arm64_sve2_narrow_run(0x4444a020, 1, 0, init_h, src_b, src_b,
+                               exp_sadalp_h, sizeof(exp_sadalp_h));
+    test_arm64_sve2_narrow_run(0x4484a020, 2, 1, init_s, src_h, src_h,
+                               exp_sadalp_s, sizeof(exp_sadalp_s));
+    test_arm64_sve2_narrow_run(0x44c4a020, 3, 2, init_d, src_s, src_s,
+                               exp_sadalp_d, sizeof(exp_sadalp_d));
+    test_arm64_sve2_narrow_run(0x4445a020, 1, 0, init_h, src_b, src_b,
+                               exp_uadalp_h, sizeof(exp_uadalp_h));
+    test_arm64_sve2_narrow_run(0x4485a020, 2, 1, init_s, src_h, src_h,
+                               exp_uadalp_s, sizeof(exp_uadalp_s));
+    test_arm64_sve2_narrow_run(0x44c5a020, 3, 2, init_d, src_s_unsigned,
+                               src_s_unsigned, exp_uadalp_d,
+                               sizeof(exp_uadalp_d));
+}
+
+static void test_arm64_sve2_halving_add_sub(void)
+{
+    const uint8_t n_b[16] = {
+        0x80, 0x7f, 0xff, 0x01, 0x40, 0xc0, 0x00, 0xfe,
+        0x55, 0xaa, 0x08, 0xf8, 0x81, 0x7e, 0x02, 0x00,
+    };
+    const uint8_t m_b[16] = {
+        0x80, 0x01, 0x02, 0xff, 0xc0, 0x40, 0xff, 0x02,
+        0xaa, 0x55, 0xf8, 0x08, 0x7f, 0x82, 0xfe, 0x00,
+    };
+    const uint16_t n_h[8] = {
+        0x8000, 0x7fff, 0xffff, 0x0001,
+        0x4000, 0xc000, 0x0000, 0xfffe,
+    };
+    const uint16_t m_h[8] = {
+        0x8000, 0x0001, 0x0002, 0xffff,
+        0xc000, 0x4000, 0xffff, 0x0002,
+    };
+    const uint32_t n_s[4] = {
+        0x80000000u, 0x7fffffffu, 0xffffffffu, 0x00000001u,
+    };
+    const uint32_t m_s[4] = {
+        0x80000000u, 0x00000001u, 0x00000002u, 0xffffffffu,
+    };
+    const uint64_t n_d[2] = {
+        0xffffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint64_t m_d[2] = {
+        0x0000000000000001ull, 0x7fffffffffffffffull,
+    };
+    const uint8_t exp_shadd_b[16] = {
+        0x80, 0x40, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00,
+        0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t exp_uhadd_b[16] = {
+        0x80, 0x40, 0x80, 0x80, 0x80, 0x80, 0x7f, 0x80,
+        0x7f, 0x7f, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00,
+    };
+    const uint8_t exp_shsub_b[16] = {
+        0x00, 0x3f, 0xfe, 0x01, 0x40, 0xc0, 0x00, 0xfe,
+        0x55, 0xaa, 0x08, 0xf8, 0x81, 0x7e, 0x02, 0x00,
+    };
+    const uint8_t exp_uhsub_b[16] = {
+        0x00, 0x3f, 0x7e, 0x81, 0xc0, 0x40, 0x80, 0x7e,
+        0xd5, 0x2a, 0x88, 0x78, 0x01, 0xfe, 0x82, 0x00,
+    };
+    const uint8_t exp_srhadd_b[16] = {
+        0x80, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t exp_urhadd_b[16] = {
+        0x80, 0x40, 0x81, 0x80, 0x80, 0x80, 0x80, 0x80,
+        0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00,
+    };
+    const uint8_t exp_shsubr_b[16] = {
+        0x00, 0xc1, 0x01, 0xff, 0xc0, 0x40, 0xff, 0x02,
+        0xaa, 0x55, 0xf8, 0x08, 0x7f, 0x82, 0xfe, 0x00,
+    };
+    const uint16_t exp_shadd_h[8] = {
+        0x8000, 0x4000, 0x0000, 0x0000,
+        0x0000, 0x0000, 0xffff, 0x0000,
+    };
+    const uint16_t exp_uhadd_h[8] = {
+        0x8000, 0x4000, 0x8000, 0x8000,
+        0x8000, 0x8000, 0x7fff, 0x8000,
+    };
+    const uint16_t exp_shsub_h[8] = {
+        0x0000, 0x3fff, 0xfffe, 0x0001,
+        0x4000, 0xc000, 0x0000, 0xfffe,
+    };
+    const uint16_t exp_uhsub_h[8] = {
+        0x0000, 0x3fff, 0x7ffe, 0x8001,
+        0xc000, 0x4000, 0x8000, 0x7ffe,
+    };
+    const uint16_t exp_srhadd_h[8] = {
+        0x8000, 0x4000, 0x0001, 0x0000,
+        0x0000, 0x0000, 0x0000, 0x0000,
+    };
+    const uint16_t exp_urhadd_h[8] = {
+        0x8000, 0x4000, 0x8001, 0x8000,
+        0x8000, 0x8000, 0x8000, 0x8000,
+    };
+    const uint16_t exp_uhsubr_h[8] = {
+        0x0000, 0xc001, 0x8001, 0x7fff,
+        0x4000, 0xc000, 0x7fff, 0x8002,
+    };
+    const uint32_t exp_shadd_s[4] = {
+        0x80000000u, 0x40000000u, 0x00000000u, 0x00000000u,
+    };
+    const uint32_t exp_uhadd_s[4] = {
+        0x80000000u, 0x40000000u, 0x80000000u, 0x80000000u,
+    };
+    const uint32_t exp_shsub_s[4] = {
+        0x00000000u, 0x3fffffffu, 0xfffffffeu, 0x00000001u,
+    };
+    const uint32_t exp_uhsub_s[4] = {
+        0x00000000u, 0x3fffffffu, 0x7ffffffeu, 0x80000001u,
+    };
+    const uint32_t exp_srhadd_s[4] = {
+        0x80000000u, 0x40000000u, 0x00000001u, 0x00000000u,
+    };
+    const uint32_t exp_urhadd_s[4] = {
+        0x80000000u, 0x40000000u, 0x80000001u, 0x80000000u,
+    };
+    const uint64_t exp_shadd_d[2] = {
+        0x0000000000000000ull, 0xffffffffffffffffull,
+    };
+    const uint64_t exp_uhadd_d[2] = {
+        0x8000000000000000ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t exp_shsub_d[2] = {
+        0xffffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint64_t exp_uhsub_d[2] = {
+        0x7fffffffffffffffull, 0x0000000000000000ull,
+    };
+    const uint64_t exp_srhadd_d[2] = {
+        0x0000000000000000ull, 0x0000000000000000ull,
+    };
+    const uint64_t exp_urhadd_d[2] = {
+        0x8000000000000000ull, 0x8000000000000000ull,
+    };
+
+    test_arm64_sve2_narrow_run(0x44108020, 0, 0, n_b, m_b, m_b,
+                               exp_shadd_b, sizeof(exp_shadd_b));
+    test_arm64_sve2_narrow_run(0x44508020, 1, 1, n_h, m_h, m_h,
+                               exp_shadd_h, sizeof(exp_shadd_h));
+    test_arm64_sve2_narrow_run(0x44908020, 2, 2, n_s, m_s, m_s,
+                               exp_shadd_s, sizeof(exp_shadd_s));
+    test_arm64_sve2_narrow_run(0x44d08020, 3, 3, n_d, m_d, m_d,
+                               exp_shadd_d, sizeof(exp_shadd_d));
+    test_arm64_sve2_narrow_run(0x44118020, 0, 0, n_b, m_b, m_b,
+                               exp_uhadd_b, sizeof(exp_uhadd_b));
+    test_arm64_sve2_narrow_run(0x44518020, 1, 1, n_h, m_h, m_h,
+                               exp_uhadd_h, sizeof(exp_uhadd_h));
+    test_arm64_sve2_narrow_run(0x44918020, 2, 2, n_s, m_s, m_s,
+                               exp_uhadd_s, sizeof(exp_uhadd_s));
+    test_arm64_sve2_narrow_run(0x44d18020, 3, 3, n_d, m_d, m_d,
+                               exp_uhadd_d, sizeof(exp_uhadd_d));
+    test_arm64_sve2_narrow_run(0x44128020, 0, 0, n_b, m_b, m_b,
+                               exp_shsub_b, sizeof(exp_shsub_b));
+    test_arm64_sve2_narrow_run(0x44528020, 1, 1, n_h, m_h, m_h,
+                               exp_shsub_h, sizeof(exp_shsub_h));
+    test_arm64_sve2_narrow_run(0x44928020, 2, 2, n_s, m_s, m_s,
+                               exp_shsub_s, sizeof(exp_shsub_s));
+    test_arm64_sve2_narrow_run(0x44d28020, 3, 3, n_d, m_d, m_d,
+                               exp_shsub_d, sizeof(exp_shsub_d));
+    test_arm64_sve2_narrow_run(0x44138020, 0, 0, n_b, m_b, m_b,
+                               exp_uhsub_b, sizeof(exp_uhsub_b));
+    test_arm64_sve2_narrow_run(0x44538020, 1, 1, n_h, m_h, m_h,
+                               exp_uhsub_h, sizeof(exp_uhsub_h));
+    test_arm64_sve2_narrow_run(0x44938020, 2, 2, n_s, m_s, m_s,
+                               exp_uhsub_s, sizeof(exp_uhsub_s));
+    test_arm64_sve2_narrow_run(0x44d38020, 3, 3, n_d, m_d, m_d,
+                               exp_uhsub_d, sizeof(exp_uhsub_d));
+    test_arm64_sve2_narrow_run(0x44148020, 0, 0, n_b, m_b, m_b,
+                               exp_srhadd_b, sizeof(exp_srhadd_b));
+    test_arm64_sve2_narrow_run(0x44548020, 1, 1, n_h, m_h, m_h,
+                               exp_srhadd_h, sizeof(exp_srhadd_h));
+    test_arm64_sve2_narrow_run(0x44948020, 2, 2, n_s, m_s, m_s,
+                               exp_srhadd_s, sizeof(exp_srhadd_s));
+    test_arm64_sve2_narrow_run(0x44d48020, 3, 3, n_d, m_d, m_d,
+                               exp_srhadd_d, sizeof(exp_srhadd_d));
+    test_arm64_sve2_narrow_run(0x44158020, 0, 0, n_b, m_b, m_b,
+                               exp_urhadd_b, sizeof(exp_urhadd_b));
+    test_arm64_sve2_narrow_run(0x44558020, 1, 1, n_h, m_h, m_h,
+                               exp_urhadd_h, sizeof(exp_urhadd_h));
+    test_arm64_sve2_narrow_run(0x44958020, 2, 2, n_s, m_s, m_s,
+                               exp_urhadd_s, sizeof(exp_urhadd_s));
+    test_arm64_sve2_narrow_run(0x44d58020, 3, 3, n_d, m_d, m_d,
+                               exp_urhadd_d, sizeof(exp_urhadd_d));
+    test_arm64_sve2_narrow_run(0x44168020, 0, 0, n_b, m_b, m_b,
+                               exp_shsubr_b, sizeof(exp_shsubr_b));
+    test_arm64_sve2_narrow_run(0x44578020, 1, 1, n_h, m_h, m_h,
+                               exp_uhsubr_h, sizeof(exp_uhsubr_h));
+}
+
+static void test_arm64_sve2_pairwise_pred(void)
+{
+    const uint8_t n_b[16] = {
+        0x80, 0x7f, 0xff, 0x01, 0x40, 0xc0, 0x00, 0xfe,
+        0x55, 0xaa, 0x08, 0xf8, 0x81, 0x7e, 0x02, 0x00,
+    };
+    const uint8_t m_b[16] = {
+        0x80, 0x01, 0x02, 0xff, 0xc0, 0x40, 0xff, 0x02,
+        0xaa, 0x55, 0xf8, 0x08, 0x7f, 0x82, 0xfe, 0x00,
+    };
+    const uint16_t n_h[8] = {
+        0x8000, 0x7fff, 0xffff, 0x0001,
+        0x4000, 0xc000, 0x0000, 0xfffe,
+    };
+    const uint16_t m_h[8] = {
+        0x8000, 0x0001, 0x0002, 0xffff,
+        0xc000, 0x4000, 0xffff, 0x0002,
+    };
+    const uint32_t n_s[4] = {
+        0x80000000u, 0x7fffffffu, 0xffffffffu, 0x00000001u,
+    };
+    const uint32_t m_s[4] = {
+        0x80000000u, 0x00000001u, 0x00000002u, 0xffffffffu,
+    };
+    const uint64_t n_d[2] = {
+        0x8000000000000000ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t m_d[2] = {
+        0xffffffffffffffffull, 0x0000000000000001ull,
+    };
+    const uint8_t exp_addp_b[16] = {
+        0xff, 0x81, 0x00, 0x01, 0x00, 0x00, 0xfe, 0x01,
+        0xff, 0xff, 0x00, 0x00, 0xff, 0x01, 0x02, 0xfe,
+    };
+    const uint8_t exp_smaxp_b[16] = {
+        0x7f, 0x01, 0x01, 0x02, 0x40, 0x40, 0x00, 0x02,
+        0x55, 0x55, 0x08, 0x08, 0x7e, 0x7f, 0x02, 0x00,
+    };
+    const uint8_t exp_umaxp_b[16] = {
+        0x80, 0x80, 0xff, 0xff, 0xc0, 0xc0, 0xfe, 0xff,
+        0xaa, 0xaa, 0xf8, 0xf8, 0x81, 0x82, 0x02, 0xfe,
+    };
+    const uint8_t exp_sminp_b[16] = {
+        0x80, 0x80, 0xff, 0xff, 0xc0, 0xc0, 0xfe, 0xff,
+        0xaa, 0xaa, 0xf8, 0xf8, 0x81, 0x82, 0x00, 0xfe,
+    };
+    const uint8_t exp_uminp_b[16] = {
+        0x7f, 0x01, 0x01, 0x02, 0x40, 0x40, 0x00, 0x02,
+        0x55, 0x55, 0x08, 0x08, 0x7e, 0x7f, 0x00, 0x00,
+    };
+    const uint16_t exp_addp_h[8] = {
+        0xffff, 0x8001, 0x0000, 0x0001,
+        0x0000, 0x0000, 0xfffe, 0x0001,
+    };
+    const uint16_t exp_smaxp_h[8] = {
+        0x7fff, 0x0001, 0x0001, 0x0002,
+        0x4000, 0x4000, 0x0000, 0x0002,
+    };
+    const uint16_t exp_umaxp_h[8] = {
+        0x8000, 0x8000, 0xffff, 0xffff,
+        0xc000, 0xc000, 0xfffe, 0xffff,
+    };
+    const uint16_t exp_sminp_h[8] = {
+        0x8000, 0x8000, 0xffff, 0xffff,
+        0xc000, 0xc000, 0xfffe, 0xffff,
+    };
+    const uint16_t exp_uminp_h[8] = {
+        0x7fff, 0x0001, 0x0001, 0x0002,
+        0x4000, 0x4000, 0x0000, 0x0002,
+    };
+    const uint32_t exp_addp_s[4] = {
+        0xffffffffu, 0x80000001u, 0x00000000u, 0x00000001u,
+    };
+    const uint32_t exp_smaxp_s[4] = {
+        0x7fffffffu, 0x00000001u, 0x00000001u, 0x00000002u,
+    };
+    const uint32_t exp_umaxp_s[4] = {
+        0x80000000u, 0x80000000u, 0xffffffffu, 0xffffffffu,
+    };
+    const uint32_t exp_sminp_s[4] = {
+        0x80000000u, 0x80000000u, 0xffffffffu, 0xffffffffu,
+    };
+    const uint32_t exp_uminp_s[4] = {
+        0x7fffffffu, 0x00000001u, 0x00000001u, 0x00000002u,
+    };
+    const uint64_t exp_addp_d[2] = {
+        0xffffffffffffffffull, 0x0000000000000000ull,
+    };
+    const uint64_t exp_smaxp_d[2] = {
+        0x7fffffffffffffffull, 0x0000000000000001ull,
+    };
+    const uint64_t exp_umaxp_d[2] = {
+        0x8000000000000000ull, 0xffffffffffffffffull,
+    };
+    const uint64_t exp_sminp_d[2] = {
+        0x8000000000000000ull, 0xffffffffffffffffull,
+    };
+    const uint64_t exp_uminp_d[2] = {
+        0x7fffffffffffffffull, 0x0000000000000001ull,
+    };
+
+    test_arm64_sve2_narrow_run(0x4411a020, 0, 0, n_b, m_b, m_b,
+                               exp_addp_b, sizeof(exp_addp_b));
+    test_arm64_sve2_narrow_run(0x4451a020, 1, 1, n_h, m_h, m_h,
+                               exp_addp_h, sizeof(exp_addp_h));
+    test_arm64_sve2_narrow_run(0x4491a020, 2, 2, n_s, m_s, m_s,
+                               exp_addp_s, sizeof(exp_addp_s));
+    test_arm64_sve2_narrow_run(0x44d1a020, 3, 3, n_d, m_d, m_d,
+                               exp_addp_d, sizeof(exp_addp_d));
+    test_arm64_sve2_narrow_run(0x4414a020, 0, 0, n_b, m_b, m_b,
+                               exp_smaxp_b, sizeof(exp_smaxp_b));
+    test_arm64_sve2_narrow_run(0x4454a020, 1, 1, n_h, m_h, m_h,
+                               exp_smaxp_h, sizeof(exp_smaxp_h));
+    test_arm64_sve2_narrow_run(0x4494a020, 2, 2, n_s, m_s, m_s,
+                               exp_smaxp_s, sizeof(exp_smaxp_s));
+    test_arm64_sve2_narrow_run(0x44d4a020, 3, 3, n_d, m_d, m_d,
+                               exp_smaxp_d, sizeof(exp_smaxp_d));
+    test_arm64_sve2_narrow_run(0x4415a020, 0, 0, n_b, m_b, m_b,
+                               exp_umaxp_b, sizeof(exp_umaxp_b));
+    test_arm64_sve2_narrow_run(0x4455a020, 1, 1, n_h, m_h, m_h,
+                               exp_umaxp_h, sizeof(exp_umaxp_h));
+    test_arm64_sve2_narrow_run(0x4495a020, 2, 2, n_s, m_s, m_s,
+                               exp_umaxp_s, sizeof(exp_umaxp_s));
+    test_arm64_sve2_narrow_run(0x44d5a020, 3, 3, n_d, m_d, m_d,
+                               exp_umaxp_d, sizeof(exp_umaxp_d));
+    test_arm64_sve2_narrow_run(0x4416a020, 0, 0, n_b, m_b, m_b,
+                               exp_sminp_b, sizeof(exp_sminp_b));
+    test_arm64_sve2_narrow_run(0x4456a020, 1, 1, n_h, m_h, m_h,
+                               exp_sminp_h, sizeof(exp_sminp_h));
+    test_arm64_sve2_narrow_run(0x4496a020, 2, 2, n_s, m_s, m_s,
+                               exp_sminp_s, sizeof(exp_sminp_s));
+    test_arm64_sve2_narrow_run(0x44d6a020, 3, 3, n_d, m_d, m_d,
+                               exp_sminp_d, sizeof(exp_sminp_d));
+    test_arm64_sve2_narrow_run(0x4417a020, 0, 0, n_b, m_b, m_b,
+                               exp_uminp_b, sizeof(exp_uminp_b));
+    test_arm64_sve2_narrow_run(0x4457a020, 1, 1, n_h, m_h, m_h,
+                               exp_uminp_h, sizeof(exp_uminp_h));
+    test_arm64_sve2_narrow_run(0x4497a020, 2, 2, n_s, m_s, m_s,
+                               exp_uminp_s, sizeof(exp_uminp_s));
+    test_arm64_sve2_narrow_run(0x44d7a020, 3, 3, n_d, m_d, m_d,
+                               exp_uminp_d, sizeof(exp_uminp_d));
+}
+
+static void test_arm64_sve2_saturating_add_sub(void)
+{
+    uc_engine *uc;
+    const char code_pred[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x3] */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\x80\xe0\x18\x25" /* ptrue p0.b,vl4 */
+        "\x20\x80\x18\x44" /* sqadd z0.b,p0/m,z0.b,z1.b */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const uint8_t n_b[16] = {
+        0x80, 0x7f, 0xff, 0x01, 0x40, 0xc0, 0x00, 0xfe,
+        0x55, 0xaa, 0x08, 0xf8, 0x81, 0x7e, 0x02, 0x00,
+    };
+    const uint8_t m_b[16] = {
+        0x80, 0x01, 0x02, 0xff, 0xc0, 0x40, 0xff, 0x02,
+        0xaa, 0x55, 0xf8, 0x08, 0x7f, 0x82, 0xfe, 0x00,
+    };
+    const uint16_t n_h[8] = {
+        0x8000, 0x7fff, 0xffff, 0x0001,
+        0x4000, 0xc000, 0x0000, 0xfffe,
+    };
+    const uint16_t m_h[8] = {
+        0x8000, 0x0001, 0x0002, 0xffff,
+        0xc000, 0x4000, 0xffff, 0x0002,
+    };
+    const uint32_t n_s[4] = {
+        0x80000000u, 0x7fffffffu, 0xffffffffu, 0x00000001u,
+    };
+    const uint32_t m_s[4] = {
+        0x80000000u, 0x00000001u, 0x00000002u, 0xffffffffu,
+    };
+    const uint64_t n_d[2] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint64_t m_d[2] = {
+        0x0000000000000001ull, 0xffffffffffffffffull,
+    };
+    const uint8_t exp_pred_b[16] = {
+        0x80, 0x7f, 0x01, 0x00, 0x40, 0xc0, 0x00, 0xfe,
+        0x55, 0xaa, 0x08, 0xf8, 0x81, 0x7e, 0x02, 0x00,
+    };
+    const uint8_t exp_sqadd_b[16] = {
+        0x80, 0x7f, 0x01, 0x00, 0x00, 0x00, 0xff, 0x00,
+        0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t exp_uqadd_b[16] = {
+        0xff, 0x80, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
+    };
+    const uint8_t exp_sqsub_b[16] = {
+        0x00, 0x7e, 0xfd, 0x02, 0x7f, 0x80, 0x01, 0xfc,
+        0x7f, 0x80, 0x10, 0xf0, 0x80, 0x7f, 0x04, 0x00,
+    };
+    const uint8_t exp_uqsub_b[16] = {
+        0x00, 0x7e, 0xfd, 0x00, 0x00, 0x80, 0x00, 0xfc,
+        0x00, 0x55, 0x00, 0xf0, 0x02, 0x00, 0x00, 0x00,
+    };
+    const uint8_t exp_suqadd_b[16] = {
+        0x00, 0x7f, 0x01, 0x7f, 0x7f, 0x00, 0x7f, 0x00,
+        0x7f, 0xff, 0x7f, 0x00, 0x00, 0x7f, 0x7f, 0x00,
+    };
+    const uint8_t exp_usqadd_b[16] = {
+        0x00, 0x80, 0xff, 0x00, 0x00, 0xff, 0x00, 0xff,
+        0x00, 0xff, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00,
+    };
+    const uint8_t exp_sqsubr_b[16] = {
+        0x00, 0x82, 0x03, 0xfe, 0x80, 0x7f, 0xff, 0x04,
+        0x80, 0x7f, 0xf0, 0x10, 0x7f, 0x80, 0xfc, 0x00,
+    };
+    const uint16_t exp_sqadd_h[8] = {
+        0x8000, 0x7fff, 0x0001, 0x0000,
+        0x0000, 0x0000, 0xffff, 0x0000,
+    };
+    const uint16_t exp_uqadd_h[8] = {
+        0xffff, 0x8000, 0xffff, 0xffff,
+        0xffff, 0xffff, 0xffff, 0xffff,
+    };
+    const uint16_t exp_sqsub_h[8] = {
+        0x0000, 0x7ffe, 0xfffd, 0x0002,
+        0x7fff, 0x8000, 0x0001, 0xfffc,
+    };
+    const uint16_t exp_uqsub_h[8] = {
+        0x0000, 0x7ffe, 0xfffd, 0x0000,
+        0x0000, 0x8000, 0x0000, 0xfffc,
+    };
+    const uint16_t exp_suqadd_h[8] = {
+        0x0000, 0x7fff, 0x0001, 0x7fff,
+        0x7fff, 0x0000, 0x7fff, 0x0000,
+    };
+    const uint16_t exp_usqadd_h[8] = {
+        0x0000, 0x8000, 0xffff, 0x0000,
+        0x0000, 0xffff, 0x0000, 0xffff,
+    };
+    const uint16_t exp_uqsubr_h[8] = {
+        0x0000, 0x0000, 0x0000, 0xfffe,
+        0x8000, 0x0000, 0xffff, 0x0000,
+    };
+    const uint32_t exp_sqadd_s[4] = {
+        0x80000000u, 0x7fffffffu, 0x00000001u, 0x00000000u,
+    };
+    const uint32_t exp_uqadd_s[4] = {
+        0xffffffffu, 0x80000000u, 0xffffffffu, 0xffffffffu,
+    };
+    const uint32_t exp_sqsub_s[4] = {
+        0x00000000u, 0x7ffffffeu, 0xfffffffdu, 0x00000002u,
+    };
+    const uint32_t exp_uqsub_s[4] = {
+        0x00000000u, 0x7ffffffeu, 0xfffffffdu, 0x00000000u,
+    };
+    const uint32_t exp_suqadd_s[4] = {
+        0x00000000u, 0x7fffffffu, 0x00000001u, 0x7fffffffu,
+    };
+    const uint32_t exp_usqadd_s[4] = {
+        0x00000000u, 0x80000000u, 0xffffffffu, 0x00000000u,
+    };
+    const uint64_t exp_sqadd_d[2] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint64_t exp_uqadd_d[2] = {
+        0x8000000000000000ull, 0xffffffffffffffffull,
+    };
+    const uint64_t exp_sqsub_d[2] = {
+        0x7ffffffffffffffeull, 0x8000000000000001ull,
+    };
+    const uint64_t exp_uqsub_d[2] = {
+        0x7ffffffffffffffeull, 0x0000000000000000ull,
+    };
+    const uint64_t exp_suqadd_d[2] = {
+        0x7fffffffffffffffull, 0x7fffffffffffffffull,
+    };
+    const uint64_t exp_usqadd_d[2] = {
+        0x8000000000000000ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t exp_sqsubr_d[2] = {
+        0x8000000000000002ull, 0x7fffffffffffffffull,
+    };
+    const uint64_t exp_uqsubr_d[2] = {
+        0x0000000000000000ull, 0x7fffffffffffffffull,
+    };
+    uint8_t got_b[16];
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x6 = 0x40200;
+    int i;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_pred,
+                    sizeof(code_pred) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, n_b, sizeof(n_b)));
+    OK(uc_mem_write(uc, x4, m_b, sizeof(m_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_pred) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_pred_b[i]);
+    }
+    OK(uc_close(uc));
+
+    test_arm64_sve2_narrow_run(0x44188020, 0, 0, n_b, m_b, m_b,
+                               exp_sqadd_b, sizeof(exp_sqadd_b));
+    test_arm64_sve2_narrow_run(0x44588020, 1, 1, n_h, m_h, m_h,
+                               exp_sqadd_h, sizeof(exp_sqadd_h));
+    test_arm64_sve2_narrow_run(0x44988020, 2, 2, n_s, m_s, m_s,
+                               exp_sqadd_s, sizeof(exp_sqadd_s));
+    test_arm64_sve2_narrow_run(0x44d88020, 3, 3, n_d, m_d, m_d,
+                               exp_sqadd_d, sizeof(exp_sqadd_d));
+    test_arm64_sve2_narrow_run(0x44198020, 0, 0, n_b, m_b, m_b,
+                               exp_uqadd_b, sizeof(exp_uqadd_b));
+    test_arm64_sve2_narrow_run(0x44598020, 1, 1, n_h, m_h, m_h,
+                               exp_uqadd_h, sizeof(exp_uqadd_h));
+    test_arm64_sve2_narrow_run(0x44998020, 2, 2, n_s, m_s, m_s,
+                               exp_uqadd_s, sizeof(exp_uqadd_s));
+    test_arm64_sve2_narrow_run(0x44d98020, 3, 3, n_d, m_d, m_d,
+                               exp_uqadd_d, sizeof(exp_uqadd_d));
+    test_arm64_sve2_narrow_run(0x441a8020, 0, 0, n_b, m_b, m_b,
+                               exp_sqsub_b, sizeof(exp_sqsub_b));
+    test_arm64_sve2_narrow_run(0x445a8020, 1, 1, n_h, m_h, m_h,
+                               exp_sqsub_h, sizeof(exp_sqsub_h));
+    test_arm64_sve2_narrow_run(0x449a8020, 2, 2, n_s, m_s, m_s,
+                               exp_sqsub_s, sizeof(exp_sqsub_s));
+    test_arm64_sve2_narrow_run(0x44da8020, 3, 3, n_d, m_d, m_d,
+                               exp_sqsub_d, sizeof(exp_sqsub_d));
+    test_arm64_sve2_narrow_run(0x441b8020, 0, 0, n_b, m_b, m_b,
+                               exp_uqsub_b, sizeof(exp_uqsub_b));
+    test_arm64_sve2_narrow_run(0x445b8020, 1, 1, n_h, m_h, m_h,
+                               exp_uqsub_h, sizeof(exp_uqsub_h));
+    test_arm64_sve2_narrow_run(0x449b8020, 2, 2, n_s, m_s, m_s,
+                               exp_uqsub_s, sizeof(exp_uqsub_s));
+    test_arm64_sve2_narrow_run(0x44db8020, 3, 3, n_d, m_d, m_d,
+                               exp_uqsub_d, sizeof(exp_uqsub_d));
+    test_arm64_sve2_narrow_run(0x441c8020, 0, 0, n_b, m_b, m_b,
+                               exp_suqadd_b, sizeof(exp_suqadd_b));
+    test_arm64_sve2_narrow_run(0x445c8020, 1, 1, n_h, m_h, m_h,
+                               exp_suqadd_h, sizeof(exp_suqadd_h));
+    test_arm64_sve2_narrow_run(0x449c8020, 2, 2, n_s, m_s, m_s,
+                               exp_suqadd_s, sizeof(exp_suqadd_s));
+    test_arm64_sve2_narrow_run(0x44dc8020, 3, 3, n_d, m_d, m_d,
+                               exp_suqadd_d, sizeof(exp_suqadd_d));
+    test_arm64_sve2_narrow_run(0x441d8020, 0, 0, n_b, m_b, m_b,
+                               exp_usqadd_b, sizeof(exp_usqadd_b));
+    test_arm64_sve2_narrow_run(0x445d8020, 1, 1, n_h, m_h, m_h,
+                               exp_usqadd_h, sizeof(exp_usqadd_h));
+    test_arm64_sve2_narrow_run(0x449d8020, 2, 2, n_s, m_s, m_s,
+                               exp_usqadd_s, sizeof(exp_usqadd_s));
+    test_arm64_sve2_narrow_run(0x44dd8020, 3, 3, n_d, m_d, m_d,
+                               exp_usqadd_d, sizeof(exp_usqadd_d));
+    test_arm64_sve2_narrow_run(0x441e8020, 0, 0, n_b, m_b, m_b,
+                               exp_sqsubr_b, sizeof(exp_sqsubr_b));
+    test_arm64_sve2_narrow_run(0x445f8020, 1, 1, n_h, m_h, m_h,
+                               exp_uqsubr_h, sizeof(exp_uqsubr_h));
+    test_arm64_sve2_narrow_run(0x44de8020, 3, 3, n_d, m_d, m_d,
+                               exp_sqsubr_d, sizeof(exp_sqsubr_d));
+    test_arm64_sve2_narrow_run(0x44df8020, 3, 3, n_d, m_d, m_d,
+                               exp_uqsubr_d, sizeof(exp_uqsubr_d));
+}
+
+static void test_arm64_sve2_int_estimate(void)
+{
+    uc_engine *uc;
+    uint8_t invalid_code[8];
+    const uint32_t src_s[4] = {
+        0x00000000u, 0x40000000u, 0x80000000u, 0xffffffffu,
+    };
+    const uint32_t exp_urecpe_s[4] = {
+        0xffffffffu, 0xffffffffu, 0xff800000u, 0x80000000u,
+    };
+    const uint32_t exp_ursqrte_s[4] = {
+        0xffffffffu, 0xff800000u, 0xb4800000u, 0x80000000u,
+    };
+
+    test_arm64_sve2_narrow_run(0x4480a020, 2, 2, src_s, src_s, src_s,
+                               exp_urecpe_s, sizeof(exp_urecpe_s));
+    test_arm64_sve2_narrow_run(0x4481a020, 2, 2, src_s, src_s, src_s,
+                               exp_ursqrte_s, sizeof(exp_ursqrte_s));
+
+    test_arm64_emit32(invalid_code, 0, 0x2518e3e0);
+    test_arm64_emit32(invalid_code, 4, 0x4440a020);
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM,
+                    (const char *)invalid_code, sizeof(invalid_code),
+                    UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_sve(uc);
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(invalid_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_variable_shift(void)
+{
+    const uint8_t var_n_b[16] = {
+        0x7f, 0x40, 0x81, 0x80, 0x55, 0xaa, 0x01, 0xff,
+        0x10, 0xf0, 0x7e, 0x02, 0xfe, 0x00, 0x33, 0xcc,
+    };
+    const uint8_t var_shift_b[16] = {
+        0x01, 0x02, 0x01, 0x01, 0xff, 0xfe, 0x07, 0x08,
+        0xf8, 0xf7, 0x00, 0x04, 0x84, 0x7f, 0xfc, 0x03,
+    };
+    const uint8_t exp_srshl_b[16] = {
+        0xfe, 0x00, 0x02, 0x00, 0x2b, 0xeb, 0x80, 0x00,
+        0x00, 0x00, 0x7e, 0x20, 0x00, 0x00, 0x03, 0x60,
+    };
+    const uint8_t exp_urshl_b[16] = {
+        0xfe, 0x00, 0x02, 0x00, 0x2b, 0x2b, 0x80, 0x00,
+        0x00, 0x00, 0x7e, 0x20, 0x00, 0x00, 0x03, 0x60,
+    };
+    const uint8_t exp_sqshl_b[16] = {
+        0x7f, 0x7f, 0x80, 0x80, 0x2a, 0xea, 0x7f, 0x80,
+        0x00, 0xff, 0x7e, 0x20, 0xff, 0x00, 0x03, 0x80,
+    };
+    const uint8_t exp_uqshl_b[16] = {
+        0xfe, 0xff, 0xff, 0xff, 0x2a, 0x2a, 0x80, 0xff,
+        0x00, 0x00, 0x7e, 0x20, 0x00, 0x00, 0x03, 0xff,
+    };
+    const uint8_t exp_sqrshl_b[16] = {
+        0x7f, 0x7f, 0x80, 0x80, 0x2b, 0xeb, 0x7f, 0x80,
+        0x00, 0x00, 0x7e, 0x20, 0x00, 0x00, 0x03, 0x80,
+    };
+    const uint8_t exp_uqrshl_b[16] = {
+        0xfe, 0xff, 0xff, 0xff, 0x2b, 0x2b, 0x80, 0xff,
+        0x00, 0x00, 0x7e, 0x20, 0x00, 0x00, 0x03, 0xff,
+    };
+    const uint16_t var_n_h[8] = {
+        0x7fff, 0x4000, 0x8001, 0x8000,
+        0x1234, 0xedcc, 0x0001, 0xffff,
+    };
+    const uint16_t var_shift_h[8] = {
+        0x0001, 0x0002, 0x0001, 0x0001,
+        0xffff, 0xfffe, 0x000f, 0x0010,
+    };
+    const uint16_t exp_srshl_h[8] = {
+        0xfffe, 0x0000, 0x0002, 0x0000,
+        0x091a, 0xfb73, 0x8000, 0x0000,
+    };
+    const uint16_t exp_urshl_h[8] = {
+        0xfffe, 0x0000, 0x0002, 0x0000,
+        0x091a, 0x3b73, 0x8000, 0x0000,
+    };
+    const uint16_t exp_sqshl_h[8] = {
+        0x7fff, 0x7fff, 0x8000, 0x8000,
+        0x091a, 0xfb73, 0x7fff, 0x8000,
+    };
+    const uint16_t exp_uqshl_h[8] = {
+        0xfffe, 0xffff, 0xffff, 0xffff,
+        0x091a, 0x3b73, 0x8000, 0xffff,
+    };
+    const uint16_t exp_sqrshl_h[8] = {
+        0x7fff, 0x7fff, 0x8000, 0x8000,
+        0x091a, 0xfb73, 0x7fff, 0x8000,
+    };
+    const uint16_t exp_uqrshl_h[8] = {
+        0xfffe, 0xffff, 0xffff, 0xffff,
+        0x091a, 0x3b73, 0x8000, 0xffff,
+    };
+    const uint32_t var_n_s[4] = {
+        0x7fffffffu, 0x40000000u, 0x80000001u, 0x80000000u,
+    };
+    const uint32_t var_shift_s[4] = {
+        0x00000001u, 0x00000002u, 0xffffffffu, 0x00000020u,
+    };
+    const uint32_t exp_srshl_s[4] = {
+        0xfffffffeu, 0x00000000u, 0xc0000001u, 0x00000000u,
+    };
+    const uint32_t exp_urshl_s[4] = {
+        0xfffffffeu, 0x00000000u, 0x40000001u, 0x00000000u,
+    };
+    const uint32_t exp_sqshl_s[4] = {
+        0x7fffffffu, 0x7fffffffu, 0xc0000000u, 0x80000000u,
+    };
+    const uint32_t exp_uqshl_s[4] = {
+        0xfffffffeu, 0xffffffffu, 0x40000000u, 0xffffffffu,
+    };
+    const uint32_t exp_sqrshl_s[4] = {
+        0x7fffffffu, 0x7fffffffu, 0xc0000001u, 0x80000000u,
+    };
+    const uint32_t exp_uqrshl_s[4] = {
+        0xfffffffeu, 0xffffffffu, 0x40000001u, 0xffffffffu,
+    };
+    const uint64_t var_n_d[2] = {
+        0x7fffffffffffffffull, 0x8000000000000000ull,
+    };
+    const uint64_t var_shift_d[2] = {
+        0x0000000000000001ull, 0xffffffffffffffffull,
+    };
+    const uint64_t exp_srshl_d[2] = {
+        0xfffffffffffffffeull, 0xc000000000000000ull,
+    };
+    const uint64_t exp_urshl_d[2] = {
+        0xfffffffffffffffeull, 0x4000000000000000ull,
+    };
+    const uint64_t exp_sqshl_d[2] = {
+        0x7fffffffffffffffull, 0xc000000000000000ull,
+    };
+    const uint64_t exp_uqshl_d[2] = {
+        0xfffffffffffffffeull, 0x4000000000000000ull,
+    };
+    const uint64_t exp_sqrshl_d[2] = {
+        0x7fffffffffffffffull, 0xc000000000000000ull,
+    };
+    const uint64_t exp_uqrshl_d[2] = {
+        0xfffffffffffffffeull, 0x4000000000000000ull,
+    };
+
+    test_arm64_sve2_narrow_run(0x44028020, 0, 0, var_n_b, var_shift_b,
+                               var_shift_b, exp_srshl_b,
+                               sizeof(exp_srshl_b));
+    test_arm64_sve2_narrow_run(0x44428020, 1, 1, var_n_h, var_shift_h,
+                               var_shift_h, exp_srshl_h,
+                               sizeof(exp_srshl_h));
+    test_arm64_sve2_narrow_run(0x44828020, 2, 2, var_n_s, var_shift_s,
+                               var_shift_s, exp_srshl_s,
+                               sizeof(exp_srshl_s));
+    test_arm64_sve2_narrow_run(0x44c28020, 3, 3, var_n_d, var_shift_d,
+                               var_shift_d, exp_srshl_d,
+                               sizeof(exp_srshl_d));
+    test_arm64_sve2_narrow_run(0x44038020, 0, 0, var_n_b, var_shift_b,
+                               var_shift_b, exp_urshl_b,
+                               sizeof(exp_urshl_b));
+    test_arm64_sve2_narrow_run(0x44438020, 1, 1, var_n_h, var_shift_h,
+                               var_shift_h, exp_urshl_h,
+                               sizeof(exp_urshl_h));
+    test_arm64_sve2_narrow_run(0x44838020, 2, 2, var_n_s, var_shift_s,
+                               var_shift_s, exp_urshl_s,
+                               sizeof(exp_urshl_s));
+    test_arm64_sve2_narrow_run(0x44c38020, 3, 3, var_n_d, var_shift_d,
+                               var_shift_d, exp_urshl_d,
+                               sizeof(exp_urshl_d));
+    test_arm64_sve2_narrow_run(0x44088020, 0, 0, var_n_b, var_shift_b,
+                               var_shift_b, exp_sqshl_b,
+                               sizeof(exp_sqshl_b));
+    test_arm64_sve2_narrow_run(0x44488020, 1, 1, var_n_h, var_shift_h,
+                               var_shift_h, exp_sqshl_h,
+                               sizeof(exp_sqshl_h));
+    test_arm64_sve2_narrow_run(0x44888020, 2, 2, var_n_s, var_shift_s,
+                               var_shift_s, exp_sqshl_s,
+                               sizeof(exp_sqshl_s));
+    test_arm64_sve2_narrow_run(0x44c88020, 3, 3, var_n_d, var_shift_d,
+                               var_shift_d, exp_sqshl_d,
+                               sizeof(exp_sqshl_d));
+    test_arm64_sve2_narrow_run(0x44098020, 0, 0, var_n_b, var_shift_b,
+                               var_shift_b, exp_uqshl_b,
+                               sizeof(exp_uqshl_b));
+    test_arm64_sve2_narrow_run(0x44498020, 1, 1, var_n_h, var_shift_h,
+                               var_shift_h, exp_uqshl_h,
+                               sizeof(exp_uqshl_h));
+    test_arm64_sve2_narrow_run(0x44898020, 2, 2, var_n_s, var_shift_s,
+                               var_shift_s, exp_uqshl_s,
+                               sizeof(exp_uqshl_s));
+    test_arm64_sve2_narrow_run(0x44c98020, 3, 3, var_n_d, var_shift_d,
+                               var_shift_d, exp_uqshl_d,
+                               sizeof(exp_uqshl_d));
+    test_arm64_sve2_narrow_run(0x440a8020, 0, 0, var_n_b, var_shift_b,
+                               var_shift_b, exp_sqrshl_b,
+                               sizeof(exp_sqrshl_b));
+    test_arm64_sve2_narrow_run(0x444a8020, 1, 1, var_n_h, var_shift_h,
+                               var_shift_h, exp_sqrshl_h,
+                               sizeof(exp_sqrshl_h));
+    test_arm64_sve2_narrow_run(0x448a8020, 2, 2, var_n_s, var_shift_s,
+                               var_shift_s, exp_sqrshl_s,
+                               sizeof(exp_sqrshl_s));
+    test_arm64_sve2_narrow_run(0x44ca8020, 3, 3, var_n_d, var_shift_d,
+                               var_shift_d, exp_sqrshl_d,
+                               sizeof(exp_sqrshl_d));
+    test_arm64_sve2_narrow_run(0x440b8020, 0, 0, var_n_b, var_shift_b,
+                               var_shift_b, exp_uqrshl_b,
+                               sizeof(exp_uqrshl_b));
+    test_arm64_sve2_narrow_run(0x444b8020, 1, 1, var_n_h, var_shift_h,
+                               var_shift_h, exp_uqrshl_h,
+                               sizeof(exp_uqrshl_h));
+    test_arm64_sve2_narrow_run(0x448b8020, 2, 2, var_n_s, var_shift_s,
+                               var_shift_s, exp_uqrshl_s,
+                               sizeof(exp_uqrshl_s));
+    test_arm64_sve2_narrow_run(0x44cb8020, 3, 3, var_n_d, var_shift_d,
+                               var_shift_d, exp_uqrshl_d,
+                               sizeof(exp_uqrshl_d));
+
+    test_arm64_sve2_narrow_run(0x44068020, 0, 0, var_shift_b, var_n_b,
+                               var_n_b, exp_srshl_b, sizeof(exp_srshl_b));
+    test_arm64_sve2_narrow_run(0x44478020, 1, 1, var_shift_h, var_n_h,
+                               var_n_h, exp_urshl_h, sizeof(exp_urshl_h));
+    test_arm64_sve2_narrow_run(0x448c8020, 2, 2, var_shift_s, var_n_s,
+                               var_n_s, exp_sqshl_s, sizeof(exp_sqshl_s));
+    test_arm64_sve2_narrow_run(0x44cd8020, 3, 3, var_shift_d, var_n_d,
+                               var_n_d, exp_uqshl_d, sizeof(exp_uqshl_d));
+    test_arm64_sve2_narrow_run(0x440e8020, 0, 0, var_shift_b, var_n_b,
+                               var_n_b, exp_sqrshl_b,
+                               sizeof(exp_sqrshl_b));
+    test_arm64_sve2_narrow_run(0x444f8020, 1, 1, var_shift_h, var_n_h,
+                               var_n_h, exp_uqrshl_h,
+                               sizeof(exp_uqrshl_h));
+}
+
+static void test_arm64_sve2_eor_adcl(void)
+{
+    uc_engine *uc;
+    const char code_eor_b[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x3] */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x20\x90\x02\x45" /* eorbt z0.b,z1.b,z2.b */
+        "\xc0\xe0\x00\xe4" /* st1b { z0.b },p0,[x6] */
+        "\xe3\xa0\x00\xa4" /* ld1b { z3.b },p0/z,[x7] */
+        "\x23\x94\x02\x45" /* eortb z3.b,z1.b,z2.b */
+        "\x03\xe1\x00\xe4"; /* st1b { z3.b },p0,[x8] */
+    const char code_eor_h[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\xa0\xa4" /* ld1h { z0.h },p0/z,[x3] */
+        "\x81\xa0\xa0\xa4" /* ld1h { z1.h },p0/z,[x4] */
+        "\xa2\xa0\xa0\xa4" /* ld1h { z2.h },p0/z,[x5] */
+        "\x20\x90\x42\x45" /* eorbt z0.h,z1.h,z2.h */
+        "\xc0\xe0\xa0\xe4" /* st1h { z0.h },p0,[x6] */
+        "\xe3\xa0\xa0\xa4" /* ld1h { z3.h },p0/z,[x7] */
+        "\x23\x94\x42\x45" /* eortb z3.h,z1.h,z2.h */
+        "\x03\xe1\xa0\xe4"; /* st1h { z3.h },p0,[x8] */
+    const char code_eor_s[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\x40\xa5" /* ld1w { z0.s },p0/z,[x3] */
+        "\x81\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x4] */
+        "\xa2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x5] */
+        "\x20\x90\x82\x45" /* eorbt z0.s,z1.s,z2.s */
+        "\xc0\xe0\x40\xe5" /* st1w { z0.s },p0,[x6] */
+        "\xe3\xa0\x40\xa5" /* ld1w { z3.s },p0/z,[x7] */
+        "\x23\x94\x82\x45" /* eortb z3.s,z1.s,z2.s */
+        "\x03\xe1\x40\xe5"; /* st1w { z3.s },p0,[x8] */
+    const char code_eor_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\xe0\xa5" /* ld1d { z0.d },p0/z,[x3] */
+        "\x81\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x4] */
+        "\xa2\xa0\xe0\xa5" /* ld1d { z2.d },p0/z,[x5] */
+        "\x20\x90\xc2\x45" /* eorbt z0.d,z1.d,z2.d */
+        "\xc0\xe0\xe0\xe5" /* st1d { z0.d },p0,[x6] */
+        "\xe3\xa0\xe0\xa5" /* ld1d { z3.d },p0/z,[x7] */
+        "\x23\x94\xc2\x45" /* eortb z3.d,z1.d,z2.d */
+        "\x03\xe1\xe0\xe5"; /* st1d { z3.d },p0,[x8] */
+    const char code_adcl_s[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\xe0\xa5" /* ld1d { z0.d },p0/z,[x3] */
+        "\x81\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x4] */
+        "\xa2\xa0\xe0\xa5" /* ld1d { z2.d },p0/z,[x5] */
+        "\x20\xd0\x02\x45" /* adclb z0.d,z1.s,z2.d */
+        "\xc0\xe0\xe0\xe5" /* st1d { z0.d },p0,[x6] */
+        "\xe3\xa0\xe0\xa5" /* ld1d { z3.d },p0/z,[x7] */
+        "\x23\xd4\x02\x45" /* adclt z3.d,z1.s,z2.d */
+        "\x03\xe1\xe0\xe5"; /* st1d { z3.d },p0,[x8] */
+    const char code_adcl_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\xa0\xe0\xa5" /* ld1d { z0.d },p0/z,[x3] */
+        "\x81\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x4] */
+        "\xa2\xa0\xe0\xa5" /* ld1d { z2.d },p0/z,[x5] */
+        "\x20\xd0\x42\x45" /* adclb z0.d,z1.d,z2.d */
+        "\xc0\xe0\xe0\xe5" /* st1d { z0.d },p0,[x6] */
+        "\xe3\xa0\xe0\xa5" /* ld1d { z3.d },p0/z,[x7] */
+        "\x23\xd4\x42\x45" /* adclt z3.d,z1.d,z2.d */
+        "\x03\xe1\xe0\xe5"; /* st1d { z3.d },p0,[x8] */
+    uint8_t init_b[16], init2_b[16], n_b[16], m_b[16];
+    uint8_t exp_eorbt_b[16], exp_eortb_b[16], got_b[16];
+    uint16_t init_h[8], init2_h[8], n_h[8], m_h[8];
+    uint16_t exp_eorbt_h[8], exp_eortb_h[8], got_h[8];
+    uint32_t init_s[4], init2_s[4], n_s[4], m_s[4];
+    uint32_t exp_eorbt_s[4], exp_eortb_s[4], got_s[4];
+    uint64_t init_d[2], init2_d[2], n_d[2], m_d[2];
+    uint64_t exp_eorbt_d[2], exp_eortb_d[2], got_d[2];
+    uint64_t adcl_acc_b[2], adcl_acc_t[2], adcl_m_s[2];
+    uint32_t adcl_n_s[4];
+    uint64_t exp_adclb_s[2], exp_adclt_s[2], got_adcl_s[2];
+    uint64_t adcl_acc_b_d[2], adcl_acc_t_d[2], adcl_n_d[2], adcl_m_d[2];
+    uint64_t exp_adclb_d[2], exp_adclt_d[2], got_adcl_d[2];
+    uint64_t x3 = 0x40000;
+    uint64_t x4 = 0x40100;
+    uint64_t x5 = 0x40200;
+    uint64_t x6 = 0x40300;
+    uint64_t x7 = 0x40400;
+    uint64_t x8 = 0x40500;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        init_b[i] = (uint8_t)(0x80 + i);
+        init2_b[i] = (uint8_t)(0xa0 + i);
+        n_b[i] = (uint8_t)(0x11 + i * 3);
+        m_b[i] = (uint8_t)(0xf0 - i * 5);
+        exp_eorbt_b[i] = init_b[i];
+        exp_eortb_b[i] = init2_b[i];
+    }
+    for (i = 0; i < 8; i++) {
+        exp_eorbt_b[2 * i] = n_b[2 * i] ^ m_b[2 * i + 1];
+        exp_eortb_b[2 * i + 1] = n_b[2 * i + 1] ^ m_b[2 * i];
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_eor_b,
+                    sizeof(code_eor_b) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, init_b, sizeof(init_b)));
+    OK(uc_mem_write(uc, x4, n_b, sizeof(n_b)));
+    OK(uc_mem_write(uc, x5, m_b, sizeof(m_b)));
+    OK(uc_mem_write(uc, x7, init2_b, sizeof(init2_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_eor_b) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_eorbt_b[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_eortb_b[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 8; i++) {
+        init_h[i] = (uint16_t)(0x8000 + i);
+        init2_h[i] = (uint16_t)(0xa000 + i);
+        n_h[i] = (uint16_t)(0x1100 + i * 0x21);
+        m_h[i] = (uint16_t)(0xf000 - i * 0x31);
+        exp_eorbt_h[i] = init_h[i];
+        exp_eortb_h[i] = init2_h[i];
+    }
+    for (i = 0; i < 4; i++) {
+        exp_eorbt_h[2 * i] = n_h[2 * i] ^ m_h[2 * i + 1];
+        exp_eortb_h[2 * i + 1] = n_h[2 * i + 1] ^ m_h[2 * i];
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_eor_h,
+                    sizeof(code_eor_h) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, init_h, sizeof(init_h)));
+    OK(uc_mem_write(uc, x4, n_h, sizeof(n_h)));
+    OK(uc_mem_write(uc, x5, m_h, sizeof(m_h)));
+    OK(uc_mem_write(uc, x7, init2_h, sizeof(init2_h)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_eor_h) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_eorbt_h[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_eortb_h[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 4; i++) {
+        init_s[i] = 0x80000000u + (uint32_t)i;
+        init2_s[i] = 0xa0000000u + (uint32_t)i;
+        n_s[i] = 0x11110000u + (uint32_t)i * 0x1111u;
+        m_s[i] = 0xf0000000u - (uint32_t)i * 0x10101u;
+        exp_eorbt_s[i] = init_s[i];
+        exp_eortb_s[i] = init2_s[i];
+    }
+    for (i = 0; i < 2; i++) {
+        exp_eorbt_s[2 * i] = n_s[2 * i] ^ m_s[2 * i + 1];
+        exp_eortb_s[2 * i + 1] = n_s[2 * i + 1] ^ m_s[2 * i];
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_eor_s,
+                    sizeof(code_eor_s) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, init_s, sizeof(init_s)));
+    OK(uc_mem_write(uc, x4, n_s, sizeof(n_s)));
+    OK(uc_mem_write(uc, x5, m_s, sizeof(m_s)));
+    OK(uc_mem_write(uc, x7, init2_s, sizeof(init2_s)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_eor_s) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == exp_eorbt_s[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == exp_eortb_s[i]);
+    }
+    OK(uc_close(uc));
+
+    init_d[0] = 0x8000000000000000ull;
+    init_d[1] = 0x8000000000000001ull;
+    init2_d[0] = 0xa000000000000000ull;
+    init2_d[1] = 0xa000000000000001ull;
+    n_d[0] = 0x0123456789abcdefull;
+    n_d[1] = 0xfedcba9876543210ull;
+    m_d[0] = 0x1111111111111111ull;
+    m_d[1] = 0x2222222222222222ull;
+    exp_eorbt_d[0] = n_d[0] ^ m_d[1];
+    exp_eorbt_d[1] = init_d[1];
+    exp_eortb_d[0] = init2_d[0];
+    exp_eortb_d[1] = n_d[1] ^ m_d[0];
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_eor_d,
+                    sizeof(code_eor_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, init_d, sizeof(init_d)));
+    OK(uc_mem_write(uc, x4, n_d, sizeof(n_d)));
+    OK(uc_mem_write(uc, x5, m_d, sizeof(m_d)));
+    OK(uc_mem_write(uc, x7, init2_d, sizeof(init2_d)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_eor_d) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_eorbt_d[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_eortb_d[i]);
+    }
+    OK(uc_close(uc));
+
+    adcl_acc_b[0] = 0x00000000fffffffeull;
+    adcl_acc_b[1] = 0x0000000000000010ull;
+    adcl_acc_t[0] = 0x0000000000000020ull;
+    adcl_acc_t[1] = 0x0000000000000030ull;
+    adcl_n_s[0] = 5;
+    adcl_n_s[1] = 0x7fffffffu;
+    adcl_n_s[2] = 0xffffffffu;
+    adcl_n_s[3] = 0x80000000u;
+    adcl_m_s[0] = 0x0000000100000000ull;
+    adcl_m_s[1] = 0;
+    exp_adclb_s[0] = 0x0000000100000004ull;
+    exp_adclb_s[1] = 0x000000010000000full;
+    exp_adclt_s[0] = 0x0000000080000020ull;
+    exp_adclt_s[1] = 0x0000000080000030ull;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_adcl_s,
+                    sizeof(code_adcl_s) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, adcl_acc_b, sizeof(adcl_acc_b)));
+    OK(uc_mem_write(uc, x4, adcl_n_s, sizeof(adcl_n_s)));
+    OK(uc_mem_write(uc, x5, adcl_m_s, sizeof(adcl_m_s)));
+    OK(uc_mem_write(uc, x7, adcl_acc_t, sizeof(adcl_acc_t)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_adcl_s) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_adcl_s, sizeof(got_adcl_s)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_adcl_s[i] == exp_adclb_s[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_adcl_s, sizeof(got_adcl_s)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_adcl_s[i] == exp_adclt_s[i]);
+    }
+    OK(uc_close(uc));
+
+    adcl_acc_b_d[0] = 0xffffffffffffffffull;
+    adcl_acc_b_d[1] = 0;
+    adcl_acc_t_d[0] = 0x10;
+    adcl_acc_t_d[1] = 0;
+    adcl_n_d[0] = 2;
+    adcl_n_d[1] = 0xfffffffffffffff0ull;
+    adcl_m_d[0] = 0;
+    adcl_m_d[1] = 1;
+    exp_adclb_d[0] = 2;
+    exp_adclb_d[1] = 1;
+    exp_adclt_d[0] = 1;
+    exp_adclt_d[1] = 1;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_adcl_d,
+                    sizeof(code_adcl_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x3, adcl_acc_b_d, sizeof(adcl_acc_b_d)));
+    OK(uc_mem_write(uc, x4, adcl_n_d, sizeof(adcl_n_d)));
+    OK(uc_mem_write(uc, x5, adcl_m_d, sizeof(adcl_m_d)));
+    OK(uc_mem_write(uc, x7, adcl_acc_t_d, sizeof(adcl_acc_t_d)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_adcl_d) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_adcl_d, sizeof(got_adcl_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_adcl_d[i] == exp_adclb_d[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_adcl_d, sizeof(got_adcl_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_adcl_d[i] == exp_adclt_d[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_bitperm(void)
+{
+    uc_engine *uc;
+    const char code_b[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x20\xb0\x02\x45" /* bext z0.b,z1.b,z2.b */
+        "\xc0\xe0\x00\xe4" /* st1b { z0.b },p0,[x6] */
+        "\x23\xb4\x02\x45" /* bdep z3.b,z1.b,z2.b */
+        "\xe3\xe0\x00\xe4" /* st1b { z3.b },p0,[x7] */
+        "\x24\xb8\x02\x45" /* bgrp z4.b,z1.b,z2.b */
+        "\x04\xe1\x00\xe4"; /* st1b { z4.b },p0,[x8] */
+    const char code_h[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xa0\xa4" /* ld1h { z1.h },p0/z,[x4] */
+        "\xa2\xa0\xa0\xa4" /* ld1h { z2.h },p0/z,[x5] */
+        "\x20\xb0\x42\x45" /* bext z0.h,z1.h,z2.h */
+        "\xc0\xe0\xa0\xe4" /* st1h { z0.h },p0,[x6] */
+        "\x23\xb4\x42\x45" /* bdep z3.h,z1.h,z2.h */
+        "\xe3\xe0\xa0\xe4" /* st1h { z3.h },p0,[x7] */
+        "\x24\xb8\x42\x45" /* bgrp z4.h,z1.h,z2.h */
+        "\x04\xe1\xa0\xe4"; /* st1h { z4.h },p0,[x8] */
+    const char code_s[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x4] */
+        "\xa2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x5] */
+        "\x20\xb0\x82\x45" /* bext z0.s,z1.s,z2.s */
+        "\xc0\xe0\x40\xe5" /* st1w { z0.s },p0,[x6] */
+        "\x23\xb4\x82\x45" /* bdep z3.s,z1.s,z2.s */
+        "\xe3\xe0\x40\xe5" /* st1w { z3.s },p0,[x7] */
+        "\x24\xb8\x82\x45" /* bgrp z4.s,z1.s,z2.s */
+        "\x04\xe1\x40\xe5"; /* st1w { z4.s },p0,[x8] */
+    const char code_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x4] */
+        "\xa2\xa0\xe0\xa5" /* ld1d { z2.d },p0/z,[x5] */
+        "\x20\xb0\xc2\x45" /* bext z0.d,z1.d,z2.d */
+        "\xc0\xe0\xe0\xe5" /* st1d { z0.d },p0,[x6] */
+        "\x23\xb4\xc2\x45" /* bdep z3.d,z1.d,z2.d */
+        "\xe3\xe0\xe0\xe5" /* st1d { z3.d },p0,[x7] */
+        "\x24\xb8\xc2\x45" /* bgrp z4.d,z1.d,z2.d */
+        "\x04\xe1\xe0\xe5"; /* st1d { z4.d },p0,[x8] */
+    uint8_t n_b[16], m_b[16], exp_bext_b[16], exp_bdep_b[16];
+    uint8_t exp_bgrp_b[16], got_b[16];
+    uint16_t n_h[8], m_h[8], exp_bext_h[8], exp_bdep_h[8];
+    uint16_t exp_bgrp_h[8], got_h[8];
+    uint32_t n_s[4], m_s[4], exp_bext_s[4], exp_bdep_s[4];
+    uint32_t exp_bgrp_s[4], got_s[4];
+    uint64_t n_d[2], m_d[2], exp_bext_d[2], exp_bdep_d[2];
+    uint64_t exp_bgrp_d[2], got_d[2];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    uint64_t x7 = 0x40300;
+    uint64_t x8 = 0x40400;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        n_b[i] = (uint8_t)(0x35 + i * 13);
+        m_b[i] = (uint8_t)(0x5a ^ (i * 17));
+        exp_bext_b[i] = (uint8_t)test_arm64_bitextract(n_b[i], m_b[i], 8);
+        exp_bdep_b[i] = (uint8_t)test_arm64_bitdeposit(n_b[i], m_b[i], 8);
+        exp_bgrp_b[i] = (uint8_t)test_arm64_bitgroup(n_b[i], m_b[i], 8);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_b,
+                    sizeof(code_b) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_b, sizeof(n_b)));
+    OK(uc_mem_write(uc, x5, m_b, sizeof(m_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_b) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_bext_b[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_bdep_b[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_bgrp_b[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 8; i++) {
+        n_h[i] = (uint16_t)(0x1357 + i * 0x1111);
+        m_h[i] = (uint16_t)(0xa55a ^ (i * 0x1234));
+        exp_bext_h[i] = (uint16_t)test_arm64_bitextract(n_h[i], m_h[i], 16);
+        exp_bdep_h[i] = (uint16_t)test_arm64_bitdeposit(n_h[i], m_h[i], 16);
+        exp_bgrp_h[i] = (uint16_t)test_arm64_bitgroup(n_h[i], m_h[i], 16);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_h,
+                    sizeof(code_h) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_h, sizeof(n_h)));
+    OK(uc_mem_write(uc, x5, m_h, sizeof(m_h)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_h) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_bext_h[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_bdep_h[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_bgrp_h[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 4; i++) {
+        n_s[i] = 0x12345678u + (uint32_t)i * 0x01020304u;
+        m_s[i] = 0x96696996u ^ ((uint32_t)i * 0x11111111u);
+        exp_bext_s[i] = (uint32_t)test_arm64_bitextract(n_s[i], m_s[i], 32);
+        exp_bdep_s[i] = (uint32_t)test_arm64_bitdeposit(n_s[i], m_s[i], 32);
+        exp_bgrp_s[i] = (uint32_t)test_arm64_bitgroup(n_s[i], m_s[i], 32);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_s,
+                    sizeof(code_s) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_s, sizeof(n_s)));
+    OK(uc_mem_write(uc, x5, m_s, sizeof(m_s)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_s) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == exp_bext_s[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == exp_bdep_s[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == exp_bgrp_s[i]);
+    }
+    OK(uc_close(uc));
+
+    n_d[0] = 0x0123456789abcdefull;
+    n_d[1] = 0xfedcba9876543210ull;
+    m_d[0] = 0x0f0f3333ccccf0f0ull;
+    m_d[1] = 0x13579bdf2468ace0ull;
+    for (i = 0; i < 2; i++) {
+        exp_bext_d[i] = test_arm64_bitextract(n_d[i], m_d[i], 64);
+        exp_bdep_d[i] = test_arm64_bitdeposit(n_d[i], m_d[i], 64);
+        exp_bgrp_d[i] = test_arm64_bitgroup(n_d[i], m_d[i], 64);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_d,
+                    sizeof(code_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_d, sizeof(n_d)));
+    OK(uc_mem_write(uc, x5, m_d, sizeof(m_d)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_d) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_bext_d[i]);
+    }
+    OK(uc_mem_read(uc, x7, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_bdep_d[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_bgrp_d[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_match_hist(void)
+{
+    uc_engine *uc;
+    const char code_match_b[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\xe4\xa0\x00\xa4" /* ld1b { z4.b },p0/z,[x7] */
+        "\x21\x80\x22\x45" /* match p1.b,p0/z,z1.b,z2.b */
+        "\xc4\xe4\x00\xe4" /* st1b { z4.b },p1,[x6] */
+        "\x32\x80\x22\x45" /* nmatch p2.b,p0/z,z1.b,z2.b */
+        "\x04\xe9\x00\xe4"; /* st1b { z4.b },p2,[x8] */
+    const char code_match_h[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xa0\xa4" /* ld1h { z1.h },p0/z,[x4] */
+        "\xa2\xa0\xa0\xa4" /* ld1h { z2.h },p0/z,[x5] */
+        "\xe4\xa0\xa0\xa4" /* ld1h { z4.h },p0/z,[x7] */
+        "\x21\x80\x62\x45" /* match p1.h,p0/z,z1.h,z2.h */
+        "\xc4\xe4\xa0\xe4" /* st1h { z4.h },p1,[x6] */
+        "\x32\x80\x62\x45" /* nmatch p2.h,p0/z,z1.h,z2.h */
+        "\x04\xe9\xa0\xe4"; /* st1h { z4.h },p2,[x8] */
+    const char code_histcnt_s[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x4] */
+        "\xa2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x5] */
+        "\x20\xc0\xa2\x45" /* histcnt z0.s,p0/z,z1.s,z2.s */
+        "\xc0\xe0\x40\xe5"; /* st1w { z0.s },p0,[x6] */
+    const char code_histcnt_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x4] */
+        "\xa2\xa0\xe0\xa5" /* ld1d { z2.d },p0/z,[x5] */
+        "\x20\xc0\xe2\x45" /* histcnt z0.d,p0/z,z1.d,z2.d */
+        "\xc0\xe0\xe0\xe5"; /* st1d { z0.d },p0,[x6] */
+    const char code_histseg[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x20\xa0\x22\x45" /* histseg z0.b,z1.b,z2.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    uint8_t n_b[16] = {
+        1, 2, 3, 4, 5, 6, 7, 8,
+        9, 10, 11, 12, 13, 14, 15, 16,
+    };
+    uint8_t m_b[16] = {
+        3, 8, 13, 0x42, 1, 1, 0xaa, 0xbb,
+        16, 7, 0x30, 0x31, 2, 2, 0xcc, 0xdd,
+    };
+    uint16_t n_h[8] = {
+        0x1001, 0x1002, 0x2222, 0x3333,
+        0x4444, 0x5555, 0x6666, 0x7777,
+    };
+    uint16_t m_h[8] = {
+        0x3333, 0x1002, 0x7777, 0x9999,
+        0x4444, 0xaaaa, 0xbbbb, 0x1001,
+    };
+    uint8_t fill_b[16], zero_b[16], got_b[16];
+    uint8_t exp_match_b[16], exp_nmatch_b[16];
+    uint16_t fill_h[8], zero_h[8], got_h[8];
+    uint16_t exp_match_h[8], exp_nmatch_h[8];
+    uint32_t n_s[4] = { 5, 7, 5, 9 };
+    uint32_t m_s[4] = { 5, 5, 7, 5 };
+    uint32_t exp_histcnt_s[4], got_s[4];
+    uint64_t n_d[2] = { 0x11, 0x22 };
+    uint64_t m_d[2] = { 0x22, 0x22 };
+    uint64_t exp_histcnt_d[2], got_d[2];
+    uint8_t n_seg[16] = {
+        1, 2, 3, 4, 5, 6, 7, 8,
+        9, 10, 1, 2, 3, 4, 5, 6,
+    };
+    uint8_t m_seg[16] = {
+        1, 1, 2, 3, 3, 3, 5, 8,
+        8, 8, 8, 10, 0xff, 0, 4, 6,
+    };
+    uint8_t exp_histseg[16];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    uint64_t x7 = 0x40300;
+    uint64_t x8 = 0x40400;
+    int i, j;
+
+    memset(fill_b, 0xff, sizeof(fill_b));
+    memset(zero_b, 0, sizeof(zero_b));
+    memset(fill_h, 0xff, sizeof(fill_h));
+    memset(zero_h, 0, sizeof(zero_h));
+
+    for (i = 0; i < 16; i++) {
+        if (test_arm64_has_u8(m_b, 16, n_b[i])) {
+            exp_match_b[i] = 0xff;
+            exp_nmatch_b[i] = 0;
+        } else {
+            exp_match_b[i] = 0;
+            exp_nmatch_b[i] = 0xff;
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_match_b,
+                    sizeof(code_match_b) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_b, sizeof(n_b)));
+    OK(uc_mem_write(uc, x5, m_b, sizeof(m_b)));
+    OK(uc_mem_write(uc, x6, zero_b, sizeof(zero_b)));
+    OK(uc_mem_write(uc, x7, fill_b, sizeof(fill_b)));
+    OK(uc_mem_write(uc, x8, zero_b, sizeof(zero_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_match_b) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_match_b[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_nmatch_b[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 8; i++) {
+        if (test_arm64_has_u16(m_h, 8, n_h[i])) {
+            exp_match_h[i] = 0xffff;
+            exp_nmatch_h[i] = 0;
+        } else {
+            exp_match_h[i] = 0;
+            exp_nmatch_h[i] = 0xffff;
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_match_h,
+                    sizeof(code_match_h) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_h, sizeof(n_h)));
+    OK(uc_mem_write(uc, x5, m_h, sizeof(m_h)));
+    OK(uc_mem_write(uc, x6, zero_h, sizeof(zero_h)));
+    OK(uc_mem_write(uc, x7, fill_h, sizeof(fill_h)));
+    OK(uc_mem_write(uc, x8, zero_h, sizeof(zero_h)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_match_h) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_match_h[i]);
+    }
+    OK(uc_mem_read(uc, x8, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == exp_nmatch_h[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 4; i++) {
+        exp_histcnt_s[i] = 0;
+        for (j = 0; j <= i; j++) {
+            if (n_s[i] == m_s[j]) {
+                exp_histcnt_s[i]++;
+            }
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_histcnt_s,
+                    sizeof(code_histcnt_s) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_s, sizeof(n_s)));
+    OK(uc_mem_write(uc, x5, m_s, sizeof(m_s)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_histcnt_s) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == exp_histcnt_s[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 2; i++) {
+        exp_histcnt_d[i] = 0;
+        for (j = 0; j <= i; j++) {
+            if (n_d[i] == m_d[j]) {
+                exp_histcnt_d[i]++;
+            }
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_histcnt_d,
+                    sizeof(code_histcnt_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_d, sizeof(n_d)));
+    OK(uc_mem_write(uc, x5, m_d, sizeof(m_d)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_histcnt_d) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_d, sizeof(got_d)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(got_d[i] == exp_histcnt_d[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 16; i++) {
+        exp_histseg[i] = 0;
+        for (j = 0; j < 16; j++) {
+            if (n_seg[i] == m_seg[j]) {
+                exp_histseg[i]++;
+            }
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_histseg,
+                    sizeof(code_histseg) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n_seg, sizeof(n_seg)));
+    OK(uc_mem_write(uc, x5, m_seg, sizeof(m_seg)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_histseg) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == exp_histseg[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_crypto_run(const char *code, size_t code_size,
+                                       const uint8_t *input0,
+                                       const uint8_t *input1,
+                                       const uint8_t *expected)
+{
+    uc_engine *uc;
+    uint8_t got[32];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    int i;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, code_size,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, input0, 32));
+    if (input1 != NULL) {
+        OK(uc_mem_write(uc, x5, input1, 32));
+    }
+    test_arm64_mte_enable_sve_vq(uc, 1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + code_size, 0, 0));
+    OK(uc_mem_read(uc, x6, got, sizeof(got)));
+    for (i = 0; i < 32; i++) {
+        TEST_CHECK(got[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_crypto(void)
+{
+    const char code_aese[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x40\xe0\x22\x45" /* aese z0.b,z2.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_aesd[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x40\xe4\x22\x45" /* aesd z0.b,z2.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_aesmc[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x4] */
+        "\x00\xe0\x20\x45" /* aesmc z0.b,z0.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_aesimc[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x4] */
+        "\x00\xe4\x20\x45" /* aesimc z0.b,z0.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_sm4e[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x80\xa0\x00\xa4" /* ld1b { z0.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x40\xe0\x23\x45" /* sm4e z0.b,z2.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_sm4ekey[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x20\xf0\x22\x45" /* sm4ekey z0.b,z1.b,z2.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_rax1[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x20\xf4\x22\x45" /* rax1 z0.d,z1.d,z2.d */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const uint8_t state[32] = {
+        0x10, 0x17, 0x1e, 0x25, 0x2c, 0x33, 0x3a, 0x41,
+        0x48, 0x4f, 0x56, 0x5d, 0x64, 0x6b, 0x72, 0x79,
+        0x80, 0x87, 0x8e, 0x95, 0x9c, 0xa3, 0xaa, 0xb1,
+        0xb8, 0xbf, 0xc6, 0xcd, 0xd4, 0xdb, 0xe2, 0xe9,
+    };
+    const uint8_t key[32] = {
+        0xa0, 0xab, 0xb6, 0x81, 0x8c, 0x97, 0xe2, 0xed,
+        0xf8, 0xc3, 0xce, 0xd9, 0x24, 0x2f, 0x3a, 0x05,
+        0x10, 0x1b, 0x66, 0x71, 0x7c, 0x47, 0x52, 0x5d,
+        0xa8, 0xb3, 0xbe, 0x89, 0x94, 0x9f, 0xea, 0xf5,
+    };
+    const uint8_t state2[32] = {
+        0x55, 0x62, 0x6f, 0x7c, 0x89, 0x96, 0xa3, 0xb0,
+        0xbd, 0xca, 0xd7, 0xe4, 0xf1, 0xfe, 0x0b, 0x18,
+        0x25, 0x32, 0x3f, 0x4c, 0x59, 0x66, 0x73, 0x80,
+        0x8d, 0x9a, 0xa7, 0xb4, 0xc1, 0xce, 0xdb, 0xe8,
+    };
+    const uint8_t key2[32] = {
+        0x33, 0x20, 0x15, 0x0a, 0x7f, 0x6c, 0x41, 0xb6,
+        0xab, 0x98, 0x8d, 0xe2, 0xd7, 0xc4, 0x39, 0x2e,
+        0x03, 0x70, 0x65, 0x5a, 0x4f, 0xbc, 0x91, 0x86,
+        0xfb, 0xe8, 0xdd, 0x32, 0x27, 0x14, 0x09, 0x7e,
+    };
+    const uint8_t exp_aese[32] = {
+        0xe7, 0x49, 0x46, 0x10, 0xe0, 0x64, 0x52, 0x49,
+        0xe7, 0x1b, 0xc2, 0x91, 0x09, 0x65, 0x61, 0x5f,
+        0x60, 0x69, 0xbc, 0x9c, 0xe1, 0xfe, 0x30, 0x69,
+        0xca, 0x1b, 0x9b, 0xce, 0x09, 0xde, 0x41, 0x1b,
+    };
+    const uint8_t exp_aesd[32] = {
+        0xfc, 0x86, 0xe2, 0xaa, 0x47, 0x78, 0xd4, 0x4f,
+        0xfc, 0x1d, 0x6f, 0x01, 0x72, 0xf0, 0x2d, 0x1d,
+        0x96, 0x86, 0xc1, 0x83, 0xa0, 0x1c, 0xbf, 0x86,
+        0x7c, 0xae, 0xc8, 0xc4, 0x72, 0x81, 0xe1, 0xae,
+    };
+    const uint8_t exp_aesmc[32] = {
+        0x22, 0x39, 0x54, 0x73, 0x76, 0x45, 0xa8, 0xff,
+        0x4a, 0x71, 0x4c, 0x7b, 0x7e, 0x5d, 0x60, 0x47,
+        0x92, 0x89, 0xa4, 0xa3, 0xc6, 0x95, 0xb8, 0xcf,
+        0xba, 0x41, 0xdc, 0x2b, 0xce, 0xad, 0xf0, 0x97,
+    };
+    const uint8_t exp_aesimc[32] = {
+        0xe1, 0x0a, 0x97, 0x40, 0x23, 0x9b, 0xfd, 0x21,
+        0x52, 0x59, 0x54, 0x53, 0x06, 0x35, 0x18, 0x2f,
+        0x4a, 0x21, 0x7c, 0x0b, 0x25, 0xe6, 0x5b, 0xbc,
+        0x39, 0xf2, 0x5f, 0x98, 0x36, 0x45, 0x08, 0x7f,
+    };
+    const uint8_t exp_sm4e[32] = {
+        0x26, 0x49, 0x13, 0xb2, 0x02, 0xa1, 0xba, 0x48,
+        0xc7, 0x98, 0x0e, 0x8a, 0xaf, 0xb6, 0x37, 0x7e,
+        0x32, 0xad, 0xb7, 0x55, 0x42, 0x0f, 0x26, 0x81,
+        0x56, 0x58, 0x9a, 0x72, 0x09, 0x9d, 0x22, 0x60,
+    };
+    const uint8_t exp_sm4ekey[32] = {
+        0xb2, 0xf4, 0xd1, 0xdc, 0x6c, 0x0b, 0x7f, 0xb4,
+        0x92, 0xcb, 0xa2, 0x6b, 0x2c, 0x7b, 0x62, 0xcc,
+        0x7d, 0x2c, 0xbf, 0x62, 0x3c, 0x1d, 0x8a, 0x50,
+        0x3e, 0x61, 0xc0, 0x1a, 0x9a, 0x4d, 0x1e, 0x30,
+    };
+    const uint8_t exp_rax1[32] = {
+        0x32, 0x22, 0x45, 0x68, 0x77, 0x4e, 0x21, 0xdc,
+        0xeb, 0xfb, 0xcc, 0x21, 0x5e, 0x77, 0x78, 0x44,
+        0x22, 0xd2, 0xf5, 0xf8, 0xc7, 0x1e, 0x50, 0x8d,
+        0x7b, 0x4b, 0x1c, 0xd1, 0x8f, 0xe6, 0xc9, 0x14,
+    };
+
+    test_arm64_sve2_crypto_run(code_aese, sizeof(code_aese) - 1, state, key,
+                               exp_aese);
+    test_arm64_sve2_crypto_run(code_aesd, sizeof(code_aesd) - 1, state, key,
+                               exp_aesd);
+    test_arm64_sve2_crypto_run(code_aesmc, sizeof(code_aesmc) - 1, state, NULL,
+                               exp_aesmc);
+    test_arm64_sve2_crypto_run(code_aesimc, sizeof(code_aesimc) - 1, state,
+                               NULL, exp_aesimc);
+    test_arm64_sve2_crypto_run(code_sm4e, sizeof(code_sm4e) - 1, state, key,
+                               exp_sm4e);
+    test_arm64_sve2_crypto_run(code_sm4ekey, sizeof(code_sm4ekey) - 1, state2,
+                               key2, exp_sm4ekey);
+    test_arm64_sve2_crypto_run(code_rax1, sizeof(code_rax1) - 1, state2, key2,
+                               exp_rax1);
+}
+
+static void test_arm64_sve2_ext(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x23\x04\x61\x05" /* ext z3.b,{z1.b,z2.b},#9 */
+        "\xc3\xe0\x00\xe4"; /* st1b { z3.b },p0,[x6] */
+    uint8_t n[16], m[16], expected[16], got[16];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        n[i] = (uint8_t)(0x10 + i);
+        m[i] = (uint8_t)(0x80 + i);
+    }
+    for (i = 0; i < 7; i++) {
+        expected[i] = n[i + 9];
+    }
+    for (i = 7; i < 16; i++) {
+        expected[i] = m[i - 7];
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, n, sizeof(n)));
+    OK(uc_mem_write(uc, x5, m, sizeof(m)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got, sizeof(got)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_splice(void)
+{
+    uc_engine *uc;
+    const char code_b[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\x80\xe0\x18\x25" /* ptrue p0.b,vl4 */
+        "\x20\x80\x2d\x05" /* splice z0.b,p0,z1.b */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc0\xe0\x00\xe4"; /* st1b { z0.b },p0,[x6] */
+    const char code_s[] =
+        "\xe0\xe3\x98\x25" /* ptrue p0.s */
+        "\x81\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x4] */
+        "\xa2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x5] */
+        "\x40\xe0\x98\x25" /* ptrue p0.s,vl2 */
+        "\x20\x80\xad\x05" /* splice z0.s,p0,z1.s */
+        "\xe0\xe3\x98\x25" /* ptrue p0.s */
+        "\xc0\xe0\x40\xe5"; /* st1w { z0.s },p0,[x6] */
+    uint8_t left_b[16], right_b[16], expected_b[16], got_b[16];
+    uint32_t left_s[4], right_s[4], expected_s[4], got_s[4];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        left_b[i] = (uint8_t)(0x10 + i);
+        right_b[i] = (uint8_t)(0x80 + i);
+        expected_b[i] = i < 4 ? left_b[i] : right_b[i - 4];
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_b,
+                    sizeof(code_b) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, left_b, sizeof(left_b)));
+    OK(uc_mem_write(uc, x5, right_b, sizeof(right_b)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_b) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_b, sizeof(got_b)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got_b[i] == expected_b[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 4; i++) {
+        left_s[i] = 0x10101010u + (uint32_t)i;
+        right_s[i] = 0x80808080u + (uint32_t)i;
+        expected_s[i] = i < 2 ? left_s[i] : right_s[i - 2];
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_s,
+                    sizeof(code_s) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, left_s, sizeof(left_s)));
+    OK(uc_mem_write(uc, x5, right_s, sizeof(right_s)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_s) - 1, 0, 0));
+    OK(uc_mem_read(uc, x6, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == expected_s[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_tbl_tbx(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xa2\xa0\x00\xa4" /* ld1b { z2.b },p0/z,[x5] */
+        "\xc4\xa0\x00\xa4" /* ld1b { z4.b },p0/z,[x6] */
+        "\x20\x28\x24\x05" /* tbl z0.b,{z1.b,z2.b},z4.b */
+        "\x00\xe1\x00\xe4" /* st1b { z0.b },p0,[x8] */
+        "\xe3\xa0\x00\xa4" /* ld1b { z3.b },p0/z,[x7] */
+        "\x23\x2c\x24\x05" /* tbx z3.b,z1.b,z4.b */
+        "\x23\xe1\x00\xe4"; /* st1b { z3.b },p0,[x9] */
+    const char code_h[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xa0\xa4" /* ld1h { z1.h },p0/z,[x4] */
+        "\xa2\xa0\xa0\xa4" /* ld1h { z2.h },p0/z,[x5] */
+        "\xc4\xa0\xa0\xa4" /* ld1h { z4.h },p0/z,[x6] */
+        "\x20\x28\x64\x05" /* tbl z0.h,{z1.h,z2.h},z4.h */
+        "\x00\xe1\xa0\xe4" /* st1h { z0.h },p0,[x8] */
+        "\xe3\xa0\xa0\xa4" /* ld1h { z3.h },p0/z,[x7] */
+        "\x23\x2c\x64\x05" /* tbx z3.h,z1.h,z4.h */
+        "\x23\xe1\xa0\xe4"; /* st1h { z3.h },p0,[x9] */
+    const char code_s[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x4] */
+        "\xa2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x5] */
+        "\xc4\xa0\x40\xa5" /* ld1w { z4.s },p0/z,[x6] */
+        "\x20\x28\xa4\x05" /* tbl z0.s,{z1.s,z2.s},z4.s */
+        "\x00\xe1\x40\xe5" /* st1w { z0.s },p0,[x8] */
+        "\xe3\xa0\x40\xa5" /* ld1w { z3.s },p0/z,[x7] */
+        "\x23\x2c\xa4\x05" /* tbx z3.s,z1.s,z4.s */
+        "\x23\xe1\x40\xe5"; /* st1w { z3.s },p0,[x9] */
+    const char code_d[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xe0\xa5" /* ld1d { z1.d },p0/z,[x4] */
+        "\xa2\xa0\xe0\xa5" /* ld1d { z2.d },p0/z,[x5] */
+        "\xc4\xa0\xe0\xa5" /* ld1d { z4.d },p0/z,[x6] */
+        "\x20\x28\xe4\x05" /* tbl z0.d,{z1.d,z2.d},z4.d */
+        "\x00\xe1\xe0\xe5" /* st1d { z0.d },p0,[x8] */
+        "\xe3\xa0\xe0\xa5" /* ld1d { z3.d },p0/z,[x7] */
+        "\x23\x2c\xe4\x05" /* tbx z3.d,z1.d,z4.d */
+        "\x23\xe1\xe0\xe5"; /* st1d { z3.d },p0,[x9] */
+    uint8_t table0[16], table1[16], indexes[16], initial[16];
+    uint8_t expected_tbl[16], expected_tbx[16], got[16];
+    uint16_t table0_h[8], table1_h[8], indexes_h[8], initial_h[8];
+    uint16_t expected_tbl_h[8], expected_tbx_h[8], got_h[8];
+    uint32_t table0_s[4], table1_s[4], indexes_s[4], initial_s[4];
+    uint32_t expected_tbl_s[4], expected_tbx_s[4], got_s[4];
+    uint64_t table0_d[4], table1_d[4], indexes_d[4], initial_d[4];
+    uint64_t expected_tbl_d[4], expected_tbx_d[4], got_d[4];
+    uint64_t x4 = 0x40000;
+    uint64_t x5 = 0x40100;
+    uint64_t x6 = 0x40200;
+    uint64_t x7 = 0x40300;
+    uint64_t x8 = 0x40400;
+    uint64_t x9 = 0x40500;
+    int i;
+
+    const uint8_t idx_values[16] = {
+        0, 1, 15, 16, 17, 31, 32, 5,
+        20, 14, 30, 40, 7, 18, 2, 29
+    };
+    const uint16_t idx_values_h[8] = { 0, 1, 7, 8, 9, 15, 16, 3 };
+    const uint32_t idx_values_s[4] = { 0, 3, 4, 8 };
+    const uint64_t idx_values_d[4] = { 0, 3, 4, 8 };
+
+    for (i = 0; i < 16; i++) {
+        table0[i] = (uint8_t)(0x10 + i);
+        table1[i] = (uint8_t)(0x80 + i);
+        indexes[i] = idx_values[i];
+        initial[i] = (uint8_t)(0xd0 + i);
+    }
+
+    for (i = 0; i < 16; i++) {
+        int index = idx_values[i];
+
+        if (index < 16) {
+            expected_tbl[i] = table0[index];
+            expected_tbx[i] = table0[index];
+        } else if (index < 32) {
+            expected_tbl[i] = table1[index - 16];
+            expected_tbx[i] = initial[i];
+        } else {
+            expected_tbl[i] = 0;
+            expected_tbx[i] = initial[i];
+        }
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, table0, sizeof(table0)));
+    OK(uc_mem_write(uc, x5, table1, sizeof(table1)));
+    OK(uc_mem_write(uc, x6, indexes, sizeof(indexes)));
+    OK(uc_mem_write(uc, x7, initial, sizeof(initial)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, x8, got, sizeof(got)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got[i] == expected_tbl[i]);
+    }
+    OK(uc_mem_read(uc, x9, got, sizeof(got)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(got[i] == expected_tbx[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 8; i++) {
+        table0_h[i] = (uint16_t)(0x1100 + i);
+        table1_h[i] = (uint16_t)(0x8800 + i);
+        indexes_h[i] = idx_values_h[i];
+        initial_h[i] = (uint16_t)(0xd000 + i);
+    }
+    for (i = 0; i < 8; i++) {
+        int index = idx_values_h[i];
+
+        if (index < 8) {
+            expected_tbl_h[i] = table0_h[index];
+            expected_tbx_h[i] = table0_h[index];
+        } else if (index < 16) {
+            expected_tbl_h[i] = table1_h[index - 8];
+            expected_tbx_h[i] = initial_h[i];
+        } else {
+            expected_tbl_h[i] = 0;
+            expected_tbx_h[i] = initial_h[i];
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_h,
+                    sizeof(code_h) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, table0_h, sizeof(table0_h)));
+    OK(uc_mem_write(uc, x5, table1_h, sizeof(table1_h)));
+    OK(uc_mem_write(uc, x6, indexes_h, sizeof(indexes_h)));
+    OK(uc_mem_write(uc, x7, initial_h, sizeof(initial_h)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_h) - 1, 0, 0));
+    OK(uc_mem_read(uc, x8, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == expected_tbl_h[i]);
+    }
+    OK(uc_mem_read(uc, x9, got_h, sizeof(got_h)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(got_h[i] == expected_tbx_h[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 4; i++) {
+        table0_s[i] = 0x11000000u + (uint32_t)i;
+        table1_s[i] = 0x88000000u + (uint32_t)i;
+        indexes_s[i] = idx_values_s[i];
+        initial_s[i] = 0xd0000000u + (uint32_t)i;
+    }
+    for (i = 0; i < 4; i++) {
+        uint32_t index = idx_values_s[i];
+
+        if (index < 4) {
+            expected_tbl_s[i] = table0_s[index];
+            expected_tbx_s[i] = table0_s[index];
+        } else if (index < 8) {
+            expected_tbl_s[i] = table1_s[index - 4];
+            expected_tbx_s[i] = initial_s[i];
+        } else {
+            expected_tbl_s[i] = 0;
+            expected_tbx_s[i] = initial_s[i];
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_s,
+                    sizeof(code_s) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, table0_s, sizeof(table0_s)));
+    OK(uc_mem_write(uc, x5, table1_s, sizeof(table1_s)));
+    OK(uc_mem_write(uc, x6, indexes_s, sizeof(indexes_s)));
+    OK(uc_mem_write(uc, x7, initial_s, sizeof(initial_s)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_s) - 1, 0, 0));
+    OK(uc_mem_read(uc, x8, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == expected_tbl_s[i]);
+    }
+    OK(uc_mem_read(uc, x9, got_s, sizeof(got_s)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_s[i] == expected_tbx_s[i]);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < 4; i++) {
+        table0_d[i] = 0x1100000000000000ull + (uint64_t)i;
+        table1_d[i] = 0x8800000000000000ull + (uint64_t)i;
+        indexes_d[i] = idx_values_d[i];
+        initial_d[i] = 0xd000000000000000ull + (uint64_t)i;
+    }
+    for (i = 0; i < 4; i++) {
+        uint64_t index = idx_values_d[i];
+
+        if (index < 4) {
+            expected_tbl_d[i] = table0_d[index];
+            expected_tbx_d[i] = table0_d[index];
+        } else if (index < 8) {
+            expected_tbl_d[i] = table1_d[index - 4];
+            expected_tbx_d[i] = initial_d[i];
+        } else {
+            expected_tbl_d[i] = 0;
+            expected_tbx_d[i] = initial_d[i];
+        }
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code_d,
+                    sizeof(code_d) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, table0_d, sizeof(table0_d)));
+    OK(uc_mem_write(uc, x5, table1_d, sizeof(table1_d)));
+    OK(uc_mem_write(uc, x6, indexes_d, sizeof(indexes_d)));
+    OK(uc_mem_write(uc, x7, initial_d, sizeof(initial_d)));
+    test_arm64_mte_enable_sve_vq(uc, 1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code_d) - 1, 0, 0));
+    OK(uc_mem_read(uc, x8, got_d, sizeof(got_d)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_d[i] == expected_tbl_d[i]);
+    }
+    OK(uc_mem_read(uc, x9, got_d, sizeof(got_d)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(got_d[i] == expected_tbx_d[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve2_ld1ro(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc1\x20\x21\xa5" /* ld1row { z1.s },p0/z,[x6,#0x20] */
+        "\xc2\x00\x27\xa5" /* ld1row { z2.s },p0/z,[x6,x7,lsl #2] */
+        "\x01\xe1\x40\xe5" /* st1w { z1.s },p0,[x8] */
+        "\x22\xe1\x40\xe5"; /* st1w { z2.s },p0,[x9] */
+    const char invalid_vl_code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc1\x20\x21\xa5"; /* ld1row { z1.s },p0/z,[x6,#0x20] */
+    uint32_t data[16];
+    uint32_t expected[2][16];
+    uint32_t fill[16];
+    uint32_t got[16];
+    uint64_t x6 = 0x40000;
+    uint64_t x7 = 8;
+    uint64_t x8 = 0x40200;
+    uint64_t x9 = 0x40300;
+    const uint64_t zcr_len[2] = { 3, 2 };
+    int i, j;
+
+    for (i = 0; i < 16; i++) {
+        data[i] = 0x1000 + i;
+        fill[i] = 0x5a5a5a5a;
+    }
+    for (i = 0; i < 16; i++) {
+        expected[0][i] = data[8 + (i & 7)];
+        if (i < 8) {
+            expected[1][i] = data[8 + i];
+        } else if (i < 12) {
+            expected[1][i] = 0;
+        } else {
+            expected[1][i] = fill[i];
+        }
+    }
+
+    for (j = 0; j < 2; j++) {
+        uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                        sizeof(code) - 1, UC_CPU_ARM64_MAX);
+        OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+        OK(uc_mem_write(uc, 0x40000, data, sizeof(data)));
+        OK(uc_mem_write(uc, x8, fill, sizeof(fill)));
+        OK(uc_mem_write(uc, x9, fill, sizeof(fill)));
+        test_arm64_mte_enable_sve_vq(uc, zcr_len[j]);
+        OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+        OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+        OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+        OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+
+        OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+        OK(uc_mem_read(uc, x8, got, sizeof(got)));
+        for (i = 0; i < 16; i++) {
+            TEST_CHECK(got[i] == expected[j][i]);
+        }
+        OK(uc_mem_read(uc, x9, got, sizeof(got)));
+        for (i = 0; i < 16; i++) {
+            TEST_CHECK(got[i] == expected[j][i]);
+        }
+
+        OK(uc_close(uc));
+    }
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, invalid_vl_code,
+                    sizeof(invalid_vl_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, data, sizeof(data)));
+    test_arm64_mte_enable_sve_vq(uc, 0);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(invalid_vl_code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+
+    test_arm64_i8mm_expect_exception(0xa52120c1, UC_CPU_ARM64_A72);
+}
+
+static void test_arm64_mte_sve_contiguous_access(void)
+{
+    uc_engine *uc;
+    const char store_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x60\x38\x20\x05" /* mov z0.b,w3 */
+        "\x80\xe0\x00\xe4"; /* st1b { z0.b },p0,[x4] */
+    const char load_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x00\xa4" /* ld1b { z1.b },p0/z,[x4] */
+        "\xc1\xe0\x00\xe4"; /* st1b { z1.b },p0,[x6] */
+    const char ldff_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x61\x38\x20\x05" /* mov z1.b,w3 */
+        "\x81\x60\x05\xa4" /* ldff1b { z1.b },p0/z,[x4,x5] */
+        "\xc1\xe0\x00\xe4"; /* st1b { z1.b },p0,[x6] */
+    const char ldnf_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\x10\xa4" /* ldnf1b { z1.b },p0/z,[x4] */
+        "\xc1\xe0\x00\xe4"; /* st1b { z1.b },p0,[x6] */
+    const char ldff_gather_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xe2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x7] */
+        "\x61\x38\xa0\x05" /* mov z1.s,w3 */
+        "\x81\x60\x02\x84" /* ldff1b { z1.s },p0/z,[x4,z2.s,uxtw] */
+        "\xc1\xe0\x40\xe5"; /* st1w { z1.s },p0,[x6] */
+    const char ld_gather_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xe2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x7] */
+        "\x61\x38\xa0\x05" /* mov z1.s,w3 */
+        "\x81\x40\x02\x84" /* ld1b { z1.s },p0/z,[x4,z2.s,uxtw] */
+        "\xc1\xe0\x40\xe5"; /* st1w { z1.s },p0,[x6] */
+    const char st_gather_code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc1\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x6] */
+        "\xe2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x7] */
+        "\x81\x80\x42\xe4"; /* st1b { z1.s },p0,[x4,z2.s,uxtw] */
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0x0c00000000000000ull;
+    uint64_t x3 = 0x5a;
+    uint64_t x4 = 0x0c00000000040000ull;
+    uint64_t x5 = 0;
+    uint64_t x6 = 0x40100;
+    uint64_t x7 = 0x40200;
+    uint8_t mem[16];
+    uint8_t expected[16];
+    uint32_t offsets[4];
+    uint32_t words[4];
+    int i;
+
+    memset(expected, 0x5a, sizeof(expected));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, store_code,
+                    sizeof(store_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(store_code) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected, sizeof(mem)) == 0);
+    OK(uc_close(uc));
+
+    memset(expected, 0xa5, sizeof(expected));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, store_code,
+                    sizeof(store_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(store_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected, sizeof(mem)) == 0);
+    OK(uc_close(uc));
+
+    for (i = 0; i < (int)sizeof(expected); i++) {
+        expected[i] = (uint8_t)(0x30 + i);
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, load_code,
+                    sizeof(load_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(load_code) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40100, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected, sizeof(mem)) == 0);
+    OK(uc_close(uc));
+
+    memset(mem, 0, sizeof(mem));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, load_code,
+                    sizeof(load_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, mem, sizeof(mem)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(load_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40100, mem, sizeof(mem)));
+    for (i = 0; i < (int)sizeof(mem); i++) {
+        TEST_CHECK(mem[i] == 0);
+    }
+    OK(uc_close(uc));
+
+    memset(expected, 0xa5, sizeof(expected));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldff_code,
+                    sizeof(ldff_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, expected, sizeof(expected)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x3 = 0;
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(ldff_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40100, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected, sizeof(mem)) == 0);
+    OK(uc_close(uc));
+
+    for (i = 0; i < (int)sizeof(expected); i++) {
+        expected[i] = (uint8_t)(0x40 + i);
+    }
+    memset(mem, 0, sizeof(mem));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldff_code,
+                    sizeof(ldff_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, mem, sizeof(mem)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0c00000000040008ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(ldff_code) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40100, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected + 8, 8) == 0);
+    for (i = 8; i < (int)sizeof(mem); i++) {
+        TEST_CHECK(mem[i] == 0);
+    }
+    OK(uc_close(uc));
+
+    memset(mem, 0xa5, sizeof(mem));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldnf_code,
+                    sizeof(ldnf_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, mem, sizeof(mem)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(ldnf_code) - 1,
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40100, mem, sizeof(mem)));
+    for (i = 0; i < (int)sizeof(mem); i++) {
+        TEST_CHECK(mem[i] == 0);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < (int)sizeof(expected); i++) {
+        expected[i] = (uint8_t)(0x60 + i);
+    }
+    for (i = 0; i < 4; i++) {
+        offsets[i] = (uint32_t)i;
+        words[i] = 0;
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ld_gather_code,
+                    sizeof(ld_gather_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x3 = 0;
+    x4 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start,
+                    code_start + sizeof(ld_gather_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(words[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+
+    memset(expected, 0xa5, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        offsets[i] = (uint32_t)i;
+        words[i] = 0xa5a5a5a5u;
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ld_gather_code,
+                    sizeof(ld_gather_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(ld_gather_code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(words[i] == 0xa5a5a5a5u);
+    }
+    OK(uc_close(uc));
+
+    memset(expected, 0xa5, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        offsets[i] = (uint32_t)i;
+        words[i] = 0x70717273u + (uint32_t)i;
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, st_gather_code,
+                    sizeof(st_gather_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start,
+                    code_start + sizeof(st_gather_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(mem[i] == (uint8_t)words[i]);
+    }
+    for (i = 4; i < (int)sizeof(mem); i++) {
+        TEST_CHECK(mem[i] == 0xa5);
+    }
+    OK(uc_close(uc));
+
+    memset(expected, 0xa5, sizeof(expected));
+    offsets[0] = 0;
+    offsets[1] = 1;
+    offsets[2] = 16;
+    offsets[3] = 17;
+    for (i = 0; i < 4; i++) {
+        words[i] = 0x80818283u + (uint32_t)i;
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, st_gather_code,
+                    sizeof(st_gather_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(st_gather_code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected, sizeof(mem)) == 0);
+    OK(uc_close(uc));
+
+    memset(expected, 0xa5, sizeof(expected));
+    for (i = 0; i < 4; i++) {
+        offsets[i] = (uint32_t)i;
+        words[i] = 0xa5a5a5a5u;
+    }
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldff_gather_code,
+                    sizeof(ldff_gather_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x3 = 0;
+    x4 = 0x0d00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(ldff_gather_code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(words[i] == 0xa5a5a5a5u);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < (int)sizeof(expected); i++) {
+        expected[i] = (uint8_t)(0x50 + i);
+    }
+    offsets[0] = 8;
+    offsets[1] = 9;
+    offsets[2] = 16;
+    offsets[3] = 17;
+    memset(words, 0, sizeof(words));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, ldff_gather_code,
+                    sizeof(ldff_gather_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    x4 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start,
+                    code_start + sizeof(ldff_gather_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    TEST_CHECK(words[0] == expected[8]);
+    TEST_CHECK(words[1] == expected[9]);
+    TEST_CHECK(words[2] == 0);
+    TEST_CHECK(words[3] == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_sve_gather_scatter_sizem1(void)
+{
+    uint8_t load_code[20];
+    uint8_t store_code[20];
+    uint8_t mem[32];
+    uint8_t expected[32];
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    uint32_t offsets[4] = { 15, 0, 2, 4 };
+    uint32_t words[4];
+    uc_engine *uc;
+    uint64_t x4 = 0x0c00000000040000ull;
+    uint64_t x6 = 0x40100;
+    uint64_t x7 = 0x40200;
+    const uint64_t tag0 = 0x0c00000000000000ull;
+    const uint64_t tag1 = 0x0d00000000000000ull;
+    int i;
+
+    test_arm64_emit32(load_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(load_code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(load_code, 8, 0xa540a0e2);  /* ld1w z2.s */
+    test_arm64_emit32(load_code, 12, 0x84824081); /* ld1h z1.s */
+    test_arm64_emit32(load_code, 16, 0xe540e0c1); /* st1w z1.s */
+
+    test_arm64_emit32(store_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(store_code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(store_code, 8, 0xa540a0c1);  /* ld1w z1.s */
+    test_arm64_emit32(store_code, 12, 0xa540a0e2); /* ld1w z2.s */
+    test_arm64_emit32(store_code, 16, 0xe4c28081); /* st1h z1.s */
+
+    for (i = 0; i < (int)sizeof(mem); i++) {
+        mem[i] = (uint8_t)(0x10 + i);
+    }
+    memset(words, 0, sizeof(words));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)load_code,
+                    sizeof(load_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag0);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(load_code),
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    TEST_CHECK(words[0] == 0x0000201f);
+    TEST_CHECK(words[1] == 0x00001110);
+    TEST_CHECK(words[2] == 0x00001312);
+    TEST_CHECK(words[3] == 0x00001514);
+    OK(uc_close(uc));
+
+    memset(words, 0xa5, sizeof(words));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)load_code,
+                    sizeof(load_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(load_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(words[i] == 0xa5a5a5a5u);
+    }
+    OK(uc_close(uc));
+
+    memset(mem, 0xa5, sizeof(mem));
+    memcpy(expected, mem, sizeof(expected));
+    words[0] = 0x11112233;
+    words[1] = 0x22224455;
+    words[2] = 0x33336677;
+    words[3] = 0x44448899;
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)store_code,
+                    sizeof(store_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag0);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(store_code),
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    TEST_CHECK(mem[15] == 0x33);
+    TEST_CHECK(mem[16] == 0x22);
+    TEST_CHECK(mem[0] == 0x55);
+    TEST_CHECK(mem[1] == 0x44);
+    TEST_CHECK(mem[2] == 0x77);
+    TEST_CHECK(mem[3] == 0x66);
+    TEST_CHECK(mem[4] == 0x99);
+    TEST_CHECK(mem[5] == 0x88);
+    OK(uc_close(uc));
+
+    memset(mem, 0xa5, sizeof(mem));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)store_code,
+                    sizeof(store_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(store_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected, sizeof(mem)) == 0);
+    OK(uc_close(uc));
+
+    memset(words, 0xa5, sizeof(words));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)load_code,
+                    sizeof(load_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, expected, sizeof(expected)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(load_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40100, words, sizeof(words)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(words[i] == 0xa5a5a5a5u);
+    }
+    OK(uc_close(uc));
+
+    memset(mem, 0xa5, sizeof(mem));
+    words[0] = 0x11112233;
+    words[1] = 0x22224455;
+    words[2] = 0x33336677;
+    words[3] = 0x44448899;
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)store_code,
+                    sizeof(store_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, words, sizeof(words)));
+    OK(uc_mem_write(uc, 0x40200, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_checks(uc, 3ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(store_code),
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    TEST_CHECK(mem[15] == 0x33);
+    TEST_CHECK(mem[16] == 0x22);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_sve_whole_register_access(void)
+{
+    uint8_t load_code[16];
+    uint8_t store_code[20];
+    uint8_t mem[32];
+    uint8_t out[16];
+    uint8_t expected[32];
+    uc_engine *uc;
+    uint64_t x4 = 0x0c00000000040008ull;
+    uint64_t x6 = 0x40100;
+    const uint64_t tag0 = 0x0c00000000000000ull;
+    const uint64_t tag1 = 0x0d00000000000000ull;
+    int i;
+
+    test_arm64_emit32(load_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(load_code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(load_code, 8, 0x85804081);  /* ldr z1,[x4] */
+    test_arm64_emit32(load_code, 12, 0xe400e0c1); /* st1b z1.b */
+
+    test_arm64_emit32(store_code, 0, 0xd9200822);  /* stg x2,[x1] */
+    test_arm64_emit32(store_code, 4, 0x2518e3e0);  /* ptrue p0.b */
+    test_arm64_emit32(store_code, 8, 0xa400a0c1);  /* ld1b z1.b */
+    test_arm64_emit32(store_code, 12, 0xe5804081); /* str z1,[x4] */
+    test_arm64_emit32(store_code, 16, 0xd503201f); /* nop */
+
+    for (i = 0; i < (int)sizeof(mem); i++) {
+        mem[i] = (uint8_t)(0x30 + i);
+    }
+    memset(out, 0, sizeof(out));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)load_code,
+                    sizeof(load_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, out, sizeof(out)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag0);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(load_code),
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40100, out, sizeof(out)));
+    TEST_CHECK(memcmp(out, mem + 8, sizeof(out)) == 0);
+    OK(uc_close(uc));
+
+    memset(out, 0xa5, sizeof(out));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)load_code,
+                    sizeof(load_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, out, sizeof(out)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(load_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40100, out, sizeof(out)));
+    for (i = 0; i < (int)sizeof(out); i++) {
+        TEST_CHECK(out[i] == 0xa5);
+    }
+    OK(uc_close(uc));
+
+    for (i = 0; i < (int)sizeof(out); i++) {
+        out[i] = (uint8_t)(0x70 + i);
+    }
+    memset(mem, 0xa5, sizeof(mem));
+    memcpy(expected, mem, sizeof(expected));
+    memcpy(expected + 8, out, sizeof(out));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)store_code,
+                    sizeof(store_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, out, sizeof(out)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag0);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_emu_start(uc, code_start + 4, code_start + sizeof(store_code),
+                    0, 0));
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    TEST_CHECK(memcmp(mem, expected, sizeof(mem)) == 0);
+    OK(uc_close(uc));
+
+    memset(mem, 0xa5, sizeof(mem));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)store_code,
+                    sizeof(store_code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, mem, sizeof(mem)));
+    OK(uc_mem_write(uc, 0x40100, out, sizeof(out)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_mte_enable_sve(uc);
+    test_arm64_mte_store_tag_at(uc, 0x40000, tag0);
+    test_arm64_mte_store_tag_at(uc, 0x40010, tag1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(uc_emu_start(uc, code_start + 4,
+                            code_start + sizeof(store_code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, mem, sizeof(mem)));
+    for (i = 0; i < (int)sizeof(mem); i++) {
+        TEST_CHECK(mem[i] == 0xa5);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve_contiguous_store_fault_no_partial(void)
+{
+    uint8_t code[12];
+    uint8_t before[8];
+    uint8_t after[8];
+    uc_engine *uc;
+    uint64_t x3 = 0x5a;
+    uint64_t x4 = 0x40ff8;
+    uc_err err;
+
+    test_arm64_emit32(code, 0, 0x2518e3e0); /* ptrue p0.b */
+    test_arm64_emit32(code, 4, 0x05203860); /* mov z0.b,w3 */
+    test_arm64_emit32(code, 8, 0xe400e080); /* st1b { z0.b },p0,[x4] */
+
+    memset(before, 0xa5, sizeof(before));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, before, sizeof(before)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0);
+    TEST_CHECK_(err == UC_ERR_WRITE_UNMAPPED, "err=%u", err);
+    OK(uc_mem_read(uc, x4, after, sizeof(after)));
+    TEST_CHECK(memcmp(after, before, sizeof(after)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve_scatter_store_fault_no_partial(void)
+{
+    const char code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\xc1\xa0\x40\xa5" /* ld1w { z1.s },p0/z,[x6] */
+        "\xe2\xa0\x40\xa5" /* ld1w { z2.s },p0/z,[x7] */
+        "\x81\x80\x42\xe4"; /* st1b { z1.s },p0,[x4,z2.s,uxtw] */
+    uint8_t before[16];
+    uint8_t after[16];
+    uint32_t words[4] = {
+        0x1111115a, 0x2222226b, 0x3333337c, 0x4444448d,
+    };
+    uint32_t offsets[4] = { 0, 0x1000, 1, 2 };
+    uc_engine *uc;
+    uint64_t x4 = 0x40000;
+    uint64_t x6 = 0x40200;
+    uint64_t x7 = 0x40300;
+    uc_err err;
+
+    memset(before, 0xa5, sizeof(before));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, before, sizeof(before)));
+    OK(uc_mem_write(uc, x6, words, sizeof(words)));
+    OK(uc_mem_write(uc, x7, offsets, sizeof(offsets)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0);
+    TEST_CHECK_(err == UC_ERR_WRITE_UNMAPPED, "err=%u", err);
+    OK(uc_mem_read(uc, 0x40000, after, sizeof(after)));
+    TEST_CHECK(memcmp(after, before, sizeof(after)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve_ldff1_split_first_element(void)
+{
+    const char code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\x60\xa5\xa4" /* ldff1h { z1.h },p0/z,[x4,x5] */
+        "\xc1\xe0\xa0\xe4"; /* st1h { z1.h },p0,[x6] */
+    uint8_t data[4] = { 0x34, 0x12, 0x78, 0x56 };
+    uint16_t expected[8];
+    uint16_t out[8];
+    uc_engine *uc;
+    uint64_t x4 = 0x40fff;
+    uint64_t x5 = 0;
+    uint64_t x6 = 0x42000;
+    uc_err err;
+
+    memset(expected, 0, sizeof(expected));
+    expected[0] = 0x1234;
+    memset(out, 0xa5, sizeof(out));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x3000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, data, sizeof(data)));
+    OK(uc_mem_write(uc, x6, out, sizeof(out)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0);
+    TEST_CHECK_(err == UC_ERR_OK, "err=%u", err);
+    OK(uc_mem_read(uc, x6, out, sizeof(out)));
+    TEST_CHECK(memcmp(out, expected, sizeof(out)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_sve_ldnf1_split_first_element(void)
+{
+    const char code[] =
+        "\xe0\xe3\x18\x25" /* ptrue p0.b */
+        "\x81\xa0\xb0\xa4" /* ldnf1h { z1.h },p0/z,[x4] */
+        "\xc1\xe0\xa0\xe4"; /* st1h { z1.h },p0,[x6] */
+    uint8_t data[4] = { 0x34, 0x12, 0x78, 0x56 };
+    uint16_t expected[8];
+    uint16_t out[8];
+    uc_engine *uc;
+    uint64_t x4 = 0x40fff;
+    uint64_t x6 = 0x42000;
+    uc_err err;
+
+    memset(expected, 0, sizeof(expected));
+    expected[0] = 0x1234;
+    memset(out, 0xa5, sizeof(out));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x3000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, x4, data, sizeof(data)));
+    OK(uc_mem_write(uc, x6, out, sizeof(out)));
+    test_arm64_mte_enable_sve(uc);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0);
+    TEST_CHECK_(err == UC_ERR_OK, "err=%u", err);
+    OK(uc_mem_read(uc, x6, out, sizeof(out)));
+    TEST_CHECK(memcmp(out, expected, sizeof(out)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_tag_split_lifecycle(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x22\x08\x20\xd9" /* stg x2,[x1] */
+        "\x23\x00\x60\xd9"; /* ldg x3,[x1] */
+    uint8_t prealloc[0x3000];
+
+    memset(prealloc, 0, sizeof(prealloc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_checks(uc, 0);
+
+    OK(uc_mem_map(uc, 0x40000, 0x3000, UC_PROT_ALL));
+    test_arm64_mte_store_tag_at(uc, 0x40000, 0x0c00000000000000ull);
+    test_arm64_mte_store_tag_at(uc, 0x41000, 0x0d00000000000000ull);
+    test_arm64_mte_store_tag_at(uc, 0x42000, 0x0e00000000000000ull);
+    OK(uc_mem_protect(uc, 0x41000, 0x1000, UC_PROT_READ | UC_PROT_WRITE));
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x40000, 0x5000) ==
+               0x0c00000000005000ull);
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x41000, 0x6000) ==
+               0x0d00000000006000ull);
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x42000, 0x7000) ==
+               0x0e00000000007000ull);
+
+    OK(uc_mem_map(uc, 0x50000, 0x3000, UC_PROT_ALL));
+    test_arm64_mte_store_tag_at(uc, 0x50000, 0x0100000000000000ull);
+    test_arm64_mte_store_tag_at(uc, 0x51000, 0x0200000000000000ull);
+    test_arm64_mte_store_tag_at(uc, 0x52000, 0x0300000000000000ull);
+    OK(uc_mem_unmap(uc, 0x51000, 0x1000));
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x50000, 0x8000) ==
+               0x0100000000008000ull);
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x52000, 0x9000) ==
+               0x0300000000009000ull);
+    OK(uc_mem_map(uc, 0x51000, 0x1000, UC_PROT_ALL));
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x51000, 0xa000) == 0xa000);
+
+    OK(uc_mem_map_ptr(uc, 0x60000, 0x3000, UC_PROT_ALL, prealloc));
+    test_arm64_mte_store_tag_at(uc, 0x60000, 0x0400000000000000ull);
+    test_arm64_mte_store_tag_at(uc, 0x61000, 0x0500000000000000ull);
+    test_arm64_mte_store_tag_at(uc, 0x62000, 0x0600000000000000ull);
+    OK(uc_mem_protect(uc, 0x61000, 0x1000, UC_PROT_READ | UC_PROT_WRITE));
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x60000, 0xb000) ==
+               0x040000000000b000ull);
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x61000, 0xc000) ==
+               0x050000000000c000ull);
+    TEST_CHECK(test_arm64_mte_load_tag_at(uc, 0x62000, 0xd000) ==
+               0x060000000000d000ull);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_stgp(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x24\x14\x00\x69" /* stgp x4,x5,[x1] */
+        "\x23\x00\x60\xd9" /* ldg  x3,[x1] */
+        "\x46\x1c\x81\x69" /* stgp x6,x7,[x2,#0x20]! */
+        "\x48\xa5\x80\x68" /* stgp x8,x9,[x10],#0x10 */
+        "\x4b\x00\x60\xd9" /* ldg  x11,[x2] */
+        "\xac\x01\x60\xd9"; /* ldg  x12,[x13] */
+    const char invalid_code[] =
+        "\x24\x14\x00\x68"; /* stgp x4,x5,[x1], invalid index */
+    uint64_t x1 = 0x0a00000000040000ull;
+    uint64_t x2 = 0x0b00000000040020ull;
+    uint64_t x3 = 0x5000;
+    uint64_t x4 = 0x1122334455667788ull;
+    uint64_t x5 = 0x8877665544332211ull;
+    uint64_t x6 = 0x0102030405060708ull;
+    uint64_t x7 = 0x8090a0b0c0d0e0f0ull;
+    uint64_t x8 = 0xfedcba9876543210ull;
+    uint64_t x9 = 0x0011223344556677ull;
+    uint64_t x10 = 0x0c00000000040080ull;
+    uint64_t x11 = 0x6000;
+    uint64_t x12 = 0x7000;
+    uint64_t x13 = 0x0c00000000040080ull;
+    uint64_t mem;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    test_arm64_mte_enable_checks(uc, 0);
+
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X9, &x9));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X10, &x10));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X11, &x11));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X12, &x12));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X13, &x13));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X10, &x10));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X11, &x11));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X12, &x12));
+    TEST_CHECK(x2 == 0x0b00000000040040ull);
+    TEST_CHECK(x3 == 0x0a00000000005000ull);
+    TEST_CHECK(x10 == 0x0c00000000040090ull);
+    TEST_CHECK(x11 == 0x0b00000000006000ull);
+    TEST_CHECK(x12 == 0x0c00000000007000ull);
+
+    OK(uc_mem_read(uc, 0x40000, &mem, sizeof(mem)));
+    TEST_CHECK(mem == 0x1122334455667788ull);
+    OK(uc_mem_read(uc, 0x40008, &mem, sizeof(mem)));
+    TEST_CHECK(mem == 0x8877665544332211ull);
+    OK(uc_mem_read(uc, 0x40040, &mem, sizeof(mem)));
+    TEST_CHECK(mem == 0x0102030405060708ull);
+    OK(uc_mem_read(uc, 0x40048, &mem, sizeof(mem)));
+    TEST_CHECK(mem == 0x8090a0b0c0d0e0f0ull);
+    OK(uc_mem_read(uc, 0x40080, &mem, sizeof(mem)));
+    TEST_CHECK(mem == 0xfedcba9876543210ull);
+    OK(uc_mem_read(uc, 0x40088, &mem, sizeof(mem)));
+    TEST_CHECK(mem == 0x0011223344556677ull);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, invalid_code,
+                    sizeof(invalid_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    test_arm64_mte_enable_checks(uc, 0);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(invalid_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_dc_zva_checked(void)
+{
+    uc_engine *uc;
+    const char zva_code[] =
+        "\x61\x74\x0b\xd5" /* dc     gva,x1 */
+        "\x21\x74\x0b\xd5" /* dc     zva,x1 */
+        "\x23\x00\x60\xd9"; /* ldg    x3,[x1] */
+    const char mismatch_code[] =
+        "\x61\x74\x0b\xd5" /* dc     gva,x1 */
+        "\x24\x74\x0b\xd5"; /* dc     zva,x4 */
+    const uint32_t TCO[5] = { 3, 3, 4, 2, 7 };
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint64_t pstate_tco = 1ULL << 25;
+    uint8_t fill[0x40];
+    uint8_t zeroed[0x40];
+    uint8_t expected_zero[0x40] = { 0 };
+    uint64_t x1 = 0x0c00000000040000ull;
+    uint64_t x3 = 0x5000;
+    uint64_t x4 = 0x0d00000000040000ull;
+
+    memset(fill, 0xaa, sizeof(fill));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, zva_code,
+                    sizeof(zva_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, fill, sizeof(fill)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(zva_code) - 1,
+                    0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_mem_read(uc, 0x40000, zeroed, sizeof(zeroed)));
+    TEST_CHECK(x3 == 0x0c00000000005000ull);
+    TEST_CHECK(memcmp(zeroed, expected_zero, sizeof(zeroed)) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, mismatch_code,
+                    sizeof(mismatch_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, fill, sizeof(fill)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(mismatch_code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_mem_read(uc, 0x40000, zeroed, sizeof(zeroed)));
+    TEST_CHECK(memcmp(zeroed, fill, sizeof(zeroed)) == 0);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, mismatch_code,
+                    sizeof(mismatch_code) - 1, UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, fill, sizeof(fill)));
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    test_arm64_pauth_cp_reg_write(uc, TCO, pstate_tco);
+    x1 = 0x0c00000000040000ull;
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_emu_start(uc, code_start,
+                    code_start + sizeof(mismatch_code) - 1, 0, 0));
+    OK(uc_mem_read(uc, 0x40000, zeroed, sizeof(zeroed)));
+    TEST_CHECK(memcmp(zeroed, expected_zero, sizeof(zeroed)) == 0);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_dc_zva_original_fault_addr(void)
+{
+    uc_engine *uc;
+    const char code[] = "\x21\x74\x0b\xd5"; /* dc zva,x1 */
+    uint64_t x1 = 0x0c00000000040020ull;
+    uint64_t invalid_addr = 0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    test_arm64_mte_enable_checks(uc, 1ULL << 40);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_WRITE_UNMAPPED);
+    OK(uc_ctl_get_invalid_addr(uc, &invalid_addr));
+    TEST_CHECK_(invalid_addr == 0x40020, "invalid_addr=0x%llx",
+                invalid_addr);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_dc_gva_gzva(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x61\x74\x0b\xd5" /* dc   gva,x1 */
+        "\x23\x00\x60\xd9" /* ldg  x3,[x1] */
+        "\x84\x74\x0b\xd5" /* dc   gzva,x4 */
+        "\x85\x00\x60\xd9"; /* ldg  x5,[x4] */
+    uint8_t fill[0x40];
+    uint8_t data[0x40];
+    uint8_t expected_zero[0x40] = { 0 };
+    uint64_t x1 = 0x0e00000000040000ull;
+    uint64_t x3 = 0x5000;
+    uint64_t x4 = 0x0f00000000040100ull;
+    uint64_t x5 = 0x6000;
+
+    memset(fill, 0xaa, sizeof(fill));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, 0x40000, fill, sizeof(fill)));
+    OK(uc_mem_write(uc, 0x40100, fill, sizeof(fill)));
+    test_arm64_mte_enable_checks(uc, 0);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_mem_read(uc, 0x40000, data, sizeof(data)));
+    TEST_CHECK(x3 == 0x0e00000000005000ull);
+    TEST_CHECK(memcmp(data, fill, sizeof(data)) == 0);
+    OK(uc_mem_read(uc, 0x40100, data, sizeof(data)));
+    TEST_CHECK(x5 == 0x0f00000000006000ull);
+    TEST_CHECK(memcmp(data, expected_zero, sizeof(data)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_dc_gva_probe(void)
+{
+    uc_engine *uc;
+    const char code[] = "\x61\x74\x0b\xd5"; /* dc gva,x1 */
+    uint64_t x1 = 0x40000;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_WRITE_UNMAPPED);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_mem_map(uc, 0x40000, 0x1000, UC_PROT_READ));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_WRITE_PROT);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_cache_ops(void)
+{
+    uc_engine *uc;
+    const char code[] =
+        "\x61\x76\x08\xd5" /* dc igvac,x1 */
+        "\x82\x76\x08\xd5" /* dc igsw,x2 */
+        "\x63\x7a\x0b\xd5"; /* dc cgvac,x3 */
+    const char old_cpu_code[] = "\x63\x7a\x0b\xd5"; /* dc cgvac,x3 */
+    uint64_t x1 = 0x40000;
+    uint64_t x2 = 0;
+    uint64_t x3 = 0x40020;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, old_cpu_code,
+                    sizeof(old_cpu_code) - 1, UC_CPU_ARM64_A72);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(old_cpu_code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_generic_timer_state_one(const uint32_t cval_reg[5],
+                                               const uint32_t ctl_reg[5])
+{
+    uc_engine *uc;
+    uint8_t code[36];
+    uint64_t x1 = UINT64_MAX;
+    uint64_t x2 = 0;
+    uint64_t x3 = 7;
+    uint64_t x4 = 0;
+    uint64_t x5 = 0;
+    uint64_t x6 = 1;
+    uint64_t x7 = 0;
+    uint64_t x8 = 0;
+    uint64_t x9 = 0;
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0, test_arm64_msr_sysreg(1, cval_reg));
+    test_arm64_emit32(code, 4, test_arm64_mrs_sysreg(2, cval_reg));
+    test_arm64_emit32(code, 8, test_arm64_msr_sysreg(3, ctl_reg));
+    test_arm64_emit32(code, 12, test_arm64_mrs_sysreg(4, ctl_reg));
+    test_arm64_emit32(code, 16, test_arm64_msr_sysreg(5, cval_reg));
+    test_arm64_emit32(code, 20, test_arm64_msr_sysreg(6, ctl_reg));
+    test_arm64_emit32(code, 24, test_arm64_mrs_sysreg(7, ctl_reg));
+    test_arm64_emit32(code, 28, test_arm64_msr_sysreg(8, ctl_reg));
+    test_arm64_emit32(code, 32, test_arm64_mrs_sysreg(9, ctl_reg));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X8, &x8));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X7, &x7));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X9, &x9));
+    TEST_CHECK(x2 == UINT64_MAX);
+    TEST_CHECK(x4 == 3);
+    TEST_CHECK(x7 == 5);
+    TEST_CHECK(x9 == 0);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_generic_timer_state(void)
+{
+    const uint32_t CNTP_CTL_EL0[5] = { 3, 3, 14, 2, 1 };
+    const uint32_t CNTP_CVAL_EL0[5] = { 3, 3, 14, 2, 2 };
+    const uint32_t CNTV_CTL_EL0[5] = { 3, 3, 14, 3, 1 };
+    const uint32_t CNTV_CVAL_EL0[5] = { 3, 3, 14, 3, 2 };
+
+    test_arm64_generic_timer_state_one(CNTP_CVAL_EL0, CNTP_CTL_EL0);
+    test_arm64_generic_timer_state_one(CNTV_CVAL_EL0, CNTV_CTL_EL0);
+}
+
+static void test_arm64_pmu_counter_delta(void)
+{
+    uc_engine *uc;
+    uint8_t code[28];
+    const uint32_t PMCR_EL0[5] = { 3, 3, 9, 12, 0 };
+    const uint32_t PMCNTENSET_EL0[5] = { 3, 3, 9, 12, 1 };
+    const uint32_t PMCCNTR_EL0[5] = { 3, 3, 9, 13, 0 };
+    const uint32_t PMEVTYPER0_EL0[5] = { 3, 3, 14, 12, 0 };
+    const uint32_t PMEVCNTR0_EL0[5] = { 3, 3, 14, 8, 0 };
+    uint64_t pmcr = 0x41;
+    uint64_t pmccntr = 0x1234567800000000ULL;
+    uint64_t pmevtyper = 0x11;
+    uint64_t pmevcntr = 0x40000000;
+    uint64_t pmcnten = (1ULL << 31) | 1;
+    uint64_t x5 = 0;
+    uint64_t x6 = 0;
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0, test_arm64_msr_sysreg(1, PMCR_EL0));
+    test_arm64_emit32(code, 4, test_arm64_msr_sysreg(2, PMCCNTR_EL0));
+    test_arm64_emit32(code, 8, test_arm64_msr_sysreg(3, PMEVTYPER0_EL0));
+    test_arm64_emit32(code, 12, test_arm64_msr_sysreg(4, PMEVCNTR0_EL0));
+    test_arm64_emit32(code, 16, test_arm64_msr_sysreg(7, PMCNTENSET_EL0));
+    test_arm64_emit32(code, 20, test_arm64_mrs_sysreg(5, PMCCNTR_EL0));
+    test_arm64_emit32(code, 24, test_arm64_mrs_sysreg(6, PMEVCNTR0_EL0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &pmcr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &pmccntr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &pmevtyper));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &pmevcntr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X7, &pmcnten));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X5, &x5));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    TEST_CHECK(x5 >= pmccntr);
+    TEST_CHECK((uint32_t)x6 >= (uint32_t)pmevcntr);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_pmu_pmuv3p5_event_counter_one(int cpu_model,
+                                                     uint64_t expected,
+                                                     bool check_lp)
+{
+    uc_engine *uc;
+    uint8_t code[16];
+    const uint32_t PMCR_EL0[5] = { 3, 3, 9, 12, 0 };
+    const uint32_t PMEVCNTR0_EL0[5] = { 3, 3, 14, 8, 0 };
+    uint64_t pmcr = 0x80;
+    uint64_t pmevcntr = 0x1234567887654321ULL;
+    uint64_t x3 = 0;
+    uint64_t x4 = 0;
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0, test_arm64_msr_sysreg(1, PMCR_EL0));
+    test_arm64_emit32(code, 4, test_arm64_mrs_sysreg(4, PMCR_EL0));
+    test_arm64_emit32(code, 8, test_arm64_msr_sysreg(2, PMEVCNTR0_EL0));
+    test_arm64_emit32(code, 12, test_arm64_mrs_sysreg(3, PMEVCNTR0_EL0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), cpu_model);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &pmcr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &pmevcntr));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X3, &x3));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X4, &x4));
+    TEST_CHECK(x3 == expected);
+    if (check_lp) {
+        TEST_CHECK((x4 & 0x80) == 0x80);
+    }
+    OK(uc_close(uc));
+}
+
+static void test_arm64_pmu_pmuv3p5_event_counter(void)
+{
+    uc_engine *uc;
+    const char code[] = "\x1f\x20\x03\xd5";
+    const uint32_t ID_AA64DFR0_EL1[5] = { 3, 0, 0, 5, 0 };
+    uint64_t dfr0;
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code,
+                    sizeof(code) - 1, UC_CPU_ARM64_MAX);
+    dfr0 = test_arm64_pauth_cp_reg_read(uc, ID_AA64DFR0_EL1);
+    TEST_CHECK(((dfr0 >> 8) & 0xf) == 6);
+    OK(uc_close(uc));
+
+    test_arm64_pmu_pmuv3p5_event_counter_one(
+        UC_CPU_ARM64_MAX, 0x1234567887654321ULL, true);
+    test_arm64_pmu_pmuv3p5_event_counter_one(
+        UC_CPU_ARM64_A72, 0x87654321, false);
+}
+
+static void test_arm64_pmu_el2_hlp_long_counter_one(uint64_t mdcr_el2,
+                                                    uint64_t expected_pmovsr)
+{
+    uc_engine *uc;
+    uint8_t code[28];
+    const uint32_t SCR_EL3[5] = { 3, 6, 1, 1, 0 };
+    const uint32_t MDCR_EL2[5] = { 3, 4, 1, 1, 1 };
+    const uint32_t PMCR_EL0[5] = { 3, 3, 9, 12, 0 };
+    const uint32_t PMCNTENSET_EL0[5] = { 3, 3, 9, 12, 1 };
+    const uint32_t PMOVSCLR_EL0[5] = { 3, 3, 9, 12, 3 };
+    const uint32_t PMSWINC_EL0[5] = { 3, 3, 9, 12, 4 };
+    const uint32_t PMEVTYPER1_EL0[5] = { 3, 3, 14, 12, 1 };
+    const uint32_t PMEVCNTR1_EL0[5] = { 3, 3, 14, 8, 1 };
+    uint64_t pmcr = 1;
+    uint64_t pmevtyper = 0;
+    uint64_t pmevcntr = UINT32_MAX;
+    uint64_t pmcnten = 1ULL << 1;
+    uint64_t x6 = 0;
+    uint64_t x7 = 0;
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0, test_arm64_msr_sysreg(1, PMCR_EL0));
+    test_arm64_emit32(code, 4, test_arm64_msr_sysreg(2, PMEVTYPER1_EL0));
+    test_arm64_emit32(code, 8, test_arm64_msr_sysreg(3, PMEVCNTR1_EL0));
+    test_arm64_emit32(code, 12, test_arm64_msr_sysreg(4, PMCNTENSET_EL0));
+    test_arm64_emit32(code, 16, test_arm64_msr_sysreg(4, PMSWINC_EL0));
+    test_arm64_emit32(code, 20, test_arm64_mrs_sysreg(6, PMOVSCLR_EL0));
+    test_arm64_emit32(code, 24, test_arm64_mrs_sysreg(7, PMEVCNTR1_EL0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SCR_EL3, 1);
+    test_arm64_pauth_cp_reg_write(uc, MDCR_EL2, mdcr_el2);
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &pmcr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X2, &pmevtyper));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X3, &pmevcntr));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X4, &pmcnten));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X6, &x6));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X7, &x7));
+    TEST_CHECK((x6 & (1ULL << 1)) == expected_pmovsr);
+    TEST_CHECK(x7 == 0x100000000ULL);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_pmu_el2_hlp_long_counter(void)
+{
+    const uint64_t mdcr_hlp_hpmn1 = (1ULL << 26) | (1ULL << 7) | 1;
+
+    test_arm64_pmu_el2_hlp_long_counter_one(4, 1ULL << 1);
+    test_arm64_pmu_el2_hlp_long_counter_one(mdcr_hlp_hpmn1, 0);
+}
+
+static uc_err test_arm64_pmu_el0_pmevcntr_run(uint64_t pmuserenr,
+                                              bool is_write,
+                                              uint64_t *x2)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    const uint32_t PMUSERENR_EL0[5] = { 3, 3, 9, 14, 0 };
+    const uint32_t PMEVCNTR0_EL0[5] = { 3, 3, 14, 8, 0 };
+    uint32_t pstate = 0;
+    uint64_t pmevcntr = 0x12345678;
+    uint64_t x1 = 0x87654321;
+    uc_err err;
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0,
+                      is_write ? test_arm64_msr_sysreg(1, PMEVCNTR0_EL0) :
+                                 test_arm64_mrs_sysreg(2, PMEVCNTR0_EL0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, PMUSERENR_EL0, pmuserenr);
+    test_arm64_pauth_cp_reg_write(uc, PMEVCNTR0_EL0, pmevcntr);
+    OK(uc_reg_write(uc, UC_ARM64_REG_PSTATE, &pstate));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0);
+    if (x2) {
+        OK(uc_reg_read(uc, UC_ARM64_REG_X2, x2));
+    }
+
+    OK(uc_close(uc));
+    return err;
+}
+
+static void test_arm64_pmu_el0_direct_counter_access(void)
+{
+    uint64_t x2 = 0;
+
+    TEST_CHECK(test_arm64_pmu_el0_pmevcntr_run(0, false, &x2) ==
+               UC_ERR_EXCEPTION);
+    TEST_CHECK(test_arm64_pmu_el0_pmevcntr_run(1ULL << 3, false, &x2) ==
+               UC_ERR_OK);
+    TEST_CHECK(x2 == 0x12345678);
+    TEST_CHECK(test_arm64_pmu_el0_pmevcntr_run(1ULL << 3, true, NULL) ==
+               UC_ERR_EXCEPTION);
+}
+
+static uc_err test_arm64_pmu_mdcr_tpm_run(uint64_t scr_el3)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    const uint32_t SCR_EL3[5] = { 3, 6, 1, 1, 0 };
+    const uint32_t MDCR_EL2[5] = { 3, 4, 1, 1, 1 };
+    const uint32_t PMCR_EL0[5] = { 3, 3, 9, 12, 0 };
+    uint64_t mdcr_tpm = 1ULL << 6;
+    uc_err err;
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0, test_arm64_mrs_sysreg(0, PMCR_EL0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SCR_EL3, scr_el3);
+    test_arm64_pauth_cp_reg_write(uc, MDCR_EL2, mdcr_tpm);
+
+    err = uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0);
+    OK(uc_close(uc));
+    return err;
+}
+
+static void test_arm64_pmu_effective_mdcr_el2(void)
+{
+    TEST_CHECK(test_arm64_pmu_mdcr_tpm_run(0) == UC_ERR_OK);
+    TEST_CHECK(test_arm64_pmu_mdcr_tpm_run(1) == UC_ERR_EXCEPTION);
+}
+
+static void test_arm64_pmu_pmcr_n_from_mdcr_el2(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    const uint32_t SCR_EL3[5] = { 3, 6, 1, 1, 0 };
+    const uint32_t MDCR_EL2[5] = { 3, 4, 1, 1, 1 };
+    const uint32_t PMCR_EL0[5] = { 3, 3, 9, 12, 0 };
+    uint32_t pstate_el1h = 5;
+    uint64_t mdcr_hpmn1 = 1;
+    uint64_t x0 = 0;
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0, test_arm64_mrs_sysreg(0, PMCR_EL0));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, MDCR_EL2, mdcr_hpmn1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_PSTATE, &pstate_el1h));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    TEST_CHECK(((x0 >> 11) & 0x1f) == 4);
+    OK(uc_close(uc));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+    test_arm64_pauth_cp_reg_write(uc, SCR_EL3, 1);
+    test_arm64_pauth_cp_reg_write(uc, MDCR_EL2, mdcr_hpmn1);
+    OK(uc_reg_write(uc, UC_ARM64_REG_PSTATE, &pstate_el1h));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X0, &x0));
+    TEST_CHECK(((x0 >> 11) & 0x1f) == 1);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_vhe_el12_aliases(void)
+{
+    uc_engine *uc;
+    uint8_t code[8];
+    uint8_t old_code[4];
+    uint32_t pstate = 9;
+    uint64_t x1 = 0x333;
+    uint64_t x2 = 0;
+    const uint32_t ID_AA64PFR0_EL1[5] = { 3, 0, 0, 4, 0 };
+    const uint32_t SCR_EL3[5] = { 3, 6, 1, 1, 0 };
+    const uint32_t HCR_EL2[5] = { 3, 4, 1, 1, 0 };
+    const uint32_t ZCR_EL1[5] = { 3, 0, 1, 2, 0 };
+    const uint32_t ZCR_EL2[5] = { 3, 4, 1, 2, 0 };
+    const uint32_t ZCR_EL12[5] = { 3, 5, 1, 2, 0 };
+    const uint32_t SMCR_EL1[5] = { 3, 0, 1, 2, 6 };
+    const uint32_t SMCR_EL2[5] = { 3, 4, 1, 2, 6 };
+    const uint32_t SMCR_EL12[5] = { 3, 5, 1, 2, 6 };
+    const uint32_t TFSR_EL1[5] = { 3, 0, 5, 6, 0 };
+    const uint32_t TFSR_EL2[5] = { 3, 4, 5, 6, 0 };
+    const uint32_t TFSR_EL12[5] = { 3, 5, 5, 6, 0 };
+    const uint32_t SCXTNUM_EL1[5] = { 3, 0, 13, 0, 7 };
+    const uint32_t SCXTNUM_EL2[5] = { 3, 4, 13, 0, 7 };
+    const uint32_t SCXTNUM_EL12[5] = { 3, 5, 13, 0, 7 };
+    const uint64_t scr = (1ULL << 0) | (1ULL << 8) | (1ULL << 10) |
+                         (1ULL << 25) | (1ULL << 26);
+    const uint64_t hcr = (1ULL << 31) | (1ULL << 34);
+
+    memset(code, 0, sizeof(code));
+    test_arm64_emit32(code, 0, test_arm64_msr_sysreg(1, TFSR_EL1));
+    test_arm64_emit32(code, 4, test_arm64_mrs_sysreg(2, TFSR_EL1));
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)code,
+                    sizeof(code), UC_CPU_ARM64_MAX);
+
+    TEST_CHECK(((test_arm64_pauth_cp_reg_read(uc, ID_AA64PFR0_EL1) >> 56) &
+                0xf) == 2);
+
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL1, 0);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL2, 1);
+    test_arm64_pauth_cp_reg_write(uc, ZCR_EL12, 2);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, ZCR_EL1) == 2);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, ZCR_EL2) == 1);
+
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL1, 0);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL2, 1);
+    test_arm64_pauth_cp_reg_write(uc, SMCR_EL12, 2);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SMCR_EL1) == 2);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SMCR_EL2) == 1);
+
+    test_arm64_pauth_cp_reg_write(uc, SCXTNUM_EL1, 0x101);
+    test_arm64_pauth_cp_reg_write(uc, SCXTNUM_EL2, 0x202);
+    test_arm64_pauth_cp_reg_write(uc, SCXTNUM_EL12, 0x303);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SCXTNUM_EL1) == 0x303);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, SCXTNUM_EL2) == 0x202);
+
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL1, 0x111);
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL2, 0x222);
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL12, 0x444);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0x444);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL2) == 0x222);
+
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL1, 0x111);
+    test_arm64_pauth_cp_reg_write(uc, TFSR_EL2, 0x222);
+    test_arm64_pauth_cp_reg_write(uc, SCR_EL3, scr);
+    test_arm64_pauth_cp_reg_write(uc, HCR_EL2, hcr);
+    OK(uc_reg_write(uc, UC_ARM64_REG_PSTATE, &pstate));
+    OK(uc_reg_write(uc, UC_ARM64_REG_X1, &x1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_ARM64_REG_X2, &x2));
+    TEST_CHECK(x2 == x1);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL1) == 0x111);
+    TEST_CHECK(test_arm64_pauth_cp_reg_read(uc, TFSR_EL2) == x1);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(old_code, 0, test_arm64_mrs_sysreg(0, TFSR_EL12));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)old_code,
+                    sizeof(old_code), UC_CPU_ARM64_A72);
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(old_code),
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+
+    test_arm64_emit32(old_code, 0, test_arm64_mrs_sysreg(0, SCXTNUM_EL12));
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, (const char *)old_code,
+                    sizeof(old_code), UC_CPU_ARM64_A72);
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(old_code),
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
+static void test_arm64_mte_requires_max(void)
+{
+    uc_engine *uc;
+    const char code[] = "\x20\x0c\x82\x91"; /* addg x0,x1,#0x20,#3 */
+
+    uc_common_setup(&uc, UC_ARCH_ARM64, UC_MODE_ARM, code, sizeof(code) - 1,
+                    UC_CPU_ARM64_A72);
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_close(uc));
+}
+
 TEST_LIST = {{"test_arm64_until", test_arm64_until},
              {"test_arm64_code_patching", test_arm64_code_patching},
              {"test_arm64_code_patching_count", test_arm64_code_patching_count},
              {"test_arm64_v8_cas", test_arm64_v8_cas},
+             {"test_arm64_lse_rcpc_unaligned",
+              test_arm64_lse_rcpc_unaligned},
+             {"test_arm64_lse_signed_minmax_byte",
+              test_arm64_lse_signed_minmax_byte},
+             {"test_arm64_lse_rcpc_id_registers",
+              test_arm64_lse_rcpc_id_registers},
+             {"test_arm64_lse_rcpc_a72_rejects",
+              test_arm64_lse_rcpc_a72_rejects},
+             {"test_arm64_dgh_hint", test_arm64_dgh_hint},
              {"test_arm64_read_sctlr", test_arm64_read_sctlr},
              {"test_arm64_hook_insn_mrs", test_arm64_hook_insn_mrs},
              {"test_arm64_hook_insn_wfi", test_arm64_hook_insn_wfi},
@@ -963,6 +12386,170 @@ TEST_LIST = {{"test_arm64_until", test_arm64_until},
              {"test_arm64_mem_prot_regress", test_arm64_mem_prot_regress},
              {"test_arm64_mem_hook_read_write", test_arm64_mem_hook_read_write},
              {"test_arm64_pc_guarantee", test_arm64_pc_guarantee},
+             {"test_arm64_sve_id_registers", test_arm64_sve_id_registers},
+             {"test_arm64_sme_foundation", test_arm64_sme_foundation},
+             {"test_arm64_sme_svlength", test_arm64_sme_svlength},
+             {"test_arm64_sme_nonstreaming_sve_ffr",
+              test_arm64_sme_nonstreaming_sve_ffr},
+             {"test_arm64_sme_nonstreaming_sve_misc",
+              test_arm64_sme_nonstreaming_sve_misc},
+             {"test_arm64_sme_zero_mova", test_arm64_sme_zero_mova},
+             {"test_arm64_sme_context_save_restore",
+              test_arm64_sme_context_save_restore},
+             {"test_arm64_sme_mova_q_horizontal",
+              test_arm64_sme_mova_q_horizontal},
+             {"test_arm64_sme_adda", test_arm64_sme_adda},
+             {"test_arm64_sme_ldstr", test_arm64_sme_ldstr},
+             {"test_arm64_sme_ldst1", test_arm64_sme_ldst1},
+             {"test_arm64_sme_ldst1_fault_no_partial",
+              test_arm64_sme_ldst1_fault_no_partial},
+             {"test_arm64_sme_psel", test_arm64_sme_psel},
+             {"test_arm64_sme_ldst1_mte", test_arm64_sme_ldst1_mte},
+             {"test_arm64_sme_imopa", test_arm64_sme_imopa},
+             {"test_arm64_sme_fpout", test_arm64_sme_fpout},
+             {"test_arm64_i8mm_advsimd", test_arm64_i8mm_advsimd},
+             {"test_arm64_bf16_advsimd", test_arm64_bf16_advsimd},
              {"test_arm64_pauth_vanilla", test_arm64_pauth_vanilla},
              {"test_arm64_pauth_ctl", test_arm64_pauth_ctl},
+             {"test_arm64_mte_register_only", test_arm64_mte_register_only},
+             {"test_arm64_mte_ata_tag_generation",
+              test_arm64_mte_ata_tag_generation},
+             {"test_arm64_mte_tag_load_store",
+              test_arm64_mte_tag_load_store},
+             {"test_arm64_mte_tag_snapshot", test_arm64_mte_tag_snapshot},
+             {"test_arm64_mte_tag_multiple", test_arm64_mte_tag_multiple},
+             {"test_arm64_mte_checked_scalar_access",
+              test_arm64_mte_checked_scalar_access},
+             {"test_arm64_mte_tco_msr_imm",
+              test_arm64_mte_tco_msr_imm},
+             {"test_arm64_mte_simd_fp_single_access",
+              test_arm64_mte_simd_fp_single_access},
+             {"test_arm64_mte_advsimd_struct_range",
+              test_arm64_mte_advsimd_struct_range},
+             {"test_arm64_mte_lse_atomic_asym_sync_no_side_effect",
+              test_arm64_mte_lse_atomic_asym_sync_no_side_effect},
+             {"test_arm64_mte_ldapr_sync_tag_check",
+              test_arm64_mte_ldapr_sync_tag_check},
+             {"test_arm64_mte_lse_cas_asym_async_side_effect",
+              test_arm64_mte_lse_cas_asym_async_side_effect},
+             {"test_arm64_mte_exclusive_asym_access",
+              test_arm64_mte_exclusive_asym_access},
+             {"test_arm64_mte_sp_addressing_tagchecked",
+             test_arm64_mte_sp_addressing_tagchecked},
+             {"test_arm64_mte_sp_writeback_tagchecked",
+              test_arm64_mte_sp_writeback_tagchecked},
+             {"test_arm64_mte_pair_sp_tagchecked",
+              test_arm64_mte_pair_sp_tagchecked},
+             {"test_arm64_mte_pac_load_sp_tagchecked",
+              test_arm64_mte_pac_load_sp_tagchecked},
+             {"test_arm64_mte_tcma0_tag_zero_unchecked",
+              test_arm64_mte_tcma0_tag_zero_unchecked},
+             {"test_arm64_mte_ldapur_stlur_unchecked",
+              test_arm64_mte_ldapur_stlur_unchecked},
+             {"test_arm64_mte_ldapur_stlur_variants_unchecked",
+              test_arm64_mte_ldapur_stlur_variants_unchecked},
+             {"test_arm64_mte_unpriv_sp_no_tag_check",
+              test_arm64_mte_unpriv_sp_no_tag_check},
+             {"test_arm64_mte_unpriv_async_tag_check",
+              test_arm64_mte_unpriv_async_tag_check},
+             {"test_arm64_mte_page_attrs", test_arm64_mte_page_attrs},
+             {"test_arm64_mte_hcr_dct", test_arm64_mte_hcr_dct},
+             {"test_arm64_mte_cross_page_fault_priority",
+              test_arm64_mte_cross_page_fault_priority},
+             {"test_arm64_mte_ata_disabled_tag_op_probe",
+              test_arm64_mte_ata_disabled_tag_op_probe},
+             {"test_arm64_sve2_non_temporal_gather_scatter",
+              test_arm64_sve2_non_temporal_gather_scatter},
+             {"test_arm64_sve2_bitwise_ternary",
+              test_arm64_sve2_bitwise_ternary},
+             {"test_arm64_sve2_xar", test_arm64_sve2_xar},
+             {"test_arm64_sve2_pmull", test_arm64_sve2_pmull},
+             {"test_arm64_sve2_mul_base", test_arm64_sve2_mul_base},
+             {"test_arm64_sve2_mul_indexed", test_arm64_sve2_mul_indexed},
+             {"test_arm64_sve2_widen_indexed",
+             test_arm64_sve2_widen_indexed},
+             {"test_arm64_sve2_widen_accumulate",
+             test_arm64_sve2_widen_accumulate},
+             {"test_arm64_sve2_abs_accumulate",
+             test_arm64_sve2_abs_accumulate},
+             {"test_arm64_sve2_cadd_sqcadd",
+              test_arm64_sve2_cadd_sqcadd},
+             {"test_arm64_sve2_sqrdmla", test_arm64_sve2_sqrdmla},
+             {"test_arm64_sve2_complex_dot",
+             test_arm64_sve2_complex_dot},
+             {"test_arm64_sve_i8mm", test_arm64_sve_i8mm},
+             {"test_arm64_sve_bf16", test_arm64_sve_bf16},
+             {"test_arm64_sve_f32mm_f64mm", test_arm64_sve_f32mm_f64mm},
+             {"test_arm64_sve2_fp_convert", test_arm64_sve2_fp_convert},
+             {"test_arm64_sve2_fp_pairwise_flogb",
+              test_arm64_sve2_fp_pairwise_flogb},
+             {"test_arm64_sve2_fmlal", test_arm64_sve2_fmlal},
+             {"test_arm64_sve2_widen_add_shift",
+              test_arm64_sve2_widen_add_shift},
+             {"test_arm64_sve2_addhn", test_arm64_sve2_addhn},
+             {"test_arm64_sve2_xtn", test_arm64_sve2_xtn},
+             {"test_arm64_sve2_shift_narrow",
+             test_arm64_sve2_shift_narrow},
+             {"test_arm64_sve2_shift_accumulate",
+              test_arm64_sve2_shift_accumulate},
+             {"test_arm64_sve2_shift_insert",
+              test_arm64_sve2_shift_insert},
+             {"test_arm64_sve2_sat_unary", test_arm64_sve2_sat_unary},
+             {"test_arm64_sve2_adalp", test_arm64_sve2_adalp},
+             {"test_arm64_sve2_halving_add_sub",
+              test_arm64_sve2_halving_add_sub},
+             {"test_arm64_sve2_pairwise_pred",
+              test_arm64_sve2_pairwise_pred},
+             {"test_arm64_sve2_saturating_add_sub",
+             test_arm64_sve2_saturating_add_sub},
+             {"test_arm64_sve2_int_estimate",
+              test_arm64_sve2_int_estimate},
+             {"test_arm64_sve2_variable_shift",
+              test_arm64_sve2_variable_shift},
+             {"test_arm64_sve2_eor_adcl", test_arm64_sve2_eor_adcl},
+             {"test_arm64_sve2_bitperm", test_arm64_sve2_bitperm},
+             {"test_arm64_sve2_match_hist", test_arm64_sve2_match_hist},
+             {"test_arm64_sve2_crypto", test_arm64_sve2_crypto},
+             {"test_arm64_sve2_ext", test_arm64_sve2_ext},
+             {"test_arm64_sve2_splice", test_arm64_sve2_splice},
+             {"test_arm64_sve2_tbl_tbx", test_arm64_sve2_tbl_tbx},
+             {"test_arm64_sve2_ld1ro", test_arm64_sve2_ld1ro},
+             {"test_arm64_mte_sve_contiguous_access",
+              test_arm64_mte_sve_contiguous_access},
+             {"test_arm64_mte_sve_gather_scatter_sizem1",
+              test_arm64_mte_sve_gather_scatter_sizem1},
+             {"test_arm64_mte_sve_whole_register_access",
+              test_arm64_mte_sve_whole_register_access},
+             {"test_arm64_sve_contiguous_store_fault_no_partial",
+              test_arm64_sve_contiguous_store_fault_no_partial},
+             {"test_arm64_sve_scatter_store_fault_no_partial",
+              test_arm64_sve_scatter_store_fault_no_partial},
+             {"test_arm64_sve_ldff1_split_first_element",
+              test_arm64_sve_ldff1_split_first_element},
+             {"test_arm64_sve_ldnf1_split_first_element",
+              test_arm64_sve_ldnf1_split_first_element},
+             {"test_arm64_mte_tag_split_lifecycle",
+              test_arm64_mte_tag_split_lifecycle},
+             {"test_arm64_mte_stgp", test_arm64_mte_stgp},
+             {"test_arm64_mte_dc_zva_checked",
+              test_arm64_mte_dc_zva_checked},
+             {"test_arm64_mte_dc_zva_original_fault_addr",
+              test_arm64_mte_dc_zva_original_fault_addr},
+             {"test_arm64_mte_dc_gva_gzva", test_arm64_mte_dc_gva_gzva},
+             {"test_arm64_mte_dc_gva_probe", test_arm64_mte_dc_gva_probe},
+             {"test_arm64_mte_cache_ops", test_arm64_mte_cache_ops},
+             {"test_arm64_generic_timer_state", test_arm64_generic_timer_state},
+             {"test_arm64_pmu_counter_delta", test_arm64_pmu_counter_delta},
+             {"test_arm64_pmu_pmuv3p5_event_counter",
+              test_arm64_pmu_pmuv3p5_event_counter},
+             {"test_arm64_pmu_el2_hlp_long_counter",
+             test_arm64_pmu_el2_hlp_long_counter},
+             {"test_arm64_pmu_el0_direct_counter_access",
+              test_arm64_pmu_el0_direct_counter_access},
+             {"test_arm64_pmu_effective_mdcr_el2",
+              test_arm64_pmu_effective_mdcr_el2},
+             {"test_arm64_pmu_pmcr_n_from_mdcr_el2",
+              test_arm64_pmu_pmcr_n_from_mdcr_el2},
+             {"test_arm64_vhe_el12_aliases", test_arm64_vhe_el12_aliases},
+             {"test_arm64_mte_requires_max", test_arm64_mte_requires_max},
              {NULL, NULL}};
diff --git a/tests/unit/test_m68k.c b/tests/unit/test_m68k.c
index e298a8cda1..9b177a0452 100644
--- a/tests/unit/test_m68k.c
+++ b/tests/unit/test_m68k.c
@@ -1,5 +1,7 @@
 #include "unicorn_test.h"
 
+#include <string.h>
+
 const uint64_t code_start = 0x1000;
 const uint64_t code_len = 0x4000;
 
@@ -13,6 +15,17 @@ static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
     OK(uc_mem_write(*uc, code_start, code, size));
 }
 
+static void setup_stack_pointers(uc_engine *uc, uint32_t ssp,
+                                 uint32_t usp, uint32_t isp)
+{
+    uint32_t sr = 0;
+
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_write(uc, UC_M68K_REG_A7, &usp));
+    OK(uc_reg_write(uc, UC_M68K_REG_CR_MSP, &ssp));
+    OK(uc_reg_write(uc, UC_M68K_REG_CR_ISP, &isp));
+}
+
 static void test_move_to_sr(void)
 {
 
@@ -60,6 +73,431 @@ static void test_sr_contains_flags(void)
     OK(uc_close(uc));
 }
 
+static void test_fetoxm1(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0xf2, 0x10, 0x54, 0x00, /* fmove.d (a0), fp0 */
+        0xf2, 0x00, 0x00, 0x08, /* fetoxm1 fp0, fp0 */
+        0xf2, 0x10, 0x74, 0x00, /* fmove.d fp0, (a0) */
+    };
+    uint8_t input[] = {
+        0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    uint8_t expected[] = {
+        0x3f, 0xfb, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd3,
+    };
+    uint8_t result[sizeof(expected)];
+    uint32_t a0 = code_start + 0x100;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68020);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_M68K_REG_A0, &a0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, a0, result, sizeof(result)));
+
+    TEST_CHECK(memcmp(result, expected, sizeof(expected)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_coldfire_macsr_to_ccr(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0xa9, 0x00, /* move.l d0, MACSR */
+        0xa9, 0xc0, /* MACSR -> CCR */
+    };
+    uint32_t d0 = 0x0f;
+    uint32_t sr = 0x2700;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_CFV4E);
+    OK(uc_reg_write(uc, UC_M68K_REG_D0, &d0));
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_SR, &sr));
+
+    TEST_CHECK(sr == 0x270e);
+
+    OK(uc_close(uc));
+}
+
+static void test_ftrapcc_false_consumes_immediate(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0xf2, 0x7a, 0x00, 0x00, 0x12, 0x34, /* ftrapf.w #$1234 */
+        0x70, 0x2a,                         /* moveq #42, d0 */
+    };
+    uint32_t d0;
+    uint32_t pc;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68020);
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_D0, &d0));
+    OK(uc_reg_read(uc, UC_M68K_REG_PC, &pc));
+
+    TEST_CHECK(d0 == 42);
+    TEST_CHECK(pc == (uint32_t)(code_start + sizeof(code)));
+
+    OK(uc_close(uc));
+}
+
+static void test_ftrapcc_true_raises(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0xf2, 0x7c, 0x00, 0x0f, /* ftrapt */
+        0x70, 0x2a,             /* moveq #42, d0 */
+    };
+    uint32_t d0 = 0;
+    uint32_t pc;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68020);
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_D0, &d0));
+    OK(uc_reg_read(uc, UC_M68K_REG_PC, &pc));
+
+    TEST_CHECK(d0 == 0);
+    TEST_CHECK(pc == (uint32_t)(code_start + 4));
+
+    OK(uc_close(uc));
+}
+
+static void test_trapcc_false_consumes_immediate(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x51, 0xfa, 0x12, 0x34, /* trapf.w #$1234 */
+        0x70, 0x2a,             /* moveq #42, d0 */
+    };
+    uint32_t d0;
+    uint32_t pc;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68020);
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_D0, &d0));
+    OK(uc_reg_read(uc, UC_M68K_REG_PC, &pc));
+
+    TEST_CHECK(d0 == 42);
+    TEST_CHECK(pc == (uint32_t)(code_start + sizeof(code)));
+
+    OK(uc_close(uc));
+}
+
+static void test_trapcc_true_raises(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x50, 0xfc, /* trapt */
+        0x70, 0x2a, /* moveq #42, d0 */
+    };
+    uint32_t d0 = 0;
+    uint32_t pc;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68020);
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_D0, &d0));
+    OK(uc_reg_read(uc, UC_M68K_REG_PC, &pc));
+
+    TEST_CHECK(d0 == 0);
+    TEST_CHECK(pc == (uint32_t)(code_start + 2));
+
+    OK(uc_close(uc));
+}
+
+static void test_m68010_rtd(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x74, 0x00, 0x00, /* rtd #0 */
+    };
+    uint8_t retaddr[] = {
+        0x00, 0x00, 0x10, 0x04,
+    };
+    uint32_t sp = code_start + 0x100;
+    uint32_t pc;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68010);
+    OK(uc_mem_write(uc, sp, retaddr, sizeof(retaddr)));
+    OK(uc_reg_write(uc, UC_M68K_REG_A7, &sp));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_PC, &pc));
+
+    TEST_CHECK(pc == (uint32_t)(code_start + sizeof(code)));
+
+    OK(uc_close(uc));
+}
+
+static void test_m68010_supervisor_uses_ssp(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x71, /* nop */
+    };
+    uint32_t ssp = code_start + 0x300;
+    uint32_t usp = code_start + 0x500;
+    uint32_t isp = code_start + 0x700;
+    uint32_t sr;
+    uint32_t a7;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68010);
+    setup_stack_pointers(uc, ssp, usp, isp);
+
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == usp);
+
+    sr = 0x2000;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == ssp);
+
+    sr = 0x3000;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == ssp);
+
+    OK(uc_close(uc));
+}
+
+static void test_m68020_supervisor_uses_isp_and_msp(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x71, /* nop */
+    };
+    uint32_t ssp = code_start + 0x300;
+    uint32_t usp = code_start + 0x500;
+    uint32_t isp = code_start + 0x700;
+    uint32_t sr;
+    uint32_t a7;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68020);
+    setup_stack_pointers(uc, ssp, usp, isp);
+
+    sr = 0x2000;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == isp);
+
+    sr = 0x3000;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == ssp);
+
+    sr = 0;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == usp);
+
+    OK(uc_close(uc));
+}
+
+static void test_m68060_supervisor_uses_isp_and_msp(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x71, /* nop */
+    };
+    uint32_t ssp = code_start + 0x300;
+    uint32_t usp = code_start + 0x500;
+    uint32_t isp = code_start + 0x700;
+    uint32_t sr;
+    uint32_t a7;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68060);
+    setup_stack_pointers(uc, ssp, usp, isp);
+
+    sr = 0x2000;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == isp);
+
+    sr = 0x3000;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == ssp);
+
+    sr = 0;
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &a7));
+    TEST_CHECK(a7 == usp);
+
+    OK(uc_close(uc));
+}
+
+static void test_m68020_movec_msp_isp(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x7b, 0x08, 0x03, /* movec d0, msp */
+        0x4e, 0x7b, 0x18, 0x04, /* movec d1, isp */
+        0x4e, 0x7a, 0x28, 0x03, /* movec msp, d2 */
+        0x4e, 0x7a, 0x38, 0x04, /* movec isp, d3 */
+    };
+    uint32_t sr = 0x2000;
+    uint32_t d0 = code_start + 0x300;
+    uint32_t d1 = code_start + 0x700;
+    uint32_t d2 = 0;
+    uint32_t d3 = 0;
+    uint32_t msp = 0;
+    uint32_t isp = 0;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68020);
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_write(uc, UC_M68K_REG_D0, &d0));
+    OK(uc_reg_write(uc, UC_M68K_REG_D1, &d1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_CR_MSP, &msp));
+    OK(uc_reg_read(uc, UC_M68K_REG_CR_ISP, &isp));
+    OK(uc_reg_read(uc, UC_M68K_REG_D2, &d2));
+    OK(uc_reg_read(uc, UC_M68K_REG_D3, &d3));
+
+    TEST_CHECK(msp == d0);
+    TEST_CHECK(isp == d1);
+    TEST_CHECK(d2 == d0);
+    TEST_CHECK(d3 == d1);
+
+    OK(uc_close(uc));
+}
+
+static void test_m68010_movec_msp_invalid(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x7b, 0x08, 0x03, /* movec d0, msp */
+    };
+    uint32_t sr = 0x2000;
+    uint32_t d0 = code_start + 0x300;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68010);
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_write(uc, UC_M68K_REG_D0, &d0));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+
+    OK(uc_close(uc));
+}
+
+static void test_m68060_movec_msp_invalid(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x7a, 0x08, 0x03, /* movec msp, d0 */
+    };
+    uint32_t sr = 0x2000;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68060);
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+
+    OK(uc_close(uc));
+}
+
+static void test_rtr(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x4e, 0x77, /* rtr */
+    };
+    uint8_t frame[] = {
+        0x00, 0x15,             /* ccr: X, Z, C */
+        0x00, 0x00, 0x10, 0x02, /* pc */
+    };
+    uint32_t sp = code_start + 0x100;
+    uint32_t sr = 0x2700;
+    uint32_t pc;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68000);
+    OK(uc_mem_write(uc, sp, frame, sizeof(frame)));
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+    OK(uc_reg_write(uc, UC_M68K_REG_A7, &sp));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_M68K_REG_PC, &pc));
+    OK(uc_reg_read(uc, UC_M68K_REG_A7, &sp));
+    OK(uc_reg_read(uc, UC_M68K_REG_SR, &sr));
+
+    TEST_CHECK(pc == (uint32_t)(code_start + sizeof(code)));
+    TEST_CHECK(sp == (uint32_t)(code_start + 0x100 + sizeof(frame)));
+    TEST_CHECK(sr == 0x2715);
+
+    OK(uc_close(uc));
+}
+
+static void test_m68010_move_from_sr_privileged(void)
+{
+    uc_engine *uc;
+    uint8_t code[] = {
+        0x40, 0xc0, /* move.w sr, d0 */
+    };
+    uint32_t sr = 0;
+
+    uc_common_setup(&uc, UC_ARCH_M68K, UC_MODE_BIG_ENDIAN, (char *)code,
+                    sizeof(code), UC_CPU_M68K_M68010);
+    OK(uc_reg_write(uc, UC_M68K_REG_SR, &sr));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+
+    OK(uc_close(uc));
+}
+
 TEST_LIST = {{"test_move_to_sr", test_move_to_sr},
              {"test_sr_contains_flags", test_sr_contains_flags},
+             {"test_fetoxm1", test_fetoxm1},
+             {"test_coldfire_macsr_to_ccr", test_coldfire_macsr_to_ccr},
+             {"test_ftrapcc_false_consumes_immediate",
+              test_ftrapcc_false_consumes_immediate},
+             {"test_ftrapcc_true_raises", test_ftrapcc_true_raises},
+             {"test_trapcc_false_consumes_immediate",
+              test_trapcc_false_consumes_immediate},
+             {"test_trapcc_true_raises", test_trapcc_true_raises},
+             {"test_m68010_rtd", test_m68010_rtd},
+             {"test_m68010_supervisor_uses_ssp",
+              test_m68010_supervisor_uses_ssp},
+             {"test_m68020_supervisor_uses_isp_and_msp",
+              test_m68020_supervisor_uses_isp_and_msp},
+             {"test_m68060_supervisor_uses_isp_and_msp",
+              test_m68060_supervisor_uses_isp_and_msp},
+             {"test_m68020_movec_msp_isp", test_m68020_movec_msp_isp},
+             {"test_m68010_movec_msp_invalid",
+              test_m68010_movec_msp_invalid},
+             {"test_m68060_movec_msp_invalid",
+              test_m68060_movec_msp_invalid},
+             {"test_rtr", test_rtr},
+             {"test_m68010_move_from_sr_privileged",
+              test_m68010_move_from_sr_privileged},
              {NULL, NULL}};
diff --git a/tests/unit/test_mips.c b/tests/unit/test_mips.c
index c53c19c1cd..0192a593be 100644
--- a/tests/unit/test_mips.c
+++ b/tests/unit/test_mips.c
@@ -1,8 +1,27 @@
 #include "unicorn_test.h"
 
+#include <string.h>
+
 const uint64_t code_start = 0x10000000;
 const uint64_t code_len = 0x4000;
 
+#define MIPS_OP_LWC2 (0x32u << 26)
+#define MIPS_OP_LDC2 (0x36u << 26)
+#define MIPS_OP_SWC2 (0x3au << 26)
+#define MIPS_OP_SDC2 (0x3eu << 26)
+
+#define MIPS_CP0_STATUS_FR (1u << 26)
+#define MIPS_CP0_STATUS_CU1 (1u << 29)
+
+static uint32_t mips_bitswap32(uint32_t value)
+{
+    value = ((value >> 1) & 0x55555555) | ((value & 0x55555555) << 1);
+    value = ((value >> 2) & 0x33333333) | ((value & 0x33333333) << 2);
+    value = ((value >> 4) & 0x0f0f0f0f) | ((value & 0x0f0f0f0f) << 4);
+
+    return value;
+}
+
 static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
                             const char *code, uint64_t size)
 {
@@ -11,6 +30,42 @@ static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
     OK(uc_mem_write(*uc, code_start, code, size));
 }
 
+static uint32_t encode_loongson_lsdc2(uint32_t major, int rt, int rs, int rd,
+                                      int offset, int op)
+{
+    return major | (rs << 21) | (rt << 16) | (rd << 11) |
+           ((offset & 0xff) << 3) | op;
+}
+
+static uint32_t encode_loongson_gslsq(uint32_t major, int rt, int rs,
+                                      int offset, int rt1)
+{
+    return major | 0x20 | (rs << 21) | (rt << 16) |
+           (((offset >> 4) & 0x1ff) << 6) | rt1;
+}
+
+static uint32_t encode_loongson_gslsq_fpr(uint32_t major, int rt, int rs,
+                                          int offset, int rt1)
+{
+    return major | 0x8020 | (rs << 21) | (rt << 16) |
+           (((offset >> 4) & 0x1ff) << 6) | rt1;
+}
+
+static uint32_t encode_loongson_gsshfls(uint32_t major, int rt, int rs,
+                                        int offset, int op)
+{
+    return major | (rs << 21) | (rt << 16) | ((offset & 0xff) << 6) | op;
+}
+
+static void enable_mips64_fpu(uc_engine *uc)
+{
+    uint64_t status;
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_CP0_STATUS, &status));
+    status |= MIPS_CP0_STATUS_FR | MIPS_CP0_STATUS_CU1;
+    OK(uc_reg_write(uc, UC_MIPS_REG_CP0_STATUS, &status));
+}
+
 static void test_mips_el_ori(void)
 {
     uc_engine *uc;
@@ -183,6 +238,115 @@ static void test_mips_mips16(void)
     OK(uc_close(uc));
 }
 
+static void test_mips_mips32r6_mode_bitswap(void)
+{
+    uc_engine *uc;
+    char code[] = "\x7c\x03\x10\x20";
+    uint32_t r2 = 0;
+    uint32_t r3 = 0x12345678;
+
+    uc_common_setup(&uc, UC_ARCH_MIPS,
+                    UC_MODE_MIPS32 | UC_MODE_BIG_ENDIAN | UC_MODE_MIPS32R6,
+                    code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &r3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_2, &r2));
+    TEST_CHECK(r2 == mips_bitswap32(r3));
+
+    OK(uc_close(uc));
+}
+
+static void test_mips_micro_mode_li16(void)
+{
+    uc_engine *uc;
+    char code[] = "\x2a\xed";
+    uint32_t r2 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_MIPS,
+                    UC_MODE_MIPS32 | UC_MODE_LITTLE_ENDIAN | UC_MODE_MICRO,
+                    code, sizeof(code) - 1);
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_2, &r2));
+    TEST_CHECK(r2 == 0x2a);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips_nanomips_model_move16(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc5\x10";
+    uint32_t r5 = 0x12345678;
+    uint32_t r6 = 0;
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS32 | UC_MODE_LITTLE_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS32_I7200));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_5, &r5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_6, &r6));
+    TEST_CHECK(r6 == r5);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips_mips3_mode_opens(void)
+{
+    uc_engine *uc;
+
+    OK(uc_open(UC_ARCH_MIPS,
+               UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN | UC_MODE_MIPS3, &uc));
+    OK(uc_close(uc));
+
+    TEST_CHECK(uc_open(UC_ARCH_MIPS,
+                       UC_MODE_MIPS32 | UC_MODE_BIG_ENDIAN | UC_MODE_MIPS3,
+                       &uc) == UC_ERR_MODE);
+}
+
+static void test_mips_msa_w_reg_roundtrip(void)
+{
+    uc_engine *uc;
+    uint8_t w0[16] = {
+        0xff, 0xee, 0xdd, 0xcc, 0xbb, 0xaa, 0x99, 0x88,
+        0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00,
+    };
+    uint8_t w7[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    uint8_t w31[16] = {
+        0xf0, 0xe1, 0xd2, 0xc3, 0xb4, 0xa5, 0x96, 0x87,
+        0x78, 0x69, 0x5a, 0x4b, 0x3c, 0x2d, 0x1e, 0x0f,
+    };
+    uint8_t out[16] = { 0 };
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS32 | UC_MODE_LITTLE_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS32_P5600));
+
+    OK(uc_reg_write(uc, UC_MIPS_REG_W0, w0));
+    OK(uc_reg_read(uc, UC_MIPS_REG_W0, out));
+    TEST_CHECK(memcmp(out, w0, sizeof(w0)) == 0);
+
+    memset(out, 0, sizeof(out));
+    OK(uc_reg_write(uc, UC_MIPS_REG_W7, w7));
+    OK(uc_reg_read(uc, UC_MIPS_REG_W7, out));
+    TEST_CHECK(memcmp(out, w7, sizeof(w7)) == 0);
+
+    memset(out, 0, sizeof(out));
+    OK(uc_reg_write(uc, UC_MIPS_REG_W31, w31));
+    OK(uc_reg_read(uc, UC_MIPS_REG_W31, out));
+    TEST_CHECK(memcmp(out, w31, sizeof(w31)) == 0);
+
+    OK(uc_close(uc));
+}
+
 static void test_mips_mips_fpr(void)
 {
     uc_engine *uc;
@@ -222,6 +386,684 @@ static void test_mips_simple_coredump_2137(void)
     OK(uc_close(uc));
 }
 
+static void test_mips64_loongson2f_status(void)
+{
+    uc_engine *uc;
+    uint64_t status = 0;
+    const uint64_t expected = (1ULL << 22) | (1ULL << 7) | (1ULL << 6) |
+                              (1ULL << 5) | (1ULL << 2);
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_LITTLE_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_2F));
+    OK(uc_reg_read(uc, UC_MIPS_REG_CP0_STATUS, &status));
+
+    TEST_CHECK((status & expected) == expected);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_dmult(void)
+{
+    uc_engine *uc;
+    uint64_t r2 = 0x100000000ull;
+    uint64_t r3 = 0x12;
+    uint64_t r4 = 0;
+    const char code[] = "\x70\x43\x20\x11";
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &r3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_4, &r4));
+    TEST_CHECK(r4 == 0x1200000000ull);
+
+    OK(uc_close(uc));
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A4000));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &r3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_4, &r4));
+    TEST_CHECK(r4 == 0x1200000000ull);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_requires_lext(void)
+{
+    uc_engine *uc;
+    uint64_t r2 = 0x100000000ull;
+    uint64_t r3 = 0x12;
+    const char code[] = "\x70\x43\x20\x11";
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_MIPS64R2_GENERIC));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &r3));
+
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_load_zero_prefetch(void)
+{
+    uc_engine *uc;
+    uint64_t r2 = 0x20000000ull;
+    const char code[] = "\xdc\x40\x00\x00";
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_close(uc));
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_MIPS64R2_GENERIC));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_READ_UNMAPPED);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_lext_lsdc2_gpr(void)
+{
+    uc_engine *uc;
+    uint64_t data_base = 0x20000000ull;
+    uint64_t index = 0x20;
+    uint64_t r8 = 0;
+    uint64_t r9 = 0;
+    uint64_t r10 = 0;
+    uint64_t r11 = 0;
+    uint64_t r12 = 0xaa;
+    uint64_t r13 = 0x1234;
+    uint64_t r14 = 0x89abcdef;
+    uint64_t r15 = 0x1122334455667788ull;
+    uint8_t data[0x50] = { 0 };
+    uint8_t result[16];
+    uint8_t expected[] = {
+        0xaa, 0x00, 0x12, 0x34, 0x89, 0xab, 0xcd, 0xef,
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+    };
+    uint32_t code[] = {
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_LDC2, 8, 2, 3, 1, 0)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_LDC2, 9, 2, 3, 3, 1)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_LDC2, 10, 2, 3, 5, 2)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_LDC2, 11, 2, 3, 9, 3)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_SDC2, 12, 2, 3, 0x30, 0)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_SDC2, 13, 2, 3, 0x32, 1)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_SDC2, 14, 2, 3, 0x34, 2)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_SDC2, 15, 2, 3, 0x38, 3)),
+    };
+
+    data[index + 1] = 0x80;
+    data[index + 3] = 0x12;
+    data[index + 4] = 0x34;
+    data[index + 5] = 0x80;
+    data[index + 8] = 0x01;
+    data[index + 9] = 0x01;
+    data[index + 10] = 0x02;
+    data[index + 11] = 0x03;
+    data[index + 12] = 0x04;
+    data[index + 13] = 0x05;
+    data[index + 14] = 0x06;
+    data[index + 15] = 0x07;
+    data[index + 16] = 0x08;
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_map(uc, data_base, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, data_base, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &data_base));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &index));
+    OK(uc_reg_write(uc, UC_MIPS_REG_12, &r12));
+    OK(uc_reg_write(uc, UC_MIPS_REG_13, &r13));
+    OK(uc_reg_write(uc, UC_MIPS_REG_14, &r14));
+    OK(uc_reg_write(uc, UC_MIPS_REG_15, &r15));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_8, &r8));
+    OK(uc_reg_read(uc, UC_MIPS_REG_9, &r9));
+    OK(uc_reg_read(uc, UC_MIPS_REG_10, &r10));
+    OK(uc_reg_read(uc, UC_MIPS_REG_11, &r11));
+    OK(uc_mem_read(uc, data_base + index + 0x30, result, sizeof(result)));
+
+    TEST_CHECK(r8 == 0xffffffffffffff80ull);
+    TEST_CHECK(r9 == 0x1234);
+    TEST_CHECK(r10 == 0xffffffff80000001ull);
+    TEST_CHECK(r11 == 0x0102030405060708ull);
+    TEST_CHECK(memcmp(result, expected, sizeof(expected)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_lext_lsdc2_requires_lext(void)
+{
+    uc_engine *uc;
+    uint64_t data_base = 0x20000000ull;
+    uint64_t index = 0;
+    uint32_t code[] = {
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_LDC2, 8, 2, 3, 0, 2)),
+    };
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_MIPS64R2_GENERIC));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_map(uc, data_base, 0x1000, UC_PROT_ALL));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &data_base));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &index));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code),
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_lext_gslsq_gpr(void)
+{
+    uc_engine *uc;
+    uint64_t data_base = 0x20000000ull;
+    uint64_t r8 = 0;
+    uint64_t r9 = 0;
+    uint64_t r10 = 0x8877665544332211ull;
+    uint64_t r11 = 0x0102030405060708ull;
+    uint8_t data[0x50] = { 0 };
+    uint8_t result[16];
+    uint8_t expected[] = {
+        0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11,
+        0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+    };
+    uint32_t code[] = {
+        BEINT32(encode_loongson_gslsq(MIPS_OP_LWC2, 8, 2, 0x10, 9)),
+        BEINT32(encode_loongson_gslsq(MIPS_OP_SWC2, 10, 2, 0x30, 11)),
+    };
+
+    data[0x10] = 0x11;
+    data[0x11] = 0x22;
+    data[0x12] = 0x33;
+    data[0x13] = 0x44;
+    data[0x14] = 0x55;
+    data[0x15] = 0x66;
+    data[0x16] = 0x77;
+    data[0x17] = 0x88;
+    data[0x18] = 0x99;
+    data[0x19] = 0xaa;
+    data[0x1a] = 0xbb;
+    data[0x1b] = 0xcc;
+    data[0x1c] = 0xdd;
+    data[0x1d] = 0xee;
+    data[0x1e] = 0xff;
+    data[0x1f] = 0x00;
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_map(uc, data_base, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, data_base, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &data_base));
+    OK(uc_reg_write(uc, UC_MIPS_REG_10, &r10));
+    OK(uc_reg_write(uc, UC_MIPS_REG_11, &r11));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_8, &r8));
+    OK(uc_reg_read(uc, UC_MIPS_REG_9, &r9));
+    OK(uc_mem_read(uc, data_base + 0x30, result, sizeof(result)));
+
+    TEST_CHECK(r8 == 0x1122334455667788ull);
+    TEST_CHECK(r9 == 0x99aabbccddeeff00ull);
+    TEST_CHECK(memcmp(result, expected, sizeof(expected)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_lext_gslsq_requires_lext(void)
+{
+    uc_engine *uc;
+    uint64_t data_base = 0x20000000ull;
+    uint32_t code[] = {
+        BEINT32(encode_loongson_gslsq(MIPS_OP_LWC2, 8, 2, 0x10, 9)),
+    };
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_MIPS64R2_GENERIC));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_map(uc, data_base, 0x1000, UC_PROT_ALL));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &data_base));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code),
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_lext_lsdc2_fpr(void)
+{
+    uc_engine *uc;
+    uint64_t data_base = 0x20000000ull;
+    uint64_t index = 0x20;
+    uint64_t f8 = 0xaaaabbbbccccdddduLL;
+    uint64_t f9 = 0;
+    uint64_t f10 = 0x12345678a1b2c3d4ull;
+    uint64_t f11 = 0x0102030405060708ull;
+    uint8_t data[0x50] = { 0 };
+    uint8_t result[16];
+    uint8_t expected[] = {
+        0xa1, 0xb2, 0xc3, 0xd4, 0x00, 0x00, 0x00, 0x00,
+        0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+    };
+    uint32_t code[] = {
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_LDC2, 8, 2, 3, 4, 6)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_LDC2, 9, 2, 3, 8, 7)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_SDC2, 10, 2, 3, 0x30, 6)),
+        BEINT32(encode_loongson_lsdc2(MIPS_OP_SDC2, 11, 2, 3, 0x38, 7)),
+    };
+
+    data[index + 4] = 0x11;
+    data[index + 5] = 0x22;
+    data[index + 6] = 0x33;
+    data[index + 7] = 0x44;
+    data[index + 8] = 0x55;
+    data[index + 9] = 0x66;
+    data[index + 10] = 0x77;
+    data[index + 11] = 0x88;
+    data[index + 12] = 0x99;
+    data[index + 13] = 0xaa;
+    data[index + 14] = 0xbb;
+    data[index + 15] = 0xcc;
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    enable_mips64_fpu(uc);
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_map(uc, data_base, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, data_base, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &data_base));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &index));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F8, &f8));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F10, &f10));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F11, &f11));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_F8, &f8));
+    OK(uc_reg_read(uc, UC_MIPS_REG_F9, &f9));
+    OK(uc_mem_read(uc, data_base + index + 0x30, result, sizeof(result)));
+
+    TEST_CHECK(f8 == 0xaaaabbbb11223344ull);
+    TEST_CHECK(f9 == 0x5566778899aabbccull);
+    TEST_CHECK(memcmp(result, expected, sizeof(expected)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_lext_gslsq_fpr(void)
+{
+    uc_engine *uc;
+    uint64_t data_base = 0x20000000ull;
+    uint64_t f8 = 0;
+    uint64_t f9 = 0;
+    uint64_t f10 = 0x8877665544332211ull;
+    uint64_t f11 = 0x0102030405060708ull;
+    uint8_t data[0x50] = { 0 };
+    uint8_t result[16];
+    uint8_t expected[] = {
+        0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11,
+        0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+    };
+    uint32_t code[] = {
+        BEINT32(encode_loongson_gslsq_fpr(MIPS_OP_LWC2, 8, 2, 0x10, 9)),
+        BEINT32(encode_loongson_gslsq_fpr(MIPS_OP_SWC2, 10, 2, 0x30, 11)),
+    };
+
+    data[0x10] = 0x11;
+    data[0x11] = 0x22;
+    data[0x12] = 0x33;
+    data[0x13] = 0x44;
+    data[0x14] = 0x55;
+    data[0x15] = 0x66;
+    data[0x16] = 0x77;
+    data[0x17] = 0x88;
+    data[0x18] = 0x99;
+    data[0x19] = 0xaa;
+    data[0x1a] = 0xbb;
+    data[0x1b] = 0xcc;
+    data[0x1c] = 0xdd;
+    data[0x1d] = 0xee;
+    data[0x1e] = 0xff;
+    data[0x1f] = 0x00;
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    enable_mips64_fpu(uc);
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_map(uc, data_base, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, data_base, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &data_base));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F10, &f10));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F11, &f11));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_F8, &f8));
+    OK(uc_reg_read(uc, UC_MIPS_REG_F9, &f9));
+    OK(uc_mem_read(uc, data_base + 0x30, result, sizeof(result)));
+
+    TEST_CHECK(f8 == 0x1122334455667788ull);
+    TEST_CHECK(f9 == 0x99aabbccddeeff00ull);
+    TEST_CHECK(memcmp(result, expected, sizeof(expected)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_lext_shifted_fpr(void)
+{
+    uc_engine *uc;
+    uint64_t data_base = 0x20000000ull;
+    uint64_t f8 = 0xaaaaaaaa55555555ull;
+    uint64_t f9 = 0xbbbbbbbb66666666ull;
+    uint64_t f10 = 0;
+    uint64_t f11 = 0;
+    uint64_t f12 = 0x00000000a1b2c3d4ull;
+    uint64_t f13 = 0x00000000b1b2b3b4ull;
+    uint64_t f14 = 0xc1c2c3c4c5c6c7c8ull;
+    uint64_t f15 = 0xd1d2d3d4d5d6d7d8ull;
+    uint8_t data[0x60] = { 0 };
+    uint8_t result[0x40];
+    uint8_t expected[0x40] = { 0 };
+    uint32_t code[] = {
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_LWC2, 8, 2, 0, 4)),
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_LWC2, 9, 2, 3, 5)),
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_LWC2, 10, 2, 8, 6)),
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_LWC2, 11, 2, 15, 7)),
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_SWC2, 12, 2, 0x20, 4)),
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_SWC2, 13, 2, 0x33, 5)),
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_SWC2, 14, 2, 0x40, 6)),
+        BEINT32(encode_loongson_gsshfls(MIPS_OP_SWC2, 15, 2, 0x57, 7)),
+    };
+
+    data[0] = 0x10;
+    data[1] = 0x20;
+    data[2] = 0x30;
+    data[3] = 0x40;
+    data[8] = 0x01;
+    data[9] = 0x02;
+    data[10] = 0x03;
+    data[11] = 0x04;
+    data[12] = 0x05;
+    data[13] = 0x06;
+    data[14] = 0x07;
+    data[15] = 0x08;
+    expected[0] = 0xa1;
+    expected[1] = 0xb2;
+    expected[2] = 0xc3;
+    expected[3] = 0xd4;
+    expected[0x10] = 0xb1;
+    expected[0x11] = 0xb2;
+    expected[0x12] = 0xb3;
+    expected[0x13] = 0xb4;
+    expected[0x20] = 0xc1;
+    expected[0x21] = 0xc2;
+    expected[0x22] = 0xc3;
+    expected[0x23] = 0xc4;
+    expected[0x24] = 0xc5;
+    expected[0x25] = 0xc6;
+    expected[0x26] = 0xc7;
+    expected[0x27] = 0xc8;
+    expected[0x30] = 0xd1;
+    expected[0x31] = 0xd2;
+    expected[0x32] = 0xd3;
+    expected[0x33] = 0xd4;
+    expected[0x34] = 0xd5;
+    expected[0x35] = 0xd6;
+    expected[0x36] = 0xd7;
+    expected[0x37] = 0xd8;
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    enable_mips64_fpu(uc);
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_map(uc, data_base, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, data_base, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &data_base));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F8, &f8));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F9, &f9));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F12, &f12));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F13, &f13));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F14, &f14));
+    OK(uc_reg_write(uc, UC_MIPS_REG_F15, &f15));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_F8, &f8));
+    OK(uc_reg_read(uc, UC_MIPS_REG_F9, &f9));
+    OK(uc_reg_read(uc, UC_MIPS_REG_F10, &f10));
+    OK(uc_reg_read(uc, UC_MIPS_REG_F11, &f11));
+    OK(uc_mem_read(uc, data_base + 0x20, result, sizeof(result)));
+
+    TEST_CHECK(f8 == 0xaaaaaaaa10203040ull);
+    TEST_CHECK(f9 == 0xbbbbbbbb10203040ull);
+    TEST_CHECK(f10 == 0x0102030405060708ull);
+    TEST_CHECK(f11 == 0x0102030405060708ull);
+    TEST_CHECK(memcmp(result, expected, sizeof(expected)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_loongson3a_pagemask(void)
+{
+    uc_engine *uc;
+    uint64_t r8 = 0x2000;
+    uint64_t r9 = 0;
+    uint64_t r10 = 0x6000;
+    uint64_t r11 = 0;
+    uint32_t code[] = {
+        BEINT32(0x40882800), /* mtc0 t0, PageMask */
+        BEINT32(0x40092800), /* mfc0 t1, PageMask */
+        BEINT32(0x408a2800), /* mtc0 t2, PageMask */
+        BEINT32(0x400b2800), /* mfc0 t3, PageMask */
+    };
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_LOONGSON_3A1000));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_reg_write(uc, UC_MIPS_REG_8, &r8));
+    OK(uc_reg_write(uc, UC_MIPS_REG_10, &r10));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_9, &r9));
+    OK(uc_reg_read(uc, UC_MIPS_REG_11, &r11));
+
+    TEST_CHECK(r9 == 0);
+    TEST_CHECK(r11 == r10);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_octeon_arithmetic(void)
+{
+    uc_engine *uc;
+    uint64_t r2 = 0xf0;
+    uint64_t r3 = 0x11;
+    uint64_t r14 = 0x10000000000000ffull;
+    uint64_t r15 = 0x0000ff0000000000ull;
+    uint64_t r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r16, r17;
+    const char code[] =
+        "\x70\x43\x20\x28" /* baddu r4,r2,r3 */
+        "\x70\x43\x28\x03" /* dmul  r5,r2,r3 */
+        "\x70\x46\x39\x3a" /* exts  r6,r2,4,7 */
+        "\x70\x67\x3c\x32" /* cins  r7,r3,16,7 */
+        "\x71\xc0\x40\x2c" /* pop   r8,r14 */
+        "\x71\xc0\x48\x2d" /* dpop  r9,r14 */
+        "\x70\x43\x50\x2a" /* seq   r10,r2,r3 */
+        "\x70\x43\x58\x2b" /* sne   r11,r2,r3 */
+        "\x70\x4c\x3c\x2e" /* seqi  r12,r2,0xf0 */
+        "\x70\x4d\x04\x6f" /* snei  r13,r2,0x11 */
+        "\x71\xf0\x3a\x3b" /* exts32 r16,r15,40,7 */
+        "\x70\x71\x3a\x33"; /* cins32 r17,r3,40,7 */
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_OCTEON68XX));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &r3));
+    OK(uc_reg_write(uc, UC_MIPS_REG_14, &r14));
+    OK(uc_reg_write(uc, UC_MIPS_REG_15, &r15));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_4, &r4));
+    OK(uc_reg_read(uc, UC_MIPS_REG_5, &r5));
+    OK(uc_reg_read(uc, UC_MIPS_REG_6, &r6));
+    OK(uc_reg_read(uc, UC_MIPS_REG_7, &r7));
+    OK(uc_reg_read(uc, UC_MIPS_REG_8, &r8));
+    OK(uc_reg_read(uc, UC_MIPS_REG_9, &r9));
+    OK(uc_reg_read(uc, UC_MIPS_REG_10, &r10));
+    OK(uc_reg_read(uc, UC_MIPS_REG_11, &r11));
+    OK(uc_reg_read(uc, UC_MIPS_REG_12, &r12));
+    OK(uc_reg_read(uc, UC_MIPS_REG_13, &r13));
+    OK(uc_reg_read(uc, UC_MIPS_REG_16, &r16));
+    OK(uc_reg_read(uc, UC_MIPS_REG_17, &r17));
+
+    TEST_CHECK(r4 == 0x1);
+    TEST_CHECK(r5 == 0xff0);
+    TEST_CHECK(r6 == 0xf);
+    TEST_CHECK(r7 == 0x110000);
+    TEST_CHECK(r8 == 8);
+    TEST_CHECK(r9 == 9);
+    TEST_CHECK(r10 == 0);
+    TEST_CHECK(r11 == 1);
+    TEST_CHECK(r12 == 1);
+    TEST_CHECK(r13 == 1);
+    TEST_CHECK(r16 == UINT64_MAX);
+    TEST_CHECK(r17 == 0x0000110000000000ull);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_octeon_bbit(void)
+{
+    uc_engine *uc;
+    uint64_t r2 = 0x10;
+    uint64_t r3 = 0;
+    uint64_t r4, r5, r6, r7, r8;
+    const char code[] =
+        "\xe8\x44\x00\x02" /* bbit1   r2,4,+2 */
+        "\x34\x04\x00\x01" /* ori     r4,zero,1 */
+        "\x34\x05\x00\x22" /* ori     r5,zero,0x22 */
+        "\xd8\x64\x00\x02" /* bbit032 r3,36,+2 */
+        "\x34\x06\x00\x02" /* ori     r6,zero,2 */
+        "\x34\x07\x00\x44" /* ori     r7,zero,0x44 */
+        "\x34\x08\x00\x55"; /* ori     r8,zero,0x55 */
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_OCTEON68XX));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &r3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_MIPS_REG_4, &r4));
+    OK(uc_reg_read(uc, UC_MIPS_REG_5, &r5));
+    OK(uc_reg_read(uc, UC_MIPS_REG_6, &r6));
+    OK(uc_reg_read(uc, UC_MIPS_REG_7, &r7));
+    OK(uc_reg_read(uc, UC_MIPS_REG_8, &r8));
+
+    TEST_CHECK(r4 == 1);
+    TEST_CHECK(r5 == 0);
+    TEST_CHECK(r6 == 2);
+    TEST_CHECK(r7 == 0);
+    TEST_CHECK(r8 == 0x55);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips64_octeon_requires_octeon(void)
+{
+    uc_engine *uc;
+    uint64_t r2 = 0xf0;
+    uint64_t r3 = 0x11;
+    const char code[] = "\x70\x43\x20\x28";
+
+    OK(uc_open(UC_ARCH_MIPS, UC_MODE_MIPS64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_MIPS64_MIPS64R2_GENERIC));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_MIPS_REG_2, &r2));
+    OK(uc_reg_write(uc, UC_MIPS_REG_3, &r3));
+
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_mips_cp0_count_compare(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x40\x88\x48\x00" /* mtc0 $t0, Count */
+        "\x40\x09\x48\x00" /* mfc0 $t1, Count */
+        "\x40\x8a\x58\x00" /* mtc0 $t2, Compare */
+        "\x40\x0b\x58\x00"; /* mfc0 $t3, Compare */
+    uint32_t t0 = 0x12345678;
+    uint32_t t1 = 0;
+    uint32_t t2 = 0xdeadbeef;
+    uint32_t t3 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_MIPS, UC_MODE_MIPS32 | UC_MODE_BIG_ENDIAN,
+                    code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_MIPS_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_MIPS_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_MIPS_REG_T1, &t1));
+    OK(uc_reg_read(uc, UC_MIPS_REG_T3, &t3));
+
+    TEST_CHECK(t1 == t0);
+    TEST_CHECK(t3 == t2);
+
+    OK(uc_close(uc));
+}
+
 TEST_LIST = {
     {"test_mips_stop_at_branch", test_mips_stop_at_branch},
     {"test_mips_stop_at_delay_slot", test_mips_stop_at_delay_slot},
@@ -229,9 +1071,41 @@ TEST_LIST = {
     {"test_mips_eb_ori", test_mips_eb_ori},
     {"test_mips_lwx_exception_issue_1314", test_mips_lwx_exception_issue_1314},
     {"test_mips_mips16", test_mips_mips16},
+    {"test_mips_mips32r6_mode_bitswap", test_mips_mips32r6_mode_bitswap},
+    {"test_mips_micro_mode_li16", test_mips_micro_mode_li16},
+    {"test_mips_nanomips_model_move16", test_mips_nanomips_model_move16},
+    {"test_mips_mips3_mode_opens", test_mips_mips3_mode_opens},
+    {"test_mips_msa_w_reg_roundtrip", test_mips_msa_w_reg_roundtrip},
     {"test_mips_mips_fpr", test_mips_mips_fpr},
     {"test_mips_stop_delay_slot_from_qiling",
      test_mips_stop_delay_slot_from_qiling},
      {"test_mips_simple_coredump_2134", test_mips_simple_coredump_2134},
      {"test_mips_simple_coredump_2137", test_mips_simple_coredump_2137},
-    {NULL, NULL}};
\ No newline at end of file
+     {"test_mips64_loongson2f_status", test_mips64_loongson2f_status},
+     {"test_mips64_loongson3a_dmult", test_mips64_loongson3a_dmult},
+     {"test_mips64_loongson3a_requires_lext",
+      test_mips64_loongson3a_requires_lext},
+     {"test_mips64_loongson3a_load_zero_prefetch",
+      test_mips64_loongson3a_load_zero_prefetch},
+     {"test_mips64_loongson3a_lext_lsdc2_gpr",
+      test_mips64_loongson3a_lext_lsdc2_gpr},
+     {"test_mips64_loongson3a_lext_lsdc2_requires_lext",
+      test_mips64_loongson3a_lext_lsdc2_requires_lext},
+     {"test_mips64_loongson3a_lext_gslsq_gpr",
+      test_mips64_loongson3a_lext_gslsq_gpr},
+     {"test_mips64_loongson3a_lext_gslsq_requires_lext",
+      test_mips64_loongson3a_lext_gslsq_requires_lext},
+     {"test_mips64_loongson3a_lext_lsdc2_fpr",
+      test_mips64_loongson3a_lext_lsdc2_fpr},
+     {"test_mips64_loongson3a_lext_gslsq_fpr",
+      test_mips64_loongson3a_lext_gslsq_fpr},
+     {"test_mips64_loongson3a_lext_shifted_fpr",
+      test_mips64_loongson3a_lext_shifted_fpr},
+     {"test_mips64_loongson3a_pagemask",
+      test_mips64_loongson3a_pagemask},
+     {"test_mips64_octeon_arithmetic", test_mips64_octeon_arithmetic},
+     {"test_mips64_octeon_bbit", test_mips64_octeon_bbit},
+     {"test_mips64_octeon_requires_octeon",
+      test_mips64_octeon_requires_octeon},
+     {"test_mips_cp0_count_compare", test_mips_cp0_count_compare},
+    {NULL, NULL}};
diff --git a/tests/unit/test_ppc.c b/tests/unit/test_ppc.c
index 4171e62c0f..950e8349c6 100644
--- a/tests/unit/test_ppc.c
+++ b/tests/unit/test_ppc.c
@@ -3,6 +3,25 @@
 const uint64_t code_start = 0x1000;
 const uint64_t code_len = 0x4000;
 
+typedef struct PpcCodeHookTrace {
+    uint64_t address[2];
+    uint32_t size[2];
+    uint32_t count;
+} PpcCodeHookTrace;
+
+static void test_ppc64_prefixed_code_hook(uc_engine *uc, uint64_t address,
+                                          uint32_t size, void *user_data)
+{
+    PpcCodeHookTrace *trace = (PpcCodeHookTrace *)user_data;
+
+    (void)uc;
+    if (trace->count < 2) {
+        trace->address[trace->count] = address;
+        trace->size[trace->count] = size;
+    }
+    trace->count++;
+}
+
 static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
                             const char *code, uint64_t size)
 {
@@ -34,7 +53,7 @@ static void test_ppc32_add(void)
     OK(uc_close(uc));
 }
 
-// https://www.ibm.com/docs/en/aix/7.2?topic=set-fadd-fa-floating-add-instruction
+/* IBM AIX fadd/floating-add instruction reference. */
 static void test_ppc32_fadd(void)
 {
     uc_engine *uc;
@@ -153,10 +172,4936 @@ static void test_ppc32_spr_mftb(void)
     TEST_CHECK(t1 != t2);
 }
 
+static void run_ppc64_power10_byte_reverse(const char *code, uint64_t expected)
+{
+    uc_engine *uc;
+    uint64_t src = 0x0123456789abcdefull;
+    uint64_t dst = 0;
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, 4));
+
+    OK(uc_reg_write(uc, UC_PPC_REG_3, &src));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_4, &dst));
+
+    TEST_CHECK(dst == expected);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_brd(void)
+{
+    run_ppc64_power10_byte_reverse("\x7c\x64\x01\x76",
+                                   0xefcdab8967452301ull);
+}
+
+static void test_ppc64_power10_brw(void)
+{
+    run_ppc64_power10_byte_reverse("\x7c\x64\x01\x36",
+                                   0x67452301efcdab89ull);
+}
+
+static void test_ppc64_power10_brh(void)
+{
+    run_ppc64_power10_byte_reverse("\x7c\x64\x01\xb6",
+                                   0x23016745ab89efcdull);
+}
+
+static void run_ppc64_power10_mask_op(const char *code, uint64_t src,
+                                      uint64_t mask, uint64_t expected)
+{
+    uc_engine *uc;
+    uint64_t dst = 0;
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, 4));
+
+    OK(uc_reg_write(uc, UC_PPC_REG_3, &src));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &mask));
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_4, &dst));
+
+    TEST_CHECK(dst == expected);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_cfuged(void)
+{
+    run_ppc64_power10_mask_op("\x7c\x64\x29\xb8", 0x0123456789abcdefull,
+                              0x00ff00ff00ff00ffull,
+                              0x014589cd2367abefull);
+}
+
+static void test_ppc64_power10_cntlzdm(void)
+{
+    run_ppc64_power10_mask_op("\x7c\x64\x28\x76", 0x0010000000000000ull,
+                              0x00ff000000000000ull, 3);
+    run_ppc64_power10_mask_op("\x7c\x64\x28\x76", 0,
+                              0x00ff000000000000ull, 8);
+}
+
+static void test_ppc64_power10_cnttzdm(void)
+{
+    run_ppc64_power10_mask_op("\x7c\x64\x2c\x76", 0x1000,
+                              0xff00, 4);
+    run_ppc64_power10_mask_op("\x7c\x64\x2c\x76", 0,
+                              0xff00, 8);
+}
+
+static void test_ppc64_power10_pdepd(void)
+{
+    run_ppc64_power10_mask_op("\x7c\x64\x29\x38", 0xb, 0xf0, 0xb0);
+}
+
+static void test_ppc64_power10_pextd(void)
+{
+    run_ppc64_power10_mask_op("\x7c\x64\x29\x78", 0xb0, 0xf0, 0xb);
+}
+
+static void test_ppc64_power10_mask_op_requires_isa310(void)
+{
+    uc_engine *uc;
+    uint64_t src = 0xb;
+    uint64_t mask = 0xf0;
+    const char code[] = "\x7c\x64\x29\x38";
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_PPC_REG_3, &src));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &mask));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_setbc(void)
+{
+    uc_engine *uc;
+    uint32_t cr1 = 0x4;
+    uint64_t r4, r5, r6, r7, r8, r9, r10, r11;
+    char code[] =
+        "\x7c\x85\x03\x00" /* setbc   r4, 5 */
+        "\x7c\xa5\x03\x40" /* setbcr  r5, 5 */
+        "\x7c\xc5\x03\x80" /* setnbc  r6, 5 */
+        "\x7c\xe5\x03\xc0" /* setnbcr r7, 5 */
+        "\x7d\x06\x03\x00" /* setbc   r8, 6 */
+        "\x7d\x26\x03\x40" /* setbcr  r9, 6 */
+        "\x7d\x46\x03\x80" /* setnbc  r10, 6 */
+        "\x7d\x66\x03\xc0"; /* setnbcr r11, 6 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_PPC_REG_CR1, &cr1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_4, &r4));
+    OK(uc_reg_read(uc, UC_PPC_REG_5, &r5));
+    OK(uc_reg_read(uc, UC_PPC_REG_6, &r6));
+    OK(uc_reg_read(uc, UC_PPC_REG_7, &r7));
+    OK(uc_reg_read(uc, UC_PPC_REG_8, &r8));
+    OK(uc_reg_read(uc, UC_PPC_REG_9, &r9));
+    OK(uc_reg_read(uc, UC_PPC_REG_10, &r10));
+    OK(uc_reg_read(uc, UC_PPC_REG_11, &r11));
+
+    TEST_CHECK(r4 == 1);
+    TEST_CHECK(r5 == 0);
+    TEST_CHECK(r6 == UINT64_MAX);
+    TEST_CHECK(r7 == 0);
+    TEST_CHECK(r8 == 0);
+    TEST_CHECK(r9 == 1);
+    TEST_CHECK(r10 == 0);
+    TEST_CHECK(r11 == UINT64_MAX);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_setbc_requires_isa310(void)
+{
+    uc_engine *uc;
+    uint32_t cr1 = 0x4;
+    const char code[] = "\x7c\x85\x03\x00";
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_PPC_REG_CR1, &cr1));
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_power10_vector_mask_op(const char *op,
+                                             const uint8_t src[16],
+                                             const uint8_t mask[16],
+                                             const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t mask_addr = code_start + 0x1010;
+    uint64_t dst_addr = code_start + 0x1020;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    int i;
+    char code[16] =
+        "\x7c\x80\x60\xce" /* lvx  v4, 0, r12 */
+        "\x7c\xa0\x68\xce" /* lvx  v5, 0, r13 */
+        "\0\0\0\0"
+        "\x7c\x60\x71\xce"; /* stvx v3, 0, r14 */
+
+    memcpy(code + 8, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+    OK(uc_mem_write(uc, mask_addr, mask, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &mask_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(dst[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vcfuged(void)
+{
+    const uint8_t src[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+    };
+    const uint8_t mask[16] = {
+        0x00, 0xff, 0x00, 0xff, 0x00, 0xff, 0x00, 0xff,
+        0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+    };
+    const uint8_t expected[16] = {
+        0x01, 0x45, 0x89, 0xcd, 0x23, 0x67, 0xab, 0xef,
+        0xfd, 0xb9, 0x75, 0x31, 0xec, 0xa8, 0x64, 0x20,
+    };
+
+    run_ppc64_power10_vector_mask_op("\x10\x64\x2d\x4d", src, mask,
+                                     expected);
+}
+
+static void test_ppc64_power10_vclzdm(void)
+{
+    const uint8_t src[16] = {
+        0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t mask[16] = {
+        0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08,
+    };
+
+    run_ppc64_power10_vector_mask_op("\x10\x64\x2f\x84", src, mask,
+                                     expected);
+}
+
+static void test_ppc64_power10_vctzdm(void)
+{
+    const uint8_t src[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t mask[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00,
+    };
+    const uint8_t expected[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08,
+    };
+
+    run_ppc64_power10_vector_mask_op("\x10\x64\x2f\xc4", src, mask,
+                                     expected);
+}
+
+static void test_ppc64_power10_vpdepd(void)
+{
+    const uint8_t src[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x34,
+    };
+    const uint8_t mask[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0xff,
+    };
+    const uint8_t expected[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x34,
+    };
+
+    run_ppc64_power10_vector_mask_op("\x10\x64\x2d\xcd", src, mask,
+                                     expected);
+}
+
+static void test_ppc64_power10_vpextd(void)
+{
+    const uint8_t src[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x34,
+    };
+    const uint8_t mask[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0xff,
+    };
+    const uint8_t expected[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x34,
+    };
+
+    run_ppc64_power10_vector_mask_op("\x10\x64\x2d\x8d", src, mask,
+                                     expected);
+}
+
+static void test_ppc64_power10_vmx_mask_materialize_extract(void)
+{
+    uc_engine *uc;
+    uint64_t b_addr = code_start + 0x1000;
+    uint64_t expand_addr = code_start + 0x1100;
+    uint64_t h_addr = code_start + 0x1200;
+    uint64_t w_addr = code_start + 0x1300;
+    uint64_t d_addr = code_start + 0x1400;
+    uint64_t q_addr = code_start + 0x1500;
+    uint64_t bmi_addr = code_start + 0x1600;
+    uint64_t msr;
+    uint64_t value;
+    uint8_t dst[16];
+    const uint64_t b_mask = 0xa55a;
+    const uint64_t h_mask = 0xa5;
+    const uint64_t w_mask = 0x0a;
+    const uint64_t d_mask = 0x02;
+    const uint64_t q_mask = 1;
+    const uint8_t expected_b[16] = {
+        0xff, 0x00, 0xff, 0x00, 0x00, 0xff, 0x00, 0xff,
+        0x00, 0xff, 0x00, 0xff, 0xff, 0x00, 0xff, 0x00,
+    };
+    const uint8_t expected_h[16] = {
+        0xff, 0xff, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00,
+        0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0xff, 0xff,
+    };
+    const uint8_t expected_w[16] = {
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_d[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_q[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t expected_bmi[16] = {
+        0xff, 0x00, 0x00, 0xff, 0x00, 0xff, 0xff, 0x00,
+        0x00, 0xff, 0xff, 0x00, 0xff, 0x00, 0x00, 0xff,
+    };
+    const char code[] =
+        "\x10\x90\x2e\x42"
+        "\x7c\x80\xd9\xce"
+        "\x10\xc8\x26\x42"
+        "\x10\xe0\x26\x42"
+        "\x7c\xe0\xe1\xce"
+        "\x10\xb1\x3e\x42"
+        "\x7c\xa0\xe9\xce"
+        "\x11\x09\x2e\x42"
+        "\x10\xd2\x4e\x42"
+        "\x7c\xc0\xf1\xce"
+        "\x11\x4a\x36\x42"
+        "\x10\xf3\x5e\x42"
+        "\x7c\xe0\xf9\xce"
+        "\x11\x8b\x3e\x42"
+        "\x11\x14\x6e\x42"
+        "\x7d\x00\x81\xce"
+        "\x11\xcc\x46\x42"
+        "\x11\x34\x96\x55"
+        "\x7d\x20\x79\xce"
+        "\x12\x58\x26\x42"
+        "\x12\x79\x26\x42"
+        "\x12\x9b\x2e\x42"
+        "\x12\xdd\x36\x42"
+        "\x12\xbe\x3e\x42";
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &b_mask));
+    OK(uc_reg_write(uc, UC_PPC_REG_7, &h_mask));
+    OK(uc_reg_write(uc, UC_PPC_REG_9, &w_mask));
+    OK(uc_reg_write(uc, UC_PPC_REG_11, &d_mask));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &q_mask));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &bmi_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &q_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_27, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_28, &expand_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_29, &h_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_30, &w_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_31, &d_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_mem_read(uc, b_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected_b, sizeof(dst)) == 0);
+    OK(uc_mem_read(uc, expand_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected_b, sizeof(dst)) == 0);
+    OK(uc_mem_read(uc, h_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected_h, sizeof(dst)) == 0);
+    OK(uc_mem_read(uc, w_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected_w, sizeof(dst)) == 0);
+    OK(uc_mem_read(uc, d_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected_d, sizeof(dst)) == 0);
+    OK(uc_mem_read(uc, q_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected_q, sizeof(dst)) == 0);
+    OK(uc_mem_read(uc, bmi_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected_bmi, sizeof(dst)) == 0);
+
+    OK(uc_reg_read(uc, UC_PPC_REG_6, &value));
+    TEST_CHECK(value == b_mask);
+    OK(uc_reg_read(uc, UC_PPC_REG_8, &value));
+    TEST_CHECK(value == h_mask);
+    OK(uc_reg_read(uc, UC_PPC_REG_10, &value));
+    TEST_CHECK(value == w_mask);
+    OK(uc_reg_read(uc, UC_PPC_REG_12, &value));
+    TEST_CHECK(value == d_mask);
+    OK(uc_reg_read(uc, UC_PPC_REG_14, &value));
+    TEST_CHECK(value == q_mask);
+    OK(uc_reg_read(uc, UC_PPC_REG_18, &value));
+    TEST_CHECK(value == 0x0800000000000000ull);
+    OK(uc_reg_read(uc, UC_PPC_REG_19, &value));
+    TEST_CHECK(value == 0x0800000000000000ull);
+    OK(uc_reg_read(uc, UC_PPC_REG_20, &value));
+    TEST_CHECK(value == 0x0800000000000000ull);
+    OK(uc_reg_read(uc, UC_PPC_REG_21, &value));
+    TEST_CHECK(value == 0x0800000000000000ull);
+    OK(uc_reg_read(uc, UC_PPC_REG_22, &value));
+    TEST_CHECK(value == 0x0800000000000000ull);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vector_mask_requires_isa310(void)
+{
+    uc_engine *uc;
+    size_t i;
+    uint64_t msr;
+    static const uint8_t code[][4] = {
+        { 0x10, 0x64, 0x2d, 0xcd },
+        { 0x10, 0x90, 0x2e, 0x42 },
+        { 0x11, 0x34, 0x96, 0x55 },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= 1ull << 25;
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void run_ppc64_power10_vmx_quad_op(const uint8_t op[4],
+                                          const uint8_t a[16],
+                                          const uint8_t b[16],
+                                          const uint8_t seed[16],
+                                          const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t seed_addr = code_start + 0x1020;
+    uint64_t dst_addr = code_start + 0x1030;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[20] = {
+        0x7c, 0x60, 0x70, 0xce,
+        0x7c, 0x80, 0x60, 0xce,
+        0x7c, 0xa0, 0x68, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 12, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+    OK(uc_mem_write(uc, seed_addr, seed, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &seed_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vmx_quad_shift_rotate(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t a[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+    };
+    const uint8_t a_neg[16] = {
+        0x81, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+    };
+    const uint8_t sh68[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0x44,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t sh4[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0x04,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t sh65[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0x41,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t rot12[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0x0c,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t mask_nm[16] = {
+        0, 0, 0, 0, 0, 0x08, 0x5f, 0x05,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t mask_mi[16] = {
+        0, 0, 0, 0, 0, 0x10, 0x4f, 0x09,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t seed[16] = {
+        0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+        0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb,
+    };
+    const uint8_t vslq[16] = {
+        0xed, 0xcb, 0xa9, 0x87, 0x65, 0x43, 0x21, 0x00,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vsrq[16] = {
+        0x00, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde,
+        0xff, 0xed, 0xcb, 0xa9, 0x87, 0x65, 0x43, 0x21,
+    };
+    const uint8_t vsraq[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xc0, 0x91, 0xa2, 0xb3, 0xc4, 0xd5, 0xe6, 0xf7,
+    };
+    const uint8_t vrlq[16] = {
+        0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xff, 0xed,
+        0xcb, 0xa9, 0x87, 0x65, 0x43, 0x21, 0x00, 0x12,
+    };
+    const uint8_t vrlqnm[16] = {
+        0x00, 0x68, 0xac, 0xf1, 0x35, 0x79, 0xbd, 0xff,
+        0xdb, 0x97, 0x53, 0x0e, 0, 0, 0, 0,
+    };
+    const uint8_t vrlqmi[16] = {
+        0xaa, 0xaa, 0xcf, 0x13, 0x57, 0x9b, 0xdf, 0xfd,
+        0xb9, 0x75, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0xbb,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\x05",
+                                  a, sh68, zero, vslq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\x05",
+                                  a, sh4, zero, vsrq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\x05",
+                                  a_neg, sh65, zero, vsraq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\x05",
+                                  a, rot12, zero, vrlq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\x45",
+                                  a, mask_nm, zero, vrlqnm);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\x45",
+                                  a, mask_mi, seed, vrlqmi);
+}
+
+static void test_ppc64_power10_vmx_doubleword_immediate_shift(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t a[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+    };
+    const uint8_t b[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+    };
+    const uint8_t vsldbi4[16] = {
+        0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xff,
+        0xed, 0xcb, 0xa9, 0x87, 0x65, 0x43, 0x21, 0x01,
+    };
+    const uint8_t vsrdbi4[16] = {
+        0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x78,
+        0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf0,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\x16",
+                                  a, b, zero, vsldbi4);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\x16",
+                                  a, b, zero, vsrdbi4);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\x16",
+                                  a, b, zero, a);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\x16",
+                                  a, b, zero, b);
+}
+
+static void run_ppc64_power10_vmx_quad_compare_dot(const uint8_t op[4],
+                                                   const uint8_t a[16],
+                                                   const uint8_t b[16],
+                                                   uint32_t expected_cr6)
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t dst_addr = code_start + 0x1020;
+    uint64_t msr;
+    uint32_t cr6 = 0;
+    uint8_t code[16] = {
+        0x7c, 0x80, 0x60, 0xce,
+        0x7c, 0xa0, 0x68, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 8, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_CR6, &cr6));
+    TEST_CHECK(cr6 == expected_cr6);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_power10_vcmpq_cr(const uint8_t op[4],
+                                       const uint8_t a[16],
+                                       const uint8_t b[16], int cr_reg,
+                                       uint32_t expected)
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t msr;
+    uint32_t cr = 0;
+    uint8_t code[12] = {
+        0x7c, 0x80, 0x60, 0xce,
+        0x7c, 0xa0, 0x68, 0xce,
+        0, 0, 0, 0,
+    };
+
+    memcpy(code + 8, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, cr_reg, &cr));
+    TEST_CHECK(cr == expected);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vmx_quad_compare(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t all_true[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t all_false[16] = { 0 };
+    const uint8_t a[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+    };
+    const uint8_t b[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x0f,
+    };
+    const uint8_t neg[16] = {
+        0x81, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\xc7",
+                                  a, a, zero, all_true);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\xc7",
+                                  a, b, zero, all_false);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\x87",
+                                  a, b, zero, all_true);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\x87",
+                                  neg, a, zero, all_false);
+    run_ppc64_power10_vmx_quad_compare_dot(
+        (const uint8_t *)"\x10\x64\x2d\xc7", a, a, 0x8);
+    run_ppc64_power10_vmx_quad_compare_dot(
+        (const uint8_t *)"\x10\x64\x2d\xc7", a, b, 0x2);
+    run_ppc64_power10_vcmpq_cr((const uint8_t *)"\x11\x04\x29\x41",
+                               neg, a, UC_PPC_REG_CR2, 0x8);
+    run_ppc64_power10_vcmpq_cr((const uint8_t *)"\x11\x84\x29\x01",
+                               neg, a, UC_PPC_REG_CR3, 0x4);
+}
+
+static void test_ppc64_power10_vmx_quad_compare_invalid(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    const uint8_t code[] = {
+        0x11, 0xa4, 0x29, 0x01,
+    };
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code),
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vmx_multiply_dword(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t a[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+    };
+    const uint8_t b[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05,
+    };
+    const uint8_t vmulesd[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8,
+    };
+    const uint8_t vmulosd[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0x0f,
+    };
+    const uint8_t vmuleud[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0x03,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8,
+    };
+    const uint8_t vmulld[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8,
+        0, 0, 0, 0, 0, 0, 0, 0x0f,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\xc8",
+                                  a, b, zero, vmulesd);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\xc8",
+                                  a, b, zero, vmulosd);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\xc8",
+                                  a, b, zero, vmuleud);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\xc8",
+                                  a, b, zero, vmulosd);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\xc9",
+                                  a, b, zero, vmulld);
+}
+
+static void test_ppc64_power10_vmx_multiply_high(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t dword_a[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+    };
+    const uint8_t dword_b[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05,
+    };
+    const uint8_t word_a[16] = {
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x00, 0x00,
+    };
+    const uint8_t word_b[16] = {
+        0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
+        0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00,
+    };
+    const uint8_t vmulhsw[16] = {
+        0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0,
+        0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0x01,
+    };
+    const uint8_t vmulhuw[16] = {
+        0, 0, 0, 0x01, 0, 0, 0, 0,
+        0, 0, 0, 0x01, 0, 0, 0, 0x01,
+    };
+    const uint8_t vmulhsd[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vmulhud[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0x03,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\x89",
+                                  word_a, word_b, zero, vmulhsw);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\x89",
+                                  word_a, word_b, zero, vmulhuw);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\xc9",
+                                  dword_a, dword_b, zero, vmulhsd);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\xc9",
+                                  dword_a, dword_b, zero, vmulhud);
+}
+
+static void test_ppc64_power10_vmx_vextsd2q(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t src[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+    };
+    const uint8_t expected[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x7b\x26\x02",
+                                  src, zero, zero, expected);
+}
+
+static void run_ppc64_power10_vmx_vextd_op(const uint8_t op[4],
+                                           const uint8_t a[16],
+                                           const uint8_t b[16],
+                                           uint64_t index,
+                                           const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t dst_addr = code_start + 0x1020;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[16] = {
+        0x7c, 0x80, 0x60, 0xce,
+        0x7c, 0xa0, 0x68, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 8, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_6, &index));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_power10_vmx_vins_gpr_op(const uint8_t op[4],
+                                              const uint8_t seed[16],
+                                              uint64_t index,
+                                              uint64_t value,
+                                              const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t seed_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1010;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[12] = {
+        0x7c, 0x60, 0x70, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 4, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, seed_addr, seed, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_6, &index));
+    OK(uc_reg_write(uc, UC_PPC_REG_7, &value));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &seed_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_power10_vmx_vins_vector_op(const uint8_t op[4],
+                                                 const uint8_t seed[16],
+                                                 const uint8_t src[16],
+                                                 uint64_t index,
+                                                 const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t seed_addr = code_start + 0x1000;
+    uint64_t src_addr = code_start + 0x1010;
+    uint64_t dst_addr = code_start + 0x1020;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[16] = {
+        0x7c, 0x60, 0x70, 0xce,
+        0x7c, 0xe0, 0x68, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 8, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, seed_addr, seed, 16));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_6, &index));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &seed_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vmx_extract_double(void)
+{
+    const uint8_t a[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t b[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    const uint8_t vextdubvlx[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0x02,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vextduhvrx[16] = {
+        0, 0, 0, 0, 0, 0, 0x1b, 0x1c,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vextduwvlx[16] = {
+        0, 0, 0, 0, 0x0e, 0x0f, 0x10, 0x11,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vextddvrx[16] = {
+        0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    run_ppc64_power10_vmx_vextd_op((const uint8_t *)"\x10\x64\x29\x98",
+                                   a, b, 2, vextdubvlx);
+    run_ppc64_power10_vmx_vextd_op((const uint8_t *)"\x10\x64\x29\x9b",
+                                   a, b, 3, vextduhvrx);
+    run_ppc64_power10_vmx_vextd_op((const uint8_t *)"\x10\x64\x29\x9c",
+                                   a, b, 14, vextduwvlx);
+    run_ppc64_power10_vmx_vextd_op((const uint8_t *)"\x10\x64\x29\x9f",
+                                   a, b, 1, vextddvrx);
+}
+
+static void test_ppc64_power10_vmx_insert_gpr(void)
+{
+    const uint8_t seed[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint64_t value = 0x1122334455667788ull;
+    const uint8_t vinsblx[16] = {
+        0x00, 0x11, 0x22, 0x88, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vinshrx[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0x77, 0x88, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vinswrx[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0x55, 0x66, 0x77, 0x88, 0xee, 0xff,
+    };
+    const uint8_t vinsdrx[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x11,
+        0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0xff,
+    };
+    const uint8_t vinsw[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x55, 0x66,
+        0x77, 0x88, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vinsd[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+    };
+
+    run_ppc64_power10_vmx_vins_gpr_op(
+        (const uint8_t *)"\x10\x66\x3a\x0f", seed, 3, value, vinsblx);
+    run_ppc64_power10_vmx_vins_gpr_op(
+        (const uint8_t *)"\x10\x66\x3b\x4f", seed, 3, value, vinshrx);
+    run_ppc64_power10_vmx_vins_gpr_op(
+        (const uint8_t *)"\x10\x66\x3b\x8f", seed, 2, value, vinswrx);
+    run_ppc64_power10_vmx_vins_gpr_op(
+        (const uint8_t *)"\x10\x66\x3b\xcf", seed, 1, value, vinsdrx);
+    run_ppc64_power10_vmx_vins_gpr_op(
+        (const uint8_t *)"\x10\x66\x38\xcf", seed, 0, value, vinsw);
+    run_ppc64_power10_vmx_vins_gpr_op(
+        (const uint8_t *)"\x10\x68\x39\xcf", seed, 0, value, vinsd);
+}
+
+static void test_ppc64_power10_vmx_insert_vector(void)
+{
+    const uint8_t seed[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t src[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+    };
+    const uint8_t vinsbvlx[16] = {
+        0x00, 0x11, 0xa7, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vinshvrx[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xa6, 0xa7, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vinswvlx[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0xa4, 0xa5, 0xa6,
+        0xa7, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vinswvrx[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xa4, 0xa5, 0xa6, 0xa7, 0xff,
+    };
+
+    run_ppc64_power10_vmx_vins_vector_op(
+        (const uint8_t *)"\x10\x66\x38\x0f", seed, src, 2, vinsbvlx);
+    run_ppc64_power10_vmx_vins_vector_op(
+        (const uint8_t *)"\x10\x66\x39\x4f", seed, src, 4, vinshvrx);
+    run_ppc64_power10_vmx_vins_vector_op(
+        (const uint8_t *)"\x10\x66\x38\x8f", seed, src, 5, vinswvlx);
+    run_ppc64_power10_vmx_vins_vector_op(
+        (const uint8_t *)"\x10\x66\x39\x8f", seed, src, 1, vinswvrx);
+}
+
+static void test_ppc64_power10_vmx_divmod_word(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t a[16] = {
+        0xff, 0xff, 0xff, 0xf6, 0x00, 0x00, 0x00, 0x14,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t b[16] = {
+        0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xfc,
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vdivsw[16] = {
+        0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xfb,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t vdivuw[16] = {
+        0x55, 0x55, 0x55, 0x52, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t vmodsw[16] = {
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vmoduw[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\x8b",
+                                  a, b, zero, vdivsw);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\x8b",
+                                  a, b, zero, vdivuw);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2f\x8b",
+                                  a, b, zero, vmodsw);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2e\x8b",
+                                  a, b, zero, vmoduw);
+}
+
+static void test_ppc64_power10_vmx_divmod_dword(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t a_signed[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf6,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t b_signed[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t a_zero[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t b_zero[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t a_unsigned[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf6,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t b_unsigned[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vdivsd[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vmodsd[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vdivsd_zero[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfb,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t vmodsd_zero[16] = { 0 };
+    const uint8_t vdivud[16] = {
+        0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x52,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t vmodud[16] = { 0 };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\xcb",
+                                  a_signed, b_signed, zero, vdivsd);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2f\xcb",
+                                  a_signed, b_signed, zero, vmodsd);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\xcb",
+                                  a_zero, b_zero, zero, vdivsd_zero);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2f\xcb",
+                                  a_zero, b_zero, zero, vmodsd_zero);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\xcb",
+                                  a_unsigned, b_unsigned, zero, vdivud);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2e\xcb",
+                                  a_unsigned, b_unsigned, zero, vmodud);
+}
+
+static void test_ppc64_power10_vmx_divide_extended(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t a_word[16] = {
+        0x00, 0x00, 0x00, 0x05, 0xff, 0xff, 0xff, 0xfb,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t b_word_signed[16] = {
+        0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04,
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t b_word_unsigned[16] = {
+        0x00, 0x00, 0x00, 0x04, 0xff, 0xff, 0xff, 0xff,
+        0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t vdivesw[16] = {
+        0x40, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vdiveuw[16] = {
+        0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xfb,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t a_dword[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfb,
+    };
+    const uint8_t b_dword[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+    };
+    const uint8_t a_dword_edge[16] = {
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t b_dword_edge[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t a_dword_unsigned[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t b_dword_unsigned[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vdivesd[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vdivesd_edge[16] = {
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t vdiveud[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t a_quad_pos[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05,
+    };
+    const uint8_t a_quad_neg[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfb,
+    };
+    const uint8_t b_quad[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+    };
+    const uint8_t vdivesq_pos[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vdivesq_neg[16] = {
+        0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\x8b",
+                                  a_word, b_word_signed, zero, vdivesw);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\x8b",
+                                  a_word, b_word_unsigned, zero, vdiveuw);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\xcb",
+                                  a_dword, b_dword, zero, vdivesd);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\xcb",
+                                  a_dword_edge, b_dword_edge, zero,
+                                  vdivesd_edge);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\xcb",
+                                  a_dword_unsigned, b_dword_unsigned, zero,
+                                  vdiveud);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\x0b",
+                                  a_quad_pos, b_quad, zero, vdivesq_pos);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2b\x0b",
+                                  a_quad_neg, b_quad, zero, vdivesq_neg);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2a\x0b",
+                                  a_quad_pos, b_quad, zero, vdivesq_pos);
+}
+
+static void test_ppc64_power10_vmx_divmod_quad(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t pos100[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64,
+    };
+    const uint8_t neg100[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x9c,
+    };
+    const uint8_t seven[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07,
+    };
+    const uint8_t int128_min[16] = {
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t neg_one[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t vdivuq[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e,
+    };
+    const uint8_t vmoduq[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
+    };
+    const uint8_t vdivsq[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf2,
+    };
+    const uint8_t vmodsq[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\x0b",
+                                  pos100, seven, zero, vdivuq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2e\x0b",
+                                  pos100, seven, zero, vmoduq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\x0b",
+                                  neg100, seven, zero, vdivsq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2f\x0b",
+                                  neg100, seven, zero, vmodsq);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\x0b",
+                                  pos100, zero, zero, pos100);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2e\x0b",
+                                  pos100, zero, zero, zero);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\x0b",
+                                  int128_min, neg_one, zero, int128_min);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x2f\x0b",
+                                  int128_min, neg_one, zero, zero);
+}
+
+static void run_ppc64_power10_vmx_vstri_op(const uint8_t op[4],
+                                           const uint8_t src[16],
+                                           uint32_t initial_cr6,
+                                           const uint8_t expected[16],
+                                           uint32_t expected_cr6)
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint32_t cr6;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[12] = {
+        0x7c, 0xa0, 0x60, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 4, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_CR6, &initial_cr6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    OK(uc_reg_read(uc, UC_PPC_REG_CR6, &cr6));
+
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+    TEST_CHECK(cr6 == expected_cr6);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vmx_string_isolate(void)
+{
+    const uint8_t vstribl_src[16] = {
+        0x11, 0x22, 0x00, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+    };
+    const uint8_t vstribr_src[16] = {
+        0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+        0x09, 0x0a, 0x0b, 0x0c, 0x00, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t vstrihl_src[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x00, 0x00, 0x55, 0x66,
+        0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee,
+    };
+    const uint8_t vstrihr_src[16] = {
+        0x11, 0x11, 0x22, 0x22, 0x33, 0x33, 0x00, 0x00,
+        0x44, 0x44, 0x55, 0x55, 0x66, 0x66, 0x77, 0x77,
+    };
+    const uint8_t vstribl_expected[16] = {
+        0x11, 0x22, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vstribr_expected[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t vstrihl_expected[16] = {
+        0x11, 0x22, 0x33, 0x44, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vstrihr_expected[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0x44, 0x44, 0x55, 0x55, 0x66, 0x66, 0x77, 0x77,
+    };
+
+    run_ppc64_power10_vmx_vstri_op((const uint8_t *)"\x10\x60\x2c\x0d",
+                                   vstribl_src, 8, vstribl_expected, 2);
+    run_ppc64_power10_vmx_vstri_op((const uint8_t *)"\x10\x61\x28\x0d",
+                                   vstribr_src, 8, vstribr_expected, 8);
+    run_ppc64_power10_vmx_vstri_op((const uint8_t *)"\x10\x62\x2c\x0d",
+                                   vstrihl_src, 8, vstrihl_expected, 2);
+    run_ppc64_power10_vmx_vstri_op((const uint8_t *)"\x10\x63\x28\x0d",
+                                   vstrihr_src, 8, vstrihr_expected, 8);
+}
+
+static void run_ppc64_power10_vmx_vclr_op(const uint8_t op[4],
+                                          uint64_t count,
+                                          const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    const uint8_t src[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    uint8_t code[12] = {
+        0x7c, 0x80, 0x60, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 4, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, src_addr, src, sizeof(src)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &count));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vmx_clear_bytes(void)
+{
+    const uint8_t vclrlb5[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vclrlb16[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t vclrrb5[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t vclrrb11[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0, 0, 0, 0, 0,
+    };
+
+    run_ppc64_power10_vmx_vclr_op((const uint8_t *)"\x10\x64\x29\x8d",
+                                  5, vclrlb5);
+    run_ppc64_power10_vmx_vclr_op((const uint8_t *)"\x10\x64\x29\x8d",
+                                  16, vclrlb16);
+    run_ppc64_power10_vmx_vclr_op((const uint8_t *)"\x10\x64\x29\xcd",
+                                  5, vclrrb5);
+    run_ppc64_power10_vmx_vclr_op((const uint8_t *)"\x10\x64\x29\xcd",
+                                  11, vclrrb11);
+}
+
+static void test_ppc64_power10_vmx_string_clear_legacy_buckets(void)
+{
+    const uint8_t zero[16] = { 0 };
+    const uint8_t a[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t b[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint8_t vmrghb[16] = {
+        0x00, 0xa0, 0x01, 0xa1, 0x02, 0xa2, 0x03, 0xa3,
+        0x04, 0xa4, 0x05, 0xa5, 0x06, 0xa6, 0x07, 0xa7,
+    };
+    const uint8_t vmrglw[16] = {
+        0x08, 0x09, 0x0a, 0x0b, 0xa8, 0xa9, 0xaa, 0xab,
+        0x0c, 0x0d, 0x0e, 0x0f, 0xac, 0xad, 0xae, 0xaf,
+    };
+
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x28\x0c",
+                                  a, b, zero, vmrghb);
+    run_ppc64_power10_vmx_quad_op((const uint8_t *)"\x10\x64\x29\x8c",
+                                  a, b, zero, vmrglw);
+}
+
+static void test_ppc64_power10_vmx_string_clear_requires_isa310(void)
+{
+    uc_engine *uc;
+    size_t i;
+    uint64_t msr;
+    static const uint8_t code[][4] = {
+        { 0x10, 0x60, 0x28, 0x0d },
+        { 0x10, 0x60, 0x2c, 0x0d },
+        { 0x10, 0x64, 0x29, 0x8d },
+        { 0x10, 0x64, 0x29, 0xcd },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= 1ull << 25;
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void run_ppc64_vmx_va_op(const uint8_t op[4], int cpu_model,
+                                const uint8_t a[16], const uint8_t b[16],
+                                const uint8_t c[16],
+                                const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t c_addr = code_start + 0x1020;
+    uint64_t dst_addr = code_start + 0x1030;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[20] = {
+        0x7c, 0x80, 0x60, 0xce,
+        0x7c, 0xa0, 0x68, 0xce,
+        0x7c, 0xc0, 0x70, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    memcpy(code + 12, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, cpu_model));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+    OK(uc_mem_write(uc, c_addr, c, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &c_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_isa300_vmx_multiply_sum_dword(void)
+{
+    const uint8_t vmsumudm_a[16] = {
+        0, 0, 0, 0, 0, 0, 0, 3,
+        0, 0, 0, 0, 0, 0, 0, 5,
+    };
+    const uint8_t vmsumudm_b[16] = {
+        0, 0, 0, 0, 0, 0, 0, 7,
+        0, 0, 0, 0, 0, 0, 0, 11,
+    };
+    const uint8_t vmsumudm_c[16] = {
+        0, 0, 0, 0, 0, 0, 0, 1,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0,
+    };
+    const uint8_t vmsumudm_expected[16] = {
+        0, 0, 0, 0, 0, 0, 0, 2,
+        0, 0, 0, 0, 0, 0, 0, 0x3c,
+    };
+    const uint8_t vmladduhm_a[16] = {
+        0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04,
+        0x00, 0x05, 0x00, 0x06, 0x00, 0x07, 0x00, 0x08,
+    };
+    const uint8_t vmladduhm_b[16] = {
+        0x00, 0x09, 0x00, 0x0a, 0x00, 0x0b, 0x00, 0x0c,
+        0x00, 0x0d, 0x00, 0x0e, 0x00, 0x0f, 0x00, 0x10,
+    };
+    const uint8_t vmladduhm_c[16] = {
+        0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02,
+        0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04,
+    };
+    const uint8_t vmladduhm_expected[16] = {
+        0x00, 0x09, 0x00, 0x15, 0x00, 0x21, 0x00, 0x32,
+        0x00, 0x41, 0x00, 0x57, 0x00, 0x69, 0x00, 0x84,
+    };
+
+    run_ppc64_vmx_va_op((const uint8_t *)"\x10\x64\x29\xa3",
+                        UC_CPU_PPC64_POWER9_V2_0, vmsumudm_a, vmsumudm_b,
+                        vmsumudm_c, vmsumudm_expected);
+    run_ppc64_vmx_va_op((const uint8_t *)"\x10\x64\x29\xa2",
+                        UC_CPU_PPC64_POWER8_V2_0, vmladduhm_a, vmladduhm_b,
+                        vmladduhm_c, vmladduhm_expected);
+}
+
+static void test_ppc64_power10_vmx_multiply_sum_carry_dword(void)
+{
+    const uint8_t all_ones[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t expected[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 2,
+    };
+
+    run_ppc64_vmx_va_op((const uint8_t *)"\x10\x64\x29\x97",
+                        UC_CPU_PPC64_POWER10_V1_0, all_ones, all_ones,
+                        all_ones, expected);
+}
+
+static void test_ppc64_isa300_vmx_multiply_sum_requires_isa300(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    const uint8_t code[] = { 0x10, 0x64, 0x29, 0xa3 };
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER8_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vmx_multiply_sum_carry_requires_isa310(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    const uint8_t code[] = { 0x10, 0x64, 0x29, 0x97 };
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 25;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_dfp_fixqq_roundtrip(const uint8_t src[16])
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    const uint8_t code[] = {
+        0x7c, 0xa0, 0x60, 0xce,
+        0xfc, 0x80, 0x2f, 0xc4,
+        0xfc, 0x61, 0x27, 0xc4,
+        0x7c, 0x60, 0x79, 0xce,
+    };
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 13);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, src, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_dfp_fixqq_roundtrip(void)
+{
+    const uint8_t positive[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
+    };
+    const uint8_t negative[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe,
+    };
+
+    run_ppc64_dfp_fixqq_roundtrip(positive);
+    run_ppc64_dfp_fixqq_roundtrip(negative);
+}
+
+static void test_ppc64_dfp_fixqq_invalid(void)
+{
+    uc_engine *uc;
+    size_t i;
+    uint64_t msr;
+    static const uint8_t code[][4] = {
+        { 0xfc, 0x62, 0x27, 0xc4 },
+        { 0xfc, 0xa0, 0x2f, 0xc4 },
+        { 0xfc, 0x61, 0x2f, 0xc4 },
+        { 0xfc, 0x61, 0x27, 0xc5 },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= (1ull << 25) | (1ull << 13);
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_power10_vmx_divmod_requires_isa310(void)
+{
+    uc_engine *uc;
+    size_t i;
+    uint64_t msr;
+    static const uint8_t code[][4] = {
+        { 0x10, 0x64, 0x28, 0x0b },
+        { 0x10, 0x64, 0x28, 0x8b },
+        { 0x10, 0x64, 0x28, 0xcb },
+        { 0x10, 0x64, 0x29, 0x0b },
+        { 0x10, 0x64, 0x29, 0x8b },
+        { 0x10, 0x64, 0x29, 0xcb },
+        { 0x10, 0x64, 0x2a, 0x0b },
+        { 0x10, 0x64, 0x2a, 0x8b },
+        { 0x10, 0x64, 0x2a, 0xcb },
+        { 0x10, 0x64, 0x2b, 0x0b },
+        { 0x10, 0x64, 0x2b, 0x8b },
+        { 0x10, 0x64, 0x2b, 0xcb },
+        { 0x10, 0x64, 0x2e, 0x0b },
+        { 0x10, 0x64, 0x2e, 0x8b },
+        { 0x10, 0x64, 0x2e, 0xcb },
+        { 0x10, 0x64, 0x2f, 0x0b },
+        { 0x10, 0x64, 0x2f, 0x8b },
+        { 0x10, 0x64, 0x2f, 0xcb },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= 1ull << 25;
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_power10_vmx_quad_requires_isa310(void)
+{
+    uc_engine *uc;
+    size_t i;
+    uint64_t msr;
+    static const uint8_t code[][4] = {
+        { 0x10, 0x64, 0x29, 0x05 },
+        { 0x10, 0x64, 0x28, 0x05 },
+        { 0x10, 0x64, 0x29, 0x16 },
+        { 0x10, 0x64, 0x29, 0xc7 },
+        { 0x11, 0x84, 0x29, 0x01 },
+        { 0x10, 0x64, 0x29, 0xc9 },
+        { 0x10, 0x64, 0x2b, 0x89 },
+        { 0x10, 0x7b, 0x26, 0x02 },
+        { 0x10, 0x64, 0x29, 0x98 },
+        { 0x10, 0x66, 0x3a, 0x0f },
+        { 0x10, 0x66, 0x38, 0xcf },
+        { 0x10, 0x66, 0x38, 0x0f },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= 1ull << 25;
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void run_ppc64_power10_vsx_8rr_3src_op(const char *op,
+                                              const uint8_t a[16],
+                                              const uint8_t b[16],
+                                              const uint8_t c[16],
+                                              const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t c_addr = code_start + 0x1020;
+    uint64_t dst_addr = code_start + 0x1030;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    int i;
+    char code[24] =
+        "\x7c\x80\x60\xce" /* lvx  v4, 0, r12 */
+        "\x7c\xa0\x68\xce" /* lvx  v5, 0, r13 */
+        "\x7c\xc0\x70\xce" /* lvx  v6, 0, r14 */
+        "\0\0\0\0\0\0\0\0"
+        "\x7c\x60\x79\xce"; /* stvx v3, 0, r15 */
+
+    memcpy(code + 12, op, 8);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+    OK(uc_mem_write(uc, c_addr, c, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &c_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(dst[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_power10_vsx_8rr_splat_op(const char *op,
+                                               const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t dst_addr = code_start + 0x1000;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    char code[12] =
+        "\0\0\0\0\0\0\0\0"
+        "\x7c\x60\x79\xce"; /* stvx v3, 0, r15 */
+
+    memcpy(code, op, 8);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static uint8_t ppc_xxeval_byte(uint8_t a, uint8_t b, uint8_t c, uint8_t imm)
+{
+    uint8_t result = 0;
+    int bit;
+
+    for (bit = 0; bit < 8; bit++) {
+        uint8_t index = (((a >> bit) & 1) << 2) |
+                        (((b >> bit) & 1) << 1) |
+                        ((c >> bit) & 1);
+
+        if (imm & (1 << (7 - index))) {
+            result |= 1 << bit;
+        }
+    }
+
+    return result;
+}
+
+static void test_ppc64_power10_xxeval(void)
+{
+    const uint8_t a[16] = {
+        0x00, 0xff, 0x55, 0xaa, 0x0f, 0xf0, 0x33, 0xcc,
+        0x11, 0x22, 0x44, 0x88, 0x7e, 0x81, 0x18, 0xe7,
+    };
+    const uint8_t b[16] = {
+        0xff, 0x00, 0xaa, 0x55, 0xf0, 0x0f, 0xcc, 0x33,
+        0x88, 0x44, 0x22, 0x11, 0x81, 0x7e, 0xe7, 0x18,
+    };
+    const uint8_t c[16] = {
+        0x3c, 0xc3, 0x5a, 0xa5, 0x96, 0x69, 0x0f, 0xf0,
+        0x12, 0x34, 0x56, 0x78, 0xfe, 0xef, 0xdc, 0xcd,
+    };
+    uint8_t expected[16];
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        expected[i] = ppc_xxeval_byte(a[i], b[i], c[i], 0x96);
+    }
+
+    run_ppc64_power10_vsx_8rr_3src_op(
+        "\x05\x00\x00\x96"
+        "\x88\x64\x29\x9f",
+        a, b, c, expected);
+}
+
+static void test_ppc64_power10_xxblendvb(void)
+{
+    const uint8_t a[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t b[16] = {
+        0xff, 0xee, 0xdd, 0xcc, 0xbb, 0xaa, 0x99, 0x88,
+        0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00,
+    };
+    const uint8_t c[16] = {
+        0x00, 0x80, 0x7f, 0xff, 0x01, 0x81, 0x40, 0xc0,
+        0x08, 0x88, 0x70, 0xf0, 0x10, 0x90, 0x20, 0xa0,
+    };
+    uint8_t expected[16];
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        expected[i] = c[i] & 0x80 ? b[i] : a[i];
+    }
+
+    run_ppc64_power10_vsx_8rr_3src_op(
+        "\x05\x00\x00\x00"
+        "\x84\x64\x29\x8f",
+        a, b, c, expected);
+}
+
+static void test_ppc64_power10_xxblendvd(void)
+{
+    const uint8_t a[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+    };
+    const uint8_t b[16] = {
+        0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+    };
+    const uint8_t c[16] = {
+        0x80, 0, 0, 0, 0, 0, 0, 0,
+        0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t expected[16] = {
+        0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+    };
+
+    run_ppc64_power10_vsx_8rr_3src_op(
+        "\x05\x00\x00\x00"
+        "\x84\x64\x29\xbf",
+        a, b, c, expected);
+}
+
+static uint8_t ppc_xxpermx_byte(const uint8_t a[16], const uint8_t b[16],
+                                uint8_t c, uint8_t uim)
+{
+    uint8_t idx;
+
+    if ((c >> 5) != uim) {
+        return 0;
+    }
+
+    idx = c & 0x1f;
+    if (idx < 16) {
+        return a[idx];
+    }
+    return b[idx - 16];
+}
+
+static void test_ppc64_power10_xxpermx(void)
+{
+    const uint8_t a[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t b[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+    };
+    const uint8_t c[16] = {
+        0x20, 0x2f, 0x30, 0x3f, 0x00, 0x40, 0x25, 0x35,
+        0x21, 0x31, 0x22, 0x32, 0x23, 0x33, 0x24, 0x34,
+    };
+    uint8_t expected[16];
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        expected[i] = ppc_xxpermx_byte(a, b, c[i], 1);
+    }
+
+    run_ppc64_power10_vsx_8rr_3src_op(
+        "\x05\x00\x00\x01"
+        "\x88\x64\x29\x8f",
+        a, b, c, expected);
+}
+
+static void test_ppc64_power10_xxspltiw(void)
+{
+    const uint8_t expected[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x11, 0x22, 0x33, 0x44,
+        0x11, 0x22, 0x33, 0x44, 0x11, 0x22, 0x33, 0x44,
+    };
+
+    run_ppc64_power10_vsx_8rr_splat_op(
+        "\x05\x00\x11\x22"
+        "\x80\x67\x33\x44",
+        expected);
+}
+
+static void test_ppc64_power10_xxspltidp(void)
+{
+    const uint8_t expected[16] = {
+        0x3f, 0xf0, 0, 0, 0, 0, 0, 0,
+        0x3f, 0xf0, 0, 0, 0, 0, 0, 0,
+    };
+
+    run_ppc64_power10_vsx_8rr_splat_op(
+        "\x05\x00\x3f\x80"
+        "\x80\x65\x00\x00",
+        expected);
+}
+
+static void test_ppc64_power10_xxsplti32dx(void)
+{
+    uc_engine *uc;
+    uint64_t dst_addr = code_start + 0x1000;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    const uint8_t expected[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+    };
+    const char code[] =
+        "\x05\x00\x11\x22"
+        "\x80\x67\x33\x44"
+        "\x05\x00\x55\x66"
+        "\x80\x63\x77\x88"
+        "\x7c\x60\x79\xce"; /* stvx v3, 0, r15 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_power10_xxgenpcv_op(const uint8_t op[4],
+                                          const uint8_t src[16],
+                                          const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[12] = {
+        0x7c, 0xa0, 0x60, 0xce,
+        0, 0, 0, 0,
+        0x7c, 0x80, 0x79, 0xce,
+    };
+
+    memcpy(&code[4], op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_xxgenpcv(void)
+{
+    const uint8_t src[16] = {
+        0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+        0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+    };
+    const uint8_t be_all[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t le_all[16] = {
+        0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
+        0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+    };
+    const uint8_t sparse[16] = {
+        0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x00, 0x00,
+        0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
+    };
+    const uint8_t be_exp_sparse[16] = {
+        0x00, 0x11, 0x12, 0x01, 0x14, 0x02, 0x16, 0x17,
+        0x18, 0x03, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x04,
+    };
+    const uint8_t be_comp_sparse[16] = {
+        0x00, 0x03, 0x05, 0x09, 0x0f, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t le_exp_sparse[16] = {
+        0x04, 0x1e, 0x1d, 0x03, 0x1b, 0x02, 0x19, 0x18,
+        0x17, 0x01, 0x15, 0x14, 0x13, 0x12, 0x11, 0x00,
+    };
+    const uint8_t le_comp_sparse[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x0f, 0x0c, 0x0a, 0x06, 0x00,
+    };
+
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x80\x2f\x29", src, be_all);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x81\x2f\x29", src, be_all);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x82\x2f\x29", src, le_all);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x83\x2f\x29", src, le_all);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x80\x2f\x2b", src, be_all);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x80\x2f\x69", src, be_all);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x80\x2f\x6b", src, be_all);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x80\x2f\x29", sparse, be_exp_sparse);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x81\x2f\x29", sparse, be_comp_sparse);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x82\x2f\x29", sparse, le_exp_sparse);
+    run_ppc64_power10_xxgenpcv_op(
+        (const uint8_t *)"\xf0\x83\x2f\x29", sparse, le_comp_sparse);
+}
+
+static void run_ppc64_power10_lxvkq_op(const uint8_t op[4],
+                                       const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t dst_addr = code_start + 0x1000;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    uint8_t code[8] = {
+        0, 0, 0, 0,
+        0x7c, 0x80, 0x79, 0xce,
+    };
+
+    memcpy(code, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_lxvkq(void)
+{
+    const uint8_t positive_two[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t negative_zero[16] = {
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t positive_inf[16] = {
+        0x7f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t dquiet_nan[16] = {
+        0x7f, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    run_ppc64_power10_lxvkq_op(
+        (const uint8_t *)"\xf0\x9f\x12\xd1", positive_two);
+    run_ppc64_power10_lxvkq_op(
+        (const uint8_t *)"\xf0\x9f\x82\xd1", negative_zero);
+    run_ppc64_power10_lxvkq_op(
+        (const uint8_t *)"\xf0\x9f\x42\xd1", positive_inf);
+    run_ppc64_power10_lxvkq_op(
+        (const uint8_t *)"\xf0\x9f\x4a\xd1", dquiet_nan);
+}
+
+static void test_ppc64_power10_xxgenpcv_lxvkq_invalid(void)
+{
+    uc_engine *uc;
+    size_t i;
+    uint64_t msr;
+    static const uint8_t code[][4] = {
+        { 0xf0, 0x84, 0x2f, 0x29 },
+        { 0xf0, 0x9f, 0x02, 0xd1 },
+        { 0xf0, 0x9f, 0x52, 0xd1 },
+        { 0xf0, 0x9f, 0xca, 0xd1 },
+        { 0xf0, 0x9e, 0x12, 0xd1 },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= (1ull << 25) | (1ull << 23);
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_power10_xxgenpcv_lxvkq_requires_isa310(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    size_t i;
+    static const uint8_t code[][4] = {
+        { 0xf0, 0x80, 0x2f, 0x29 },
+        { 0xf0, 0x9f, 0x12, 0xd1 },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= (1ull << 25) | (1ull << 23);
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void run_ppc64_power10_xvtlsbb(const uint8_t src[16],
+                                      uint32_t expected)
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t msr;
+    uint32_t cr2 = 0;
+    const char code[] =
+        "\x7c\x80\x60\xce" /* lvx     v4, 0, r12 */
+        "\xf1\x02\x27\x6e"; /* xvtlsbb cr2, vs36 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_CR2, &cr2));
+    TEST_CHECK(cr2 == expected);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_xvtlsbb(void)
+{
+    const uint8_t all_false[16] = { 0 };
+    const uint8_t all_true[16] = {
+        1, 3, 5, 7, 9, 0x0b, 0x0d, 0x0f,
+        0x11, 0x13, 0x15, 0x17, 0x19, 0x1b, 0x1d, 0x1f,
+    };
+    const uint8_t mixed[16] = {
+        0, 1, 2, 3, 4, 5, 6, 7,
+        8, 9, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+
+    run_ppc64_power10_xvtlsbb(all_false, 2);
+    run_ppc64_power10_xvtlsbb(all_true, 8);
+    run_ppc64_power10_xvtlsbb(mixed, 0);
+}
+
+static void ppc64_store_be32(uint8_t dest[4], uint32_t value)
+{
+    dest[0] = value >> 24;
+    dest[1] = value >> 16;
+    dest[2] = value >> 8;
+    dest[3] = value;
+}
+
+static uint64_t ppc64_load_be64(const uint8_t src[8])
+{
+    uint64_t value = 0;
+    int i;
+
+    for (i = 0; i < 8; i++) {
+        value = (value << 8) | src[i];
+    }
+
+    return value;
+}
+
+static uint32_t ppc64_vgnb_opcode(int rt, int vrb, int n)
+{
+    return (0x04u << 26) | ((uint32_t)rt << 21) |
+           ((uint32_t)n << 16) | ((uint32_t)vrb << 11) | 0x4ccu;
+}
+
+static uint32_t ppc64_x_opcode(int rt, int ra, int rb, int opc2, int opc3,
+                               int rc)
+{
+    return (0x1fu << 26) | ((uint32_t)rt << 21) |
+           ((uint32_t)ra << 16) | ((uint32_t)rb << 11) |
+           ((uint32_t)opc3 << 6) | ((uint32_t)opc2 << 1) |
+           (uint32_t)rc;
+}
+
+static uint32_t ppc64_fp_xo4_opcode(int rt, int opc4, int rb)
+{
+    return (0x3fu << 26) | ((uint32_t)rt << 21) |
+           ((uint32_t)opc4 << 16) | ((uint32_t)rb << 11) |
+           (0x12u << 6) | (0x07u << 1);
+}
+
+static uint32_t ppc64_slbiag_opcode(int rs, int l)
+{
+    return ppc64_x_opcode(rs, l, 0, 0x12, 0x1a, 0);
+}
+
+static uint64_t ppc64_vgnb_ref(const uint8_t src[16], int n)
+{
+    static const uint64_t mask[6][5] = {
+        {
+            0xAAAAAAAAAAAAAAAAULL, 0xccccccccccccccccULL,
+            0xf0f0f0f0f0f0f0f0ULL, 0xff00ff00ff00ff00ULL,
+            0xffff0000ffff0000ULL
+        },
+        {
+            0x9249249249249249ULL, 0xC30C30C30C30C30CULL,
+            0xF00F00F00F00F00FULL, 0xFF0000FF0000FF00ULL,
+            0xFFFF00000000FFFFULL
+        },
+        {
+            0x8888888888888888ULL, 0,
+            0xf000f000f000f000ULL, 0, 0xFFFF000000000000ULL
+        },
+        {
+            0x8421084210842108ULL, 0, 0xF0000F0000F0000FULL, 0, 0
+        },
+        {
+            0x8208208208208208ULL, 0, 0xF00000F00000F000ULL, 0, 0
+        },
+        {
+            0x8102040810204081ULL, 0, 0xF000000F000000F0ULL, 0, 0
+        }
+    };
+    uint64_t hi = ppc64_load_be64(src);
+    uint64_t lo = ppc64_load_be64(src + 8);
+    uint64_t m;
+    int i;
+    int nbits = (64 + n - 1) / n;
+    int sh;
+
+    lo <<= n * nbits - 64;
+
+    for (i = 0, sh = n - 1; i < 5; i++, sh <<= 1) {
+        m = mask[n - 2][i];
+        if (m) {
+            hi &= m;
+            lo &= m;
+        }
+        if (sh < 64) {
+            hi = (hi << sh) | hi;
+            lo = (lo << sh) | lo;
+        }
+    }
+
+    m = ~(~0ULL >> nbits);
+    hi &= m;
+    lo &= m;
+    lo >>= nbits;
+
+    return hi | lo;
+}
+
+static void run_ppc64_power10_vgnb(int n, const uint8_t src[16])
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t msr;
+    uint64_t r3;
+    uint32_t op = ppc64_vgnb_opcode(3, 4, n);
+    uint8_t code[8] = {
+        0x7c, 0x80, 0x60, 0xce,
+        0, 0, 0, 0,
+    };
+
+    ppc64_store_be32(code + 4, op);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_3, &r3));
+    TEST_CHECK(r3 == ppc64_vgnb_ref(src, n));
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vgnb(void)
+{
+    const uint8_t src[16] = {
+        0x81, 0x42, 0x24, 0x18, 0xff, 0x00, 0x99, 0x66,
+        0x3c, 0xc3, 0x5a, 0xa5, 0x01, 0x80, 0x7e, 0xe7,
+    };
+
+    run_ppc64_power10_vgnb(2, src);
+    run_ppc64_power10_vgnb(3, src);
+    run_ppc64_power10_vgnb(7, src);
+}
+
+static void test_ppc64_power10_vgnb_undefined_no_change(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    uint64_t r3;
+    uint64_t initial = 0x0123456789abcdefULL;
+    int i;
+
+    for (i = 0; i < 2; i++) {
+        uint32_t op = ppc64_vgnb_opcode(3, 4, i);
+        uint8_t code[4];
+
+        ppc64_store_be32(code, op);
+
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= (1ull << 25) | (1ull << 23);
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+        OK(uc_reg_write(uc, UC_PPC_REG_3, &initial));
+
+        OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+        OK(uc_reg_read(uc, UC_PPC_REG_3, &r3));
+        TEST_CHECK(r3 == initial);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_power10_vgnb_invalid(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    uint32_t op = ppc64_vgnb_opcode(3, 4, 2) | 0x00080000u;
+    uint8_t code[4];
+
+    ppc64_store_be32(code, op);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vgnb_requires_isa310(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    uint32_t op = ppc64_vgnb_opcode(3, 4, 2);
+    uint8_t code[4];
+
+    ppc64_store_be32(code, op);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static uint64_t ppc64_addg6s_ref(uint64_t a, uint64_t b)
+{
+    uint64_t carry = 0;
+    uint64_t t = 0;
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        t += (a >> (i * 4)) & 0xf;
+        t += (b >> (i * 4)) & 0xf;
+        t = (t & 0x10) != 0;
+        carry |= t << (i * 4);
+    }
+
+    carry ^= 0x1111111111111111ULL;
+    return carry * 6;
+}
+
+static void test_ppc64_isa206_bcd_addg6s(void)
+{
+    uc_engine *uc;
+    uint64_t r3 = 0x0901090109010901ULL;
+    uint64_t r4 = 0x0109010901090109ULL;
+    uint64_t r5;
+    uint64_t r6;
+    uint32_t op0 = ppc64_x_opcode(5, 3, 4, 0x0a, 0x02, 0);
+    uint32_t op1 = ppc64_x_opcode(6, 3, 4, 0x0a, 0x12, 0);
+    uint8_t code[8];
+
+    ppc64_store_be32(code, op0);
+    ppc64_store_be32(code + 4, op1);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER7_V2_3));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_reg_write(uc, UC_PPC_REG_3, &r3));
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_5, &r5));
+    OK(uc_reg_read(uc, UC_PPC_REG_6, &r6));
+    TEST_CHECK(r5 == ppc64_addg6s_ref(r3, r4));
+    TEST_CHECK(r6 == ppc64_addg6s_ref(r3, r4));
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_isa206_bcd_convert(void)
+{
+    uc_engine *uc;
+    uint64_t r3;
+    uint64_t r4 = 0x0000000100000001ULL;
+    uint64_t r5;
+    uint64_t r6;
+    uint64_t r7 = 0x0009876500012345ULL;
+    uint32_t cdtbcd_r3_r4 = ppc64_x_opcode(4, 3, 0, 0x1a, 0x08, 0);
+    uint32_t cbcdtd_r5_r7 = ppc64_x_opcode(7, 5, 0, 0x1a, 0x09, 0);
+    uint32_t cdtbcd_r6_r5 = ppc64_x_opcode(5, 6, 0, 0x1a, 0x08, 0);
+    uint8_t code[12];
+
+    ppc64_store_be32(code, cdtbcd_r3_r4);
+    ppc64_store_be32(code + 4, cbcdtd_r5_r7);
+    ppc64_store_be32(code + 8, cdtbcd_r6_r5);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER7_V2_3));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &r4));
+    OK(uc_reg_write(uc, UC_PPC_REG_7, &r7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_3, &r3));
+    OK(uc_reg_read(uc, UC_PPC_REG_5, &r5));
+    OK(uc_reg_read(uc, UC_PPC_REG_6, &r6));
+    TEST_CHECK(r3 == r4);
+    TEST_CHECK(r5 != 0);
+    TEST_CHECK(r6 == r7);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_isa206_bcd_requires_bcda(void)
+{
+    uc_engine *uc;
+    uint32_t code[] = {
+        ppc64_x_opcode(5, 3, 4, 0x0a, 0x02, 0),
+        ppc64_x_opcode(4, 3, 0, 0x1a, 0x08, 0),
+        ppc64_x_opcode(4, 3, 0, 0x1a, 0x09, 0),
+    };
+    uint8_t insn[4];
+    size_t i;
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        ppc64_store_be32(insn, code[i]);
+
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER5_V2_1));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, insn, sizeof(insn)));
+
+        TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(insn),
+                                0, 0) == UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_isa300_slbiag(void)
+{
+    uc_engine *uc;
+    uint64_t r3 = 0x1000000000000000ull;
+    uint64_t r4 = 0x2000000000000000ull;
+    uint64_t pc = 0;
+    uint32_t code[] = {
+        ppc64_slbiag_opcode(3, 0),
+        ppc64_slbiag_opcode(4, 1),
+    };
+    uint8_t insn[8];
+
+    ppc64_store_be32(insn, code[0]);
+    ppc64_store_be32(insn + 4, code[1]);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, insn, sizeof(insn)));
+    OK(uc_reg_write(uc, UC_PPC_REG_3, &r3));
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(insn), 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_PC, &pc));
+    TEST_CHECK(pc == code_start + sizeof(insn));
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_isa300_slbiag_exceptions(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    uint32_t insns[] = {
+        ppc64_slbiag_opcode(3, 0),
+        ppc64_slbiag_opcode(3, 0),
+        ppc64_slbiag_opcode(3, 0) | 0x00020000u,
+        ppc64_slbiag_opcode(3, 0) | 0x00000800u,
+    };
+    uc_cpu_ppc64 models[] = {
+        UC_CPU_PPC64_POWER8_V2_0,
+        UC_CPU_PPC64_POWER9_V2_0,
+        UC_CPU_PPC64_POWER9_V2_0,
+        UC_CPU_PPC64_POWER9_V2_0,
+    };
+    bool pr[] = { false, true, false, false };
+    uint8_t code[4];
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        ppc64_store_be32(code, insns[i]);
+
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, models[i]));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+        if (pr[i]) {
+            OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+            msr |= 1ull << 14;
+            OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+        }
+
+        TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code),
+                                0, 0) == UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_isa300_mffscdrn(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    uint32_t fpscr = 0xff;
+    uint64_t f4;
+    uint64_t f5;
+    uint64_t f6;
+    uint64_t f7;
+    uint64_t f8 = 5ull << 32;
+    uint64_t mode = fpscr;
+    uint32_t code[] = {
+        ppc64_fp_xo4_opcode(4, 0x14, 8),
+        ppc64_fp_xo4_opcode(5, 0x15, 2),
+        ppc64_fp_xo4_opcode(6, 0x00, 0),
+        ppc64_fp_xo4_opcode(7, 0x18, 0),
+    };
+    uint8_t insn[16];
+    size_t i;
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        ppc64_store_be32(insn + i * 4, code[i]);
+    }
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, insn, sizeof(insn)));
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 13;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_FPSCR, &fpscr));
+    OK(uc_reg_write(uc, UC_PPC_REG_FPR8, &f8));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(insn), 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_FPR4, &f4));
+    OK(uc_reg_read(uc, UC_PPC_REG_FPR5, &f5));
+    OK(uc_reg_read(uc, UC_PPC_REG_FPR6, &f6));
+    OK(uc_reg_read(uc, UC_PPC_REG_FPR7, &f7));
+
+    TEST_CHECK(f4 == mode);
+    TEST_CHECK(f5 == ((5ull << 32) | mode));
+    TEST_CHECK(f6 == ((2ull << 32) | mode));
+    TEST_CHECK(f7 == ((2ull << 32) | mode));
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_isa300_mffscdrn_requires_isa300(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    uint32_t code[] = {
+        ppc64_fp_xo4_opcode(4, 0x01, 0),
+        ppc64_fp_xo4_opcode(4, 0x18, 0),
+        ppc64_fp_xo4_opcode(4, 0x14, 8),
+        ppc64_fp_xo4_opcode(5, 0x15, 2),
+        ppc64_fp_xo4_opcode(4, 0x16, 8),
+        ppc64_fp_xo4_opcode(5, 0x17, 2),
+    };
+    uint8_t insn[4];
+    size_t i;
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        ppc64_store_be32(insn, code[i]);
+
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER8_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, insn, sizeof(insn)));
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= 1ull << 13;
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(insn),
+                                0, 0) == UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_power10_vsx_bf16_convert(void)
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1010;
+    uint64_t msr;
+    const uint8_t src[16] = {
+        0x3f, 0x80, 0x00, 0x00,
+        0xc0, 0x00, 0x00, 0x00,
+        0x40, 0x40, 0x00, 0x00,
+        0x3f, 0x00, 0x00, 0x00,
+    };
+    uint8_t dst[16] = { 0 };
+    const char code[] =
+        "\x7c\xa0\x60\xce" /* lvx          v5,0,r12 */
+        "\xf0\x91\x2f\x6f" /* xvcvspbf16  vs36,vs37 */
+        "\xf0\xb0\x27\x6f" /* xvcvbf16spn vs37,vs36 */
+        "\x7c\xa0\x69\xce"; /* stvx         v5,0,r13 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, src_addr, src, sizeof(src)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, src, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_mma_xxsetaccz(void)
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint8_t dst[64] = { 0xff };
+    uint8_t expected[64] = { 0 };
+    const uint8_t src[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10,
+    };
+    const char code[] =
+        "\x7c\x00\x66\x98" /* lxvd2x    vs0,0,r12 */
+        "\x7c\x20\x6e\x98" /* lxvd2x    vs1,0,r13 */
+        "\x7c\x40\x76\x98" /* lxvd2x    vs2,0,r14 */
+        "\x7c\x60\x7e\x98" /* lxvd2x    vs3,0,r15 */
+        "\x7c\x03\x01\x62" /* xxsetaccz acc0 */
+        "\x7c\x00\x87\x98" /* stxvd2x   vs0,0,r16 */
+        "\x7c\x20\x8f\x98" /* stxvd2x   vs1,0,r17 */
+        "\x7c\x40\x97\x98" /* stxvd2x   vs2,0,r18 */
+        "\x7c\x60\x9f\x98"; /* stxvd2x   vs3,0,r19 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, src_addr, src, sizeof(src)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_18, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_19, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, code_start + 0x1100, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_mma_acc_moves(void)
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint8_t dst[64] = { 0 };
+    uint8_t src[64];
+    size_t i;
+    const char code[] =
+        "\x7c\x00\x66\x98" /* lxvd2x  vs0,0,r12 */
+        "\x7c\x20\x6e\x98" /* lxvd2x  vs1,0,r13 */
+        "\x7c\x40\x76\x98" /* lxvd2x  vs2,0,r14 */
+        "\x7c\x60\x7e\x98" /* lxvd2x  vs3,0,r15 */
+        "\x7c\x00\x01\x62" /* xxmfacc acc0 */
+        "\x7c\x01\x01\x62" /* xxmtacc acc0 */
+        "\x7c\x00\x87\x98" /* stxvd2x vs0,0,r16 */
+        "\x7c\x20\x8f\x98" /* stxvd2x vs1,0,r17 */
+        "\x7c\x40\x97\x98" /* stxvd2x vs2,0,r18 */
+        "\x7c\x60\x9f\x98"; /* stxvd2x vs3,0,r19 */
+
+    for (i = 0; i < sizeof(src); i++) {
+        src[i] = (uint8_t)(0x13 + i * 9);
+    }
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, src_addr, src, sizeof(src)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    src_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &src_addr));
+    src_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &src_addr));
+    src_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_18, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_19, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, code_start + 0x1100, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, src, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_power10_mma_ger_op(const char *op,
+                                         const uint8_t a[16],
+                                         const uint8_t b[16],
+                                         const uint8_t expected[64])
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint8_t dst[64] = { 0 };
+    char code[28] =
+        "\x7c\xa0\x60\xce" /* lvx  v5,0,r12 */
+        "\x7c\xc0\x68\xce" /* lvx  v6,0,r13 */
+        "\0\0\0\0"
+        "\x7c\x00\x77\x98" /* stxvd2x vs0,0,r14 */
+        "\x7c\x20\x7f\x98" /* stxvd2x vs1,0,r15 */
+        "\x7c\x40\x87\x98" /* stxvd2x vs2,0,r16 */
+        "\x7c\x60\x8f\x98"; /* stxvd2x vs3,0,r17 */
+
+    memcpy(code + 8, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, code_start + 0x1100, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_mma_integer_ger(void)
+{
+    const uint8_t a[16] = {
+        0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+        0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+    };
+    const uint8_t b[16] = {
+        0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+        0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+    };
+    const uint8_t expected[64] = {
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08,
+    };
+
+    run_ppc64_power10_mma_ger_op("\xec\x05\x30\x1e", a, b, expected);
+}
+
+static void test_ppc64_power10_mma_f32_ger(void)
+{
+    const uint8_t a[16] = {
+        0x3f, 0x80, 0x00, 0x00, 0x3f, 0x80, 0x00, 0x00,
+        0x3f, 0x80, 0x00, 0x00, 0x3f, 0x80, 0x00, 0x00,
+    };
+    const uint8_t b[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected[64] = {
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+    };
+
+    run_ppc64_power10_mma_ger_op("\xec\x05\x30\xde", a, b, expected);
+}
+
+static void test_ppc64_power10_mma_prefixed_f32_ger(void)
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t dst_addr = code_start + 0x1100;
+    uint64_t msr;
+    uint8_t dst[64] = { 0 };
+    const uint8_t a[16] = {
+        0x3f, 0x80, 0x00, 0x00, 0x3f, 0x80, 0x00, 0x00,
+        0x3f, 0x80, 0x00, 0x00, 0x3f, 0x80, 0x00, 0x00,
+    };
+    const uint8_t b[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected[64] = {
+        0x40, 0x00, 0x00, 0x00, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const char code[] =
+        "\x7c\xa0\x60\xce" /* lvx      v5,0,r12 */
+        "\x7c\xc0\x68\xce" /* lvx      v6,0,r13 */
+        "\x07\x90\x00\x88" /* pmx prefix, xmsk=8, ymsk=8 */
+        "\xec\x05\x30\xde" /* pmxvf32ger acc0,vs37,vs38 */
+        "\x7c\x00\x77\x98" /* stxvd2x vs0,0,r14 */
+        "\x7c\x20\x7f\x98" /* stxvd2x vs1,0,r15 */
+        "\x7c\x40\x87\x98" /* stxvd2x vs2,0,r16 */
+        "\x7c\x60\x8f\x98"; /* stxvd2x vs3,0,r17 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &dst_addr));
+    dst_addr += 16;
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, code_start + 0x1100, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_mma_requires_isa310(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    static const uint8_t code[][4] = {
+        { 0x7c, 0x00, 0x01, 0x62 },
+        { 0x7c, 0x01, 0x01, 0x62 },
+        { 0x7c, 0x03, 0x01, 0x62 },
+        { 0xec, 0x05, 0x30, 0x1e },
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= (1ull << 25) | (1ull << 23);
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void run_ppc64_vsx_qp_madd_op(const char *op,
+                                     const uint8_t addend[16],
+                                     const uint8_t a[16],
+                                     const uint8_t b[16],
+                                     const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t addend_addr = code_start + 0x1000;
+    uint64_t a_addr = code_start + 0x1010;
+    uint64_t b_addr = code_start + 0x1020;
+    uint64_t dst_addr = code_start + 0x1030;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    char code[20] =
+        "\x7c\x80\x60\xce" /* lvx  v4,0,r12 */
+        "\x7c\xa0\x68\xce" /* lvx  v5,0,r13 */
+        "\x7c\xc0\x70\xce" /* lvx  v6,0,r14 */
+        "\0\0\0\0"
+        "\x7c\x80\x79\xce"; /* stvx v4,0,r15 */
+
+    memcpy(code + 12, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, addend_addr, addend, 16));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &addend_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_vsx_qp_xform_op(const char *op,
+                                      const uint8_t a[16],
+                                      const uint8_t b[16],
+                                      const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint64_t a_addr = code_start + 0x1000;
+    uint64_t b_addr = code_start + 0x1010;
+    uint64_t dst_addr = code_start + 0x1020;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    char code[16] =
+        "\x7c\xa0\x60\xce" /* lvx  v5,0,r12 */
+        "\x7c\xc0\x68\xce" /* lvx  v6,0,r13 */
+        "\0\0\0\0"
+        "\x7c\x80\x71\xce"; /* stvx v4,0,r14 */
+
+    memcpy(code + 8, op, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, a_addr, a, 16));
+    OK(uc_mem_write(uc, b_addr, b, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &a_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void run_ppc64_vsx_qp_convert_roundtrip(const char *to_qp,
+                                               const char *from_qp,
+                                               const uint8_t src[16])
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t dst_addr = code_start + 0x1010;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    char code[16] =
+        "\x7c\xa0\x60\xce" /* lvx  v5,0,r12 */
+        "\0\0\0\0"
+        "\0\0\0\0"
+        "\x7c\xc0\x69\xce"; /* stvx v6,0,r13 */
+
+    memcpy(code + 4, to_qp, 4);
+    memcpy(code + 8, from_qp, 4);
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+    OK(uc_mem_write(uc, src_addr, src, 16));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+    TEST_CHECK(memcmp(dst, src, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_vsx_qp_multiply_add(void)
+{
+    const uint8_t qp_zero[16] = { 0 };
+    const uint8_t qp_one[16] = {
+        0x3f, 0xff, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t qp_two[16] = {
+        0x40, 0x00, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t qp_negative_two[16] = {
+        0xc0, 0x00, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    run_ppc64_vsx_qp_madd_op("\xfc\x85\x33\x08", qp_zero, qp_one,
+                             qp_two, qp_two);
+    run_ppc64_vsx_qp_madd_op("\xfc\x85\x33\x09", qp_zero, qp_one,
+                             qp_two, qp_two);
+    run_ppc64_vsx_qp_madd_op("\xfc\x85\x33\x48", qp_zero, qp_one,
+                             qp_two, qp_two);
+    run_ppc64_vsx_qp_madd_op("\xfc\x85\x33\x88", qp_zero, qp_one,
+                             qp_two, qp_negative_two);
+    run_ppc64_vsx_qp_madd_op("\xfc\x85\x33\xc8", qp_zero, qp_one,
+                             qp_two, qp_negative_two);
+}
+
+static void test_ppc64_power10_vsx_qp_compare_minmax(void)
+{
+    const uint8_t qp_one[16] = {
+        0x3f, 0xff, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t qp_two[16] = {
+        0x40, 0x00, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t false_mask[16] = { 0 };
+    const uint8_t true_mask[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+
+    run_ppc64_vsx_qp_xform_op("\xfc\x85\x30\x88", qp_one, qp_one,
+                              true_mask);
+    run_ppc64_vsx_qp_xform_op("\xfc\x85\x31\x88", qp_two, qp_one,
+                              true_mask);
+    run_ppc64_vsx_qp_xform_op("\xfc\x85\x31\xc8", qp_one, qp_two,
+                              false_mask);
+    run_ppc64_vsx_qp_xform_op("\xfc\x85\x35\x48", qp_one, qp_two,
+                              qp_two);
+    run_ppc64_vsx_qp_xform_op("\xfc\x85\x35\xc8", qp_one, qp_two,
+                              qp_one);
+}
+
+static void test_ppc64_power10_vsx_qp_convert(void)
+{
+    const uint8_t unsigned_two[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 2,
+    };
+    const uint8_t signed_negative_two[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe,
+    };
+
+    run_ppc64_vsx_qp_convert_roundtrip("\xfc\x83\x2e\x88",
+                                       "\xfc\xc0\x26\x88",
+                                       unsigned_two);
+    run_ppc64_vsx_qp_convert_roundtrip("\xfc\x8b\x2e\x88",
+                                       "\xfc\xc8\x26\x88",
+                                       signed_negative_two);
+}
+
+static void test_ppc64_vsx_qp_multiply_add_requires_isa300(void)
+{
+    uc_engine *uc;
+    uint64_t msr;
+    const char code[] = "\xfc\x85\x33\x08";
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER8_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_vsx_requires_isa310(void)
+{
+    uc_engine *uc;
+    uint64_t src_addr = code_start + 0x1000;
+    uint64_t msr;
+    const uint8_t zero[16] = { 0 };
+    static const char code[][8] = {
+        {
+            "\x7c\x80\x60\xce"
+            "\xf1\x02\x27\x6e"
+        },
+        {
+            "\x7c\xa0\x60\xce"
+            "\xf0\x91\x2f\x6f"
+        },
+        {
+            "\x7c\x80\x60\xce"
+            "\xf0\x90\x27\x6f"
+        },
+        {
+            "\x7c\x80\x60\xce"
+            "\xfc\x85\x30\x88"
+        },
+        {
+            "\x7c\x80\x60\xce"
+            "\xfc\x80\x26\x88"
+        },
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+        OK(uc_mem_write(uc, src_addr, zero, 16));
+
+        OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+        msr |= (1ull << 25) | (1ull << 23);
+        OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+        OK(uc_reg_write(uc, UC_PPC_REG_12, &src_addr));
+
+        TEST_CHECK(uc_emu_start(uc, code_start,
+                                code_start + sizeof(code[i]), 0, 0) ==
+                   UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void setup_ppc64_hash(uc_engine **uc, const char *code, size_t size)
+{
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, uc));
+    OK(uc_ctl_set_cpu_model(*uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(*uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(*uc, code_start, code, size));
+}
+
+static void test_ppc64_power10_hash_store_check(void)
+{
+    uc_engine *uc;
+    uint64_t base = code_start + 0x1008;
+    uint64_t rb = 0x0123456789abcdefull;
+    uint8_t stored[8] = { 0 };
+    const uint8_t expected[8] = {
+        0xdc, 0x02, 0xcd, 0xe3, 0xd6, 0x91, 0x1f, 0xba,
+    };
+    const char code[] =
+        "\x7f\xe4\x2d\xa5" /* hashst  -8(r4), r5 */
+        "\x7f\xe4\x2d\xe5"; /* hashchk -8(r4), r5 */
+
+    setup_ppc64_hash(&uc, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &base));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &rb));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, base - 8, stored, sizeof(stored)));
+    TEST_CHECK(memcmp(stored, expected, sizeof(stored)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_hash_key_spr(void)
+{
+    uc_engine *uc;
+    uint64_t base = code_start + 0x1008;
+    uint64_t rb = 0x0123456789abcdefull;
+    uint64_t key = 0x1122334455667788ull;
+    uint8_t stored[8] = { 0 };
+    const uint8_t expected[8] = {
+        0x96, 0xbf, 0xa7, 0x47, 0xe0, 0x45, 0x85, 0xe2,
+    };
+    const char code[] =
+        "\x7c\xd4\x73\xa6" /* mtspr HASHKEYR, r6 */
+        "\x7f\xe4\x2d\xa5"; /* hashst -8(r4), r5 */
+
+    setup_ppc64_hash(&uc, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &base));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &rb));
+    OK(uc_reg_write(uc, UC_PPC_REG_6, &key));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, base - 8, stored, sizeof(stored)));
+    TEST_CHECK(memcmp(stored, expected, sizeof(stored)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_hash_check_mismatch(void)
+{
+    uc_engine *uc;
+    uint64_t base = code_start + 0x1008;
+    uint64_t rb = 0x1123456789abcdefull;
+    const uint8_t stored[8] = {
+        0xdc, 0x02, 0xcd, 0xe3, 0xd6, 0x91, 0x1f, 0xba,
+    };
+    const char code[] = "\x7f\xe4\x2d\xe5"; /* hashchk -8(r4), r5 */
+
+    setup_ppc64_hash(&uc, code, sizeof(code) - 1);
+    OK(uc_mem_write(uc, base - 8, stored, sizeof(stored)));
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &base));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &rb));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_hash_privileged(void)
+{
+    uc_engine *uc;
+    uint64_t base = code_start + 0x1008;
+    uint64_t rb = 0x0123456789abcdefull;
+    uint8_t stored[8] = { 0 };
+    const uint8_t expected[8] = {
+        0xdc, 0x02, 0xcd, 0xe3, 0xd6, 0x91, 0x1f, 0xba,
+    };
+    const char code[] =
+        "\x7f\xe4\x2d\x25" /* hashstp  -8(r4), r5 */
+        "\x7f\xe4\x2d\x65"; /* hashchkp -8(r4), r5 */
+
+    setup_ppc64_hash(&uc, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &base));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &rb));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, base - 8, stored, sizeof(stored)));
+    TEST_CHECK(memcmp(stored, expected, sizeof(stored)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_hash_privileged_requires_sv(void)
+{
+    uc_engine *uc;
+    uint64_t base = code_start + 0x1008;
+    uint64_t rb = 0x0123456789abcdefull;
+    uint64_t msr;
+    const char code[] = "\x7f\xe4\x2d\x25"; /* hashstp -8(r4), r5 */
+
+    setup_ppc64_hash(&uc, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_PPC_REG_4, &base));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &rb));
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 14;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_hash_ra0_invalid(void)
+{
+    uc_engine *uc;
+    uint64_t rb = 0x0123456789abcdefull;
+    const char code[] = "\x7f\xe0\x2d\xa5"; /* hashst -8(0), r5 */
+
+    setup_ppc64_hash(&uc, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &rb));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power9_hash_nop(void)
+{
+    uc_engine *uc;
+    uint64_t rb = 0x0123456789abcdefull;
+    uint64_t pc = 0;
+    const char code[] = "\x7f\xe0\x2d\xa5"; /* hashst -8(0), r5 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_PPC_REG_5, &rb));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_PC, &pc));
+    TEST_CHECK(pc == code_start + sizeof(code) - 1);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x20000;
+    uint64_t r5, r6, r7, r8;
+    uint64_t r9 = 0x1122334455667788ull;
+    uint64_t r10 = 0xfedcba9876543210ull;
+    uint8_t store_w[4] = { 0 };
+    uint8_t store_d[8] = { 0 };
+    const uint8_t pc_relative_src[4] = { 0xa5, 0xb6, 0xc7, 0xd8 };
+    const uint8_t src[16] = {
+        0x89, 0xab, 0xcd, 0xef,
+        0x80, 0x00, 0x00, 0x01,
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+    };
+    const char code[] =
+        "\x06\x10\x00\x00\x80\xa0\x00\x80" /* plwz r5,0x80(0),1 */
+        "\x06\x00\x00\x02\x80\xc0\x00\x00" /* plwz r6,0x20000(0) */
+        "\x04\x00\x00\x02\xa4\xe0\x00\x04" /* plwa r7,0x20004(0) */
+        "\x04\x00\x00\x02\xe5\x00\x00\x08" /* pld  r8,0x20008(0) */
+        "\x06\x00\x00\x02\x91\x20\x00\x10" /* pstw r9,0x20010(0) */
+        "\x04\x00\x00\x02\xf5\x40\x00\x18"; /* pstd r10,0x20018(0) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, code_start + 0x80, pc_relative_src,
+                    sizeof(pc_relative_src)));
+    OK(uc_mem_write(uc, data, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_PPC_REG_9, &r9));
+    OK(uc_reg_write(uc, UC_PPC_REG_10, &r10));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_5, &r5));
+    OK(uc_reg_read(uc, UC_PPC_REG_6, &r6));
+    OK(uc_reg_read(uc, UC_PPC_REG_7, &r7));
+    OK(uc_reg_read(uc, UC_PPC_REG_8, &r8));
+    OK(uc_mem_read(uc, data + 0x10, store_w, sizeof(store_w)));
+    OK(uc_mem_read(uc, data + 0x18, store_d, sizeof(store_d)));
+
+    TEST_CHECK(r5 == 0xa5b6c7d8);
+    TEST_CHECK(r6 == 0x89abcdef);
+    TEST_CHECK(r7 == 0xffffffff80000001ull);
+    TEST_CHECK(r8 == 0x0123456789abcdefull);
+    TEST_CHECK(memcmp(store_w, "\x55\x66\x77\x88", sizeof(store_w)) == 0);
+    TEST_CHECK(memcmp(store_d, "\xfe\xdc\xba\x98\x76\x54\x32\x10",
+                      sizeof(store_d)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_byte_half_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x40000;
+    uint64_t r21, r22, r23, r24;
+    uint64_t r25 = 0x1122334455667788ull;
+    uint64_t r26 = 0xaabbccddeeff0099ull;
+    uint8_t store_b[1] = { 0 };
+    uint8_t store_h[2] = { 0 };
+    const uint8_t pc_relative_src[1] = { 0xee };
+    const uint8_t src[8] = {
+        0x7f, 0x00, 0x80, 0x01,
+        0x80, 0x02, 0x00, 0x00,
+    };
+    const char code[] =
+        "\x06\x00\x00\x04\x8a\xa0\x00\x00" /* plbz r21,0x40000(0) */
+        "\x06\x00\x00\x04\xa2\xc0\x00\x02" /* plhz r22,0x40002(0) */
+        "\x06\x00\x00\x04\xaa\xe0\x00\x04" /* plha r23,0x40004(0) */
+        "\x06\x10\x00\x00\x8b\x00\x00\x88" /* plbz r24,0x88(0),1 */
+        "\x06\x00\x00\x04\x9b\x20\x00\x10" /* pstb r25,0x40010(0) */
+        "\x06\x00\x00\x04\xb3\x40\x00\x12"; /* psth r26,0x40012(0) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, code_start + 0xa0, pc_relative_src,
+                    sizeof(pc_relative_src)));
+    OK(uc_mem_write(uc, data, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_PPC_REG_25, &r25));
+    OK(uc_reg_write(uc, UC_PPC_REG_26, &r26));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_21, &r21));
+    OK(uc_reg_read(uc, UC_PPC_REG_22, &r22));
+    OK(uc_reg_read(uc, UC_PPC_REG_23, &r23));
+    OK(uc_reg_read(uc, UC_PPC_REG_24, &r24));
+    OK(uc_mem_read(uc, data + 0x10, store_b, sizeof(store_b)));
+    OK(uc_mem_read(uc, data + 0x12, store_h, sizeof(store_h)));
+
+    TEST_CHECK(r21 == 0x7f);
+    TEST_CHECK(r22 == 0x8001);
+    TEST_CHECK(r23 == 0xffffffffffff8002ull);
+    TEST_CHECK(r24 == 0xee);
+    TEST_CHECK(store_b[0] == 0x88);
+    TEST_CHECK(memcmp(store_h, "\x00\x99", sizeof(store_h)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_quad_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x50000;
+    uint64_t r12, r13;
+    uint64_t r14 = 0x1122334455667788ull;
+    uint64_t r15 = 0x99aabbccddeeff00ull;
+    uint8_t stored[16] = { 0 };
+    const uint8_t src[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+    };
+    const uint8_t expected[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+    };
+    const char code[] =
+        "\x04\x00\x00\x05\xe1\x80\x00\x00" /* plq  r12,0x50000(0) */
+        "\x04\x00\x00\x05\xf1\xc0\x00\x10"; /* pstq r14,0x50010(0) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, data, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &r14));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &r15));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_12, &r12));
+    OK(uc_reg_read(uc, UC_PPC_REG_13, &r13));
+    OK(uc_mem_read(uc, data + 0x10, stored, sizeof(stored)));
+
+    TEST_CHECK(r12 == 0x0123456789abcdefull);
+    TEST_CHECK(r13 == 0xfedcba9876543210ull);
+    TEST_CHECK(memcmp(stored, expected, sizeof(stored)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_quad_ra_rt_invalid(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x50000;
+    const char code[] =
+        "\x04\x00\x00\x05\xe2\x10\x00\x00"; /* plq r16,0x50000(r16) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &data));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_float_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x60000;
+    uint64_t msr;
+    uint64_t f4, f5, f8;
+    uint64_t f6 = 0x3ff8000000000000ull;
+    uint64_t f7 = 0xc010000000000000ull;
+    uint8_t store_s[4] = { 0 };
+    uint8_t store_d[8] = { 0 };
+    const uint8_t pc_relative_src[4] = { 0x40, 0x00, 0x00, 0x00 };
+    const uint8_t src[16] = {
+        0x3f, 0x80, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00,
+    };
+    const char code[] =
+        "\x06\x00\x00\x06\xc0\x80\x00\x00" /* plfs  f4,0x60000(0) */
+        "\x06\x00\x00\x06\xc8\xa0\x00\x08" /* plfd  f5,0x60008(0) */
+        "\x06\x00\x00\x06\xd0\xc0\x00\x10" /* pstfs f6,0x60010(0) */
+        "\x06\x00\x00\x06\xd8\xe0\x00\x18" /* pstfd f7,0x60018(0) */
+        "\x06\x10\x00\x00\xc1\x00\x00\x88"; /* plfs  f8,0x88(0),1 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, code_start + 0xa8, pc_relative_src,
+                    sizeof(pc_relative_src)));
+    OK(uc_mem_write(uc, data, src, sizeof(src)));
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= 1ull << 13;
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_FPR6, &f6));
+    OK(uc_reg_write(uc, UC_PPC_REG_FPR7, &f7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_FPR4, &f4));
+    OK(uc_reg_read(uc, UC_PPC_REG_FPR5, &f5));
+    OK(uc_reg_read(uc, UC_PPC_REG_FPR8, &f8));
+    OK(uc_mem_read(uc, data + 0x10, store_s, sizeof(store_s)));
+    OK(uc_mem_read(uc, data + 0x18, store_d, sizeof(store_d)));
+
+    TEST_CHECK(f4 == 0x3ff0000000000000ull);
+    TEST_CHECK(f5 == 0xc000000000000000ull);
+    TEST_CHECK(f8 == 0x4000000000000000ull);
+    TEST_CHECK(memcmp(store_s, "\x3f\xc0\x00\x00", sizeof(store_s)) == 0);
+    TEST_CHECK(memcmp(store_d, "\xc0\x10\x00\x00\x00\x00\x00\x00",
+                      sizeof(store_d)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_vsx_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x70000;
+    uint64_t src2_addr = data + 0x30;
+    uint64_t dst1_addr = data + 0x40;
+    uint64_t msr;
+    uint8_t dst1[16] = { 0 };
+    uint8_t dst2[16] = { 0 };
+    const uint8_t src1[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t src2[16] = {
+        0xff, 0xee, 0xdd, 0xcc, 0xbb, 0xaa, 0x99, 0x88,
+        0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00,
+    };
+    const char code[] =
+        "\x04\x00\x00\x07\xcc\x80\x00\x00" /* plxv  vs36,0x70000(0) */
+        "\x7c\x80\x79\xce"                 /* stvx  v4,0,r15 */
+        "\x7c\xa0\x70\xce"                 /* lvx   v5,0,r14 */
+        "\x04\x00\x00\x07\xdc\xa0\x00\x20"; /* pstxv vs37,0x70020(0) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, data, src1, sizeof(src1)));
+    OK(uc_mem_write(uc, src2_addr, src2, sizeof(src2)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &src2_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst1_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst1_addr, dst1, sizeof(dst1)));
+    OK(uc_mem_read(uc, data + 0x20, dst2, sizeof(dst2)));
+
+    TEST_CHECK(memcmp(dst1, src1, sizeof(dst1)) == 0);
+    TEST_CHECK(memcmp(dst2, src2, sizeof(dst2)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_isa300_lxvwsx(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0xd0000;
+    uint64_t dst_addr = data + 0x100;
+    uint64_t msr;
+    uint8_t dst[16] = { 0 };
+    const uint8_t src[4] = { 0x12, 0x34, 0x56, 0x78 };
+    const uint8_t expected[16] = {
+        0x12, 0x34, 0x56, 0x78, 0x12, 0x34, 0x56, 0x78,
+        0x12, 0x34, 0x56, 0x78, 0x12, 0x34, 0x56, 0x78,
+    };
+    const char code[] =
+        "\x7c\x80\x62\xd9" /* lxvwsx vs36,0,r12 */
+        "\x7c\x80\x69\xce"; /* stvx   v4,0,r13 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, data, src, sizeof(src)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &data));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &dst_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_addr, dst, sizeof(dst)));
+
+    TEST_CHECK(memcmp(dst, expected, sizeof(dst)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_isa300_lxvwsx_requires_isa300(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0xd0000;
+    uint64_t msr;
+    const char code[] = "\x7c\x80\x62\xd9";
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER8_V2_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &data));
+
+    TEST_CHECK(uc_emu_start(uc, code_start,
+                            code_start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_vsx_scalar_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x80000;
+    uint64_t src_xsd_addr = data + 0x30;
+    uint64_t src_xssp_addr = data + 0x40;
+    uint64_t dst_xsd_addr = data + 0x50;
+    uint64_t dst_xssp_addr = data + 0x60;
+    uint64_t msr;
+    uint8_t dst_xsd[16] = { 0 };
+    uint8_t dst_xssp[16] = { 0 };
+    uint8_t store_xsd[8] = { 0 };
+    uint8_t store_xssp[4] = { 0 };
+    const uint8_t src_load[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t src_xsd[16] = {
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+        0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00, 0x11,
+    };
+    const uint8_t src_xssp[16] = {
+        0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc,
+    };
+    const uint8_t scalar_xsd_expected[16] = {
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t scalar_xssp_expected[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const char code[] =
+        "\x04\x00\x00\x08\xa8\x80\x00\x00" /* plxsd   vs36,0x80000(0) */
+        "\x7c\x80\x79\xce"                 /* stvx    v4,0,r15 */
+        "\x04\x00\x00\x08\xac\xa0\x00\x08" /* plxssp  vs37,0x80008(0) */
+        "\x7c\xa0\x89\xce"                 /* stvx    v5,0,r17 */
+        "\x7c\xc0\x70\xce"                 /* lvx     v6,0,r14 */
+        "\x04\x00\x00\x08\xb8\xc0\x00\x20" /* pstxsd  vs38,0x80020(0) */
+        "\x7c\xe0\x98\xce"                 /* lvx     v7,0,r19 */
+        "\x04\x00\x00\x08\xbc\xe0\x00\x28"; /* pstxssp vs39,0x80028(0) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, data, src_load, sizeof(src_load)));
+    OK(uc_mem_write(uc, src_xsd_addr, src_xsd, sizeof(src_xsd)));
+    OK(uc_mem_write(uc, src_xssp_addr, src_xssp, sizeof(src_xssp)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &src_xsd_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_xsd_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &dst_xssp_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_19, &src_xssp_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_xsd_addr, dst_xsd, sizeof(dst_xsd)));
+    OK(uc_mem_read(uc, dst_xssp_addr, dst_xssp, sizeof(dst_xssp)));
+    OK(uc_mem_read(uc, data + 0x20, store_xsd, sizeof(store_xsd)));
+    OK(uc_mem_read(uc, data + 0x28, store_xssp, sizeof(store_xssp)));
+
+    TEST_CHECK(memcmp(dst_xsd, scalar_xsd_expected, sizeof(dst_xsd)) == 0);
+    TEST_CHECK(memcmp(dst_xssp, scalar_xssp_expected,
+                      sizeof(dst_xssp)) == 0);
+    TEST_CHECK(memcmp(store_xsd, src_xsd, sizeof(store_xsd)) == 0);
+    TEST_CHECK(memcmp(store_xssp, "\x3f\xc0\x00\x00",
+                      sizeof(store_xssp)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_xvp_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x90000;
+    uint64_t src_v6_addr = data + 0x80;
+    uint64_t src_v7_addr = data + 0x90;
+    uint64_t dst_v4_addr = data + 0xa0;
+    uint64_t dst_v5_addr = data + 0xb0;
+    uint64_t msr;
+    uint8_t dst_v4[16] = { 0 };
+    uint8_t dst_v5[16] = { 0 };
+    uint8_t store_pair[32] = { 0 };
+    const uint8_t load_pair[32] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+        0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+        0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+    };
+    const uint8_t store_v6[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    const uint8_t store_v7[16] = {
+        0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+        0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+    };
+    const uint8_t store_expected[32] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+        0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+        0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+    };
+    const char code[] =
+        "\x18\xb2\x00\x00" /* lxvp  vs36,0(r18) */
+        "\x7c\x80\x79\xce" /* stvx  v4,0,r15 */
+        "\x7c\xa0\x81\xce" /* stvx  v5,0,r16 */
+        "\x7c\xc0\x70\xce" /* lvx   v6,0,r14 */
+        "\x7c\xe0\x88\xce" /* lvx   v7,0,r17 */
+        "\x18\xf2\x00\x41"; /* stxvp vs38,0x40(r18) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, data, load_pair, sizeof(load_pair)));
+    OK(uc_mem_write(uc, src_v6_addr, store_v6, sizeof(store_v6)));
+    OK(uc_mem_write(uc, src_v7_addr, store_v7, sizeof(store_v7)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &src_v6_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_v4_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &dst_v5_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &src_v7_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_18, &data));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_v4_addr, dst_v4, sizeof(dst_v4)));
+    OK(uc_mem_read(uc, dst_v5_addr, dst_v5, sizeof(dst_v5)));
+    OK(uc_mem_read(uc, data + 0x40, store_pair, sizeof(store_pair)));
+
+    TEST_CHECK(memcmp(dst_v4, load_pair, sizeof(dst_v4)) == 0);
+    TEST_CHECK(memcmp(dst_v5, load_pair + 16, sizeof(dst_v5)) == 0);
+    TEST_CHECK(memcmp(store_pair, store_expected, sizeof(store_pair)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_xvpx_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0xa0000;
+    uint64_t load_addr = data;
+    uint64_t src_v10_addr = data + 0x80;
+    uint64_t src_v11_addr = data + 0x90;
+    uint64_t store_addr = data + 0x40;
+    uint64_t dst_v8_addr = data + 0xa0;
+    uint64_t dst_v9_addr = data + 0xb0;
+    uint64_t msr;
+    uint8_t dst_v8[16] = { 0 };
+    uint8_t dst_v9[16] = { 0 };
+    uint8_t store_pair[32] = { 0 };
+    const uint8_t load_pair[32] = {
+        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+        0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+        0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+        0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
+    };
+    const uint8_t store_v10[16] = {
+        0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+        0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+    };
+    const uint8_t store_v11[16] = {
+        0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+        0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+    };
+    const uint8_t store_expected[32] = {
+        0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+        0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+        0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+        0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+    };
+    const char code[] =
+        "\x7d\x20\x62\x9a" /* lxvpx  vs40,0,r12 */
+        "\x7d\x00\x69\xce" /* stvx   v8,0,r13 */
+        "\x7d\x20\x79\xce" /* stvx   v9,0,r15 */
+        "\x7d\x40\x70\xce" /* lvx    v10,0,r14 */
+        "\x7d\x60\x88\xce" /* lvx    v11,0,r17 */
+        "\x7d\x60\x83\x9a"; /* stxvpx vs42,0,r16 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, load_addr, load_pair, sizeof(load_pair)));
+    OK(uc_mem_write(uc, src_v10_addr, store_v10, sizeof(store_v10)));
+    OK(uc_mem_write(uc, src_v11_addr, store_v11, sizeof(store_v11)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &load_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &dst_v8_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &src_v10_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_v9_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &store_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &src_v11_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_v8_addr, dst_v8, sizeof(dst_v8)));
+    OK(uc_mem_read(uc, dst_v9_addr, dst_v9, sizeof(dst_v9)));
+    OK(uc_mem_read(uc, store_addr, store_pair, sizeof(store_pair)));
+
+    TEST_CHECK(memcmp(dst_v8, load_pair, sizeof(dst_v8)) == 0);
+    TEST_CHECK(memcmp(dst_v9, load_pair + 16, sizeof(dst_v9)) == 0);
+    TEST_CHECK(memcmp(store_pair, store_expected, sizeof(store_pair)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_lxvr_stxvr(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0xc0000;
+    uint64_t load_b_addr = data;
+    uint64_t load_h_addr = data + 0x10;
+    uint64_t load_w_addr = data + 0x20;
+    uint64_t load_d_addr = data + 0x30;
+    uint64_t store_b_addr = data + 0x40;
+    uint64_t store_h_addr = data + 0x42;
+    uint64_t store_w_addr = data + 0x44;
+    uint64_t store_d_addr = data + 0x48;
+    uint64_t seed_b_addr = data + 0x200;
+    uint64_t seed_h_addr = data + 0x210;
+    uint64_t seed_w_addr = data + 0x220;
+    uint64_t seed_d_addr = data + 0x230;
+    uint64_t dst_b_addr = data + 0x100;
+    uint64_t dst_h_addr = data + 0x110;
+    uint64_t dst_w_addr = data + 0x120;
+    uint64_t dst_d_addr = data + 0x130;
+    uint64_t msr;
+    uint8_t dst_b[16] = { 0 };
+    uint8_t dst_h[16] = { 0 };
+    uint8_t dst_w[16] = { 0 };
+    uint8_t dst_d[16] = { 0 };
+    uint8_t store_b[1] = { 0 };
+    uint8_t store_h[2] = { 0 };
+    uint8_t store_w[4] = { 0 };
+    uint8_t store_d[8] = { 0 };
+    const uint8_t load_b[1] = { 0x11 };
+    const uint8_t load_h[2] = { 0x22, 0x33 };
+    const uint8_t load_w[4] = { 0x44, 0x55, 0x66, 0x77 };
+    const uint8_t load_d[8] = {
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const uint8_t seed_b[16] = {
+        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+        0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xfe,
+    };
+    const uint8_t seed_h[16] = {
+        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0xab, 0xcd,
+    };
+    const uint8_t seed_w[16] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0x0e, 0x0f, 0x10, 0x11, 0x12, 0x34, 0x56, 0x78,
+    };
+    const uint8_t seed_d[16] = {
+        0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+        0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe, 0xba, 0xbe,
+    };
+    const uint8_t expected_b[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x11,
+    };
+    const uint8_t expected_h[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x22, 0x33,
+    };
+    const uint8_t expected_w[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x44, 0x55, 0x66, 0x77,
+    };
+    const uint8_t expected_d[16] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    const char code[] =
+        "\x7c\x80\x60\x1b" /* lxvrbx  vs36,0,r12 */
+        "\x7c\x80\xa1\xce" /* stvx    v4,0,r20 */
+        "\x7c\xa0\x68\x5b" /* lxvrhx  vs37,0,r13 */
+        "\x7c\xa0\xa9\xce" /* stvx    v5,0,r21 */
+        "\x7c\xc0\x70\x9b" /* lxvrwx  vs38,0,r14 */
+        "\x7c\xc0\xb1\xce" /* stvx    v6,0,r22 */
+        "\x7c\xe0\x78\xdb" /* lxvrdx  vs39,0,r15 */
+        "\x7c\xe0\xb9\xce" /* stvx    v7,0,r23 */
+        "\x7d\x00\x80\xce" /* lvx     v8,0,r16 */
+        "\x7d\x00\xc1\x1b" /* stxvrbx vs40,0,r24 */
+        "\x7d\x20\x88\xce" /* lvx     v9,0,r17 */
+        "\x7d\x20\xc9\x5b" /* stxvrhx vs41,0,r25 */
+        "\x7d\x40\x90\xce" /* lvx     v10,0,r18 */
+        "\x7d\x40\xd1\x9b" /* stxvrwx vs42,0,r26 */
+        "\x7d\x60\x98\xce" /* lvx     v11,0,r19 */
+        "\x7d\x60\xd9\xdb"; /* stxvrdx vs43,0,r27 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, load_b_addr, load_b, sizeof(load_b)));
+    OK(uc_mem_write(uc, load_h_addr, load_h, sizeof(load_h)));
+    OK(uc_mem_write(uc, load_w_addr, load_w, sizeof(load_w)));
+    OK(uc_mem_write(uc, load_d_addr, load_d, sizeof(load_d)));
+    OK(uc_mem_write(uc, seed_b_addr, seed_b, sizeof(seed_b)));
+    OK(uc_mem_write(uc, seed_h_addr, seed_h, sizeof(seed_h)));
+    OK(uc_mem_write(uc, seed_w_addr, seed_w, sizeof(seed_w)));
+    OK(uc_mem_write(uc, seed_d_addr, seed_d, sizeof(seed_d)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &load_b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_13, &load_h_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &load_w_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &load_d_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &seed_b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &seed_h_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_18, &seed_w_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_19, &seed_d_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_20, &dst_b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_21, &dst_h_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_22, &dst_w_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_23, &dst_d_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_24, &store_b_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_25, &store_h_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_26, &store_w_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_27, &store_d_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_b_addr, dst_b, sizeof(dst_b)));
+    OK(uc_mem_read(uc, dst_h_addr, dst_h, sizeof(dst_h)));
+    OK(uc_mem_read(uc, dst_w_addr, dst_w, sizeof(dst_w)));
+    OK(uc_mem_read(uc, dst_d_addr, dst_d, sizeof(dst_d)));
+    OK(uc_mem_read(uc, store_b_addr, store_b, sizeof(store_b)));
+    OK(uc_mem_read(uc, store_h_addr, store_h, sizeof(store_h)));
+    OK(uc_mem_read(uc, store_w_addr, store_w, sizeof(store_w)));
+    OK(uc_mem_read(uc, store_d_addr, store_d, sizeof(store_d)));
+
+    TEST_CHECK(memcmp(dst_b, expected_b, sizeof(dst_b)) == 0);
+    TEST_CHECK(memcmp(dst_h, expected_h, sizeof(dst_h)) == 0);
+    TEST_CHECK(memcmp(dst_w, expected_w, sizeof(dst_w)) == 0);
+    TEST_CHECK(memcmp(dst_d, expected_d, sizeof(dst_d)) == 0);
+    TEST_CHECK(store_b[0] == 0xfe);
+    TEST_CHECK(memcmp(store_h, "\xab\xcd", sizeof(store_h)) == 0);
+    TEST_CHECK(memcmp(store_w, "\x12\x34\x56\x78", sizeof(store_w)) == 0);
+    TEST_CHECK(memcmp(store_d, "\xde\xad\xbe\xef\xca\xfe\xba\xbe",
+                      sizeof(store_d)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_vsx_pair_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0xb0000;
+    uint64_t src_v6_addr = data + 0x80;
+    uint64_t src_v7_addr = data + 0x90;
+    uint64_t dst_v4_addr = data + 0xa0;
+    uint64_t dst_v5_addr = data + 0xb0;
+    uint64_t msr;
+    uint8_t dst_v4[16] = { 0 };
+    uint8_t dst_v5[16] = { 0 };
+    uint8_t store_pair[32] = { 0 };
+    const uint8_t load_pair[32] = {
+        0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
+        0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+        0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+        0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+    };
+    const uint8_t store_v6[16] = {
+        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+        0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+    };
+    const uint8_t store_v7[16] = {
+        0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+        0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
+    };
+    const uint8_t store_expected[32] = {
+        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+        0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+        0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+        0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
+    };
+    const char code[] =
+        "\x04\x00\x00\x0b\xe8\xa0\x00\x00" /* plxvp  vs36,0xb0000(0) */
+        "\x7c\x80\x79\xce"                 /* stvx   v4,0,r15 */
+        "\x7c\xa0\x81\xce"                 /* stvx   v5,0,r16 */
+        "\x7c\xc0\x70\xce"                 /* lvx    v6,0,r14 */
+        "\x7c\xe0\x88\xce"                 /* lvx    v7,0,r17 */
+        "\x04\x00\x00\x0b\xf8\xe0\x00\x40"; /* pstxvp vs38,0xb0040(0) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, data, load_pair, sizeof(load_pair)));
+    OK(uc_mem_write(uc, src_v6_addr, store_v6, sizeof(store_v6)));
+    OK(uc_mem_write(uc, src_v7_addr, store_v7, sizeof(store_v7)));
+
+    OK(uc_reg_read(uc, UC_PPC_REG_MSR, &msr));
+    msr |= (1ull << 25) | (1ull << 23);
+    OK(uc_reg_write(uc, UC_PPC_REG_MSR, &msr));
+    OK(uc_reg_write(uc, UC_PPC_REG_14, &src_v6_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &dst_v4_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_16, &dst_v5_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_17, &src_v7_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_mem_read(uc, dst_v4_addr, dst_v4, sizeof(dst_v4)));
+    OK(uc_mem_read(uc, dst_v5_addr, dst_v5, sizeof(dst_v5)));
+    OK(uc_mem_read(uc, data + 0x40, store_pair, sizeof(store_pair)));
+
+    TEST_CHECK(memcmp(dst_v4, load_pair, sizeof(dst_v4)) == 0);
+    TEST_CHECK(memcmp(dst_v5, load_pair + 16, sizeof(dst_v5)) == 0);
+    TEST_CHECK(memcmp(store_pair, store_expected, sizeof(store_pair)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_negative_load_store(void)
+{
+    uc_engine *uc;
+    uint64_t data = 0x30000;
+    uint64_t base = data + 0x100;
+    uint64_t r16, r17, r18;
+    uint64_t r19 = 0xaabbccddeeff0011ull;
+    uint64_t r20 = 0x0123456789abcdefull;
+    uint8_t store_w[4] = { 0 };
+    uint8_t store_d[8] = { 0 };
+    const uint8_t src[16] = {
+        0x01, 0x23, 0x45, 0x67,
+        0x80, 0x00, 0x00, 0x01,
+        0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+    };
+    const char code[] =
+        "\x06\x03\xff\xff\x82\x0f\xff\x00" /* plwz r16,-0x100(r15) */
+        "\x04\x03\xff\xff\xa6\x2f\xff\x04" /* plwa r17,-0xfc(r15) */
+        "\x04\x03\xff\xff\xe6\x4f\xff\x08" /* pld  r18,-0xf8(r15) */
+        "\x06\x03\xff\xff\x92\x6f\xff\x20" /* pstw r19,-0xe0(r15) */
+        "\x04\x03\xff\xff\xf6\x8f\xff\x28"; /* pstd r20,-0xd8(r15) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_map(uc, data, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, data, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_PPC_REG_15, &base));
+    OK(uc_reg_write(uc, UC_PPC_REG_19, &r19));
+    OK(uc_reg_write(uc, UC_PPC_REG_20, &r20));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_16, &r16));
+    OK(uc_reg_read(uc, UC_PPC_REG_17, &r17));
+    OK(uc_reg_read(uc, UC_PPC_REG_18, &r18));
+    OK(uc_mem_read(uc, data + 0x20, store_w, sizeof(store_w)));
+    OK(uc_mem_read(uc, data + 0x28, store_d, sizeof(store_d)));
+
+    TEST_CHECK(r16 == 0x01234567);
+    TEST_CHECK(r17 == 0xffffffff80000001ull);
+    TEST_CHECK(r18 == 0x1122334455667788ull);
+    TEST_CHECK(memcmp(store_w, "\xee\xff\x00\x11", sizeof(store_w)) == 0);
+    TEST_CHECK(memcmp(store_d, "\x01\x23\x45\x67\x89\xab\xcd\xef",
+                      sizeof(store_d)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_paddi(void)
+{
+    uc_engine *uc;
+    uint64_t r11, r12, r13, r14, r15;
+    const char code[] =
+        "\x06\x00\x00\x01\x39\x6c\x23\x45" /* paddi r11,r12,0x12345 */
+        "\x06\x00\x01\x23\x39\xa0\x45\x67" /* pli   r13,0x1234567 */
+        "\x06\x10\x00\x00\x39\xc0\x00\x20" /* paddi r14,0,0x20,1 */
+        "\x06\x03\xff\xff\x39\xe0\xff\xff"; /* pli   r15,-1 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    r12 = 0x100000000ull;
+    OK(uc_reg_write(uc, UC_PPC_REG_12, &r12));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_11, &r11));
+    OK(uc_reg_read(uc, UC_PPC_REG_13, &r13));
+    OK(uc_reg_read(uc, UC_PPC_REG_14, &r14));
+    OK(uc_reg_read(uc, UC_PPC_REG_15, &r15));
+
+    TEST_CHECK(r11 == 0x100012345ull);
+    TEST_CHECK(r13 == 0x1234567);
+    TEST_CHECK(r14 == code_start + 0x10 + 0x20);
+    TEST_CHECK(r15 == 0xffffffffffffffffull);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_r_requires_ra0(void)
+{
+    uc_engine *uc;
+    uint64_t r1 = 1;
+    uint64_t pc = 0;
+    const char code[] =
+        "\x06\x10\x00\x00\x39\xc1\x00\x20"; /* paddi r14,r1,0x20,1 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_PPC_REG_1, &r1));
+
+    TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                            0, 0) == UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_PPC_REG_PC, &pc));
+    TEST_CHECK(pc == code_start + sizeof(code) - 1);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_pnop(void)
+{
+    uc_engine *uc;
+    uint64_t r3 = 0;
+    uint64_t r4 = 0;
+    uint64_t pc = 0;
+    const char code[] =
+        "\x07\x00\x00\x00\x38\x80\x00\x09" /* pnop, valid addi suffix */
+        "\x38\x60\x00\x07"; /* addi r3,0,7 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_PPC_REG_3, &r3));
+    OK(uc_reg_read(uc, UC_PPC_REG_4, &r4));
+    OK(uc_reg_read(uc, UC_PPC_REG_PC, &pc));
+
+    TEST_CHECK(r3 == 7);
+    TEST_CHECK(r4 == 0);
+    TEST_CHECK(pc == code_start + sizeof(code) - 1);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_pnop_invalid_suffix(void)
+{
+    uc_engine *uc;
+    size_t i;
+    static const uint8_t suffixes[][4] = {
+        { 0x40, 0x00, 0x00, 0x00 }, /* bc */
+        { 0x48, 0x00, 0x00, 0x00 }, /* b */
+        { 0x44, 0x00, 0x00, 0x02 }, /* sc */
+        { 0x44, 0x00, 0x00, 0x01 }, /* scv */
+        { 0x4c, 0x00, 0x00, 0x20 }, /* bclr */
+        { 0x4c, 0x00, 0x04, 0x20 }, /* bcctr */
+        { 0x4c, 0x00, 0x04, 0x60 }, /* bctar */
+        { 0x4c, 0x00, 0x01, 0x24 }, /* rfebb */
+        { 0x4c, 0x00, 0x00, 0xa4 }, /* rfscv */
+        { 0x4c, 0x00, 0x00, 0x24 }, /* rfid */
+        { 0x4c, 0x00, 0x02, 0x24 }, /* hrfid */
+        { 0x4c, 0x00, 0x02, 0x64 }, /* urfid */
+        { 0x4c, 0x00, 0x02, 0xe4 }, /* stop */
+        { 0x7c, 0x00, 0x01, 0x24 }, /* mtmsr L=0 */
+        { 0x7c, 0x00, 0x01, 0x64 }, /* mtmsrd L=0 */
+        { 0x00, 0x00, 0x02, 0x00 }, /* attn */
+    };
+    uint8_t code[8] = { 0x07, 0x00, 0x00, 0x00 };
+
+    for (i = 0; i < sizeof(suffixes) / sizeof(suffixes[0]); i++) {
+        memcpy(&code[4], suffixes[i], sizeof(suffixes[i]));
+
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code, sizeof(code)));
+
+        TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code),
+                                0, 0) == UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_power10_prefixed_boundary(void)
+{
+    uc_engine *uc;
+    uint64_t pc = 0;
+    uint64_t start = code_start + 0x3c;
+    const char code[] =
+        "\x06\x00\x00\x02\x80\xc0\x00\x00"; /* plwz r6,0x20000(0) */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, start, code, sizeof(code) - 1));
+
+    TEST_CHECK(uc_emu_start(uc, start, start + sizeof(code) - 1, 0, 0) ==
+               UC_ERR_EXCEPTION);
+    OK(uc_reg_read(uc, UC_PPC_REG_PC, &pc));
+    TEST_CHECK(pc == start + sizeof(code) - 1);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power10_prefixed_hook_size(void)
+{
+    uc_engine *uc;
+    uc_hook hook;
+    PpcCodeHookTrace trace = { 0 };
+    const char code[] =
+        "\x07\x00\x00\x00\x60\x00\x00\x00" /* pnop */
+        "\x38\x60\x00\x07"; /* addi r3,0,7 */
+
+    OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER10_V1_0));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_hook_add(uc, &hook, UC_HOOK_CODE, test_ppc64_prefixed_code_hook,
+                   &trace, 1, 0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    TEST_CHECK(trace.count == 2);
+    TEST_CHECK(trace.address[0] == code_start);
+    TEST_CHECK(trace.size[0] == 8);
+    TEST_CHECK(trace.address[1] == code_start + 8);
+    TEST_CHECK(trace.size[1] == 4);
+
+    OK(uc_close(uc));
+}
+
+static void test_ppc64_power9_prefixed_rejected(void)
+{
+    uc_engine *uc;
+    size_t i;
+    static const uint8_t code[][8] = {
+        {
+            0x06, 0x00, 0x00, 0x02,
+            0x80, 0xc0, 0x00, 0x00,
+        },
+        {
+            0x06, 0x00, 0x00, 0x04,
+            0x8a, 0xa0, 0x00, 0x00,
+        },
+        {
+            0x04, 0x00, 0x00, 0x05,
+            0xe1, 0x80, 0x00, 0x00,
+        },
+        {
+            0x06, 0x00, 0x00, 0x06,
+            0xc0, 0x80, 0x00, 0x00,
+        },
+        {
+            0x04, 0x00, 0x00, 0x07,
+            0xcc, 0x80, 0x00, 0x00,
+        },
+        {
+            0x04, 0x00, 0x00, 0x08,
+            0xa8, 0x80, 0x00, 0x00,
+        },
+        {
+            0x04, 0x00, 0x00, 0x09,
+            0xe8, 0xa0, 0x00, 0x00,
+        },
+        {
+            0x07, 0x00, 0x00, 0x00,
+            0x60, 0x00, 0x00, 0x00,
+        },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code[i]),
+                                0, 0) == UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_ppc64_power9_xvp_rejected(void)
+{
+    uc_engine *uc;
+    size_t i;
+    static const uint8_t code[][4] = {
+        {
+            0x18, 0xa0, 0x00, 0x00,
+        },
+        {
+            0x7d, 0x20, 0x62, 0x9a,
+        },
+        {
+            0x7c, 0x80, 0x60, 0x1b,
+        },
+        {
+            0xf0, 0x80, 0x2f, 0x29,
+        },
+        {
+            0xf0, 0x9f, 0x12, 0xd1,
+        },
+    };
+
+    for (i = 0; i < sizeof(code) / sizeof(code[0]); i++) {
+        OK(uc_open(UC_ARCH_PPC, UC_MODE_64 | UC_MODE_BIG_ENDIAN, &uc));
+        OK(uc_ctl_set_cpu_model(uc, UC_CPU_PPC64_POWER9_V2_0));
+        OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+        OK(uc_mem_write(uc, code_start, code[i], sizeof(code[i])));
+
+        TEST_CHECK(uc_emu_start(uc, code_start, code_start + sizeof(code[i]),
+                                0, 0) == UC_ERR_EXCEPTION);
+
+        OK(uc_close(uc));
+    }
+}
+
 TEST_LIST = {{"test_ppc32_add", test_ppc32_add},
              {"test_ppc32_fadd", test_ppc32_fadd},
              {"test_ppc32_sc", test_ppc32_sc},
              {"test_ppc32_cr", test_ppc32_cr},
              {"test_ppc32_spr_time", test_ppc32_spr_time},
              {"test_ppc32_spr_mftb", test_ppc32_spr_mftb},
+             {"test_ppc64_power10_brd", test_ppc64_power10_brd},
+             {"test_ppc64_power10_brw", test_ppc64_power10_brw},
+             {"test_ppc64_power10_brh", test_ppc64_power10_brh},
+             {"test_ppc64_power10_cfuged", test_ppc64_power10_cfuged},
+             {"test_ppc64_power10_cntlzdm", test_ppc64_power10_cntlzdm},
+             {"test_ppc64_power10_cnttzdm", test_ppc64_power10_cnttzdm},
+             {"test_ppc64_power10_pdepd", test_ppc64_power10_pdepd},
+             {"test_ppc64_power10_pextd", test_ppc64_power10_pextd},
+             {"test_ppc64_power10_mask_op_requires_isa310",
+              test_ppc64_power10_mask_op_requires_isa310},
+             {"test_ppc64_power10_setbc", test_ppc64_power10_setbc},
+             {"test_ppc64_power10_setbc_requires_isa310",
+              test_ppc64_power10_setbc_requires_isa310},
+             {"test_ppc64_power10_vcfuged", test_ppc64_power10_vcfuged},
+             {"test_ppc64_power10_vclzdm", test_ppc64_power10_vclzdm},
+             {"test_ppc64_power10_vctzdm", test_ppc64_power10_vctzdm},
+             {"test_ppc64_power10_vpdepd", test_ppc64_power10_vpdepd},
+             {"test_ppc64_power10_vpextd", test_ppc64_power10_vpextd},
+             {"test_ppc64_power10_vmx_mask_materialize_extract",
+              test_ppc64_power10_vmx_mask_materialize_extract},
+             {"test_ppc64_power10_vector_mask_requires_isa310",
+              test_ppc64_power10_vector_mask_requires_isa310},
+             {"test_ppc64_power10_vmx_quad_shift_rotate",
+              test_ppc64_power10_vmx_quad_shift_rotate},
+             {"test_ppc64_power10_vmx_doubleword_immediate_shift",
+              test_ppc64_power10_vmx_doubleword_immediate_shift},
+             {"test_ppc64_power10_vmx_quad_compare",
+              test_ppc64_power10_vmx_quad_compare},
+             {"test_ppc64_power10_vmx_quad_compare_invalid",
+              test_ppc64_power10_vmx_quad_compare_invalid},
+             {"test_ppc64_power10_vmx_multiply_dword",
+              test_ppc64_power10_vmx_multiply_dword},
+             {"test_ppc64_power10_vmx_multiply_high",
+              test_ppc64_power10_vmx_multiply_high},
+             {"test_ppc64_power10_vmx_vextsd2q",
+              test_ppc64_power10_vmx_vextsd2q},
+             {"test_ppc64_power10_vmx_extract_double",
+             test_ppc64_power10_vmx_extract_double},
+             {"test_ppc64_power10_vmx_insert_gpr",
+              test_ppc64_power10_vmx_insert_gpr},
+             {"test_ppc64_power10_vmx_insert_vector",
+              test_ppc64_power10_vmx_insert_vector},
+             {"test_ppc64_power10_vmx_divmod_word",
+              test_ppc64_power10_vmx_divmod_word},
+             {"test_ppc64_power10_vmx_divmod_dword",
+              test_ppc64_power10_vmx_divmod_dword},
+             {"test_ppc64_power10_vmx_divide_extended",
+             test_ppc64_power10_vmx_divide_extended},
+             {"test_ppc64_power10_vmx_divmod_quad",
+              test_ppc64_power10_vmx_divmod_quad},
+             {"test_ppc64_power10_vmx_string_isolate",
+              test_ppc64_power10_vmx_string_isolate},
+             {"test_ppc64_power10_vmx_clear_bytes",
+              test_ppc64_power10_vmx_clear_bytes},
+             {"test_ppc64_power10_vmx_string_clear_legacy_buckets",
+             test_ppc64_power10_vmx_string_clear_legacy_buckets},
+             {"test_ppc64_power10_vmx_string_clear_requires_isa310",
+              test_ppc64_power10_vmx_string_clear_requires_isa310},
+             {"test_ppc64_isa300_vmx_multiply_sum_dword",
+             test_ppc64_isa300_vmx_multiply_sum_dword},
+             {"test_ppc64_power10_vmx_multiply_sum_carry_dword",
+              test_ppc64_power10_vmx_multiply_sum_carry_dword},
+             {"test_ppc64_isa300_vmx_multiply_sum_requires_isa300",
+              test_ppc64_isa300_vmx_multiply_sum_requires_isa300},
+             {"test_ppc64_power10_vmx_multiply_sum_carry_requires_isa310",
+              test_ppc64_power10_vmx_multiply_sum_carry_requires_isa310},
+             {"test_ppc64_dfp_fixqq_roundtrip",
+              test_ppc64_dfp_fixqq_roundtrip},
+             {"test_ppc64_dfp_fixqq_invalid",
+              test_ppc64_dfp_fixqq_invalid},
+             {"test_ppc64_power10_vmx_divmod_requires_isa310",
+              test_ppc64_power10_vmx_divmod_requires_isa310},
+             {"test_ppc64_power10_vmx_quad_requires_isa310",
+              test_ppc64_power10_vmx_quad_requires_isa310},
+             {"test_ppc64_power10_xxeval", test_ppc64_power10_xxeval},
+             {"test_ppc64_power10_xxblendvb", test_ppc64_power10_xxblendvb},
+             {"test_ppc64_power10_xxblendvd", test_ppc64_power10_xxblendvd},
+             {"test_ppc64_power10_xxpermx", test_ppc64_power10_xxpermx},
+             {"test_ppc64_power10_xxspltiw", test_ppc64_power10_xxspltiw},
+             {"test_ppc64_power10_xxspltidp", test_ppc64_power10_xxspltidp},
+             {"test_ppc64_power10_xxsplti32dx",
+              test_ppc64_power10_xxsplti32dx},
+             {"test_ppc64_power10_xxgenpcv", test_ppc64_power10_xxgenpcv},
+             {"test_ppc64_power10_lxvkq", test_ppc64_power10_lxvkq},
+             {"test_ppc64_power10_xxgenpcv_lxvkq_invalid",
+              test_ppc64_power10_xxgenpcv_lxvkq_invalid},
+             {"test_ppc64_power10_xxgenpcv_lxvkq_requires_isa310",
+              test_ppc64_power10_xxgenpcv_lxvkq_requires_isa310},
+             {"test_ppc64_power10_xvtlsbb", test_ppc64_power10_xvtlsbb},
+             {"test_ppc64_power10_vgnb", test_ppc64_power10_vgnb},
+             {"test_ppc64_power10_vgnb_undefined_no_change",
+              test_ppc64_power10_vgnb_undefined_no_change},
+             {"test_ppc64_power10_vgnb_invalid",
+              test_ppc64_power10_vgnb_invalid},
+             {"test_ppc64_power10_vgnb_requires_isa310",
+              test_ppc64_power10_vgnb_requires_isa310},
+             {"test_ppc64_isa206_bcd_addg6s",
+              test_ppc64_isa206_bcd_addg6s},
+             {"test_ppc64_isa206_bcd_convert",
+              test_ppc64_isa206_bcd_convert},
+             {"test_ppc64_isa206_bcd_requires_bcda",
+              test_ppc64_isa206_bcd_requires_bcda},
+             {"test_ppc64_isa300_slbiag",
+              test_ppc64_isa300_slbiag},
+             {"test_ppc64_isa300_slbiag_exceptions",
+              test_ppc64_isa300_slbiag_exceptions},
+             {"test_ppc64_isa300_mffscdrn",
+              test_ppc64_isa300_mffscdrn},
+             {"test_ppc64_isa300_mffscdrn_requires_isa300",
+              test_ppc64_isa300_mffscdrn_requires_isa300},
+             {"test_ppc64_power10_vsx_bf16_convert",
+              test_ppc64_power10_vsx_bf16_convert},
+             {"test_ppc64_power10_mma_xxsetaccz",
+              test_ppc64_power10_mma_xxsetaccz},
+             {"test_ppc64_power10_mma_acc_moves",
+              test_ppc64_power10_mma_acc_moves},
+             {"test_ppc64_power10_mma_integer_ger",
+              test_ppc64_power10_mma_integer_ger},
+             {"test_ppc64_power10_mma_f32_ger",
+              test_ppc64_power10_mma_f32_ger},
+             {"test_ppc64_power10_mma_prefixed_f32_ger",
+              test_ppc64_power10_mma_prefixed_f32_ger},
+             {"test_ppc64_power10_mma_requires_isa310",
+              test_ppc64_power10_mma_requires_isa310},
+             {"test_ppc64_vsx_qp_multiply_add",
+              test_ppc64_vsx_qp_multiply_add},
+             {"test_ppc64_power10_vsx_qp_compare_minmax",
+              test_ppc64_power10_vsx_qp_compare_minmax},
+             {"test_ppc64_power10_vsx_qp_convert",
+              test_ppc64_power10_vsx_qp_convert},
+             {"test_ppc64_vsx_qp_multiply_add_requires_isa300",
+              test_ppc64_vsx_qp_multiply_add_requires_isa300},
+             {"test_ppc64_power10_vsx_requires_isa310",
+              test_ppc64_power10_vsx_requires_isa310},
+             {"test_ppc64_power10_xvp_load_store",
+              test_ppc64_power10_xvp_load_store},
+             {"test_ppc64_power10_xvpx_load_store",
+              test_ppc64_power10_xvpx_load_store},
+             {"test_ppc64_power10_lxvr_stxvr",
+              test_ppc64_power10_lxvr_stxvr},
+             {"test_ppc64_power10_hash_store_check",
+              test_ppc64_power10_hash_store_check},
+             {"test_ppc64_power10_hash_key_spr",
+              test_ppc64_power10_hash_key_spr},
+             {"test_ppc64_power10_hash_check_mismatch",
+              test_ppc64_power10_hash_check_mismatch},
+             {"test_ppc64_power10_hash_privileged",
+              test_ppc64_power10_hash_privileged},
+             {"test_ppc64_power10_hash_privileged_requires_sv",
+              test_ppc64_power10_hash_privileged_requires_sv},
+             {"test_ppc64_power10_hash_ra0_invalid",
+              test_ppc64_power10_hash_ra0_invalid},
+             {"test_ppc64_power9_hash_nop", test_ppc64_power9_hash_nop},
+             {"test_ppc64_power10_prefixed_load_store",
+              test_ppc64_power10_prefixed_load_store},
+             {"test_ppc64_power10_prefixed_byte_half_load_store",
+              test_ppc64_power10_prefixed_byte_half_load_store},
+             {"test_ppc64_power10_prefixed_quad_load_store",
+              test_ppc64_power10_prefixed_quad_load_store},
+             {"test_ppc64_power10_prefixed_quad_ra_rt_invalid",
+              test_ppc64_power10_prefixed_quad_ra_rt_invalid},
+             {"test_ppc64_power10_prefixed_float_load_store",
+              test_ppc64_power10_prefixed_float_load_store},
+             {"test_ppc64_power10_prefixed_vsx_load_store",
+              test_ppc64_power10_prefixed_vsx_load_store},
+             {"test_ppc64_isa300_lxvwsx", test_ppc64_isa300_lxvwsx},
+             {"test_ppc64_isa300_lxvwsx_requires_isa300",
+              test_ppc64_isa300_lxvwsx_requires_isa300},
+             {"test_ppc64_power10_prefixed_vsx_scalar_load_store",
+              test_ppc64_power10_prefixed_vsx_scalar_load_store},
+             {"test_ppc64_power10_prefixed_vsx_pair_load_store",
+              test_ppc64_power10_prefixed_vsx_pair_load_store},
+             {"test_ppc64_power10_prefixed_negative_load_store",
+              test_ppc64_power10_prefixed_negative_load_store},
+             {"test_ppc64_power10_prefixed_paddi",
+              test_ppc64_power10_prefixed_paddi},
+             {"test_ppc64_power10_prefixed_r_requires_ra0",
+              test_ppc64_power10_prefixed_r_requires_ra0},
+             {"test_ppc64_power10_pnop", test_ppc64_power10_pnop},
+             {"test_ppc64_power10_pnop_invalid_suffix",
+              test_ppc64_power10_pnop_invalid_suffix},
+             {"test_ppc64_power10_prefixed_boundary",
+              test_ppc64_power10_prefixed_boundary},
+             {"test_ppc64_power10_prefixed_hook_size",
+              test_ppc64_power10_prefixed_hook_size},
+             {"test_ppc64_power9_prefixed_rejected",
+              test_ppc64_power9_prefixed_rejected},
+             {"test_ppc64_power9_xvp_rejected",
+              test_ppc64_power9_xvp_rejected},
              {NULL, NULL}};
diff --git a/tests/unit/test_riscv.c b/tests/unit/test_riscv.c
index 86c633eb83..1e2a7999da 100644
--- a/tests/unit/test_riscv.c
+++ b/tests/unit/test_riscv.c
@@ -2,6 +2,59 @@
 
 const uint64_t code_start = 0x1000;
 const uint64_t code_len = 0x4000;
+const uint64_t riscv_data_start = 0x8000;
+
+#define RISCV_MSTATUS_VS_INITIAL 0x200
+#define RISCV_MSTATUS_VS_DIRTY 0x600
+#define RISCV_MSTATUS_FS_INITIAL 0x2000
+#define RISCV32_MSTATUS_SD 0x80000000u
+#define RISCV64_MSTATUS_SD 0x8000000000000000ull
+#define RISCV64_VTYPE_VILL 0x8000000000000000ull
+#define RISCV_CSR_FFLAGS 0x001
+#define RISCV_CSR_VSTART 0x008
+#define RISCV_CSR_VXSAT 0x009
+#define RISCV_CSR_VXRM 0x00a
+#define RISCV_CSR_VSATP 0x280
+#define RISCV_CSR_STIMECMP 0x14d
+#define RISCV_CSR_MSTATUS 0x300
+#define RISCV_CSR_PMPCFG0 0x3a0
+#define RISCV_CSR_PMPADDR0 0x3b0
+#define RISCV_CSR_HSTATUS 0x600
+#define RISCV_CSR_HGATP 0x680
+#define RISCV_MSTATUS_MXR 0x00080000ull
+#define RISCV_PMPCFG_R 0x01
+#define RISCV_PMPCFG_X 0x04
+#define RISCV_PMPCFG_A_NA4 0x10
+#define RISCV_PMPCFG_A_NAPOT 0x18
+#define RISCV_EXCP_INST_ACCESS_FAULT 0x1
+#define RISCV_EXCP_ILLEGAL_INST 0x2
+#define RISCV_EXCP_LOAD_ACCESS_FAULT 0x5
+#define RISCV_EXCP_STORE_AMO_ACCESS_FAULT 0x7
+#define RISCV_EXCP_S_ECALL 0x9
+#define RISCV_EXCP_VS_ECALL 0xa
+#define RISCV_EXCP_M_ECALL 0xb
+#define RISCV_EXCP_INST_GUEST_PAGE_FAULT 0x14
+#define RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT 0x15
+#define RISCV_EXCP_VIRT_INSTRUCTION_FAULT 0x16
+#define RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT 0x17
+#define RISCV_HSTATUS_VSBE 0x00000020ull
+#define RISCV_HSTATUS_SPVP 0x00000100ull
+#define RISCV_HSTATUS_HU 0x00000200ull
+#define RISCV_HSTATUS_VTW 0x00200000ull
+#define RISCV_HSTATUS_VSXL 0x300000000ull
+#define RISCV_HSTATUS_VSXL_RV64 0x200000000ull
+#define RISCV64_SATP_MODE_SV39 0x8000000000000000ull
+#define RISCV_PTE_V 0x001ull
+#define RISCV_PTE_R 0x002ull
+#define RISCV_PTE_X 0x008ull
+#define RISCV_PTE_A 0x040ull
+#define RISCV_PTE_D 0x080ull
+
+static uint32_t riscv_encode_addi(uint32_t rd, uint32_t rs1, int32_t imm)
+{
+    return (((uint32_t)imm & 0xfff) << 20) | ((rs1 & 0x1f) << 15) |
+           ((rd & 0x1f) << 7) | 0x13;
+}
 
 static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
                             const char *code, uint64_t size)
@@ -11,6 +64,54 @@ static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
     OK(uc_mem_write(*uc, code_start, code, size));
 }
 
+static void uc_common_setup_model(uc_engine **uc, uc_arch arch, uc_mode mode,
+                                  const char *code, uint64_t size,
+                                  int cpu_model)
+{
+    OK(uc_open(arch, mode, uc));
+    OK(uc_ctl_set_cpu_model(*uc, cpu_model));
+    OK(uc_mem_map(*uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(*uc, code_start, code, size));
+}
+
+static void test_riscv_block_count_cb(uc_engine *uc, uint64_t address,
+                                      uint32_t size, void *user_data)
+{
+    uint32_t *count = (uint32_t *)user_data;
+
+    (*count)++;
+}
+
+static void riscv32_enable_vector_state(uc_engine *uc)
+{
+    uint32_t mstatus = RISCV_MSTATUS_VS_INITIAL;
+
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+}
+
+static void riscv32_enable_vector_fp_state(uc_engine *uc)
+{
+    uint32_t mstatus = RISCV_MSTATUS_VS_INITIAL |
+                       RISCV_MSTATUS_FS_INITIAL;
+
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+}
+
+static void riscv64_enable_vector_state(uc_engine *uc)
+{
+    uint64_t mstatus = RISCV_MSTATUS_VS_INITIAL;
+
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+}
+
+static void riscv64_enable_vector_fp_state(uc_engine *uc)
+{
+    uint64_t mstatus = RISCV_MSTATUS_VS_INITIAL |
+                       RISCV_MSTATUS_FS_INITIAL;
+
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+}
+
 static void test_riscv32_nop(void)
 {
     uc_engine *uc;
@@ -55,6 +156,62 @@ static void test_riscv64_nop(void)
     OK(uc_close(uc));
 }
 
+static void test_riscv32_zihintpause(void)
+{
+    uc_engine *uc;
+    uc_hook hook;
+    uint8_t code[] = {
+        0x0f, 0x00, 0x00, 0x01, /* pause */
+        0x93, 0x02, 0x10, 0x00, /* addi t0, zero, 1 */
+    };
+    uint32_t block_count = 0;
+    uint32_t t0 = 0;
+    uint32_t pc = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, (char *)code,
+                    sizeof(code));
+    OK(uc_hook_add(uc, &hook, UC_HOOK_BLOCK, test_riscv_block_count_cb,
+                   &block_count, 1, 0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_PC, &pc));
+
+    TEST_CHECK(t0 == 1);
+    TEST_CHECK(pc == (uint32_t)(code_start + sizeof(code)));
+    TEST_CHECK(block_count == 2);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_zihintpause(void)
+{
+    uc_engine *uc;
+    uc_hook hook;
+    uint8_t code[] = {
+        0x0f, 0x00, 0x00, 0x01, /* pause */
+        0x93, 0x02, 0x10, 0x00, /* addi t0, zero, 1 */
+    };
+    uint32_t block_count = 0;
+    uint64_t t0 = 0;
+    uint64_t pc = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, (char *)code,
+                    sizeof(code));
+    OK(uc_hook_add(uc, &hook, UC_HOOK_BLOCK, test_riscv_block_count_cb,
+                   &block_count, 1, 0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_PC, &pc));
+
+    TEST_CHECK(t0 == 1);
+    TEST_CHECK(pc == code_start + sizeof(code));
+    TEST_CHECK(block_count == 2);
+
+    OK(uc_close(uc));
+}
+
 static void test_riscv32_until_pc_update(void)
 {
     uc_engine *uc;
@@ -484,6 +641,297 @@ static void test_riscv64_fmv_w_x_nanbox(void)
     OK(uc_close(uc));
 }
 
+static void test_riscv64_fmin_fmax_snan(void)
+{
+    static const struct {
+        const char *name;
+        const char code[4];
+        uint64_t lhs;
+        uint64_t rhs;
+        uint64_t expected;
+    } cases[] = {
+        {"fmin.s", "\x53\x85\xc5\x28", 0xffffffff7fa00000ULL,
+         0xffffffff3fc00000ULL, 0xffffffff3fc00000ULL},
+        {"fmax.s", "\x53\x95\xc5\x28", 0xffffffff7fa00000ULL,
+         0xffffffff3fc00000ULL, 0xffffffff3fc00000ULL},
+        {"fmin.d", "\x53\x85\xc5\x2a", 0x7ff4000000000000ULL,
+         0x3ff8000000000000ULL, 0x3ff8000000000000ULL},
+        {"fmax.d", "\x53\x95\xc5\x2a", 0x7ff4000000000000ULL,
+         0x3ff8000000000000ULL, 0x3ff8000000000000ULL},
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+        uc_engine *uc;
+        uint64_t dst = 0;
+        uint64_t fflags = 0;
+        uint64_t lhs = cases[i].lhs;
+        uint64_t mstatus = 0x6000;
+        uint64_t rhs = cases[i].rhs;
+
+        uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, cases[i].code,
+                        sizeof(cases[i].code));
+        OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+        OK(uc_reg_write(uc, UC_RISCV_REG_F11, &lhs));
+        OK(uc_reg_write(uc, UC_RISCV_REG_F12, &rhs));
+
+        OK(uc_emu_start(uc, code_start, code_start + sizeof(cases[i].code),
+                        0, 1));
+        OK(uc_reg_read(uc, UC_RISCV_REG_F10, &dst));
+        OK(uc_reg_read(uc, UC_RISCV_REG_FFLAGS, &fflags));
+
+        TEST_CHECK_(dst == cases[i].expected, "%s result", cases[i].name);
+        TEST_CHECK_(fflags == 0x10, "%s invalid flag", cases[i].name);
+
+        OK(uc_close(uc));
+    }
+}
+
+static void test_riscv32_zfh_load_store_move(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x87\x10\x05\x00" /* flh f1, 0(a0) */
+        "\xd3\x82\x00\xe4" /* fmv.x.h t0, f1 */
+        "\x53\x01\x03\xf4" /* fmv.h.x f2, t1 */
+        "\x27\x11\x25\x00"; /* fsh f2, 2(a0) */
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t t0 = 0;
+    uint32_t t1 = 0x3e00;
+    uint64_t f1 = 0;
+    uint64_t f2 = 0;
+    uint64_t mstatus = 0x6000;
+    uint8_t input[4] = { 0x00, 0xbc, 0x00, 0x00 };
+    uint8_t output[4] = { 0 };
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code,
+                    sizeof(code) - 1);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 4));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F1, &f1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F2, &f2));
+    OK(uc_mem_read(uc, a0, output, sizeof(output)));
+
+    TEST_CHECK(t0 == 0xffffbc00);
+    TEST_CHECK(f1 == 0xffffffffffffbc00ULL);
+    TEST_CHECK(f2 == 0xffffffffffff3e00ULL);
+    TEST_CHECK(output[2] == 0x00);
+    TEST_CHECK(output[3] == 0x3e);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_zfh_load_store_move(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x87\x10\x05\x00" /* flh f1, 0(a0) */
+        "\xd3\x82\x00\xe4" /* fmv.x.h t0, f1 */
+        "\x53\x01\x03\xf4" /* fmv.h.x f2, t1 */
+        "\x27\x11\x25\x00"; /* fsh f2, 2(a0) */
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t t0 = 0;
+    uint64_t t1 = 0x3e00;
+    uint64_t f1 = 0;
+    uint64_t f2 = 0;
+    uint64_t mstatus = 0x6000;
+    uint8_t input[4] = { 0x00, 0xbc, 0x00, 0x00 };
+    uint8_t output[4] = { 0 };
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 4));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F1, &f1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F2, &f2));
+    OK(uc_mem_read(uc, a0, output, sizeof(output)));
+
+    TEST_CHECK(t0 == 0xffffffffffffbc00ULL);
+    TEST_CHECK(f1 == 0xffffffffffffbc00ULL);
+    TEST_CHECK(f2 == 0xffffffffffff3e00ULL);
+    TEST_CHECK(output[2] == 0x00);
+    TEST_CHECK(output[3] == 0x3e);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_zfh_arith_compare_class(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x53\x82\x20\x04" /* fadd.h f4, f1, f2 */
+        "\xd3\x02\x32\x14" /* fmul.h f5, f4, f3 */
+        "\x53\x83\x20\x2c" /* fmin.h f6, f1, f2 */
+        "\x53\x85\x20\x24" /* fsgnj.h f10, f1, f2 */
+        "\xd3\x95\x10\x24" /* fsgnjn.h f11, f1, f1 */
+        "\x53\xa6\xb5\x24" /* fsgnjx.h f12, f11, f11 */
+        "\xc3\x84\x20\x1c" /* fmadd.h f9, f1, f2, f3 */
+        "\xd3\x92\x20\xa4" /* flt.h t0, f1, f2 */
+        "\x53\xa3\x20\xa4" /* feq.h t1, f1, f2 */
+        "\xd3\x93\x00\xe4"; /* fclass.h t2, f1 */
+    uint64_t f1 = 0xffffffffffff3c00ULL;
+    uint64_t f2 = 0xffffffffffff4000ULL;
+    uint64_t f3 = 0xffffffffffff3800ULL;
+    uint64_t f4 = 0;
+    uint64_t f5 = 0;
+    uint64_t f6 = 0;
+    uint64_t f9 = 0;
+    uint64_t f10 = 0;
+    uint64_t f11 = 0;
+    uint64_t f12 = 0;
+    uint64_t t0 = 0;
+    uint64_t t1 = 0;
+    uint64_t t2 = 0;
+    uint64_t mstatus = 0x6000;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F2, &f2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F3, &f3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 10));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_F4, &f4));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F5, &f5));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F6, &f6));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F9, &f9));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F10, &f10));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F11, &f11));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F12, &f12));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T2, &t2));
+
+    TEST_CHECK(f4 == 0xffffffffffff4200ULL);
+    TEST_CHECK(f5 == 0xffffffffffff3e00ULL);
+    TEST_CHECK(f6 == 0xffffffffffff3c00ULL);
+    TEST_CHECK(f9 == 0xffffffffffff4100ULL);
+    TEST_CHECK(f10 == 0xffffffffffff3c00ULL);
+    TEST_CHECK(f11 == 0xffffffffffffbc00ULL);
+    TEST_CHECK(f12 == 0xffffffffffff3c00ULL);
+    TEST_CHECK(t0 == 1);
+    TEST_CHECK(t1 == 0);
+    TEST_CHECK(t2 == 0x40);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_zfh_conversions(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\xd3\x81\x02\xd4" /* fcvt.h.w f3, t0 */
+        "\x53\x83\x01\xc4" /* fcvt.w.h t1, f3 */
+        "\x53\x82\x00\x44" /* fcvt.h.s f4, f1 */
+        "\xd3\x02\x22\x40" /* fcvt.s.h f5, f4 */
+        "\x53\x03\x11\x44" /* fcvt.h.d f6, f2 */
+        "\xd3\x03\x23\x42"; /* fcvt.d.h f7, f6 */
+    uint64_t f1 = 0xffffffff3fc00000ULL;
+    uint64_t f2 = 0x4008000000000000ULL;
+    uint64_t f3 = 0;
+    uint64_t f4 = 0;
+    uint64_t f5 = 0;
+    uint64_t f6 = 0;
+    uint64_t f7 = 0;
+    uint32_t t0 = 2;
+    uint32_t t1 = 0;
+    uint64_t mstatus = 0x6000;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code,
+                    sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F2, &f2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 6));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F3, &f3));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F4, &f4));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F5, &f5));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F6, &f6));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F7, &f7));
+
+    TEST_CHECK(t1 == 2);
+    TEST_CHECK(f3 == 0xffffffffffff4000ULL);
+    TEST_CHECK(f4 == 0xffffffffffff3e00ULL);
+    TEST_CHECK(f5 == 0xffffffff3fc00000ULL);
+    TEST_CHECK(f6 == 0xffffffffffff4200ULL);
+    TEST_CHECK(f7 == 0x4008000000000000ULL);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_zfh_conversions(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\xd3\x81\x02\xd4" /* fcvt.h.w f3, t0 */
+        "\x53\x83\x01\xc4" /* fcvt.w.h t1, f3 */
+        "\x53\x82\x00\x44" /* fcvt.h.s f4, f1 */
+        "\xd3\x02\x22\x40" /* fcvt.s.h f5, f4 */
+        "\x53\x03\x11\x44" /* fcvt.h.d f6, f2 */
+        "\xd3\x03\x23\x42" /* fcvt.d.h f7, f6 */
+        "\xd3\x03\x23\xc4" /* fcvt.l.h t2, f6 */
+        "\x53\x84\x23\xd4"; /* fcvt.h.l f8, t2 */
+    uint64_t f1 = 0xffffffff3fc00000ULL;
+    uint64_t f2 = 0x4008000000000000ULL;
+    uint64_t f3 = 0;
+    uint64_t f4 = 0;
+    uint64_t f5 = 0;
+    uint64_t f6 = 0;
+    uint64_t f7 = 0;
+    uint64_t f8 = 0;
+    uint64_t t0 = 2;
+    uint64_t t1 = 0;
+    uint64_t t2 = 0;
+    uint64_t mstatus = 0x6000;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F2, &f2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 8));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F3, &f3));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F4, &f4));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F5, &f5));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F6, &f6));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F7, &f7));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F8, &f8));
+
+    TEST_CHECK(t1 == 2);
+    TEST_CHECK(t2 == 3);
+    TEST_CHECK(f3 == 0xffffffffffff4000ULL);
+    TEST_CHECK(f4 == 0xffffffffffff3e00ULL);
+    TEST_CHECK(f5 == 0xffffffff3fc00000ULL);
+    TEST_CHECK(f6 == 0xffffffffffff4200ULL);
+    TEST_CHECK(f7 == 0x4008000000000000ULL);
+    TEST_CHECK(f8 == 0xffffffffffff4200ULL);
+
+    OK(uc_close(uc));
+}
+
 static void test_riscv64_code_patching(void)
 {
     uc_engine *uc;
@@ -908,9 +1356,9889 @@ static void test_riscv_priv(void)
     TEST_ASSERT(reg_value == 0);
 }
 
+static uint32_t riscv_encode_r(uint32_t funct7, uint32_t rs2, uint32_t rs1,
+                               uint32_t funct3, uint32_t rd, uint32_t opcode)
+{
+    return ((funct7 & 0x7f) << 25) | ((rs2 & 0x1f) << 20) |
+           ((rs1 & 0x1f) << 15) | ((funct3 & 0x7) << 12) |
+           ((rd & 0x1f) << 7) | (opcode & 0x7f);
+}
+
+static uint32_t riscv_encode_i(uint32_t imm, uint32_t rs1, uint32_t funct3,
+                               uint32_t rd, uint32_t opcode)
+{
+    return ((imm & 0xfff) << 20) | ((rs1 & 0x1f) << 15) |
+           ((funct3 & 0x7) << 12) | ((rd & 0x1f) << 7) |
+           (opcode & 0x7f);
+}
+
+static uint32_t riscv_encode_s(uint32_t imm, uint32_t rs2, uint32_t rs1,
+                               uint32_t funct3, uint32_t opcode)
+{
+    return (((imm >> 5) & 0x7f) << 25) | ((rs2 & 0x1f) << 20) |
+           ((rs1 & 0x1f) << 15) | ((funct3 & 0x7) << 12) |
+           ((imm & 0x1f) << 7) | (opcode & 0x7f);
+}
+
+static uint32_t riscv_encode_csr(uint32_t csr, uint32_t rs1,
+                                 uint32_t funct3, uint32_t rd)
+{
+    return riscv_encode_i(csr, rs1, funct3, rd, 0x73);
+}
+
+static uint32_t riscv_encode_k_aes(uint32_t funct5, uint32_t shamt,
+                                   uint32_t rs2, uint32_t rs1, uint32_t rd)
+{
+    uint32_t funct7 = ((shamt >> 3) << 5) | (funct5 & 0x1f);
+
+    return riscv_encode_r(funct7, rs2, rs1, 0, rd, 0x33);
+}
+
+static uint32_t riscv_encode_rvv_op(uint32_t funct6, uint32_t vm,
+                                    uint32_t vs2, uint32_t rs1,
+                                    uint32_t funct3, uint32_t vd)
+{
+    return ((funct6 & 0x3f) << 26) | ((vm & 1) << 25) |
+           ((vs2 & 0x1f) << 20) | ((rs1 & 0x1f) << 15) |
+           ((funct3 & 0x7) << 12) | ((vd & 0x1f) << 7) | 0x57;
+}
+
+static uint32_t riscv_encode_rvv_vsetvli(uint32_t rd, uint32_t rs1,
+                                         uint32_t zimm)
+{
+    return ((zimm & 0x7ff) << 20) | ((rs1 & 0x1f) << 15) |
+           (7 << 12) | ((rd & 0x1f) << 7) | 0x57;
+}
+
+static uint32_t riscv_encode_rvv_ldst(int is_store, uint32_t width,
+                                      uint32_t vm, uint32_t rs1,
+                                      uint32_t reg)
+{
+    return ((vm & 1) << 25) | ((rs1 & 0x1f) << 15) |
+           ((width & 0x7) << 12) | ((reg & 0x1f) << 7) |
+           (is_store ? 0x27 : 0x07);
+}
+
+static uint32_t riscv_encode_rvv_segment_ldst(int is_store, uint32_t width,
+                                              uint32_t vm, uint32_t rs1,
+                                              uint32_t reg, uint32_t nf)
+{
+    return riscv_encode_rvv_ldst(is_store, width, vm, rs1, reg) |
+           (((nf - 1) & 0x7) << 29);
+}
+
+static uint32_t riscv_encode_rvv_stride_ldst(int is_store, uint32_t width,
+                                             uint32_t vm, uint32_t rs1,
+                                             uint32_t rs2, uint32_t reg,
+                                             uint32_t nf)
+{
+    return (((nf - 1) & 0x7) << 29) | (2 << 26) | ((vm & 1) << 25) |
+           ((rs2 & 0x1f) << 20) | ((rs1 & 0x1f) << 15) |
+           ((width & 0x7) << 12) | ((reg & 0x1f) << 7) |
+           (is_store ? 0x27 : 0x07);
+}
+
+static uint32_t riscv_encode_rvv_index_ldst(int is_store, uint32_t width,
+                                            uint32_t mop, uint32_t vm,
+                                            uint32_t rs1, uint32_t rs2,
+                                            uint32_t reg, uint32_t nf)
+{
+    return (((nf - 1) & 0x7) << 29) | ((mop & 0x7) << 26) |
+           ((vm & 1) << 25) | ((rs2 & 0x1f) << 20) |
+           ((rs1 & 0x1f) << 15) | ((width & 0x7) << 12) |
+           ((reg & 0x1f) << 7) | (is_store ? 0x27 : 0x07);
+}
+
+static uint32_t riscv_encode_rvv_ff_load(uint32_t width, uint32_t vm,
+                                         uint32_t rs1, uint32_t reg,
+                                         uint32_t nf)
+{
+    return (((nf - 1) & 0x7) << 29) | ((vm & 1) << 25) |
+           (0x10 << 20) | ((rs1 & 0x1f) << 15) |
+           ((width & 0x7) << 12) | ((reg & 0x1f) << 7) | 0x07;
+}
+
+static uint32_t riscv_rvv_whole_nf_code(uint32_t nf)
+{
+    switch (nf) {
+    case 1:
+        return 0;
+    case 2:
+        return 1;
+    case 4:
+        return 3;
+    case 8:
+        return 7;
+    default:
+        return 0;
+    }
+}
+
+static uint32_t riscv_encode_rvv_whole_ldst(int is_store, uint32_t width,
+                                            uint32_t rs1, uint32_t reg,
+                                            uint32_t nf)
+{
+    return (riscv_rvv_whole_nf_code(nf) << 29) | (1 << 25) |
+           (0x08 << 20) | ((rs1 & 0x1f) << 15) |
+           ((width & 0x7) << 12) | ((reg & 0x1f) << 7) |
+           (is_store ? 0x27 : 0x07);
+}
+
+static uint32_t riscv_encode_rvv_mask_ldst(int is_store, uint32_t rs1,
+                                           uint32_t reg)
+{
+    return (1 << 25) | (0x0b << 20) | ((rs1 & 0x1f) << 15) |
+           ((reg & 0x1f) << 7) | (is_store ? 0x27 : 0x07);
+}
+
+static uint32_t riscv_encode_rvv_vmvnr(uint32_t nf, uint32_t rs2,
+                                       uint32_t rd)
+{
+    return (0x27 << 26) | (1 << 25) |
+           ((rs2 & 0x1f) << 20) |
+           (riscv_rvv_whole_nf_code(nf) << 15) | (3 << 12) |
+           ((rd & 0x1f) << 7) | 0x57;
+}
+
+static uint32_t riscv_encode_rvh_load(uint32_t funct7, uint32_t selector,
+                                      uint32_t rs1, uint32_t rd)
+{
+    return ((funct7 & 0x7f) << 25) | ((selector & 0x1f) << 20) |
+           ((rs1 & 0x1f) << 15) | (4 << 12) | ((rd & 0x1f) << 7) |
+           0x73;
+}
+
+static uint32_t riscv_encode_rvh_store(uint32_t funct7, uint32_t rs2,
+                                       uint32_t rs1)
+{
+    return ((funct7 & 0x7f) << 25) | ((rs2 & 0x1f) << 20) |
+           ((rs1 & 0x1f) << 15) | (4 << 12) | 0x73;
+}
+
+static void riscv_insn_to_code(uint8_t code[4], uint32_t insn)
+{
+    code[0] = insn;
+    code[1] = insn >> 8;
+    code[2] = insn >> 16;
+    code[3] = insn >> 24;
+}
+
+static void test_riscv64_sstc_stimecmp(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_csr(RISCV_CSR_STIMECMP, 5, 1, 0),
+        riscv_encode_csr(RISCV_CSR_STIMECMP, 0, 2, 6),
+    };
+    uint8_t code[sizeof(insns)];
+    uint64_t t0 = 0x1122334455667788ull;
+    uint64_t t1 = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == t0);
+    OK(uc_close(uc));
+}
+
+static void run_riscv32_zbb_case(uint32_t insn, uint32_t rs1,
+                                 uint32_t rs2, int rd, uint32_t expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t actual = 0;
+
+    riscv_insn_to_code(code, insn);
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, (const char *)code,
+                    sizeof(code));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &rs1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X7, &rs2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X0 + rd, &actual));
+    TEST_CHECK(actual == expected);
+    OK(uc_close(uc));
+}
+
+static void run_riscv64_zbb_case(uint32_t insn, uint64_t rs1,
+                                 uint64_t rs2, int rd, uint64_t expected)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t actual = 0;
+
+    riscv_insn_to_code(code, insn);
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, (const char *)code,
+                    sizeof(code));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &rs1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X7, &rs2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X0 + rd, &actual));
+    TEST_CHECK(actual == expected);
+    OK(uc_close(uc));
+}
+
+static void run_riscv_insn_illegal(uc_mode mode, uint32_t insn)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+
+    riscv_insn_to_code(code, insn);
+    uc_common_setup(&uc, UC_ARCH_RISCV, mode, (const char *)code,
+                    sizeof(code));
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv_model_insn_illegal(uc_mode mode, int cpu_model,
+                                         uint32_t insn)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+
+    riscv_insn_to_code(code, insn);
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, mode, (const char *)code,
+                          sizeof(code), cpu_model);
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv_non_fp_model_insn_illegal(uc_mode mode, int cpu_model,
+                                                uint32_t insn)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t mstatus = RISCV_MSTATUS_FS_INITIAL;
+    uint8_t data[8] = { 0 };
+
+    riscv_insn_to_code(code, insn);
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, mode, (const char *)code,
+                          sizeof(code), cpu_model);
+    OK(uc_mem_write(uc, a0, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_MSTATUS, &mstatus));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv_rvv_non_vector_model_illegal(uc_mode mode,
+                                                   int cpu_model)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t t0 = 4;
+
+    riscv_insn_to_code(code, riscv_encode_rvv_vsetvli(0, 5, 0xc0));
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, mode, (const char *)code,
+                          sizeof(code), cpu_model);
+    if (mode == UC_MODE_RISCV64) {
+        riscv64_enable_vector_state(uc);
+    } else {
+        riscv32_enable_vector_state(uc);
+    }
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv_rvv_non_vector_model_data_illegal(uc_mode mode,
+                                                        int cpu_model)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t vl = 4;
+    uint64_t vtype = 0xc0;
+
+    riscv_insn_to_code(code, riscv_encode_rvv_ldst(0, 0, 1, 10, 1));
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, mode, (const char *)code,
+                          sizeof(code), cpu_model);
+    if (mode == UC_MODE_RISCV64) {
+        riscv64_enable_vector_state(uc);
+    } else {
+        riscv32_enable_vector_state(uc);
+    }
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_VTYPE, &vtype));
+    OK(uc_reg_write(uc, UC_RISCV_REG_VL, &vl));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv64_rvv_illegal_vtype(uint32_t insn, uint32_t zimm)
+{
+    uc_engine *uc;
+    uint8_t code[2 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, zimm),
+        insn,
+    };
+    uint64_t t0 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv64_rvv_illegal(uint32_t insn)
+{
+    run_riscv64_rvv_illegal_vtype(insn, 0xc0);
+}
+
+static void run_riscv64_rvv_illegal_vstart_vl0(uint32_t insn)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc0),
+        riscv_encode_csr(RISCV_CSR_VSTART, 6, 1, 0),
+        insn,
+    };
+    uint64_t t0 = 0;
+    uint64_t t1 = 1;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv64_rvv_fp_illegal_vtype(uint32_t insn, uint32_t zimm)
+{
+    uc_engine *uc;
+    uint8_t code[2 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, zimm),
+        insn,
+    };
+    uint64_t t0 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_riscv_svinval_case(uc_mode mode)
+{
+    uc_engine *uc;
+    uint8_t code[12];
+    uint32_t insns[] = {
+        0x16208073, /* sinval.vma x1, x2 */
+        0x18000073, /* sfence.w.inval */
+        0x18100073, /* sfence.inval.ir */
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, mode, (const char *)code,
+                    sizeof(code));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_svinval(void)
+{
+    run_riscv_svinval_case(UC_MODE_RISCV32);
+}
+
+static void test_riscv64_svinval(void)
+{
+    run_riscv_svinval_case(UC_MODE_RISCV64);
+}
+
+static void test_riscv_svinval_hinval_requires_rvh(void)
+{
+    run_riscv_insn_illegal(UC_MODE_RISCV32, 0x26208073);
+    run_riscv_insn_illegal(UC_MODE_RISCV32, 0x66208073);
+    run_riscv_insn_illegal(UC_MODE_RISCV64, 0x26208073);
+    run_riscv_insn_illegal(UC_MODE_RISCV64, 0x66208073);
+}
+
+static void test_riscv_svinval_requires_s(void)
+{
+    uint32_t insns[] = {
+        0x16208073, /* sinval.vma x1, x2 */
+        0x18000073, /* sfence.w.inval */
+        0x18100073, /* sfence.inval.ir */
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        run_riscv_model_insn_illegal(UC_MODE_RISCV32,
+                                     UC_CPU_RISCV32_SIFIVE_E31, insns[i]);
+        run_riscv_model_insn_illegal(UC_MODE_RISCV64,
+                                     UC_CPU_RISCV64_SIFIVE_E51, insns[i]);
+    }
+}
+
+static void test_riscv32_rvh_hlv_hsv(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvh_load(0x30, 0, 6, 5),
+        riscv_encode_rvh_load(0x30, 1, 6, 8),
+        riscv_encode_rvh_load(0x32, 0, 7, 9),
+        riscv_encode_rvh_load(0x32, 1, 7, 10),
+        riscv_encode_rvh_load(0x34, 0, 11, 12),
+        riscv_encode_rvh_store(0x31, 13, 14),
+        riscv_encode_rvh_store(0x33, 15, 16),
+        riscv_encode_rvh_store(0x35, 17, 18),
+        riscv_encode_rvh_load(0x32, 3, 19, 20),
+        riscv_encode_rvh_load(0x34, 3, 21, 22),
+    };
+    uint8_t code[sizeof(insns)];
+    uint8_t data[0x40] = {
+        [0x00] = 0x80,
+        [0x04] = 0x80, [0x05] = 0xff,
+        [0x08] = 0x78, [0x09] = 0x56, [0x0a] = 0x34, [0x0b] = 0x80,
+        [0x20] = 0xcd, [0x21] = 0xab,
+        [0x24] = 0xef, [0x25] = 0xcd, [0x26] = 0xab, [0x27] = 0x89,
+    };
+    uint8_t stored[12] = { 0 };
+    uint32_t regs[23] = { 0 };
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV32_BASE32);
+    OK(uc_mem_map(uc, riscv_data_start, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, riscv_data_start, data, sizeof(data)));
+
+    regs[6] = (uint32_t)riscv_data_start;
+    regs[7] = (uint32_t)(riscv_data_start + 0x04);
+    regs[11] = (uint32_t)(riscv_data_start + 0x08);
+    regs[13] = 0xaabbccdd;
+    regs[14] = (uint32_t)(riscv_data_start + 0x10);
+    regs[15] = 0x11223344;
+    regs[16] = (uint32_t)(riscv_data_start + 0x14);
+    regs[17] = 0x55667788;
+    regs[18] = (uint32_t)(riscv_data_start + 0x18);
+    regs[19] = (uint32_t)(riscv_data_start + 0x20);
+    regs[21] = (uint32_t)(riscv_data_start + 0x24);
+
+    for (i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) {
+        if (regs[i] != 0) {
+            OK(uc_reg_write(uc, UC_RISCV_REG_X0 + (int)i, &regs[i]));
+        }
+    }
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X5, &regs[5]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X8, &regs[8]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X9, &regs[9]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X10, &regs[10]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X12, &regs[12]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X20, &regs[20]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X22, &regs[22]));
+    OK(uc_mem_read(uc, riscv_data_start + 0x10, stored, sizeof(stored)));
+
+    TEST_CHECK(regs[5] == 0xffffff80u);
+    TEST_CHECK(regs[8] == 0x80);
+    TEST_CHECK(regs[9] == 0xffffff80u);
+    TEST_CHECK(regs[10] == 0xff80);
+    TEST_CHECK(regs[12] == 0x80345678u);
+    TEST_CHECK(regs[20] == 0xabcd);
+    TEST_CHECK(regs[22] == 0x89abcdefu);
+    TEST_CHECK(stored[0] == 0xdd);
+    TEST_CHECK(stored[4] == 0x44);
+    TEST_CHECK(stored[5] == 0x33);
+    TEST_CHECK(stored[8] == 0x88);
+    TEST_CHECK(stored[9] == 0x77);
+    TEST_CHECK(stored[10] == 0x66);
+    TEST_CHECK(stored[11] == 0x55);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvh_hlv_hsv(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvh_load(0x30, 0, 6, 5),
+        riscv_encode_rvh_load(0x34, 1, 7, 8),
+        riscv_encode_rvh_load(0x36, 0, 9, 10),
+        riscv_encode_rvh_store(0x37, 11, 12),
+        riscv_encode_rvh_load(0x34, 3, 13, 14),
+    };
+    uint8_t code[sizeof(insns)];
+    uint8_t data[0x40] = {
+        [0x00] = 0x80,
+        [0x08] = 0xef, [0x09] = 0xcd, [0x0a] = 0xab, [0x0b] = 0x89,
+        [0x10] = 0x08, [0x11] = 0x07, [0x12] = 0x06, [0x13] = 0x05,
+        [0x14] = 0x04, [0x15] = 0x03, [0x16] = 0x02, [0x17] = 0x01,
+        [0x28] = 0xef, [0x29] = 0xcd, [0x2a] = 0xab, [0x2b] = 0x89,
+    };
+    uint8_t stored[8] = { 0 };
+    uint64_t regs[15] = { 0 };
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_mem_map(uc, riscv_data_start, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, riscv_data_start, data, sizeof(data)));
+
+    regs[6] = riscv_data_start;
+    regs[7] = riscv_data_start + 0x08;
+    regs[9] = riscv_data_start + 0x10;
+    regs[11] = 0x8877665544332211ull;
+    regs[12] = riscv_data_start + 0x20;
+    regs[13] = riscv_data_start + 0x28;
+
+    for (i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) {
+        if (regs[i] != 0) {
+            OK(uc_reg_write(uc, UC_RISCV_REG_X0 + (int)i, &regs[i]));
+        }
+    }
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X5, &regs[5]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X8, &regs[8]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X10, &regs[10]));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X14, &regs[14]));
+    OK(uc_mem_read(uc, riscv_data_start + 0x20, stored, sizeof(stored)));
+
+    TEST_CHECK(regs[5] == 0xffffffffffffff80ull);
+    TEST_CHECK(regs[8] == 0x89abcdefull);
+    TEST_CHECK(regs[10] == 0x0102030405060708ull);
+    TEST_CHECK(regs[14] == 0x89abcdefull);
+    TEST_CHECK(stored[0] == 0x11);
+    TEST_CHECK(stored[1] == 0x22);
+    TEST_CHECK(stored[2] == 0x33);
+    TEST_CHECK(stored[3] == 0x44);
+    TEST_CHECK(stored[4] == 0x55);
+    TEST_CHECK(stored[5] == 0x66);
+    TEST_CHECK(stored[6] == 0x77);
+    TEST_CHECK(stored[7] == 0x88);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvh_hlvx_vs_stage_xonly_mxr(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_csr(RISCV_CSR_VSATP, 5, 1, 0),
+        riscv_encode_csr(RISCV_CSR_HSTATUS, 6, 1, 0),
+        riscv_encode_csr(RISCV_CSR_MSTATUS, 7, 1, 0),
+        riscv_encode_rvh_load(0x34, 3, 13, 14),
+    };
+    uint8_t code[sizeof(insns)];
+    uint8_t data[4] = { 0xef, 0xcd, 0xab, 0x89 };
+    uint64_t pt_root = 0x20000;
+    uint64_t pt_l1 = 0x21000;
+    uint64_t pt_l0 = 0x22000;
+    uint64_t va = riscv_data_start;
+    uint64_t root_pte = LEINT64((pt_l1 >> 2) | RISCV_PTE_V);
+    uint64_t l1_pte = LEINT64((pt_l0 >> 2) | RISCV_PTE_V);
+    uint64_t leaf_pte = LEINT64((riscv_data_start >> 2) | RISCV_PTE_V |
+                                RISCV_PTE_X | RISCV_PTE_A | RISCV_PTE_D);
+    uint64_t vsatp = RISCV64_SATP_MODE_SV39 | (pt_root >> 12);
+    uint64_t hstatus = RISCV_HSTATUS_SPVP;
+    uint64_t mstatus = RISCV_MSTATUS_MXR;
+    uint64_t value = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_mem_map(uc, riscv_data_start, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_map(uc, pt_root, 0x3000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, riscv_data_start, data, sizeof(data)));
+    OK(uc_mem_write(uc, pt_root, &root_pte, sizeof(root_pte)));
+    OK(uc_mem_write(uc, pt_l1, &l1_pte, sizeof(l1_pte)));
+    OK(uc_mem_write(uc, pt_l0 + 8 * 8, &leaf_pte, sizeof(leaf_pte)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X5, &vsatp));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &hstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X7, &mstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X13, &va));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X14, &value));
+    TEST_CHECK(value == 0x89abcdefull);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_requires_h(void)
+{
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_rvh_load(0x30, 0, 6, 5));
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_rvh_load(0x30, 0, 6, 5));
+}
+
+static void test_riscv_rvh_requires_hlsx(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t addr = riscv_data_start;
+    uint64_t priv = 0;
+
+    riscv_insn_to_code(code, riscv_encode_rvh_load(0x30, 0, 6, 5));
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_mem_map(uc, riscv_data_start, 0x1000, UC_PROT_ALL));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &addr));
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_hstatus_layout(void)
+{
+    uc_engine *uc;
+    uint32_t code = riscv_encode_addi(0, 0, 0);
+    uint64_t hstatus = RISCV_HSTATUS_HU | RISCV_HSTATUS_SPVP |
+                       RISCV_HSTATUS_VSBE;
+    uint64_t actual = 0;
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)&code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_reg_write(uc, UC_RISCV_REG_HSTATUS, &hstatus));
+    OK(uc_reg_read(uc, UC_RISCV_REG_HSTATUS, &actual));
+
+    TEST_CHECK((actual & RISCV_HSTATUS_HU) != 0);
+    TEST_CHECK((actual & RISCV_HSTATUS_SPVP) != 0);
+    TEST_CHECK((actual & RISCV_HSTATUS_VSBE) == 0);
+    TEST_CHECK((actual & RISCV_HSTATUS_VSXL) == RISCV_HSTATUS_VSXL_RV64);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_hedeleg_mask(void)
+{
+    uc_engine *uc;
+    uint32_t code = riscv_encode_addi(0, 0, 0);
+    uint64_t hedeleg = UINT64_MAX;
+    uint64_t actual = 0;
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)&code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_reg_write(uc, UC_RISCV_REG_HEDELEG, &hedeleg));
+    OK(uc_reg_read(uc, UC_RISCV_REG_HEDELEG, &actual));
+
+    TEST_CHECK((actual & (1ull << RISCV_EXCP_ILLEGAL_INST)) != 0);
+    TEST_CHECK((actual & (1ull << RISCV_EXCP_S_ECALL)) == 0);
+    TEST_CHECK((actual & (1ull << RISCV_EXCP_VS_ECALL)) == 0);
+    TEST_CHECK((actual & (1ull << RISCV_EXCP_M_ECALL)) == 0);
+    TEST_CHECK((actual & (1ull << RISCV_EXCP_INST_GUEST_PAGE_FAULT)) == 0);
+    TEST_CHECK((actual & (1ull << RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT)) == 0);
+    TEST_CHECK((actual & (1ull << RISCV_EXCP_VIRT_INSTRUCTION_FAULT)) == 0);
+    TEST_CHECK((actual &
+                (1ull << RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_hu_allows_u_mode(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint8_t data = 0x7f;
+    uint64_t addr = riscv_data_start;
+    uint64_t hstatus = RISCV_HSTATUS_HU;
+    uint64_t priv = 0;
+    uint64_t value = 0;
+
+    riscv_insn_to_code(code, riscv_encode_rvh_load(0x30, 0, 6, 5));
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_mem_map(uc, riscv_data_start, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, riscv_data_start, &data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &addr));
+    OK(uc_reg_write(uc, UC_RISCV_REG_HSTATUS, &hstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_X5, &value));
+    TEST_CHECK(value == data);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_hu_tb_flags(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint8_t data = 0x7f;
+    uint64_t addr = riscv_data_start;
+    uint64_t hstatus = RISCV_HSTATUS_HU;
+    uint64_t priv = 0;
+    uint64_t value = 0;
+
+    riscv_insn_to_code(code, riscv_encode_rvh_load(0x30, 0, 6, 5));
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_mem_map(uc, riscv_data_start, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, riscv_data_start, &data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &addr));
+    OK(uc_reg_write(uc, UC_RISCV_REG_HSTATUS, &hstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    hstatus = 0;
+    value = 0;
+    priv = 3;
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+    OK(uc_reg_write(uc, UC_RISCV_REG_HSTATUS, &hstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X5, &value));
+    priv = 0;
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+
+    OK(uc_close(uc));
+}
+
+typedef struct {
+    uint32_t count;
+    uint32_t intno;
+} RiscvIntrCapture;
+
+static void test_riscv_intr_capture_cb(uc_engine *uc, uint32_t intno,
+                                       void *data)
+{
+    RiscvIntrCapture *capture = (RiscvIntrCapture *)data;
+
+    capture->count++;
+    capture->intno = intno;
+    uc_emu_stop(uc);
+}
+
+static uint64_t riscv_pmp_napot_addr(uint64_t base, uint64_t size)
+{
+    return (base >> 2) | (((size / 2) - 1) >> 2);
+}
+
+static void run_riscv64_pmp_na4_data_access(uint32_t access_insn,
+                                            uint64_t access_addr,
+                                            RiscvIntrCapture *capture,
+                                            uint64_t *result)
+{
+    uc_engine *uc;
+    uc_hook hook;
+    uint32_t insns[] = {
+        riscv_encode_csr(RISCV_CSR_PMPADDR0, 5, 1, 0),
+        riscv_encode_csr(RISCV_CSR_PMPADDR0 + 1, 6, 1, 0),
+        riscv_encode_csr(RISCV_CSR_PMPCFG0, 7, 1, 0),
+        access_insn,
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t data = 0x12345678;
+    uint64_t code_pmpaddr = riscv_pmp_napot_addr(code_start, 0x1000);
+    uint64_t data_pmpaddr = riscv_data_start >> 2;
+    uint64_t pmpcfg = ((RISCV_PMPCFG_R | RISCV_PMPCFG_A_NA4) << 8) |
+                      (RISCV_PMPCFG_X | RISCV_PMPCFG_A_NAPOT);
+    uint64_t priv = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_SIFIVE_U54);
+    OK(uc_mem_map(uc, riscv_data_start, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, riscv_data_start, &data, sizeof(data)));
+    OK(uc_hook_add(uc, &hook, UC_HOOK_INTR, test_riscv_intr_capture_cb,
+                   capture, 1, 0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X5, &code_pmpaddr));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &data_pmpaddr));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X7, &pmpcfg));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X8, &access_addr));
+
+    OK(uc_emu_start(uc, code_start, code_start + 3 * 4, 0, 0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+    OK(uc_emu_start(uc, code_start + 3 * 4, code_start + sizeof(code), 0, 0));
+    if (result != NULL) {
+        OK(uc_reg_read(uc, UC_RISCV_REG_X9, result));
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_pmp_na4_load(void)
+{
+    RiscvIntrCapture capture = { 0 };
+    uint32_t load = riscv_encode_i(0, 8, 2, 9, 0x03);
+    uint64_t value = 0;
+
+    run_riscv64_pmp_na4_data_access(load, riscv_data_start, &capture,
+                                    &value);
+    TEST_CHECK(capture.count == 0);
+    TEST_CHECK(value == 0x12345678);
+}
+
+static void test_riscv64_pmp_na4_rejects_outside(void)
+{
+    RiscvIntrCapture capture = { 0 };
+    uint32_t load = riscv_encode_i(0, 8, 2, 9, 0x03);
+
+    run_riscv64_pmp_na4_data_access(load, riscv_data_start + 4, &capture,
+                                    NULL);
+    TEST_CHECK(capture.count == 1);
+    TEST_CHECK(capture.intno == RISCV_EXCP_LOAD_ACCESS_FAULT);
+}
+
+static void test_riscv64_pmp_na4_rejects_store(void)
+{
+    RiscvIntrCapture capture = { 0 };
+    uint32_t store = riscv_encode_s(0, 9, 8, 2, 0x23);
+
+    run_riscv64_pmp_na4_data_access(store, riscv_data_start, &capture, NULL);
+    TEST_CHECK(capture.count == 1);
+    TEST_CHECK(capture.intno == RISCV_EXCP_STORE_AMO_ACCESS_FAULT);
+}
+
+static void run_riscv64_rvh_virtual_instruction(uint32_t insn)
+{
+    uc_engine *uc;
+    uc_hook hook;
+    RiscvIntrCapture capture = { 0 };
+    uint8_t code[4];
+    uint64_t priv = 5;
+
+    riscv_insn_to_code(code, insn);
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_hook_add(uc, &hook, UC_HOOK_INTR, test_riscv_intr_capture_cb,
+                   &capture, 1, 0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    TEST_CHECK(capture.count == 1);
+    TEST_CHECK(capture.intno == RISCV_EXCP_VIRT_INSTRUCTION_FAULT);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_virtual_instruction_fault(void)
+{
+    run_riscv64_rvh_virtual_instruction(riscv_encode_rvh_load(0x30, 0, 6, 5));
+    run_riscv64_rvh_virtual_instruction(0x22000073);
+    run_riscv64_rvh_virtual_instruction(0x62000073);
+}
+
+static void test_riscv64_rvh_indirect_g_stage_fault(void)
+{
+    uc_engine *uc;
+    uc_hook hook;
+    RiscvIntrCapture capture = { 0 };
+    uint32_t insns[] = {
+        riscv_encode_csr(RISCV_CSR_VSATP, 5, 1, 0),
+        riscv_encode_csr(RISCV_CSR_HGATP, 6, 1, 0),
+        riscv_encode_csr(RISCV_CSR_HSTATUS, 7, 1, 0),
+        riscv_encode_rvh_load(0x34, 0, 13, 14),
+    };
+    uint8_t code[sizeof(insns)];
+    uint64_t vs_pt_root = 0x20000;
+    uint64_t g_pt_root = 0x30000;
+    uint64_t invalid_pte = 0;
+    uint64_t va = riscv_data_start;
+    uint64_t vsatp = RISCV64_SATP_MODE_SV39 | (vs_pt_root >> 12);
+    uint64_t hgatp = RISCV64_SATP_MODE_SV39 | (g_pt_root >> 12);
+    uint64_t hstatus = RISCV_HSTATUS_SPVP;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_mem_map(uc, g_pt_root, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, g_pt_root, &invalid_pte, sizeof(invalid_pte)));
+    OK(uc_hook_add(uc, &hook, UC_HOOK_INTR, test_riscv_intr_capture_cb,
+                   &capture, 1, 0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X5, &vsatp));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &hgatp));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X7, &hstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X13, &va));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    TEST_CHECK(capture.count == 1);
+    TEST_CHECK(capture.intno == RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv_virtual_wfi_fault(void)
+{
+    uc_engine *uc;
+    uc_hook hook;
+    RiscvIntrCapture capture = { 0 };
+    uint32_t insn = 0x10500073;
+    uint8_t code[4];
+    uint64_t priv = 5;
+    uint64_t hstatus = RISCV_HSTATUS_VTW;
+
+    riscv_insn_to_code(code, insn);
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_hook_add(uc, &hook, UC_HOOK_INTR, test_riscv_intr_capture_cb,
+                   &capture, 1, 0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_HSTATUS, &hstatus));
+    OK(uc_reg_write(uc, UC_RISCV_REG_PRIV, &priv));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    TEST_CHECK(capture.count == 1);
+    TEST_CHECK(capture.intno == RISCV_EXCP_VIRT_INSTRUCTION_FAULT);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_hfence(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        0x22000073,
+        0x62000073,
+    };
+    uint8_t code[sizeof(insns)];
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup_model(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                          (const char *)code, sizeof(code),
+                          UC_CPU_RISCV64_BASE64);
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+    OK(uc_close(uc));
+}
+
+static void test_riscv_rvh_hfence_requires_h(void)
+{
+    uint32_t insns[] = {
+        0x22000073,
+        0x62000073,
+    };
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        run_riscv_model_insn_illegal(UC_MODE_RISCV32,
+                                     UC_CPU_RISCV32_SIFIVE_E31, insns[i]);
+        run_riscv_model_insn_illegal(UC_MODE_RISCV64,
+                                     UC_CPU_RISCV64_SIFIVE_E51, insns[i]);
+    }
+}
+
+static void test_riscv64_rv128_opcodes_rejected(void)
+{
+    uint32_t lq = riscv_encode_i(0, 6, 2, 5, 0x0f);
+    uint32_t sq = riscv_encode_s(0, 5, 6, 4, 0x23);
+    uint32_t addd = riscv_encode_r(0, 7, 6, 0, 5, 0x7b);
+    uint32_t muld = riscv_encode_r(1, 7, 6, 0, 5, 0x7b);
+
+    run_riscv_insn_illegal(UC_MODE_RISCV64, lq);
+    run_riscv_insn_illegal(UC_MODE_RISCV64, sq);
+    run_riscv_insn_illegal(UC_MODE_RISCV64, addd);
+    run_riscv_insn_illegal(UC_MODE_RISCV64, muld);
+}
+
+static void test_riscv32_xventanacondops(void)
+{
+    uint32_t vt_maskc = riscv_encode_r(0x00, 7, 6, 6, 5, 0x7b);
+    uint32_t vt_maskcn = riscv_encode_r(0x00, 7, 6, 7, 5, 0x7b);
+
+    run_riscv32_zbb_case(vt_maskc, 0xa5a55a5a, 1, 5, 0xa5a55a5a);
+    run_riscv32_zbb_case(vt_maskc, 0xa5a55a5a, 0, 5, 0);
+    run_riscv32_zbb_case(vt_maskcn, 0x55aa1234, 0, 5, 0x55aa1234);
+    run_riscv32_zbb_case(vt_maskcn, 0x55aa1234, 1, 5, 0);
+}
+
+static void test_riscv64_xventanacondops(void)
+{
+    uint32_t vt_maskc = riscv_encode_r(0x00, 7, 6, 6, 5, 0x7b);
+    uint32_t vt_maskcn = riscv_encode_r(0x00, 7, 6, 7, 5, 0x7b);
+
+    run_riscv64_zbb_case(vt_maskc, 0xfedcba9876543210ull, 1, 5,
+                         0xfedcba9876543210ull);
+    run_riscv64_zbb_case(vt_maskc, 0xfedcba9876543210ull, 0, 5, 0);
+    run_riscv64_zbb_case(vt_maskcn, 0x0123456789abcdefull, 0, 5,
+                         0x0123456789abcdefull);
+    run_riscv64_zbb_case(vt_maskcn, 0x0123456789abcdefull, 1, 5, 0);
+}
+
+static void run_riscv32_seed_case(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint32_t source = 0;
+    uint32_t actual = 0;
+
+    riscv_insn_to_code(code, riscv_encode_i(0x015, 6, 1, 5, 0x73));
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, (const char *)code,
+                    sizeof(code));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &source));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X5, &actual));
+    TEST_CHECK((actual & 0xc0000000u) == 0x80000000u);
+    OK(uc_close(uc));
+}
+
+static void run_riscv64_seed_case(void)
+{
+    uc_engine *uc;
+    uint8_t code[4];
+    uint64_t source = 0;
+    uint64_t actual = 0;
+
+    riscv_insn_to_code(code, riscv_encode_i(0x015, 6, 1, 5, 0x73));
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, (const char *)code,
+                    sizeof(code));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &source));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X5, &actual));
+    TEST_CHECK((actual & 0xc0000000ull) == 0x80000000ull);
+    TEST_CHECK((actual >> 32) == 0);
+    OK(uc_close(uc));
+}
+
+static uint64_t riscv_ref_clmul(uint64_t rs1, uint64_t rs2, int bits)
+{
+    uint64_t mask = bits == 64 ? ~0ull : 0xffffffffull;
+    uint64_t result = 0;
+    int i;
+
+    rs1 &= mask;
+    rs2 &= mask;
+
+    for (i = 0; i < bits; i++) {
+        if ((rs2 >> i) & 1) {
+            result ^= rs1 << i;
+        }
+    }
+
+    return result & mask;
+}
+
+static uint64_t riscv_ref_clmulr(uint64_t rs1, uint64_t rs2, int bits)
+{
+    uint64_t mask = bits == 64 ? ~0ull : 0xffffffffull;
+    uint64_t result = 0;
+    int i;
+
+    rs1 &= mask;
+    rs2 &= mask;
+
+    for (i = 0; i < bits; i++) {
+        if ((rs2 >> i) & 1) {
+            result ^= rs1 >> (bits - i - 1);
+        }
+    }
+
+    return result & mask;
+}
+
+static uint64_t riscv_ref_brev8(uint64_t value, int bits)
+{
+    uint64_t result = 0;
+    int byte;
+    int bit;
+
+    for (byte = 0; byte < bits; byte += 8) {
+        uint64_t src = (value >> byte) & 0xff;
+        uint64_t dst = 0;
+
+        for (bit = 0; bit < 8; bit++) {
+            dst |= ((src >> bit) & 1) << (7 - bit);
+        }
+        result |= dst << byte;
+    }
+
+    return bits == 64 ? result : result & 0xffffffffull;
+}
+
+static uint64_t riscv_ref_pack(uint64_t rs1, uint64_t rs2, int bits)
+{
+    int half = bits / 2;
+    uint64_t half_mask = (1ull << half) - 1;
+
+    return (rs1 & half_mask) | ((rs2 & half_mask) << half);
+}
+
+static uint64_t riscv_ref_packh(uint64_t rs1, uint64_t rs2)
+{
+    return (rs1 & 0xff) | ((rs2 & 0xff) << 8);
+}
+
+static uint64_t riscv_ref_packw(uint64_t rs1, uint64_t rs2)
+{
+    uint32_t word = (rs1 & 0xffff) | ((rs2 & 0xffff) << 16);
+
+    return (uint64_t)(int64_t)(int32_t)word;
+}
+
+static uint32_t riscv_ref_shuf_stage(uint32_t src, uint32_t mask_l,
+                                     uint32_t mask_r, int shift)
+{
+    uint32_t x = src & ~(mask_l | mask_r);
+
+    x |= ((src << shift) & mask_l) | ((src >> shift) & mask_r);
+    return x;
+}
+
+static uint32_t riscv_ref_unzip(uint32_t value)
+{
+    value = riscv_ref_shuf_stage(value, 0x44444444u, 0x22222222u, 1);
+    value = riscv_ref_shuf_stage(value, 0x30303030u, 0x0c0c0c0cu, 2);
+    value = riscv_ref_shuf_stage(value, 0x0f000f00u, 0x00f000f0u, 4);
+    value = riscv_ref_shuf_stage(value, 0x00ff0000u, 0x0000ff00u, 8);
+    return value;
+}
+
+static uint32_t riscv_ref_zip(uint32_t value)
+{
+    value = riscv_ref_shuf_stage(value, 0x00ff0000u, 0x0000ff00u, 8);
+    value = riscv_ref_shuf_stage(value, 0x0f000f00u, 0x00f000f0u, 4);
+    value = riscv_ref_shuf_stage(value, 0x30303030u, 0x0c0c0c0cu, 2);
+    value = riscv_ref_shuf_stage(value, 0x44444444u, 0x22222222u, 1);
+    return value;
+}
+
+static uint64_t riscv_ref_xperm(uint64_t rs1, uint64_t rs2, int bits,
+                                int sz_log2)
+{
+    uint64_t result = 0;
+    uint64_t sz = 1ull << sz_log2;
+    uint64_t mask = (1ull << sz) - 1;
+    uint64_t pos;
+    int i;
+
+    for (i = 0; i < bits; i += sz) {
+        pos = ((rs2 >> i) & mask) << sz_log2;
+        if (pos < (uint64_t)bits) {
+            result |= ((rs1 >> pos) & mask) << i;
+        }
+    }
+
+    return bits == 64 ? result : result & 0xffffffffull;
+}
+
+static uint32_t riscv_rotl32(uint32_t value, unsigned int shift)
+{
+    return (value << shift) | (value >> (32 - shift));
+}
+
+static uint32_t riscv_rotr32(uint32_t value, unsigned int shift)
+{
+    return (value >> shift) | (value << (32 - shift));
+}
+
+static uint64_t riscv_rotl64(uint64_t value, unsigned int shift)
+{
+    return (value << shift) | (value >> (64 - shift));
+}
+
+static uint64_t riscv_rotr64(uint64_t value, unsigned int shift)
+{
+    return (value >> shift) | (value << (64 - shift));
+}
+
+static uint64_t riscv_sext32(uint32_t value)
+{
+    return (uint64_t)(int64_t)(int32_t)value;
+}
+
+static uint32_t riscv_ref_sha256(uint32_t value, int op)
+{
+    switch (op) {
+    case 0:
+        return riscv_rotr32(value, 2) ^ riscv_rotr32(value, 13) ^
+               riscv_rotr32(value, 22);
+    case 1:
+        return riscv_rotr32(value, 6) ^ riscv_rotr32(value, 11) ^
+               riscv_rotr32(value, 25);
+    case 2:
+        return riscv_rotr32(value, 7) ^ riscv_rotr32(value, 18) ^
+               (value >> 3);
+    default:
+        return riscv_rotr32(value, 17) ^ riscv_rotr32(value, 19) ^
+               (value >> 10);
+    }
+}
+
+static uint64_t riscv_ref_sha512_rv32(uint32_t rs1, uint32_t rs2, int op)
+{
+    uint64_t value = ((uint64_t)rs2 << 32) | rs1;
+    uint64_t result;
+
+    switch (op) {
+    case 0:
+        result = riscv_rotl64(value, 25) ^ riscv_rotl64(value, 30) ^
+                 riscv_rotr64(value, 28);
+        break;
+    case 1:
+        result = riscv_rotl64(value, 23) ^ riscv_rotr64(value, 14) ^
+                 riscv_rotr64(value, 18);
+        break;
+    case 2:
+        result = riscv_rotr64(value, 1) ^ riscv_rotr64(value, 7) ^
+                 riscv_rotr64(value, 8);
+        break;
+    case 3:
+        result = riscv_rotr64(value, 1) ^ ((uint32_t)value >> 7) ^
+                 riscv_rotr64(value, 8);
+        break;
+    case 4:
+        result = riscv_rotl64(value, 3) ^ riscv_rotr64(value, 6) ^
+                 riscv_rotr64(value, 19);
+        break;
+    default:
+        result = riscv_rotl64(value, 3) ^ ((uint32_t)value >> 6) ^
+                 riscv_rotr64(value, 19);
+        break;
+    }
+
+    return result & 0xffffffffull;
+}
+
+static uint64_t riscv_ref_sha512_rv64(uint64_t value, int op)
+{
+    switch (op) {
+    case 0:
+        return riscv_rotr64(value, 28) ^ riscv_rotr64(value, 34) ^
+               riscv_rotr64(value, 39);
+    case 1:
+        return riscv_rotr64(value, 14) ^ riscv_rotr64(value, 18) ^
+               riscv_rotr64(value, 41);
+    case 2:
+        return riscv_rotr64(value, 1) ^ riscv_rotr64(value, 8) ^
+               (value >> 7);
+    default:
+        return riscv_rotr64(value, 19) ^ riscv_rotr64(value, 61) ^
+               (value >> 6);
+    }
+}
+
+static uint32_t riscv_ref_sm3(uint32_t value, int op)
+{
+    if (op == 0) {
+        return value ^ riscv_rotl32(value, 9) ^ riscv_rotl32(value, 17);
+    }
+
+    return value ^ riscv_rotl32(value, 15) ^ riscv_rotl32(value, 23);
+}
+
+static void test_riscv32_zba(void)
+{
+    run_riscv32_zbb_case(riscv_encode_r(0x10, 7, 6, 2, 5, 0x33),
+                         3, 5, 5, 11);
+    run_riscv32_zbb_case(riscv_encode_r(0x10, 7, 6, 4, 5, 0x33),
+                         3, 5, 5, 17);
+    run_riscv32_zbb_case(riscv_encode_r(0x10, 7, 6, 6, 5, 0x33),
+                         3, 5, 5, 29);
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_r(0x04, 7, 6, 0, 5, 0x3b));
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_i(0x081, 6, 1, 5, 0x1b));
+}
+
+static void test_riscv64_zba(void)
+{
+    run_riscv64_zbb_case(riscv_encode_r(0x10, 7, 6, 2, 5, 0x33),
+                         3, 5, 5, 11);
+    run_riscv64_zbb_case(riscv_encode_r(0x10, 7, 6, 4, 5, 0x33),
+                         3, 5, 5, 17);
+    run_riscv64_zbb_case(riscv_encode_r(0x10, 7, 6, 6, 5, 0x33),
+                         3, 5, 5, 29);
+    run_riscv64_zbb_case(riscv_encode_r(0x04, 7, 6, 0, 5, 0x3b),
+                         0xffffffff00000002ull, 5, 5, 7);
+    run_riscv64_zbb_case(riscv_encode_r(0x10, 7, 6, 2, 5, 0x3b),
+                         0xffffffff00000002ull, 5, 5, 9);
+    run_riscv64_zbb_case(riscv_encode_r(0x10, 7, 6, 4, 5, 0x3b),
+                         0xffffffff00000002ull, 5, 5, 13);
+    run_riscv64_zbb_case(riscv_encode_r(0x10, 7, 6, 6, 5, 0x3b),
+                         0xffffffff00000002ull, 5, 5, 21);
+    run_riscv64_zbb_case(riscv_encode_i(0x081, 6, 1, 5, 0x1b),
+                         0xffffffff80000001ull, 0, 5, 0x100000002ull);
+    run_riscv64_zbb_case(riscv_encode_i(0x09f, 6, 1, 5, 0x1b),
+                         0xffffffff80000001ull, 0, 5,
+                         0x4000000080000000ull);
+    run_riscv64_zbb_case(riscv_encode_i(0x0a0, 6, 1, 5, 0x1b),
+                         0xffffffff80000001ull, 0, 5,
+                         0x8000000100000000ull);
+    run_riscv64_zbb_case(riscv_encode_i(0x0bf, 6, 1, 5, 0x1b),
+                         1, 0, 5, 0x8000000000000000ull);
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_i(0x0c0, 6, 1, 5, 0x1b));
+}
+
+static void test_riscv32_zbc(void)
+{
+    uint32_t rs1 = 0x12345678u;
+    uint32_t rs2 = 0x00f0f00fu;
+
+    run_riscv32_zbb_case(riscv_encode_r(0x05, 7, 6, 1, 5, 0x33),
+                         rs1, rs2, 5,
+                         riscv_ref_clmul(rs1, rs2, 32));
+    run_riscv32_zbb_case(riscv_encode_r(0x05, 7, 6, 2, 5, 0x33),
+                         rs1, rs2, 5,
+                         riscv_ref_clmulr(rs1, rs2, 32));
+    run_riscv32_zbb_case(riscv_encode_r(0x05, 7, 6, 3, 5, 0x33),
+                         rs1, rs2, 5,
+                         riscv_ref_clmulr(rs1, rs2, 32) >> 1);
+}
+
+static void test_riscv64_zbc(void)
+{
+    uint64_t rs1 = 0x0123456789abcdefull;
+    uint64_t rs2 = 0xf0f00f0f33333333ull;
+
+    run_riscv64_zbb_case(riscv_encode_r(0x05, 7, 6, 1, 5, 0x33),
+                         rs1, rs2, 5,
+                         riscv_ref_clmul(rs1, rs2, 64));
+    run_riscv64_zbb_case(riscv_encode_r(0x05, 7, 6, 2, 5, 0x33),
+                         rs1, rs2, 5,
+                         riscv_ref_clmulr(rs1, rs2, 64));
+    run_riscv64_zbb_case(riscv_encode_r(0x05, 7, 6, 3, 5, 0x33),
+                         rs1, rs2, 5,
+                         riscv_ref_clmulr(rs1, rs2, 64) >> 1);
+}
+
+static void test_riscv32_zbkb(void)
+{
+    uint32_t rs1 = 0x12345678u;
+    uint32_t rs2 = 0x89abcdefu;
+    uint32_t value = 0x01234567u;
+
+    run_riscv32_zbb_case(riscv_encode_i(0x687, 6, 5, 5, 0x13),
+                         value, 0, 5, riscv_ref_brev8(value, 32));
+    run_riscv32_zbb_case(riscv_encode_r(0x04, 7, 6, 4, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_pack(rs1, rs2, 32));
+    run_riscv32_zbb_case(riscv_encode_r(0x04, 7, 6, 7, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_packh(rs1, rs2));
+    run_riscv32_zbb_case(riscv_encode_i(0x08f, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_zip(value));
+    run_riscv32_zbb_case(riscv_encode_i(0x08f, 6, 5, 5, 0x13),
+                         value, 0, 5, riscv_ref_unzip(value));
+}
+
+static void test_riscv64_zbkb(void)
+{
+    uint64_t rs1 = 0x123456789abcdef0ull;
+    uint64_t rs2 = 0x0fedcba987658321ull;
+    uint64_t value = 0x0123456789abcdefull;
+
+    run_riscv64_zbb_case(riscv_encode_i(0x687, 6, 5, 5, 0x13),
+                         value, 0, 5, riscv_ref_brev8(value, 64));
+    run_riscv64_zbb_case(riscv_encode_r(0x04, 7, 6, 4, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_pack(rs1, rs2, 64));
+    run_riscv64_zbb_case(riscv_encode_r(0x04, 7, 6, 7, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_packh(rs1, rs2));
+    run_riscv64_zbb_case(riscv_encode_r(0x04, 7, 6, 4, 5, 0x3b),
+                         rs1, rs2, 5, riscv_ref_packw(rs1, rs2));
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_i(0x08f, 6, 1, 5, 0x13));
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_i(0x08f, 6, 5, 5, 0x13));
+}
+
+static void test_riscv32_zbkx(void)
+{
+    uint32_t rs1 = 0xfedcba98u;
+    uint32_t rs2 = 0x01234567u;
+
+    run_riscv32_zbb_case(riscv_encode_r(0x14, 7, 6, 2, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_xperm(rs1, rs2, 32, 2));
+    run_riscv32_zbb_case(riscv_encode_r(0x14, 7, 6, 4, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_xperm(rs1, rs2, 32, 3));
+}
+
+static void test_riscv64_zbkx(void)
+{
+    uint64_t rs1 = 0xfedcba9876543210ull;
+    uint64_t rs2 = 0x0123456789abcdefull;
+
+    run_riscv64_zbb_case(riscv_encode_r(0x14, 7, 6, 2, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_xperm(rs1, rs2, 64, 2));
+    run_riscv64_zbb_case(riscv_encode_r(0x14, 7, 6, 4, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_xperm(rs1, rs2, 64, 3));
+}
+
+static void test_riscv32_zknh_sha256(void)
+{
+    uint32_t value = 0x89abcdefu;
+
+    run_riscv32_zbb_case(riscv_encode_i(0x100, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha256(value, 0));
+    run_riscv32_zbb_case(riscv_encode_i(0x101, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha256(value, 1));
+    run_riscv32_zbb_case(riscv_encode_i(0x102, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha256(value, 2));
+    run_riscv32_zbb_case(riscv_encode_i(0x103, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha256(value, 3));
+}
+
+static void test_riscv64_zknh_sha256(void)
+{
+    uint32_t value = 0x89abcdefu;
+    uint64_t reg_value = 0xffffffff89abcdefull;
+
+    run_riscv64_zbb_case(riscv_encode_i(0x100, 6, 1, 5, 0x13),
+                         reg_value, 0, 5,
+                         riscv_sext32(riscv_ref_sha256(value, 0)));
+    run_riscv64_zbb_case(riscv_encode_i(0x101, 6, 1, 5, 0x13),
+                         reg_value, 0, 5,
+                         riscv_sext32(riscv_ref_sha256(value, 1)));
+    run_riscv64_zbb_case(riscv_encode_i(0x102, 6, 1, 5, 0x13),
+                         reg_value, 0, 5,
+                         riscv_sext32(riscv_ref_sha256(value, 2)));
+    run_riscv64_zbb_case(riscv_encode_i(0x103, 6, 1, 5, 0x13),
+                         reg_value, 0, 5,
+                         riscv_sext32(riscv_ref_sha256(value, 3)));
+}
+
+static void test_riscv32_zknh_sha512(void)
+{
+    uint32_t rs1 = 0x89abcdefu;
+    uint32_t rs2 = 0x01234567u;
+
+    run_riscv32_zbb_case(riscv_encode_r(0x28, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_sha512_rv32(rs1, rs2, 0));
+    run_riscv32_zbb_case(riscv_encode_r(0x29, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_sha512_rv32(rs1, rs2, 1));
+    run_riscv32_zbb_case(riscv_encode_r(0x2a, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_sha512_rv32(rs1, rs2, 2));
+    run_riscv32_zbb_case(riscv_encode_r(0x2e, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_sha512_rv32(rs1, rs2, 3));
+    run_riscv32_zbb_case(riscv_encode_r(0x2b, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_sha512_rv32(rs1, rs2, 4));
+    run_riscv32_zbb_case(riscv_encode_r(0x2f, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, riscv_ref_sha512_rv32(rs1, rs2, 5));
+}
+
+static void test_riscv64_zknh_sha512(void)
+{
+    uint64_t value = 0x0123456789abcdefull;
+
+    run_riscv64_zbb_case(riscv_encode_i(0x104, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha512_rv64(value, 0));
+    run_riscv64_zbb_case(riscv_encode_i(0x105, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha512_rv64(value, 1));
+    run_riscv64_zbb_case(riscv_encode_i(0x106, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha512_rv64(value, 2));
+    run_riscv64_zbb_case(riscv_encode_i(0x107, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sha512_rv64(value, 3));
+}
+
+static void test_riscv_zksh_sm3(void)
+{
+    uint32_t value = 0x89abcdefu;
+    uint64_t reg_value = 0xffffffff89abcdefull;
+
+    run_riscv32_zbb_case(riscv_encode_i(0x108, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sm3(value, 0));
+    run_riscv32_zbb_case(riscv_encode_i(0x109, 6, 1, 5, 0x13),
+                         value, 0, 5, riscv_ref_sm3(value, 1));
+    run_riscv64_zbb_case(riscv_encode_i(0x108, 6, 1, 5, 0x13),
+                         reg_value, 0, 5,
+                         riscv_sext32(riscv_ref_sm3(value, 0)));
+    run_riscv64_zbb_case(riscv_encode_i(0x109, 6, 1, 5, 0x13),
+                         reg_value, 0, 5,
+                         riscv_sext32(riscv_ref_sm3(value, 1)));
+}
+
+static void test_riscv32_zkne_aes(void)
+{
+    uint32_t rs1 = 0x11223344u;
+    uint32_t rs2 = 0x89abcdefu;
+
+    run_riscv32_zbb_case(riscv_encode_k_aes(0x11, 0, 7, 6, 5),
+                         rs1, rs2, 5, 0x1122339bu);
+    run_riscv32_zbb_case(riscv_encode_k_aes(0x13, 16, 7, 6, 5),
+                         rs1, rs2, 5, 0x73e69526u);
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_k_aes(0x11, 0, 7, 6, 5));
+}
+
+static void test_riscv32_zknd_aes(void)
+{
+    uint32_t rs1 = 0x11223344u;
+    uint32_t rs2 = 0x89abcdefu;
+
+    run_riscv32_zbb_case(riscv_encode_k_aes(0x15, 8, 7, 6, 5),
+                         rs1, rs2, 5, 0x1122b344u);
+    run_riscv32_zbb_case(riscv_encode_k_aes(0x17, 24, 7, 6, 5),
+                         rs1, rs2, 5, 0xdafef567u);
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_r(0x19, 7, 6, 0, 5, 0x33));
+}
+
+static void test_riscv64_zkne_zknd_aes(void)
+{
+    uint64_t rs1 = 0x0011223344556677ull;
+    uint64_t rs2 = 0x8899aabbccddeeffull;
+
+    run_riscv64_zbb_case(riscv_encode_r(0x19, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, 0x1bee28c3c4c193f5ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x1b, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, 0xae01a110c5a8545aull);
+    run_riscv64_zbb_case(riscv_encode_r(0x1d, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, 0x27f9d36652c96202ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x1f, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, 0x460a644350878da1ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x3f, 7, 6, 0, 5, 0x33),
+                         rs1, rs2, 5, 0x44556677ccccccccull);
+    run_riscv64_zbb_case(riscv_encode_i(0x310, 6, 1, 5, 0x13),
+                         rs1, 0, 5, 0xc3638292c3638292ull);
+    run_riscv64_zbb_case(riscv_encode_i(0x31a, 6, 1, 5, 0x13),
+                         rs1, 0, 5, 0x638293c3638293c3ull);
+    run_riscv64_zbb_case(riscv_encode_i(0x300, 6, 1, 5, 0x13),
+                         rs1, 0, 5, 0xeebbcc99aaff88ddull);
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_i(0x31b, 6, 1, 5, 0x13));
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_i(0x300, 6, 1, 5, 0x13));
+}
+
+static void test_riscv_zksed_sm4(void)
+{
+    uint32_t rs1 = 0x11223344u;
+    uint32_t rs2 = 0x89abcdefu;
+    uint64_t rs1_64 = 0xffffffff11223344ull;
+    uint64_t rs2_64 = 0xffffffff89abcdefull;
+
+    run_riscv32_zbb_case(riscv_encode_k_aes(0x18, 0, 7, 6, 5),
+                         rs1, rs2, 5, 0x0330b5d0u);
+    run_riscv32_zbb_case(riscv_encode_k_aes(0x1a, 24, 7, 6, 5),
+                         rs1, rs2, 5, 0xe6c2ad3fu);
+    run_riscv64_zbb_case(riscv_encode_k_aes(0x18, 0, 7, 6, 5),
+                         rs1_64, rs2_64, 5, 0x000000000330b5d0ull);
+    run_riscv64_zbb_case(riscv_encode_k_aes(0x1a, 24, 7, 6, 5),
+                         rs1_64, rs2_64, 5, 0xffffffffe6c2ad3full);
+}
+
+static void test_riscv_zkr_seed(void)
+{
+    run_riscv32_seed_case();
+    run_riscv64_seed_case();
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_i(0x015, 0, 2, 5, 0x73));
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_i(0x015, 0, 2, 5, 0x73));
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_i(0x015, 0, 7, 5, 0x73));
+}
+
+static void test_riscv32_zbb_unary(void)
+{
+    run_riscv32_zbb_case(riscv_encode_i(0x600, 6, 1, 5, 0x13),
+                         0, 0, 5, 32);
+    run_riscv32_zbb_case(riscv_encode_i(0x601, 6, 1, 5, 0x13),
+                         0, 0, 5, 32);
+    run_riscv32_zbb_case(riscv_encode_i(0x602, 6, 1, 5, 0x13),
+                         0xffffffffu, 0, 5, 32);
+    run_riscv32_zbb_case(riscv_encode_i(0x604, 6, 1, 5, 0x13),
+                         0x80, 0, 5, 0xffffff80u);
+    run_riscv32_zbb_case(riscv_encode_i(0x605, 6, 1, 5, 0x13),
+                         0x8001, 0, 5, 0xffff8001u);
+    run_riscv32_zbb_case(riscv_encode_r(0x04, 0, 6, 4, 5, 0x33),
+                         0xffff8001u, 0, 5, 0x8001);
+    run_riscv32_zbb_case(riscv_encode_i(0x287, 6, 5, 5, 0x13),
+                         0x01008000, 0, 5, 0xff00ff00u);
+    run_riscv32_zbb_case(riscv_encode_i(0x698, 6, 5, 5, 0x13),
+                         0x11223344, 0, 5, 0x44332211);
+}
+
+static void test_riscv64_zbb_unary(void)
+{
+    run_riscv64_zbb_case(riscv_encode_i(0x600, 6, 1, 5, 0x13),
+                         0, 0, 5, 64);
+    run_riscv64_zbb_case(riscv_encode_i(0x601, 6, 1, 5, 0x13),
+                         0, 0, 5, 64);
+    run_riscv64_zbb_case(riscv_encode_i(0x602, 6, 1, 5, 0x13),
+                         0xffffffffffffffffull, 0, 5, 64);
+    run_riscv64_zbb_case(riscv_encode_i(0x604, 6, 1, 5, 0x13),
+                         0x80, 0, 5, 0xffffffffffffff80ull);
+    run_riscv64_zbb_case(riscv_encode_i(0x605, 6, 1, 5, 0x13),
+                         0x8001, 0, 5, 0xffffffffffff8001ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x04, 0, 6, 4, 5, 0x3b),
+                         0xffffffffffff8001ull, 0, 5, 0x8001);
+    run_riscv64_zbb_case(riscv_encode_i(0x287, 6, 5, 5, 0x13),
+                         0x0100800000000001ull, 0, 5,
+                         0xff00ff00000000ffull);
+    run_riscv64_zbb_case(riscv_encode_i(0x6b8, 6, 5, 5, 0x13),
+                         0x1122334455667788ull, 0, 5,
+                         0x8877665544332211ull);
+}
+
+static void test_riscv32_zbb_binary(void)
+{
+    run_riscv32_zbb_case(riscv_encode_r(0x20, 7, 6, 7, 5, 0x33),
+                         0xff00ff00u, 0x0f0f0f0fu, 5, 0xf000f000u);
+    run_riscv32_zbb_case(riscv_encode_r(0x20, 7, 6, 6, 5, 0x33),
+                         0x0000ff00u, 0x00ff00ffu, 5, 0xff00ff00u);
+    run_riscv32_zbb_case(riscv_encode_r(0x20, 7, 6, 4, 5, 0x33),
+                         0xaaaaaaaau, 0xffff0000u, 5, 0xaaaa5555u);
+    run_riscv32_zbb_case(riscv_encode_r(0x05, 7, 6, 4, 5, 0x33),
+                         0x80000000u, 0x7fffffffu, 5, 0x80000000u);
+    run_riscv32_zbb_case(riscv_encode_r(0x05, 7, 6, 6, 5, 0x33),
+                         0x80000000u, 0x7fffffffu, 5, 0x7fffffffu);
+    run_riscv32_zbb_case(riscv_encode_r(0x05, 7, 6, 5, 5, 0x33),
+                         0x80000000u, 0x7fffffffu, 5, 0x7fffffffu);
+    run_riscv32_zbb_case(riscv_encode_r(0x05, 7, 6, 7, 5, 0x33),
+                         0x80000000u, 0x7fffffffu, 5, 0x80000000u);
+}
+
+static void test_riscv64_zbb_binary(void)
+{
+    run_riscv64_zbb_case(riscv_encode_r(0x20, 7, 6, 7, 5, 0x33),
+                         0xff00ff00ff00ff00ull, 0x0f0f0f0f0f0f0f0full,
+                         5, 0xf000f000f000f000ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x20, 7, 6, 6, 5, 0x33),
+                         0x000000000000ff00ull, 0x00ff00ff00ff00ffull,
+                         5, 0xff00ff00ff00ff00ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x20, 7, 6, 4, 5, 0x33),
+                         0xaaaaaaaaaaaaaaaaull, 0xffff0000ffff0000ull,
+                         5, 0xaaaa5555aaaa5555ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x05, 7, 6, 4, 5, 0x33),
+                         0x8000000000000000ull, 0x7fffffffffffffffull,
+                         5, 0x8000000000000000ull);
+    run_riscv64_zbb_case(riscv_encode_r(0x05, 7, 6, 6, 5, 0x33),
+                         0x8000000000000000ull, 0x7fffffffffffffffull,
+                         5, 0x7fffffffffffffffull);
+    run_riscv64_zbb_case(riscv_encode_r(0x05, 7, 6, 5, 5, 0x33),
+                         0x8000000000000000ull, 0x7fffffffffffffffull,
+                         5, 0x7fffffffffffffffull);
+    run_riscv64_zbb_case(riscv_encode_r(0x05, 7, 6, 7, 5, 0x33),
+                         0x8000000000000000ull, 0x7fffffffffffffffull,
+                         5, 0x8000000000000000ull);
+}
+
+static void test_riscv32_zbb_rotate(void)
+{
+    run_riscv32_zbb_case(riscv_encode_r(0x30, 7, 6, 1, 5, 0x33),
+                         1, 33, 5, 2);
+    run_riscv32_zbb_case(riscv_encode_r(0x30, 7, 6, 5, 5, 0x33),
+                         2, 33, 5, 1);
+    run_riscv32_zbb_case(riscv_encode_i(0x61f, 6, 5, 5, 0x13),
+                         1, 0, 5, 2);
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_i(0x620, 6, 5, 5, 0x13));
+}
+
+static void test_riscv64_zbb_rotate(void)
+{
+    run_riscv64_zbb_case(riscv_encode_r(0x30, 7, 6, 1, 5, 0x33),
+                         1, 65, 5, 2);
+    run_riscv64_zbb_case(riscv_encode_r(0x30, 7, 6, 5, 5, 0x33),
+                         2, 65, 5, 1);
+    run_riscv64_zbb_case(riscv_encode_i(0x63f, 6, 5, 5, 0x13),
+                         1, 0, 5, 2);
+}
+
+static void test_riscv64_zbb_word(void)
+{
+    run_riscv64_zbb_case(riscv_encode_i(0x600, 6, 1, 5, 0x1b),
+                         0xffff000000008000ull, 0, 5, 16);
+    run_riscv64_zbb_case(riscv_encode_i(0x601, 6, 1, 5, 0x1b),
+                         0xffff000080000000ull, 0, 5, 31);
+    run_riscv64_zbb_case(riscv_encode_i(0x602, 6, 1, 5, 0x1b),
+                         0xffff0000f0f00000ull, 0, 5, 8);
+    run_riscv64_zbb_case(riscv_encode_r(0x30, 7, 6, 1, 5, 0x3b),
+                         0x80000001ull, 1, 5, 3);
+    run_riscv64_zbb_case(riscv_encode_r(0x30, 7, 6, 5, 5, 0x3b),
+                         1, 1, 5, 0xffffffff80000000ull);
+    run_riscv64_zbb_case(riscv_encode_i(0x601, 6, 5, 5, 0x1b),
+                         1, 0, 5, 0xffffffff80000000ull);
+}
+
+static void test_riscv_zbb_illegal_encodings(void)
+{
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_i(0x6b8, 6, 5, 5, 0x13));
+    run_riscv_insn_illegal(UC_MODE_RISCV32,
+                           riscv_encode_r(0x30, 7, 6, 5, 5, 0x3b));
+    run_riscv_insn_illegal(UC_MODE_RISCV64,
+                           riscv_encode_i(0x698, 6, 5, 5, 0x13));
+}
+
+static void run_riscv32_zbs_reg_case(const char *code, uint32_t rs1,
+                                     uint32_t rs2, int rd, uint32_t expected)
+{
+    uc_engine *uc;
+    uint32_t actual = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code, 4);
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &rs1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X7, &rs2));
+
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X0 + rd, &actual));
+    TEST_CHECK(actual == expected);
+    OK(uc_close(uc));
+}
+
+static void run_riscv64_zbs_reg_case(const char *code, uint64_t rs1,
+                                     uint64_t rs2, int rd, uint64_t expected)
+{
+    uc_engine *uc;
+    uint64_t actual = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code, 4);
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &rs1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_X7, &rs2));
+
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X0 + rd, &actual));
+    TEST_CHECK(actual == expected);
+    OK(uc_close(uc));
+}
+
+static void run_riscv32_zbs_imm_case(const char *code, uint32_t rs1, int rd,
+                                     uint32_t expected)
+{
+    uc_engine *uc;
+    uint32_t actual = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code, 4);
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &rs1));
+
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X0 + rd, &actual));
+    TEST_CHECK(actual == expected);
+    OK(uc_close(uc));
+}
+
+static void run_riscv64_zbs_imm_case(const char *code, uint64_t rs1, int rd,
+                                     uint64_t expected)
+{
+    uc_engine *uc;
+    uint64_t actual = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code, 4);
+    OK(uc_reg_write(uc, UC_RISCV_REG_X6, &rs1));
+
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_X0 + rd, &actual));
+    TEST_CHECK(actual == expected);
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_zbs_register(void)
+{
+    const char bset[] = "\xb3\x12\x73\x28";
+    const char bclr[] = "\x33\x14\x73\x48";
+    const char binv[] = "\xb3\x14\x73\x68";
+    const char bext[] = "\x33\x55\x73\x48";
+
+    run_riscv32_zbs_reg_case(bset, 0, 33, 5, 2);
+    run_riscv32_zbs_reg_case(bclr, 0xffffffffu, 33, 8, 0xfffffffdu);
+    run_riscv32_zbs_reg_case(binv, 0, 33, 9, 2);
+    run_riscv32_zbs_reg_case(bext, 2, 33, 10, 1);
+}
+
+static void test_riscv64_zbs_register(void)
+{
+    const char bset[] = "\xb3\x12\x73\x28";
+    const char bclr[] = "\x33\x14\x73\x48";
+    const char binv[] = "\xb3\x14\x73\x68";
+    const char bext[] = "\x33\x55\x73\x48";
+
+    run_riscv64_zbs_reg_case(bset, 0, 65, 5, 2);
+    run_riscv64_zbs_reg_case(bclr, 0xffffffffffffffffull, 65, 8,
+                             0xfffffffffffffffdull);
+    run_riscv64_zbs_reg_case(binv, 0, 65, 9, 2);
+    run_riscv64_zbs_reg_case(bext, 2, 65, 10, 1);
+}
+
+static void test_riscv32_zbs_immediate(void)
+{
+    const char bseti[] = "\x93\x12\x13\x28";
+    const char bclri[] = "\x13\x14\x33\x48";
+    const char binvi[] = "\x93\x14\x43\x68";
+    const char bexti[] = "\x13\x55\x53\x48";
+    const char bseti_sh32[] = "\x93\x12\x03\x2a";
+    uc_engine *uc;
+
+    run_riscv32_zbs_imm_case(bseti, 0, 5, 2);
+    run_riscv32_zbs_imm_case(bclri, 0xffffffffu, 8, 0xfffffff7u);
+    run_riscv32_zbs_imm_case(binvi, 0, 9, 0x10);
+    run_riscv32_zbs_imm_case(bexti, 0x20, 10, 1);
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, bseti_sh32, 4);
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_zbs_immediate(void)
+{
+    const char bseti[] = "\x93\x12\x13\x28";
+    const char bclri[] = "\x13\x14\x33\x48";
+    const char binvi[] = "\x93\x14\x43\x68";
+    const char bexti[] = "\x13\x55\x53\x48";
+    const char bseti_sh63[] = "\x93\x12\xf3\x2b";
+
+    run_riscv64_zbs_imm_case(bseti, 0, 5, 2);
+    run_riscv64_zbs_imm_case(bclri, 0xffffffffffffffffull, 8,
+                             0xfffffffffffffff7ull);
+    run_riscv64_zbs_imm_case(binvi, 0, 9, 0x10);
+    run_riscv64_zbs_imm_case(bexti, 0x20, 10, 1);
+    run_riscv64_zbs_imm_case(bseti_sh63, 0, 5, 0x8000000000000000ull);
+}
+
+static void test_riscv32_rvv_vsetvli_csrs(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf5\x05\x0c"
+        "\x73\x26\x00\xc2"
+        "\xf3\x26\x10\xc2"
+        "\x73\x27\x20\xc2"
+        "\xf3\x27\x00\x30";
+    uint32_t a0 = 0;
+    uint32_t a1 = 5;
+    uint32_t a2 = 0;
+    uint32_t a3 = 0;
+    uint32_t a4 = 0;
+    uint32_t a5 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code,
+                    sizeof(code) - 1);
+    riscv32_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A5, &a5));
+
+    TEST_CHECK(a0 == 5);
+    TEST_CHECK(a2 == 5);
+    TEST_CHECK(a3 == 0xc0);
+    TEST_CHECK(a4 == 16);
+    TEST_CHECK(a5 == (RISCV32_MSTATUS_SD | RISCV_MSTATUS_VS_DIRTY));
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_vsetvli_csrs(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf5\x05\x0c"
+        "\x73\x26\x00\xc2"
+        "\xf3\x26\x10\xc2"
+        "\x73\x27\x20\xc2"
+        "\xf3\x27\x00\x30";
+    uint64_t a0 = 0;
+    uint64_t a1 = 5;
+    uint64_t a2 = 0;
+    uint64_t a3 = 0;
+    uint64_t a4 = 0;
+    uint64_t a5 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A5, &a5));
+
+    TEST_CHECK(a0 == 5);
+    TEST_CHECK(a2 == 5);
+    TEST_CHECK(a3 == 0xc0);
+    TEST_CHECK(a4 == 16);
+    TEST_CHECK(a5 == (RISCV64_MSTATUS_SD | RISCV_MSTATUS_VS_DIRTY));
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_vsetvl_clamp(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x93\x02\x00\x0c"
+        "\x57\xf5\x55\x80"
+        "\x73\x26\x00\xc2";
+    uint64_t a0 = 0;
+    uint64_t a1 = 32;
+    uint64_t a2 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A2, &a2));
+
+    TEST_CHECK(a0 == 16);
+    TEST_CHECK(a2 == 16);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_vsetivli(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf5\x03\xcc"
+        "\x73\x26\x00\xc2"
+        "\xf3\x26\x10\xc2";
+    uint32_t a0 = 0;
+    uint32_t a2 = 0;
+    uint32_t a3 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code,
+                    sizeof(code) - 1);
+    riscv32_enable_vector_state(uc);
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A3, &a3));
+
+    TEST_CHECK(a0 == 7);
+    TEST_CHECK(a2 == 7);
+    TEST_CHECK(a3 == 0xc0);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_vle8_vadd_vv_vse8(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf0\x05\x0c"
+        "\x87\x00\x05\x02"
+        "\x07\x81\x05\x02"
+        "\xd7\x81\x20\x02"
+        "\xa7\x01\x06\x02";
+    uint8_t input_a[] = {
+        0x00, 0x01, 0x02, 0x7f, 0x80, 0xfe, 0xff, 0x10,
+        0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0xf0,
+    };
+    uint8_t input_b[] = {
+        0x01, 0x02, 0x03, 0x01, 0x80, 0x03, 0x02, 0xf0,
+        0xe0, 0xd0, 0xc0, 0xb0, 0xa0, 0x90, 0x80, 0x20,
+    };
+    uint8_t expected[] = {
+        0x01, 0x03, 0x05, 0x80, 0x00, 0x01, 0x01, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,
+    };
+    uint8_t output[sizeof(expected)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input_a, sizeof(input_a)));
+    OK(uc_mem_write(uc, a1, input_b, sizeof(input_b)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_vle8_vadd_vv_vse8(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf0\x05\x0c"
+        "\x87\x00\x05\x02"
+        "\x07\x81\x05\x02"
+        "\xd7\x81\x20\x02"
+        "\xa7\x01\x06\x02";
+    uint8_t input_a[] = {
+        0x00, 0x01, 0x7f, 0x80, 0xfe, 0xff, 0x10, 0xf0,
+    };
+    uint8_t input_b[] = {
+        0x01, 0x02, 0x01, 0x80, 0x03, 0x02, 0xf0, 0x20,
+    };
+    uint8_t expected[] = {
+        0x01, 0x03, 0x80, 0x00, 0x01, 0x01, 0x00, 0x10,
+    };
+    uint8_t output[sizeof(expected)] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code,
+                    sizeof(code) - 1);
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input_a, sizeof(input_a)));
+    OK(uc_mem_write(uc, a1, input_b, sizeof(input_b)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_vmv_vi_vadd_vi_vse32(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf0\x05\x0d"
+        "\xd7\xb0\x0f\x5e"
+        "\x57\xb1\x12\x02"
+        "\x27\x61\x05\x02";
+    uint32_t expected[] = { 4, 4, 4, 4 };
+    uint32_t output[sizeof(expected) / sizeof(expected[0])] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = 4;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_mem_read(uc, a0, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_tail_agnostic_vmv(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x93\x05\x40\x00"
+        "\x57\xf0\x05\x0c"
+        "\xd7\xb0\x03\x5e"
+        "\x93\x05\x80\x00"
+        "\x57\xf0\x05\x0c"
+        "\xa7\x00\x05\x02";
+    uint8_t expected[] = {
+        0x07, 0x07, 0x07, 0x07, 0xff, 0xff, 0xff, 0xff,
+    };
+    uint8_t output[sizeof(expected)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_mem_read(uc, a0, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_vle16_vsub_vse16(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf0\x85\x0c"
+        "\x87\x50\x05\x02"
+        "\x07\xd1\x05\x02"
+        "\xd7\x81\x20\x0a"
+        "\xa7\x51\x06\x02";
+    uint16_t input_a[] = {
+        0x0001, 0x0002, 0x0100, 0xffff,
+        0x8000, 0x0001, 0x1234, 0x0100,
+    };
+    uint16_t input_b[] = {
+        0x0003, 0x0001, 0x0101, 0x0000,
+        0x7fff, 0xffff, 0x2234, 0x00ff,
+    };
+    uint16_t expected[] = {
+        0x0002, 0xffff, 0x0001, 0x0001,
+        0xffff, 0xfffe, 0x1000, 0xffff,
+    };
+    uint16_t output[sizeof(expected) / sizeof(expected[0])] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code,
+                    sizeof(code) - 1);
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input_a, sizeof(input_a)));
+    OK(uc_mem_write(uc, a1, input_b, sizeof(input_b)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_vle64_moves_logic_compare(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf0\x85\x0d"
+        "\x87\x70\x05\x02"
+        "\x57\x81\x00\x5e"
+        "\xd7\xc1\x06\x5e"
+        "\x57\x80\x21\x66"
+        "\x27\x80\x07\x02"
+        "\x57\x82\x21\x2e"
+        "\xd7\xb2\x47\x26"
+        "\x57\x43\x57\x2a"
+        "\x27\x73\x06\x02";
+    uint64_t scalar = 0xff00ff00ff00fff0ull;
+    uint64_t input[] = {
+        0x000000000000000full,
+        0xff00ff00ff00fff0ull,
+    };
+    uint64_t expected[] = {
+        0x000000000000001full,
+        0x0000000000000010ull,
+    };
+    uint64_t output[sizeof(expected) / sizeof(expected[0])] = { 0 };
+    uint8_t expected_mask[] = { 0xfd, 0xff };
+    uint8_t mask_output[sizeof(expected_mask)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = 2;
+    uint64_t a2 = code_start + 0x1100;
+    uint64_t a3 = scalar;
+    uint64_t a4 = 0x10;
+    uint64_t a5 = code_start + 0x1200;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_mem_read(uc, a5, mask_output, sizeof(mask_output)));
+    for (i = 0; i < sizeof(expected_mask); i++) {
+        TEST_CHECK(mask_output[i] == expected_mask[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_scalar_moves_compare(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf0\x85\x0c"
+        "\x87\x50\x05\x02"
+        "\xd7\x26\x10\x42"
+        "\x57\x61\x07\x42"
+        "\x27\x51\x08\x02"
+        "\x57\xc0\x17\x7e"
+        "\x27\x00\x06\x02";
+    uint16_t input[] = {
+        0xff80, 0x0000, 0x0001, 0x0002,
+        0xffff, 0x7fff, 0x8000, 0x0005,
+    };
+    uint16_t expected_move[] = {
+        0x1234, 0x0000, 0x0000, 0x0000,
+        0x0000, 0x0000, 0x0000, 0x0000,
+    };
+    uint16_t move_output[sizeof(expected_move) /
+                         sizeof(expected_move[0])] = { 0 };
+    uint8_t expected_mask[] = {
+        0xac, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    uint8_t mask_output[sizeof(expected_mask)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = 8;
+    uint64_t a2 = code_start + 0x1100;
+    uint64_t a3 = 0;
+    uint64_t a4 = 0x1234;
+    uint64_t a5 = 0;
+    uint64_t a6 = code_start + 0x1200;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A3, &a3));
+    TEST_CHECK(a3 == 0xffffffffffffff80ull);
+
+    OK(uc_mem_read(uc, a6, move_output, sizeof(move_output)));
+    for (i = 0; i < sizeof(expected_move) / sizeof(expected_move[0]); i++) {
+        TEST_CHECK(move_output[i] == expected_move[i]);
+    }
+
+    OK(uc_mem_read(uc, a2, mask_output, sizeof(mask_output)));
+    for (i = 0; i < sizeof(expected_mask); i++) {
+        TEST_CHECK(mask_output[i] == expected_mask[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_minmax(void)
+{
+    uc_engine *uc;
+    uint8_t code[11 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 14, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x04, 1, 1, 2, 0, 3),
+        riscv_encode_rvv_op(0x07, 1, 1, 2, 0, 4),
+        riscv_encode_rvv_op(0x05, 1, 1, 5, 4, 5),
+        riscv_encode_rvv_op(0x06, 1, 1, 6, 4, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 5),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 6),
+    };
+    uint32_t input_a[] = {
+        0xffffffffu, 0x80000000u, 5, 0x7fffffffu,
+    };
+    uint32_t input_b[] = {
+        1, 0xffffffffu, 6, 0x80000000u,
+    };
+    uint32_t expected_minu[] = {
+        1, 0x80000000u, 5, 0x7fffffffu,
+    };
+    uint32_t expected_max[] = {
+        1, 0xffffffffu, 6, 0x7fffffffu,
+    };
+    uint32_t expected_min_scalar[] = {
+        0xfffffffeu, 0x80000000u, 0xfffffffeu, 0xfffffffeu,
+    };
+    uint32_t expected_maxu_scalar[] = {
+        0xffffffffu, 0x80000001u, 0x80000001u, 0x80000001u,
+    };
+    uint32_t out_minu[4] = { 0 };
+    uint32_t out_max[4] = { 0 };
+    uint32_t out_min_scalar[4] = { 0 };
+    uint32_t out_maxu_scalar[4] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    uint32_t a3 = code_start + 0x1300;
+    uint32_t a5 = code_start + 0x1400;
+    uint32_t a6 = code_start + 0x1500;
+    uint32_t a4 = 4;
+    uint32_t t0 = 0xfffffffeu;
+    uint32_t t1 = 0x80000001u;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, (const char *)code,
+                    sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input_a, sizeof(input_a)));
+    OK(uc_mem_write(uc, a1, input_b, sizeof(input_b)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_minu, sizeof(out_minu)));
+    OK(uc_mem_read(uc, a3, out_max, sizeof(out_max)));
+    OK(uc_mem_read(uc, a5, out_min_scalar, sizeof(out_min_scalar)));
+    OK(uc_mem_read(uc, a6, out_maxu_scalar, sizeof(out_maxu_scalar)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_minu[i] == expected_minu[i]);
+        TEST_CHECK(out_max[i] == expected_max[i]);
+        TEST_CHECK(out_min_scalar[i] == expected_min_scalar[i]);
+        TEST_CHECK(out_maxu_scalar[i] == expected_maxu_scalar[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_reverse_subtract(void)
+{
+    uc_engine *uc;
+    uint8_t code[21 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 29, 0),
+        riscv_encode_rvv_op(0x03, 1, 1, 6, 4, 2),
+        riscv_encode_rvv_ldst(1, 0, 1, 11, 2),
+        riscv_encode_rvv_op(0x03, 1, 1, 0x1f, 3, 3),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 3),
+        riscv_encode_rvv_op(0x03, 0, 1, 6, 4, 4),
+        riscv_encode_rvv_ldst(1, 0, 1, 30, 4),
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 1),
+        riscv_encode_rvv_op(0x03, 1, 1, 6, 4, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 2),
+        riscv_encode_rvv_op(0x03, 1, 1, 0x10, 3, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 3),
+        riscv_encode_rvv_vsetvli(0, 7, 0xd8),
+        riscv_encode_rvv_ldst(0, 7, 1, 16, 1),
+        riscv_encode_rvv_op(0x03, 1, 1, 6, 4, 2),
+        riscv_encode_rvv_ldst(1, 7, 1, 17, 2),
+        riscv_encode_rvv_op(0x03, 1, 1, 0x1f, 3, 3),
+        riscv_encode_rvv_ldst(1, 7, 1, 28, 3),
+    };
+    uint8_t e8_src[] = { 1u, 2u, 0u, 0xffu };
+    uint8_t mask_src[] = { 0x05u, 0, 0, 0 };
+    uint32_t e32_src[] = { 1u, 0x20u, 0xffffffffu, 0x80000000u };
+    uint64_t e64_src[] = { 1ull, 0xffffffffffffffffull };
+    uint8_t expected_e8_vx[] = { 0x0fu, 0x0eu, 0x10u, 0x11u };
+    uint8_t expected_e8_vi[] = { 0xfeu, 0xfdu, 0xffu, 0u };
+    uint8_t expected_e8_mask[] = { 0x0fu, 0xffu, 0x10u, 0xffu };
+    uint32_t expected_e32_vx[] = {
+        0x0fu, 0xfffffff0u, 0x11u, 0x80000010u,
+    };
+    uint32_t expected_e32_vi[] = {
+        0xffffffefu, 0xffffffd0u, 0xfffffff1u, 0x7ffffff0u,
+    };
+    uint64_t expected_e64_vx[] = {
+        0x10000000full, 0x100000011ull,
+    };
+    uint64_t expected_e64_vi[] = {
+        0xfffffffffffffffeull, 0,
+    };
+    uint8_t out_e8_vx[4] = { 0 };
+    uint8_t out_e8_vi[4] = { 0 };
+    uint8_t out_e8_mask[4] = { 0 };
+    uint32_t out_e32_vx[4] = { 0 };
+    uint32_t out_e32_vi[4] = { 0 };
+    uint64_t out_e64_vx[2] = { 0 };
+    uint64_t out_e64_vi[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t1 = 0x100000010ull;
+    uint64_t t2 = 2;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, e8_src, sizeof(e8_src)));
+    OK(uc_mem_write(uc, a3, e32_src, sizeof(e32_src)));
+    OK(uc_mem_write(uc, a6, e64_src, sizeof(e64_src)));
+    OK(uc_mem_write(uc, t4, mask_src, sizeof(mask_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_e8_vx, sizeof(out_e8_vx)));
+    OK(uc_mem_read(uc, a2, out_e8_vi, sizeof(out_e8_vi)));
+    OK(uc_mem_read(uc, t5, out_e8_mask, sizeof(out_e8_mask)));
+    OK(uc_mem_read(uc, a4, out_e32_vx, sizeof(out_e32_vx)));
+    OK(uc_mem_read(uc, a5, out_e32_vi, sizeof(out_e32_vi)));
+    OK(uc_mem_read(uc, a7, out_e64_vx, sizeof(out_e64_vx)));
+    OK(uc_mem_read(uc, t3, out_e64_vi, sizeof(out_e64_vi)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_e8_vx[i] == expected_e8_vx[i]);
+        TEST_CHECK(out_e8_vi[i] == expected_e8_vi[i]);
+        TEST_CHECK(out_e8_mask[i] == expected_e8_mask[i]);
+        TEST_CHECK(out_e32_vx[i] == expected_e32_vx[i]);
+        TEST_CHECK(out_e32_vi[i] == expected_e32_vi[i]);
+    }
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_e64_vx[i] == expected_e64_vx[i]);
+        TEST_CHECK(out_e64_vi[i] == expected_e64_vi[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_reverse_subtract(void)
+{
+    uc_engine *uc;
+    uint8_t code[6 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_op(0x03, 1, 1, 6, 4, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x03, 1, 1, 0x1f, 3, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 3),
+    };
+    uint32_t src[] = { 1u, 0xffffffffu };
+    uint32_t expected_vx[] = { 0x0fu, 0x11u };
+    uint32_t expected_vi[] = { 0xfffffffeu, 0 };
+    uint32_t out_vx[2] = { 0 };
+    uint32_t out_vi[2] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    uint32_t t0 = 2;
+    uint32_t t1 = 0x10;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_vx, sizeof(out_vx)));
+    OK(uc_mem_read(uc, a2, out_vi, sizeof(out_vi)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_vx[i] == expected_vx[i]);
+        TEST_CHECK(out_vi[i] == expected_vi[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_slide(void)
+{
+    uc_engine *uc;
+    uint8_t code[22 * 4];
+    uint32_t insns[22] = {
+        riscv_encode_rvv_vsetvli(0, 6, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 3),
+        riscv_encode_rvv_op(0x0e, 1, 1, 2, 3, 3),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 3),
+        riscv_encode_rvv_ldst(0, 0, 1, 30, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 6),
+        riscv_encode_rvv_op(0x0e, 0, 1, 28, 4, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 29, 6),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 5),
+        riscv_encode_rvv_op(0x0f, 1, 5, 28, 4, 5),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 5),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 7),
+        riscv_encode_rvv_op(0x0f, 1, 7, 2, 3, 7),
+        riscv_encode_rvv_ldst(1, 0, 1, 14, 7),
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 15, 1),
+        riscv_encode_rvv_op(0x0e, 1, 1, 7, 6, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 2),
+    };
+    uint32_t tail_insns[] = {
+        riscv_encode_rvv_ldst(0, 6, 1, 15, 4),
+        riscv_encode_rvv_op(0x0f, 1, 4, 7, 6, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 4),
+    };
+    uint8_t src_e8[] = {
+        1, 2, 3, 4, 5, 6, 7, 8,
+        9, 10, 11, 12, 13, 14, 15, 16,
+    };
+    uint8_t init_e8[] = {
+        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+    };
+    uint8_t mask_src[] = { 0x55, 0x55, 0, 0 };
+    uint8_t expected_up_vi[] = {
+        0xa0, 0xa1, 1, 2, 3, 4, 5, 6,
+        7, 8, 9, 10, 11, 12, 13, 14,
+    };
+    uint8_t expected_up_vx_masked[] = {
+        0xa0, 0xff, 2, 0xff, 4, 0xff, 6, 0xff,
+        8, 0xff, 10, 0xff, 12, 0xff, 14, 0xff,
+    };
+    uint8_t expected_down_vx[] = {
+        2, 3, 4, 5, 6, 7, 8, 9,
+        10, 11, 12, 13, 14, 15, 16, 0,
+    };
+    uint8_t expected_down_vi[] = {
+        3, 4, 5, 6, 7, 8, 9, 10,
+        11, 12, 13, 14, 15, 16, 0, 0,
+    };
+    uint32_t src_e32[] = {
+        0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+    };
+    uint32_t expected_slide1up[] = {
+        0x9abcdef0u, 0x11111111u, 0x22222222u, 0x33333333u,
+    };
+    uint32_t expected_slide1down[] = {
+        0x22222222u, 0x33333333u, 0x44444444u, 0x9abcdef0u,
+    };
+    uint8_t out_up_vi[16] = { 0 };
+    uint8_t out_up_vx_masked[16] = { 0 };
+    uint8_t out_down_vx[16] = { 0 };
+    uint8_t out_down_vi[16] = { 0 };
+    uint32_t out_slide1up[4] = { 0 };
+    uint32_t out_slide1down[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t1 = 16;
+    uint64_t t2 = 0x123456789abcdef0ull;
+    uint64_t t3 = 1;
+    uint64_t t4 = code_start + 0x1800;
+    uint64_t t5 = code_start + 0x1900;
+    size_t i;
+
+    memcpy(&insns[19], tail_insns, sizeof(tail_insns));
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, src_e8, sizeof(src_e8)));
+    OK(uc_mem_write(uc, a1, init_e8, sizeof(init_e8)));
+    OK(uc_mem_write(uc, a5, src_e32, sizeof(src_e32)));
+    OK(uc_mem_write(uc, t5, mask_src, sizeof(mask_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_up_vi, sizeof(out_up_vi)));
+    OK(uc_mem_read(uc, t4, out_up_vx_masked, sizeof(out_up_vx_masked)));
+    OK(uc_mem_read(uc, a3, out_down_vx, sizeof(out_down_vx)));
+    OK(uc_mem_read(uc, a4, out_down_vi, sizeof(out_down_vi)));
+    OK(uc_mem_read(uc, a6, out_slide1up, sizeof(out_slide1up)));
+    OK(uc_mem_read(uc, a7, out_slide1down, sizeof(out_slide1down)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(out_up_vi[i] == expected_up_vi[i]);
+        TEST_CHECK(out_up_vx_masked[i] == expected_up_vx_masked[i]);
+        TEST_CHECK(out_down_vx[i] == expected_down_vx[i]);
+        TEST_CHECK(out_down_vi[i] == expected_down_vi[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_slide1up[i] == expected_slide1up[i]);
+        TEST_CHECK(out_slide1down[i] == expected_slide1down[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_slide(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 14, 2),
+        riscv_encode_rvv_op(0x0e, 1, 1, 1, 3, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 11, 2),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 3),
+        riscv_encode_rvv_op(0x0f, 1, 3, 6, 6, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+    };
+    uint32_t src[] = { 0x11111111u, 0x22222222u };
+    uint32_t init[] = { 0xaaaaaaaa, 0xbbbbbbbb };
+    uint32_t expected_up[] = { 0xaaaaaaaa, 0x11111111u };
+    uint32_t expected_down1[] = { 0x22222222u, 0x87654321u };
+    uint32_t out_up[2] = { 0 };
+    uint32_t out_down1[2] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a3 = code_start + 0x1300;
+    uint32_t a4 = code_start + 0x1400;
+    uint32_t t0 = 2;
+    uint32_t t1 = 0x87654321u;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, src, sizeof(src)));
+    OK(uc_mem_write(uc, a4, init, sizeof(init)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_up, sizeof(out_up)));
+    OK(uc_mem_read(uc, a3, out_down1, sizeof(out_down1)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_up[i] == expected_up[i]);
+        TEST_CHECK(out_down1[i] == expected_down1[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_slide_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x0e, 1, 2, 6, 4, 2));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x0e, 1, 2, 2, 3, 2));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x0e, 1, 2, 6, 6, 2));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x0f, 0, 1, 6, 4, 0));
+}
+
+static void test_riscv64_rvv_gather_compress(void)
+{
+    uc_engine *uc;
+    uint8_t code[20 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 12, 2),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_op(0x0c, 1, 1, 2, 0, 3),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 3),
+        riscv_encode_rvv_op(0x0c, 1, 1, 7, 4, 4),
+        riscv_encode_rvv_ldst(1, 0, 1, 14, 4),
+        riscv_encode_rvv_op(0x0c, 1, 1, 0x1f, 3, 5),
+        riscv_encode_rvv_ldst(1, 0, 1, 15, 5),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 0),
+        riscv_encode_rvv_op(0x0c, 0, 1, 2, 0, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 30, 6),
+        riscv_encode_rvv_ldst(0, 5, 1, 17, 10),
+        riscv_encode_rvv_op(0x0e, 1, 1, 10, 0, 8),
+        riscv_encode_rvv_ldst(1, 0, 1, 18, 8),
+        riscv_encode_rvv_vsetvli(0, 7, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 24, 1),
+        riscv_encode_rvv_mask_ldst(0, 25, 2),
+        riscv_encode_rvv_op(0x17, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 26, 3),
+    };
+    uint8_t src_e8[] = {
+        10, 20, 30, 40, 50, 60, 70, 80,
+        90, 100, 110, 120, 130, 140, 150, 160,
+    };
+    uint8_t index_e8[] = {
+        0, 3, 15, 16, 1, 2, 4, 5,
+        6, 7, 8, 9, 10, 11, 12, 13,
+    };
+    uint8_t mask_src[] = { 0x0b, 0, 0, 0 };
+    uint16_t index_e16[] = { 0, 3, 16, 1, 2, 15, 4, 5 };
+    uint32_t compress_src[] = {
+        0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u,
+    };
+    uint32_t compress_mask[] = { 0x0du, 0, 0, 0 };
+    uint8_t expected_vv[] = {
+        10, 40, 160, 0, 20, 30, 50, 60,
+        70, 80, 90, 100, 110, 120, 130, 140,
+    };
+    uint8_t expected_vx[] = {
+        50, 50, 50, 50, 50, 50, 50, 50,
+        50, 50, 50, 50, 50, 50, 50, 50,
+    };
+    uint8_t expected_vi[] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    uint8_t expected_masked[] = {
+        10, 40, 0xff, 0, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    uint8_t expected_ei16[] = {
+        10, 40, 0, 20, 30, 160, 50, 60,
+    };
+    uint32_t expected_compress[] = {
+        0x11111111u, 0x33333333u, 0x44444444u, 0xffffffffu,
+    };
+    uint8_t out_vv[16] = { 0 };
+    uint8_t out_vx[16] = { 0 };
+    uint8_t out_vi[16] = { 0 };
+    uint8_t out_masked[16] = { 0 };
+    uint8_t out_ei16[8] = { 0 };
+    uint32_t out_compress[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t s2 = code_start + 0x1800;
+    uint64_t s8 = code_start + 0x1d00;
+    uint64_t s9 = code_start + 0x1e00;
+    uint64_t s10 = code_start + 0x1f00;
+    uint64_t t0 = 16;
+    uint64_t t2 = 4;
+    uint64_t t5 = code_start + 0x2100;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_src, sizeof(mask_src)));
+    OK(uc_mem_write(uc, a1, src_e8, sizeof(src_e8)));
+    OK(uc_mem_write(uc, a2, index_e8, sizeof(index_e8)));
+    OK(uc_mem_write(uc, a7, index_e16, sizeof(index_e16)));
+    OK(uc_mem_write(uc, s8, compress_src, sizeof(compress_src)));
+    OK(uc_mem_write(uc, s9, compress_mask, sizeof(compress_mask)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S2, &s2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S8, &s8));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S9, &s9));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S10, &s10));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vv, sizeof(out_vv)));
+    OK(uc_mem_read(uc, a4, out_vx, sizeof(out_vx)));
+    OK(uc_mem_read(uc, a5, out_vi, sizeof(out_vi)));
+    OK(uc_mem_read(uc, t5, out_masked, sizeof(out_masked)));
+    OK(uc_mem_read(uc, s2, out_ei16, sizeof(out_ei16)));
+    OK(uc_mem_read(uc, s10, out_compress, sizeof(out_compress)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(out_vv[i] == expected_vv[i]);
+        TEST_CHECK(out_vx[i] == expected_vx[i]);
+        TEST_CHECK(out_vi[i] == expected_vi[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(out_ei16[i] == expected_ei16[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_compress[i] == expected_compress[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_gather_compress(void)
+{
+    uc_engine *uc;
+    uint8_t code[10 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 2),
+        riscv_encode_rvv_op(0x0c, 1, 1, 2, 0, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 3),
+        riscv_encode_rvv_vsetvli(0, 6, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 13, 4),
+        riscv_encode_rvv_mask_ldst(0, 10, 5),
+        riscv_encode_rvv_op(0x17, 1, 4, 5, 2, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 7, 6),
+    };
+    uint32_t src[] = { 0x11111111u, 0x22222222u };
+    uint32_t idx[] = { 1, 4 };
+    uint32_t expected_gather[] = { 0x22222222u, 0 };
+    uint8_t mask_src[] = { 0x0b, 0, 0, 0 };
+    uint8_t compress_src[] = { 10, 20, 30, 40 };
+    uint8_t expected_compress[] = { 10, 20, 40, 0xff };
+    uint32_t out_gather[2] = { 0 };
+    uint8_t out_compress[4] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    uint32_t a3 = code_start + 0x1300;
+    uint32_t a4 = code_start + 0x1400;
+    uint32_t t0 = 2;
+    uint32_t t1 = 4;
+    uint32_t t2 = code_start + 0x1600;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_src, sizeof(mask_src)));
+    OK(uc_mem_write(uc, a1, src, sizeof(src)));
+    OK(uc_mem_write(uc, a2, idx, sizeof(idx)));
+    OK(uc_mem_write(uc, a3, compress_src, sizeof(compress_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a4, out_gather, sizeof(out_gather)));
+    OK(uc_mem_read(uc, t2, out_compress, sizeof(out_compress)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_gather[i] == expected_gather[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_compress[i] == expected_compress[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_whole_register_move(void)
+{
+    uc_engine *uc;
+    uint8_t code[11 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc0),
+        riscv_encode_rvv_whole_ldst(0, 0, 10, 1, 1),
+        riscv_encode_rvv_vmvnr(1, 1, 2),
+        riscv_encode_rvv_ldst(1, 0, 1, 11, 2),
+        riscv_encode_rvv_vmvnr(1, 1, 4),
+        riscv_encode_rvv_vmvnr(2, 4, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 7),
+        riscv_encode_rvv_ldst(1, 0, 1, 14, 8),
+        riscv_encode_rvv_vmvnr(4, 8, 12),
+        riscv_encode_rvv_vmvnr(8, 16, 24),
+    };
+    uint8_t src[] = {
+        1, 2, 3, 4, 5, 6, 7, 8,
+        9, 10, 11, 12, 13, 14, 15, 16,
+    };
+    uint8_t out_v2[16] = { 0 };
+    uint8_t out_v6[16] = { 0 };
+    uint8_t out_v7[16] = { 0 };
+    uint8_t out_v8[16] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t t0 = 16;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_v2, sizeof(out_v2)));
+    OK(uc_mem_read(uc, a2, out_v6, sizeof(out_v6)));
+    OK(uc_mem_read(uc, a3, out_v7, sizeof(out_v7)));
+    OK(uc_mem_read(uc, a4, out_v8, sizeof(out_v8)));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(out_v2[i] == src[i]);
+        TEST_CHECK(out_v6[i] == src[i]);
+        TEST_CHECK(out_v7[i] == 0);
+        TEST_CHECK(out_v8[i] == 0);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_gather_compress_move_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x0c, 1, 2, 3, 0, 2));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x0c, 1, 3, 2, 0, 2));
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x0e, 1, 2, 1, 0, 2), 0xc8);
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x17, 1, 2, 0, 2, 2));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x17, 0, 1, 0, 2, 0));
+    run_riscv64_rvv_illegal_vstart_vl0(
+        riscv_encode_rvv_op(0x17, 1, 1, 2, 2, 3));
+    run_riscv64_rvv_illegal(riscv_encode_rvv_vmvnr(2, 2, 3));
+}
+
+static void test_riscv64_rvv_vmerge(void)
+{
+    uc_engine *uc;
+    uint8_t code[11 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 0),
+        riscv_encode_rvv_op(0x18, 1, 0, 1, 3, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 12, 2),
+        riscv_encode_rvv_op(0x17, 0, 1, 2, 0, 3),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 3),
+        riscv_encode_rvv_op(0x17, 0, 1, 0x1d, 3, 4),
+        riscv_encode_rvv_ldst(1, 0, 1, 14, 4),
+        riscv_encode_rvv_op(0x17, 0, 1, 5, 4, 5),
+        riscv_encode_rvv_ldst(1, 0, 1, 15, 5),
+    };
+    uint8_t mask_src[] = { 1, 0, 1, 0, 0, 1, 0, 1 };
+    uint8_t low[] = { 10, 20, 30, 40, 50, 60, 70, 80 };
+    uint8_t high[] = { 101, 102, 103, 104, 105, 106, 107, 108 };
+    uint8_t expected_vvm[] = { 101, 20, 103, 40, 50, 106, 70, 108 };
+    uint8_t expected_vim[] = { 0xfd, 20, 0xfd, 40, 50, 0xfd, 70, 0xfd };
+    uint8_t expected_vxm[] = { 0xaa, 20, 0xaa, 40, 50, 0xaa, 70, 0xaa };
+    uint8_t out_vvm[8] = { 0 };
+    uint8_t out_vim[8] = { 0 };
+    uint8_t out_vxm[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 8;
+    uint64_t t0 = 0xaa;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, (const char *)code,
+                    sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_src, sizeof(mask_src)));
+    OK(uc_mem_write(uc, a1, low, sizeof(low)));
+    OK(uc_mem_write(uc, a2, high, sizeof(high)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vvm, sizeof(out_vvm)));
+    OK(uc_mem_read(uc, a4, out_vim, sizeof(out_vim)));
+    OK(uc_mem_read(uc, a5, out_vxm, sizeof(out_vxm)));
+    for (i = 0; i < sizeof(expected_vvm); i++) {
+        TEST_CHECK(out_vvm[i] == expected_vvm[i]);
+        TEST_CHECK(out_vim[i] == expected_vim[i]);
+        TEST_CHECK(out_vxm[i] == expected_vxm[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_shift_vv(void)
+{
+    uc_engine *uc;
+    uint8_t code[9 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xd8),
+        riscv_encode_rvv_ldst(0, 7, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 7, 1, 11, 2),
+        riscv_encode_rvv_op(0x25, 1, 1, 2, 0, 3),
+        riscv_encode_rvv_op(0x28, 1, 1, 2, 0, 4),
+        riscv_encode_rvv_op(0x29, 1, 1, 2, 0, 5),
+        riscv_encode_rvv_ldst(1, 7, 1, 12, 3),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+    };
+    uint32_t store_sra = riscv_encode_rvv_ldst(1, 7, 1, 14, 5);
+    uint64_t input[] = {
+        0x8000000000000001ull, 0xf000000000000000ull,
+    };
+    uint64_t shifts[] = { 4, 68 };
+    uint64_t expected_sll[] = { 0x0000000000000010ull, 0 };
+    uint64_t expected_srl[] = {
+        0x0800000000000000ull, 0x0f00000000000000ull,
+    };
+    uint64_t expected_sra[] = {
+        0xf800000000000000ull, 0xff00000000000000ull,
+    };
+    uint64_t out_sll[2] = { 0 };
+    uint64_t out_srl[2] = { 0 };
+    uint64_t out_sra[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a6 = 2;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+    riscv_insn_to_code(&code[8 * 4], store_sra);
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, (const char *)code,
+                    sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_mem_write(uc, a1, shifts, sizeof(shifts)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_sll, sizeof(out_sll)));
+    OK(uc_mem_read(uc, a3, out_srl, sizeof(out_srl)));
+    OK(uc_mem_read(uc, a4, out_sra, sizeof(out_sra)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_sll[i] == expected_sll[i]);
+        TEST_CHECK(out_srl[i] == expected_srl[i]);
+        TEST_CHECK(out_sra[i] == expected_sra[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_shift_vx_vi(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 14, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 1),
+        riscv_encode_rvv_op(0x25, 1, 1, 5, 4, 2),
+        riscv_encode_rvv_op(0x28, 1, 1, 15, 3, 3),
+        riscv_encode_rvv_op(0x29, 1, 1, 2, 3, 4),
+        riscv_encode_rvv_ldst(1, 0, 1, 11, 2),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 3),
+    };
+    uint32_t store_sra = riscv_encode_rvv_ldst(1, 0, 1, 13, 4);
+    uint8_t input[] = { 0x01, 0x7f, 0x80, 0xff, 0x10, 0x40, 0xc0, 0x81 };
+    uint8_t expected_sll[] = {
+        0x02, 0xfe, 0x00, 0xfe, 0x20, 0x80, 0x80, 0x02,
+    };
+    uint8_t expected_srl[] = {
+        0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
+    };
+    uint8_t expected_sra[] = {
+        0x00, 0x1f, 0xe0, 0xff, 0x04, 0x10, 0xf0, 0xe0,
+    };
+    uint8_t out_sll[8] = { 0 };
+    uint8_t out_srl[8] = { 0 };
+    uint8_t out_sra[8] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    uint32_t a3 = code_start + 0x1300;
+    uint32_t a4 = 8;
+    uint32_t t0 = 9;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+    riscv_insn_to_code(&code[7 * 4], store_sra);
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, (const char *)code,
+                    sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_sll, sizeof(out_sll)));
+    OK(uc_mem_read(uc, a2, out_srl, sizeof(out_srl)));
+    OK(uc_mem_read(uc, a3, out_sra, sizeof(out_sra)));
+    for (i = 0; i < sizeof(expected_sll); i++) {
+        TEST_CHECK(out_sll[i] == expected_sll[i]);
+        TEST_CHECK(out_srl[i] == expected_srl[i]);
+        TEST_CHECK(out_sra[i] == expected_sra[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_mask_load_store(void)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 12, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_mask_ldst(1, 11, 0),
+    };
+    uint8_t mask_input[] = { 0xad, 0x02 };
+    uint8_t mask_output[] = { 0, 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = 10;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, (const char *)code,
+                    sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_input, sizeof(mask_input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, mask_output, sizeof(mask_output)));
+    for (i = 0; i < sizeof(mask_input); i++) {
+        TEST_CHECK(mask_output[i] == mask_input[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_masked_unit_stride(void)
+{
+    uc_engine *uc;
+    uint8_t code[5 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 0, 0, 11, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 1),
+        riscv_encode_rvv_ldst(1, 0, 0, 13, 1),
+    };
+    uint8_t mask_input[] = { 0xad };
+    uint8_t input[] = { 10, 20, 30, 40, 50, 60, 70, 80 };
+    uint8_t expected_loaded[] = { 10, 0xff, 30, 40, 0xff, 60, 0xff, 80 };
+    uint8_t expected_masked[] = { 10, 0xee, 30, 40, 0xee, 60, 0xee, 80 };
+    uint8_t out_loaded[8] = { 0 };
+    uint8_t out_masked[8] = {
+        0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+    };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a6 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, (const char *)code,
+                    sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_input, sizeof(mask_input)));
+    OK(uc_mem_write(uc, a1, input, sizeof(input)));
+    OK(uc_mem_write(uc, a3, out_masked, sizeof(out_masked)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_loaded, sizeof(out_loaded)));
+    OK(uc_mem_read(uc, a3, out_masked, sizeof(out_masked)));
+    for (i = 0; i < sizeof(expected_loaded); i++) {
+        TEST_CHECK(out_loaded[i] == expected_loaded[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_unit_stride_fault_vstart(void)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 11, 1),
+    };
+    uint8_t first_source[] = { 0x11, 0x22 };
+    uint8_t overwritten_source[] = { 0xaa, 0xbb };
+    uint8_t second_source[] = { 0x33, 0x44 };
+    uint8_t expected[] = { 0x11, 0x22, 0x33, 0x44 };
+    uint8_t output[4] = { 0 };
+    uint64_t a0 = code_start + code_len - sizeof(first_source);
+    uint64_t a1 = code_start + 0x2000;
+    uint64_t a6 = 4;
+    uint64_t pc;
+    uint64_t vstart;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, first_source, sizeof(first_source)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    uc_assert_err(UC_ERR_READ_UNMAPPED,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_PC, &pc));
+    OK(uc_reg_read(uc, UC_RISCV_REG_VSTART, &vstart));
+    TEST_CHECK(pc == code_start + 4);
+    TEST_CHECK(vstart == 2);
+
+    OK(uc_mem_write(uc, a0, overwritten_source,
+                    sizeof(overwritten_source)));
+    OK(uc_mem_map(uc, code_start + code_len, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start + code_len, second_source,
+                    sizeof(second_source)));
+
+    OK(uc_emu_start(uc, pc, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, a1, output, sizeof(output)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_VSTART, &vstart));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+    TEST_CHECK(vstart == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_strided_fault_vstart(void)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_stride_ldst(0, 0, 1, 10, 5, 1, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 11, 1),
+    };
+    uint8_t first_source[] = { 0x11, 0x22 };
+    uint8_t overwritten_source[] = { 0xaa, 0xbb };
+    uint8_t second_source[] = { 0x33, 0x44 };
+    uint8_t expected[] = { 0x11, 0x22, 0x33, 0x44 };
+    uint8_t output[4] = { 0 };
+    uint64_t a0 = code_start + code_len - 4;
+    uint64_t a1 = code_start + 0x2000;
+    uint64_t a6 = 4;
+    uint64_t t0 = 2;
+    uint64_t pc;
+    uint64_t vstart;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, &first_source[0], sizeof(first_source[0])));
+    OK(uc_mem_write(uc, a0 + 2, &first_source[1], sizeof(first_source[1])));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    uc_assert_err(UC_ERR_READ_UNMAPPED,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_PC, &pc));
+    OK(uc_reg_read(uc, UC_RISCV_REG_VSTART, &vstart));
+    TEST_CHECK(pc == code_start + 4);
+    TEST_CHECK(vstart == 2);
+
+    OK(uc_mem_write(uc, a0, &overwritten_source[0],
+                    sizeof(overwritten_source[0])));
+    OK(uc_mem_write(uc, a0 + 2, &overwritten_source[1],
+                    sizeof(overwritten_source[1])));
+    OK(uc_mem_map(uc, code_start + code_len, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start + code_len, &second_source[0],
+                    sizeof(second_source[0])));
+    OK(uc_mem_write(uc, code_start + code_len + 2, &second_source[1],
+                    sizeof(second_source[1])));
+
+    OK(uc_emu_start(uc, pc, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, a1, output, sizeof(output)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_VSTART, &vstart));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+    TEST_CHECK(vstart == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_indexed_fault_vstart(void)
+{
+    uc_engine *uc;
+    uint8_t code[4 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 12, 1),
+        riscv_encode_rvv_index_ldst(0, 0, 1, 1, 10, 1, 2, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 11, 2),
+    };
+    uint8_t indexes[] = { 0, 2, 4, 6 };
+    uint8_t first_source[] = { 0x11, 0x22 };
+    uint8_t overwritten_source[] = { 0xaa, 0xbb };
+    uint8_t second_source[] = { 0x33, 0x44 };
+    uint8_t expected[] = { 0x11, 0x22, 0x33, 0x44 };
+    uint8_t output[4] = { 0 };
+    uint64_t a0 = code_start + code_len - 4;
+    uint64_t a1 = code_start + 0x2000;
+    uint64_t a2 = code_start + 0x2100;
+    uint64_t a6 = 4;
+    uint64_t pc;
+    uint64_t vstart;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, &first_source[0], sizeof(first_source[0])));
+    OK(uc_mem_write(uc, a0 + 2, &first_source[1], sizeof(first_source[1])));
+    OK(uc_mem_write(uc, a2, indexes, sizeof(indexes)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    uc_assert_err(UC_ERR_READ_UNMAPPED,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_PC, &pc));
+    OK(uc_reg_read(uc, UC_RISCV_REG_VSTART, &vstart));
+    TEST_CHECK(pc == code_start + 8);
+    TEST_CHECK(vstart == 2);
+
+    OK(uc_mem_write(uc, a0, &overwritten_source[0],
+                    sizeof(overwritten_source[0])));
+    OK(uc_mem_write(uc, a0 + 2, &overwritten_source[1],
+                    sizeof(overwritten_source[1])));
+    OK(uc_mem_map(uc, code_start + code_len, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start + code_len, &second_source[0],
+                    sizeof(second_source[0])));
+    OK(uc_mem_write(uc, code_start + code_len + 2, &second_source[1],
+                    sizeof(second_source[1])));
+
+    OK(uc_emu_start(uc, pc, code_start + sizeof(code), 0, 0));
+    OK(uc_mem_read(uc, a1, output, sizeof(output)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_VSTART, &vstart));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+    TEST_CHECK(vstart == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_unit_stride_segment_memory(void)
+{
+    uc_engine *uc;
+    uint8_t code[7 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_segment_ldst(0, 5, 1, 10, 2, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 11, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 3),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 4),
+        riscv_encode_rvv_ldst(0, 5, 1, 14, 5),
+        riscv_encode_rvv_segment_ldst(1, 5, 1, 15, 4, 2),
+    };
+    uint16_t segment_input[] = {
+        0x101, 0x201, 0x102, 0x202, 0x103, 0x203,
+    };
+    uint16_t store_field0[] = {
+        0x111, 0x222, 0x333,
+    };
+    uint16_t store_field1[] = {
+        0xaaa, 0xbbb, 0xccc,
+    };
+    uint16_t expected_field0[] = {
+        0x101, 0x102, 0x103,
+    };
+    uint16_t expected_field1[] = {
+        0x201, 0x202, 0x203,
+    };
+    uint16_t expected_segment_output[] = {
+        0x111, 0xaaa, 0x222, 0xbbb, 0x333, 0xccc,
+    };
+    uint16_t out_field0[3] = { 0 };
+    uint16_t out_field1[3] = { 0 };
+    uint16_t segment_output[6] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 3;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, segment_input, sizeof(segment_input)));
+    OK(uc_mem_write(uc, a3, store_field0, sizeof(store_field0)));
+    OK(uc_mem_write(uc, a4, store_field1, sizeof(store_field1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_field0, sizeof(out_field0)));
+    OK(uc_mem_read(uc, a2, out_field1, sizeof(out_field1)));
+    OK(uc_mem_read(uc, a5, segment_output, sizeof(segment_output)));
+    for (i = 0; i < sizeof(expected_field0) / sizeof(expected_field0[0]);
+         i++) {
+        TEST_CHECK(out_field0[i] == expected_field0[i]);
+        TEST_CHECK(out_field1[i] == expected_field1[i]);
+    }
+    for (i = 0;
+         i < sizeof(expected_segment_output) /
+             sizeof(expected_segment_output[0]);
+         i++) {
+        TEST_CHECK(segment_output[i] == expected_segment_output[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_masked_unit_stride_segment(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 17, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_segment_ldst(0, 0, 0, 11, 2, 2),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 2),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 3),
+        riscv_encode_rvv_ldst(0, 0, 1, 14, 4),
+        riscv_encode_rvv_ldst(0, 0, 1, 15, 5),
+        riscv_encode_rvv_segment_ldst(1, 0, 0, 16, 4, 2),
+    };
+    uint8_t mask_input[] = { 0xad };
+    uint8_t segment_input[] = {
+        10, 20, 11, 21, 12, 22, 13, 23,
+        14, 24, 15, 25, 16, 26, 17, 27,
+    };
+    uint8_t store_field0[] = {
+        1, 2, 3, 4, 5, 6, 7, 8,
+    };
+    uint8_t store_field1[] = {
+        11, 12, 13, 14, 15, 16, 17, 18,
+    };
+    uint8_t expected_field0[] = {
+        10, 0xff, 12, 13, 0xff, 15, 0xff, 17,
+    };
+    uint8_t expected_field1[] = {
+        20, 0xff, 22, 23, 0xff, 25, 0xff, 27,
+    };
+    uint8_t segment_output[16] = {
+        0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+        0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+    };
+    uint8_t expected_segment_output[] = {
+        1, 11, 0xee, 0xee, 3, 13, 4, 14,
+        0xee, 0xee, 6, 16, 0xee, 0xee, 8, 18,
+    };
+    uint8_t out_field0[8] = { 0 };
+    uint8_t out_field1[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_input, sizeof(mask_input)));
+    OK(uc_mem_write(uc, a1, segment_input, sizeof(segment_input)));
+    OK(uc_mem_write(uc, a4, store_field0, sizeof(store_field0)));
+    OK(uc_mem_write(uc, a5, store_field1, sizeof(store_field1)));
+    OK(uc_mem_write(uc, a6, segment_output, sizeof(segment_output)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_field0, sizeof(out_field0)));
+    OK(uc_mem_read(uc, a3, out_field1, sizeof(out_field1)));
+    OK(uc_mem_read(uc, a6, segment_output, sizeof(segment_output)));
+    for (i = 0; i < sizeof(expected_field0); i++) {
+        TEST_CHECK(out_field0[i] == expected_field0[i]);
+        TEST_CHECK(out_field1[i] == expected_field1[i]);
+    }
+    for (i = 0; i < sizeof(expected_segment_output); i++) {
+        TEST_CHECK(segment_output[i] == expected_segment_output[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_unit_stride_segment_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_segment_ldst(0, 0, 0, 10, 0, 2));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_segment_ldst(0, 0, 1, 10, 31, 2));
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_segment_ldst(0, 7, 1, 10, 0, 2), 0xc0);
+}
+
+static void test_riscv64_rvv_strided_load_store(void)
+{
+    uc_engine *uc;
+    uint8_t code[5 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_stride_ldst(0, 5, 1, 10, 5, 1, 1),
+        riscv_encode_rvv_ldst(1, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_stride_ldst(1, 5, 1, 13, 6, 2, 1),
+    };
+    uint16_t gather_input[] = {
+        0x1010, 0xeeee, 0x2020, 0xeeee,
+        0x3030, 0xeeee, 0x4040, 0xeeee,
+    };
+    uint16_t store_input[] = {
+        0x5151, 0x6262, 0x7373, 0x8484,
+    };
+    uint16_t expected_gather[] = {
+        0x1010, 0x2020, 0x3030, 0x4040,
+    };
+    uint16_t scatter[8] = {
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+    };
+    uint16_t expected_scatter[] = {
+        0x5151, 0xeeee, 0x6262, 0xeeee,
+        0x7373, 0xeeee, 0x8484, 0xeeee,
+    };
+    uint16_t out_gather[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a6 = 4;
+    uint64_t t0 = 4;
+    uint64_t t1 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, gather_input, sizeof(gather_input)));
+    OK(uc_mem_write(uc, a2, store_input, sizeof(store_input)));
+    OK(uc_mem_write(uc, a3, scatter, sizeof(scatter)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_gather, sizeof(out_gather)));
+    OK(uc_mem_read(uc, a3, scatter, sizeof(scatter)));
+    for (i = 0; i < sizeof(expected_gather) / sizeof(expected_gather[0]);
+         i++) {
+        TEST_CHECK(out_gather[i] == expected_gather[i]);
+    }
+    for (i = 0; i < sizeof(expected_scatter) / sizeof(expected_scatter[0]);
+         i++) {
+        TEST_CHECK(scatter[i] == expected_scatter[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_negative_stride_load(void)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_stride_ldst(0, 5, 1, 10, 5, 1, 1),
+        riscv_encode_rvv_ldst(1, 5, 1, 11, 1),
+    };
+    uint16_t input[] = {
+        0x1111, 0x2222, 0x3333, 0x4444,
+    };
+    uint16_t expected[] = {
+        0x4444, 0x3333, 0x2222, 0x1111,
+    };
+    uint16_t output[4] = { 0 };
+    uint64_t data = code_start + 0x1000;
+    uint64_t a0 = data + 3 * sizeof(input[0]);
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a6 = 4;
+    uint64_t t0 = (uint64_t)-2;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, data, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_masked_strided_memory(void)
+{
+    uc_engine *uc;
+    uint8_t code[6 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_stride_ldst(0, 0, 0, 11, 5, 1, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 13, 2),
+        riscv_encode_rvv_stride_ldst(1, 0, 0, 14, 5, 2, 1),
+    };
+    uint8_t mask_input[] = { 0xad };
+    uint8_t gather_input[] = {
+        10, 11, 20, 21, 30, 31, 40, 41,
+        50, 51, 60, 61, 70, 71, 80, 81,
+    };
+    uint8_t store_input[] = {
+        1, 2, 3, 4, 5, 6, 7, 8,
+    };
+    uint8_t expected_loaded[] = {
+        10, 0xff, 30, 40, 0xff, 60, 0xff, 80,
+    };
+    uint8_t scatter[16] = {
+        0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+        0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+    };
+    uint8_t expected_scatter[] = {
+        1, 0xee, 0xee, 0xee, 3, 0xee, 4, 0xee,
+        0xee, 0xee, 6, 0xee, 0xee, 0xee, 8, 0xee,
+    };
+    uint8_t out_loaded[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a6 = 8;
+    uint64_t t0 = 2;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_input, sizeof(mask_input)));
+    OK(uc_mem_write(uc, a1, gather_input, sizeof(gather_input)));
+    OK(uc_mem_write(uc, a3, store_input, sizeof(store_input)));
+    OK(uc_mem_write(uc, a4, scatter, sizeof(scatter)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_loaded, sizeof(out_loaded)));
+    OK(uc_mem_read(uc, a4, scatter, sizeof(scatter)));
+    for (i = 0; i < sizeof(expected_loaded); i++) {
+        TEST_CHECK(out_loaded[i] == expected_loaded[i]);
+    }
+    for (i = 0; i < sizeof(expected_scatter); i++) {
+        TEST_CHECK(scatter[i] == expected_scatter[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_strided_segment_memory(void)
+{
+    uc_engine *uc;
+    uint8_t code[7 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_stride_ldst(0, 5, 1, 10, 5, 2, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 11, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 3),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 4),
+        riscv_encode_rvv_ldst(0, 5, 1, 14, 5),
+        riscv_encode_rvv_stride_ldst(1, 5, 1, 15, 5, 4, 2),
+    };
+    uint16_t segment_input[12] = {
+        0x101, 0x201, 0xeeee, 0xeeee,
+        0x102, 0x202, 0xeeee, 0xeeee,
+        0x103, 0x203, 0xeeee, 0xeeee,
+    };
+    uint16_t store_field0[] = {
+        0x111, 0x222, 0x333,
+    };
+    uint16_t store_field1[] = {
+        0xaaa, 0xbbb, 0xccc,
+    };
+    uint16_t expected_field0[] = {
+        0x101, 0x102, 0x103,
+    };
+    uint16_t expected_field1[] = {
+        0x201, 0x202, 0x203,
+    };
+    uint16_t segment_output[12] = {
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+    };
+    uint16_t expected_segment_output[] = {
+        0x111, 0xaaa, 0xeeee, 0xeeee,
+        0x222, 0xbbb, 0xeeee, 0xeeee,
+        0x333, 0xccc, 0xeeee, 0xeeee,
+    };
+    uint16_t out_field0[3] = { 0 };
+    uint16_t out_field1[3] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 3;
+    uint64_t t0 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, segment_input, sizeof(segment_input)));
+    OK(uc_mem_write(uc, a3, store_field0, sizeof(store_field0)));
+    OK(uc_mem_write(uc, a4, store_field1, sizeof(store_field1)));
+    OK(uc_mem_write(uc, a5, segment_output, sizeof(segment_output)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_field0, sizeof(out_field0)));
+    OK(uc_mem_read(uc, a2, out_field1, sizeof(out_field1)));
+    OK(uc_mem_read(uc, a5, segment_output, sizeof(segment_output)));
+    for (i = 0; i < sizeof(expected_field0) / sizeof(expected_field0[0]);
+         i++) {
+        TEST_CHECK(out_field0[i] == expected_field0[i]);
+        TEST_CHECK(out_field1[i] == expected_field1[i]);
+    }
+    for (i = 0;
+         i < sizeof(expected_segment_output) /
+             sizeof(expected_segment_output[0]);
+         i++) {
+        TEST_CHECK(segment_output[i] == expected_segment_output[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_strided_memory_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_stride_ldst(0, 0, 0, 10, 5, 0, 1));
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_stride_ldst(0, 0, 1, 10, 5, 1, 1), 0xc1);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_stride_ldst(0, 7, 1, 10, 5, 0, 2), 0xc0);
+}
+
+static void test_riscv64_rvv_indexed_load_store(void)
+{
+    uc_engine *uc;
+    uint8_t code[10 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 1),
+        riscv_encode_rvv_index_ldst(0, 0, 1, 1, 11, 1, 2, 1),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 2),
+        riscv_encode_rvv_index_ldst(0, 0, 3, 1, 11, 1, 3, 1),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 3),
+        riscv_encode_rvv_ldst(0, 5, 1, 14, 4),
+        riscv_encode_rvv_index_ldst(1, 0, 1, 1, 15, 1, 4, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 17, 5),
+        riscv_encode_rvv_index_ldst(1, 0, 3, 1, 6, 1, 5, 1),
+    };
+    uint8_t offsets[] = { 0, 4, 8, 12 };
+    uint16_t gather_input[] = {
+        0x1010, 0xeeee, 0x2020, 0xeeee,
+        0x3030, 0xeeee, 0x4040, 0xeeee,
+    };
+    uint16_t store_input[] = {
+        0x5151, 0x6262, 0x7373, 0x8484,
+    };
+    uint16_t ordered_store_input[] = {
+        0x9191, 0xa2a2, 0xb3b3, 0xc4c4,
+    };
+    uint16_t expected_gather[] = {
+        0x1010, 0x2020, 0x3030, 0x4040,
+    };
+    uint16_t expected_scatter[] = {
+        0x5151, 0xeeee, 0x6262, 0xeeee,
+        0x7373, 0xeeee, 0x8484, 0xeeee,
+    };
+    uint16_t expected_ordered_scatter[] = {
+        0x9191, 0xeeee, 0xa2a2, 0xeeee,
+        0xb3b3, 0xeeee, 0xc4c4, 0xeeee,
+    };
+    uint16_t out_unordered[4] = { 0 };
+    uint16_t out_ordered[4] = { 0 };
+    uint16_t scatter[8] = {
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+    };
+    uint16_t ordered_scatter[8] = {
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+    };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 4;
+    uint64_t a7 = code_start + 0x1600;
+    uint64_t t1 = code_start + 0x1700;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, offsets, sizeof(offsets)));
+    OK(uc_mem_write(uc, a1, gather_input, sizeof(gather_input)));
+    OK(uc_mem_write(uc, a4, store_input, sizeof(store_input)));
+    OK(uc_mem_write(uc, a5, scatter, sizeof(scatter)));
+    OK(uc_mem_write(uc, a7, ordered_store_input,
+                    sizeof(ordered_store_input)));
+    OK(uc_mem_write(uc, t1, ordered_scatter, sizeof(ordered_scatter)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_unordered, sizeof(out_unordered)));
+    OK(uc_mem_read(uc, a3, out_ordered, sizeof(out_ordered)));
+    OK(uc_mem_read(uc, a5, scatter, sizeof(scatter)));
+    OK(uc_mem_read(uc, t1, ordered_scatter, sizeof(ordered_scatter)));
+    for (i = 0; i < sizeof(expected_gather) / sizeof(expected_gather[0]);
+         i++) {
+        TEST_CHECK(out_unordered[i] == expected_gather[i]);
+        TEST_CHECK(out_ordered[i] == expected_gather[i]);
+    }
+    for (i = 0; i < sizeof(expected_scatter) / sizeof(expected_scatter[0]);
+         i++) {
+        TEST_CHECK(scatter[i] == expected_scatter[i]);
+        TEST_CHECK(ordered_scatter[i] == expected_ordered_scatter[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_indexed_mixed_widths(void)
+{
+    uc_engine *uc;
+    uint8_t code[12 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_ldst(0, 5, 1, 10, 2),
+        riscv_encode_rvv_index_ldst(0, 5, 1, 1, 11, 2, 4, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 4),
+        riscv_encode_rvv_vsetvli(0, 16, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 1),
+        riscv_encode_rvv_index_ldst(0, 6, 1, 1, 14, 1, 2, 1),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 2),
+        riscv_encode_rvv_vsetvli(0, 16, 0xd8),
+        riscv_encode_rvv_ldst(0, 7, 1, 17, 1),
+        riscv_encode_rvv_index_ldst(0, 7, 1, 1, 5, 1, 2, 1),
+        riscv_encode_rvv_ldst(1, 7, 1, 6, 2),
+    };
+    uint16_t offsets16[] = { 0, 2, 4, 6 };
+    uint32_t offsets32[] = { 0, 8, 16, 24 };
+    uint64_t offsets64[] = { 0, 16, 32, 48 };
+    uint8_t data8[] = {
+        10, 0xee, 20, 0xee, 30, 0xee, 40, 0xee,
+    };
+    uint32_t data32[] = {
+        0x10101010, 0xeeeeeeee, 0x20202020, 0xeeeeeeee,
+        0x30303030, 0xeeeeeeee, 0x40404040, 0xeeeeeeee,
+    };
+    uint64_t data64[] = {
+        0x1111111111111111ull, 0xeeeeeeeeeeeeeeeeull,
+        0x2222222222222222ull, 0xeeeeeeeeeeeeeeeeull,
+        0x3333333333333333ull, 0xeeeeeeeeeeeeeeeeull,
+        0x4444444444444444ull, 0xeeeeeeeeeeeeeeeeull,
+    };
+    uint8_t expected8[] = { 10, 20, 30, 40 };
+    uint32_t expected32[] = {
+        0x10101010, 0x20202020, 0x30303030, 0x40404040,
+    };
+    uint64_t expected64[] = {
+        0x1111111111111111ull, 0x2222222222222222ull,
+    };
+    uint8_t out8[4] = { 0 };
+    uint32_t out32[4] = { 0 };
+    uint64_t out64[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 4;
+    uint64_t a7 = code_start + 0x1600;
+    uint64_t t0 = code_start + 0x1700;
+    uint64_t t1 = code_start + 0x1800;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, offsets16, sizeof(offsets16)));
+    OK(uc_mem_write(uc, a1, data8, sizeof(data8)));
+    OK(uc_mem_write(uc, a3, offsets32, sizeof(offsets32)));
+    OK(uc_mem_write(uc, a4, data32, sizeof(data32)));
+    OK(uc_mem_write(uc, a7, offsets64, sizeof(offsets64)));
+    OK(uc_mem_write(uc, t0, data64, sizeof(data64)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out8, sizeof(out8)));
+    OK(uc_mem_read(uc, a5, out32, sizeof(out32)));
+    OK(uc_mem_read(uc, t1, out64, sizeof(out64)));
+    for (i = 0; i < sizeof(expected8); i++) {
+        TEST_CHECK(out8[i] == expected8[i]);
+    }
+    for (i = 0; i < sizeof(expected32) / sizeof(expected32[0]); i++) {
+        TEST_CHECK(out32[i] == expected32[i]);
+    }
+    for (i = 0; i < sizeof(expected64) / sizeof(expected64[0]); i++) {
+        TEST_CHECK(out64[i] == expected64[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_masked_indexed_memory(void)
+{
+    uc_engine *uc;
+    uint8_t code[7 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_index_ldst(0, 0, 1, 0, 12, 1, 2, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 2),
+        riscv_encode_rvv_ldst(0, 0, 1, 14, 3),
+        riscv_encode_rvv_index_ldst(1, 0, 1, 0, 15, 1, 3, 1),
+    };
+    uint8_t mask_input[] = { 0xad };
+    uint8_t offsets[] = { 0, 2, 4, 6, 8, 10, 12, 14 };
+    uint8_t gather_input[] = {
+        10, 0xee, 20, 0xee, 30, 0xee, 40, 0xee,
+        50, 0xee, 60, 0xee, 70, 0xee, 80, 0xee,
+    };
+    uint8_t expected_loaded[] = {
+        10, 0xff, 30, 40, 0xff, 60, 0xff, 80,
+    };
+    uint8_t store_input[] = {
+        1, 2, 3, 4, 5, 6, 7, 8,
+    };
+    uint8_t scatter[16] = {
+        0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+        0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+    };
+    uint8_t expected_scatter[] = {
+        1, 0xee, 0xee, 0xee, 3, 0xee, 4, 0xee,
+        0xee, 0xee, 6, 0xee, 0xee, 0xee, 8, 0xee,
+    };
+    uint8_t out_loaded[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_input, sizeof(mask_input)));
+    OK(uc_mem_write(uc, a1, offsets, sizeof(offsets)));
+    OK(uc_mem_write(uc, a2, gather_input, sizeof(gather_input)));
+    OK(uc_mem_write(uc, a4, store_input, sizeof(store_input)));
+    OK(uc_mem_write(uc, a5, scatter, sizeof(scatter)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_loaded, sizeof(out_loaded)));
+    OK(uc_mem_read(uc, a5, scatter, sizeof(scatter)));
+    for (i = 0; i < sizeof(expected_loaded); i++) {
+        TEST_CHECK(out_loaded[i] == expected_loaded[i]);
+    }
+    for (i = 0; i < sizeof(expected_scatter); i++) {
+        TEST_CHECK(scatter[i] == expected_scatter[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_indexed_segment_memory(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 1),
+        riscv_encode_rvv_index_ldst(0, 0, 1, 1, 11, 1, 2, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 3),
+        riscv_encode_rvv_ldst(0, 5, 1, 14, 4),
+        riscv_encode_rvv_ldst(0, 5, 1, 15, 5),
+        riscv_encode_rvv_index_ldst(1, 0, 1, 1, 17, 1, 4, 2),
+    };
+    uint8_t offsets[] = { 0, 8, 16 };
+    uint16_t segment_input[12] = {
+        0x101, 0x201, 0xeeee, 0xeeee,
+        0x102, 0x202, 0xeeee, 0xeeee,
+        0x103, 0x203, 0xeeee, 0xeeee,
+    };
+    uint16_t store_fields[] = {
+        0x111, 0x222, 0x333, 0xaaa, 0xbbb, 0xccc,
+    };
+    uint16_t expected_field0[] = {
+        0x101, 0x102, 0x103,
+    };
+    uint16_t expected_field1[] = {
+        0x201, 0x202, 0x203,
+    };
+    uint16_t segment_output[12] = {
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+    };
+    uint16_t expected_segment_output[] = {
+        0x111, 0xaaa, 0xeeee, 0xeeee,
+        0x222, 0xbbb, 0xeeee, 0xeeee,
+        0x333, 0xccc, 0xeeee, 0xeeee,
+    };
+    uint16_t out_field0[3] = { 0 };
+    uint16_t out_field1[3] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = a4 + 3 * sizeof(store_fields[0]);
+    uint64_t a6 = 3;
+    uint64_t a7 = code_start + 0x1500;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, offsets, sizeof(offsets)));
+    OK(uc_mem_write(uc, a1, segment_input, sizeof(segment_input)));
+    OK(uc_mem_write(uc, a4, store_fields, sizeof(store_fields)));
+    OK(uc_mem_write(uc, a7, segment_output, sizeof(segment_output)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_field0, sizeof(out_field0)));
+    OK(uc_mem_read(uc, a3, out_field1, sizeof(out_field1)));
+    OK(uc_mem_read(uc, a7, segment_output, sizeof(segment_output)));
+    for (i = 0; i < sizeof(expected_field0) / sizeof(expected_field0[0]);
+         i++) {
+        TEST_CHECK(out_field0[i] == expected_field0[i]);
+        TEST_CHECK(out_field1[i] == expected_field1[i]);
+    }
+    for (i = 0;
+         i < sizeof(expected_segment_output) /
+             sizeof(expected_segment_output[0]);
+         i++) {
+        TEST_CHECK(segment_output[i] == expected_segment_output[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_indexed_memory_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_index_ldst(0, 0, 1, 0, 10, 5, 0, 1));
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_index_ldst(0, 7, 1, 1, 10, 1, 8, 1), 0xc0);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_index_ldst(0, 0, 1, 1, 10, 1, 1, 1), 0xc8);
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_index_ldst(0, 0, 1, 1, 10, 1, 31, 2));
+}
+
+static void test_riscv64_rvv_fault_only_first_segment(void)
+{
+    uc_engine *uc;
+    uint8_t code[4 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_ff_load(5, 1, 10, 2, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 11, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 3),
+    };
+    uint16_t segment_input[] = {
+        0x101, 0x201, 0x102, 0x202, 0x103, 0x203,
+    };
+    uint16_t expected_field0[] = {
+        0x101, 0x102, 0x103,
+    };
+    uint16_t expected_field1[] = {
+        0x201, 0x202, 0x203,
+    };
+    uint16_t out_field0[3] = { 0 };
+    uint16_t out_field1[3] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a6 = 3;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, segment_input, sizeof(segment_input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, out_field0, sizeof(out_field0)));
+    OK(uc_mem_read(uc, a2, out_field1, sizeof(out_field1)));
+    for (i = 0; i < sizeof(expected_field0) / sizeof(expected_field0[0]);
+         i++) {
+        TEST_CHECK(out_field0[i] == expected_field0[i]);
+        TEST_CHECK(out_field1[i] == expected_field1[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_masked_fault_only_first(void)
+{
+    uc_engine *uc;
+    uint8_t code[4 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ff_load(0, 0, 11, 1, 1),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 1),
+    };
+    uint8_t mask_input[] = { 0xad };
+    uint8_t input[] = {
+        10, 20, 30, 40, 50, 60, 70, 80,
+    };
+    uint8_t expected[] = {
+        10, 0xff, 30, 40, 0xff, 60, 0xff, 80,
+    };
+    uint8_t output[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a6 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask_input, sizeof(mask_input)));
+    OK(uc_mem_write(uc, a1, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_fault_only_first_partial_fault(void)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_ff_load(5, 1, 10, 1, 1),
+        riscv_encode_rvv_ldst(1, 5, 1, 11, 1),
+    };
+    uint16_t input[] = {
+        0x1111, 0x2222,
+    };
+    uint16_t output[4] = {
+        0xeeee, 0xeeee, 0xeeee, 0xeeee,
+    };
+    uint16_t expected[] = {
+        0x1111, 0x2222, 0xeeee, 0xeeee,
+    };
+    uint64_t a0 = code_start + code_len - sizeof(input);
+    uint64_t a1 = code_start + 0x1000;
+    uint64_t a6 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_mem_write(uc, a1, output, sizeof(output)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected) / sizeof(expected[0]); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_fault_only_first_first_fault(void)
+{
+    uc_engine *uc;
+    uint8_t code[2 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_ff_load(5, 1, 10, 1, 1),
+    };
+    uint64_t a0 = code_start + code_len;
+    uint64_t a6 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    uc_assert_err(UC_ERR_READ_UNMAPPED,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_fault_only_first_illegal(void)
+{
+    run_riscv64_rvv_illegal(riscv_encode_rvv_ff_load(0, 0, 10, 0, 1));
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_ff_load(7, 1, 10, 0, 2), 0xc0);
+}
+
+static void test_riscv64_rvv_whole_register_single(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_whole_ldst(0, 0, 10, 1, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 11, 1, 1),
+        riscv_encode_rvv_whole_ldst(0, 5, 12, 2, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 13, 2, 1),
+        riscv_encode_rvv_whole_ldst(0, 6, 14, 3, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 15, 3, 1),
+        riscv_encode_rvv_whole_ldst(0, 7, 16, 4, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 17, 4, 1),
+    };
+    uint8_t input8[16];
+    uint8_t input16[16];
+    uint8_t input32[16];
+    uint8_t input64[16];
+    uint8_t output8[16] = { 0 };
+    uint8_t output16[16] = { 0 };
+    uint8_t output32[16] = { 0 };
+    uint8_t output64[16] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    size_t i;
+
+    for (i = 0; i < sizeof(input8); i++) {
+        input8[i] = (uint8_t)(0x10 + i);
+        input16[i] = (uint8_t)(0x30 + i);
+        input32[i] = (uint8_t)(0x50 + i);
+        input64[i] = (uint8_t)(0x70 + i);
+    }
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input8, sizeof(input8)));
+    OK(uc_mem_write(uc, a2, input16, sizeof(input16)));
+    OK(uc_mem_write(uc, a4, input32, sizeof(input32)));
+    OK(uc_mem_write(uc, a6, input64, sizeof(input64)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, output8, sizeof(output8)));
+    OK(uc_mem_read(uc, a3, output16, sizeof(output16)));
+    OK(uc_mem_read(uc, a5, output32, sizeof(output32)));
+    OK(uc_mem_read(uc, a7, output64, sizeof(output64)));
+    for (i = 0; i < sizeof(input8); i++) {
+        TEST_CHECK(output8[i] == input8[i]);
+        TEST_CHECK(output16[i] == input16[i]);
+        TEST_CHECK(output32[i] == input32[i]);
+        TEST_CHECK(output64[i] == input64[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_whole_register_groups(void)
+{
+    uc_engine *uc;
+    uint8_t code[6 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_whole_ldst(0, 0, 10, 2, 2),
+        riscv_encode_rvv_whole_ldst(1, 0, 11, 2, 2),
+        riscv_encode_rvv_whole_ldst(0, 5, 12, 4, 4),
+        riscv_encode_rvv_whole_ldst(1, 0, 13, 4, 4),
+        riscv_encode_rvv_whole_ldst(0, 6, 14, 8, 8),
+        riscv_encode_rvv_whole_ldst(1, 0, 15, 8, 8),
+    };
+    uint8_t input2[32];
+    uint8_t input4[64];
+    uint8_t input8[128];
+    uint8_t output2[32] = { 0 };
+    uint8_t output4[64] = { 0 };
+    uint8_t output8[128] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1200;
+    uint64_t a2 = code_start + 0x1400;
+    uint64_t a3 = code_start + 0x1800;
+    uint64_t a4 = code_start + 0x1c00;
+    uint64_t a5 = code_start + 0x2400;
+    size_t i;
+
+    for (i = 0; i < sizeof(input8); i++) {
+        if (i < sizeof(input2)) {
+            input2[i] = (uint8_t)(0x20 + i);
+        }
+        if (i < sizeof(input4)) {
+            input4[i] = (uint8_t)(0x60 + i);
+        }
+        input8[i] = (uint8_t)(0xa0 + i);
+    }
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input2, sizeof(input2)));
+    OK(uc_mem_write(uc, a2, input4, sizeof(input4)));
+    OK(uc_mem_write(uc, a4, input8, sizeof(input8)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, output2, sizeof(output2)));
+    OK(uc_mem_read(uc, a3, output4, sizeof(output4)));
+    OK(uc_mem_read(uc, a5, output8, sizeof(output8)));
+    for (i = 0; i < sizeof(input8); i++) {
+        if (i < sizeof(input2)) {
+            TEST_CHECK(output2[i] == input2[i]);
+        }
+        if (i < sizeof(input4)) {
+            TEST_CHECK(output4[i] == input4[i]);
+        }
+        TEST_CHECK(output8[i] == input8[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_whole_register_ignores_vl_vill(void)
+{
+    uc_engine *uc;
+    uint8_t code[6 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_whole_ldst(0, 0, 10, 1, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 11, 1, 1),
+        riscv_encode_rvv_vsetvli(0, 17, 0x400),
+        riscv_encode_rvv_whole_ldst(0, 0, 12, 2, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 13, 2, 1),
+    };
+    uint8_t input_vl0[16];
+    uint8_t input_vill[16];
+    uint8_t output_vl0[16] = { 0 };
+    uint8_t output_vill[16] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a6 = 0;
+    uint64_t a7 = 5;
+    size_t i;
+
+    for (i = 0; i < sizeof(input_vl0); i++) {
+        input_vl0[i] = (uint8_t)(0x40 + i);
+        input_vill[i] = (uint8_t)(0x80 + i);
+    }
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input_vl0, sizeof(input_vl0)));
+    OK(uc_mem_write(uc, a2, input_vill, sizeof(input_vill)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, output_vl0, sizeof(output_vl0)));
+    OK(uc_mem_read(uc, a3, output_vill, sizeof(output_vill)));
+    for (i = 0; i < sizeof(input_vl0); i++) {
+        TEST_CHECK(output_vl0[i] == input_vl0[i]);
+        TEST_CHECK(output_vill[i] == input_vill[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_whole_register_vstart(void)
+{
+    uc_engine *uc;
+    uint8_t code[5 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_whole_ldst(0, 0, 10, 1, 1),
+        riscv_encode_csr(RISCV_CSR_VSTART, 5, 1, 0),
+        riscv_encode_rvv_whole_ldst(0, 0, 11, 1, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 12, 1, 1),
+        riscv_encode_csr(RISCV_CSR_VSTART, 0, 2, 6),
+    };
+    uint8_t seed[16];
+    uint8_t source[16];
+    uint8_t expected[16];
+    uint8_t output[16] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t t0 = 8;
+    uint64_t t1 = 0xffff;
+    size_t i;
+
+    for (i = 0; i < sizeof(seed); i++) {
+        seed[i] = (uint8_t)(0x10 + i);
+        source[i] = (uint8_t)(0x80 + i);
+        expected[i] = i < 8 ? seed[i] : source[i];
+    }
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, seed, sizeof(seed)));
+    OK(uc_mem_write(uc, a1, source, sizeof(source)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+    TEST_CHECK(t1 == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_whole_register_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_whole_ldst(0, 0, 10, 1, 2));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_whole_ldst(0, 5, 10, 2, 4));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_whole_ldst(1, 0, 10, 4, 8));
+}
+
+static void test_riscv32_rvv_whole_register_smoke(void)
+{
+    uc_engine *uc;
+    uint8_t code[2 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_whole_ldst(0, 0, 10, 1, 1),
+        riscv_encode_rvv_whole_ldst(1, 0, 11, 1, 1),
+    };
+    uint8_t input[16];
+    uint8_t output[16] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    size_t i;
+
+    for (i = 0; i < sizeof(input); i++) {
+        input[i] = (uint8_t)(0x30 + i);
+    }
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, output, sizeof(output)));
+    for (i = 0; i < sizeof(input); i++) {
+        TEST_CHECK(output[i] == input[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_arith(void)
+{
+    uc_engine *uc;
+    uint8_t code[19 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x02, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_rvv_op(0x24, 1, 1, 2, 1, 5),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 5),
+        riscv_encode_rvv_op(0x20, 1, 1, 2, 1, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 6),
+        riscv_encode_rvv_op(0x00, 1, 1, 1, 5, 7),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 7),
+        riscv_encode_rvv_op(0x02, 1, 1, 1, 5, 8),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 8),
+        riscv_encode_rvv_op(0x24, 1, 1, 1, 5, 9),
+        riscv_encode_rvv_ldst(1, 6, 1, 28, 9),
+        riscv_encode_rvv_op(0x20, 1, 1, 1, 5, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 10),
+    };
+    uint32_t src2[] = {
+        0x41000000u, 0x40800000u, 0xc0c00000u, 0x41100000u,
+    };
+    uint32_t src1[] = {
+        0x40000000u, 0xc0000000u, 0x40400000u, 0x00000000u,
+    };
+    uint32_t expected_add_vv[] = {
+        0x41200000u, 0x40000000u, 0xc0400000u, 0x41100000u,
+    };
+    uint32_t expected_sub_vv[] = {
+        0x40c00000u, 0x40c00000u, 0xc1100000u, 0x41100000u,
+    };
+    uint32_t expected_mul_vv[] = {
+        0x41800000u, 0xc1000000u, 0xc1900000u, 0x00000000u,
+    };
+    uint32_t expected_div_vv[] = {
+        0x40800000u, 0xc0000000u, 0xc0000000u, 0x7f800000u,
+    };
+    uint32_t expected_add_vf[] = {
+        0x41200000u, 0x40c00000u, 0xc0800000u, 0x41300000u,
+    };
+    uint32_t expected_sub_vf[] = {
+        0x40c00000u, 0x40000000u, 0xc1000000u, 0x40e00000u,
+    };
+    uint32_t expected_mul_vf[] = {
+        0x41800000u, 0x41000000u, 0xc1400000u, 0x41900000u,
+    };
+    uint32_t expected_div_vf[] = {
+        0x40800000u, 0x40000000u, 0xc0400000u, 0x40900000u,
+    };
+    uint32_t out_add_vv[4] = { 0 };
+    uint32_t out_sub_vv[4] = { 0 };
+    uint32_t out_mul_vv[4] = { 0 };
+    uint32_t out_div_vv[4] = { 0 };
+    uint32_t out_add_vf[4] = { 0 };
+    uint32_t out_sub_vf[4] = { 0 };
+    uint32_t out_mul_vf[4] = { 0 };
+    uint32_t out_div_vf[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t f1 = 0xffffffff40000000ull;
+    uint64_t fflags = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_add_vv, sizeof(out_add_vv)));
+    OK(uc_mem_read(uc, a3, out_sub_vv, sizeof(out_sub_vv)));
+    OK(uc_mem_read(uc, a4, out_mul_vv, sizeof(out_mul_vv)));
+    OK(uc_mem_read(uc, a5, out_div_vv, sizeof(out_div_vv)));
+    OK(uc_mem_read(uc, a6, out_add_vf, sizeof(out_add_vf)));
+    OK(uc_mem_read(uc, a7, out_sub_vf, sizeof(out_sub_vf)));
+    OK(uc_mem_read(uc, t3, out_mul_vf, sizeof(out_mul_vf)));
+    OK(uc_mem_read(uc, t4, out_div_vf, sizeof(out_div_vf)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_FFLAGS, &fflags));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_add_vv[i] == expected_add_vv[i]);
+        TEST_CHECK(out_sub_vv[i] == expected_sub_vv[i]);
+        TEST_CHECK(out_mul_vv[i] == expected_mul_vv[i]);
+        TEST_CHECK(out_div_vv[i] == expected_div_vv[i]);
+        TEST_CHECK(out_add_vf[i] == expected_add_vf[i]);
+        TEST_CHECK(out_sub_vf[i] == expected_sub_vf[i]);
+        TEST_CHECK(out_mul_vf[i] == expected_mul_vf[i]);
+        TEST_CHECK(out_div_vf[i] == expected_div_vf[i]);
+    }
+    TEST_CHECK(fflags == 0x08);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_minmax_sign(void)
+{
+    enum {
+        FP32_RSUB_VF,
+        FP32_RDIV_VF,
+        FP32_MIN_VV,
+        FP32_MIN_VF,
+        FP32_MAX_VV,
+        FP32_MAX_VF,
+        FP32_SGNJ_VV,
+        FP32_SGNJ_VF,
+        FP32_SGNJN_VV,
+        FP32_SGNJN_VF,
+        FP32_SGNJX_VV,
+        FP32_SGNJX_VF,
+        FP32_OPS,
+    };
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x27, 1, 1, 1, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x21, 1, 1, 1, 5, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x04, 1, 1, 2, 1, 5),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 5),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x04, 1, 1, 1, 5, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 6),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x06, 1, 1, 2, 1, 7),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 7),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x06, 1, 1, 1, 5, 8),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 8),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x08, 1, 1, 2, 1, 9),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 9),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x08, 1, 1, 1, 5, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 10),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x09, 1, 1, 2, 1, 11),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 11),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x09, 1, 1, 1, 5, 12),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 12),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x0a, 1, 1, 2, 1, 13),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 13),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_op(0x0a, 1, 1, 1, 5, 14),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 14),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t src2[] = {
+        0x40800000u, 0xbf800000u, 0x7f800001u, 0x41000000u,
+    };
+    uint32_t src1[] = {
+        0x3f800000u, 0xc0400000u, 0x40a00000u, 0xc0c00000u,
+    };
+    uint32_t expected[FP32_OPS][4] = {
+        { 0xc0c00000u, 0xbf800000u, 0x7fc00000u, 0xc1200000u },
+        { 0xbf000000u, 0x40000000u, 0x7fc00000u, 0xbe800000u },
+        { 0x3f800000u, 0xc0400000u, 0x40a00000u, 0xc0c00000u },
+        { 0xc0000000u, 0xc0000000u, 0xc0000000u, 0xc0000000u },
+        { 0x40800000u, 0xbf800000u, 0x40a00000u, 0x41000000u },
+        { 0x40800000u, 0xbf800000u, 0xc0000000u, 0x41000000u },
+        { 0x40800000u, 0xbf800000u, 0x7f800001u, 0xc1000000u },
+        { 0xc0800000u, 0xbf800000u, 0xff800001u, 0xc1000000u },
+        { 0xc0800000u, 0x3f800000u, 0xff800001u, 0x41000000u },
+        { 0x40800000u, 0x3f800000u, 0x7f800001u, 0x41000000u },
+        { 0x40800000u, 0x3f800000u, 0x7f800001u, 0xc1000000u },
+        { 0xc0800000u, 0x3f800000u, 0xff800001u, 0xc1000000u },
+    };
+    const char *names[FP32_OPS] = {
+        "vfrsub.vf", "vfrdiv.vf", "vfmin.vv", "vfmin.vf",
+        "vfmax.vv", "vfmax.vf", "vfsgnj.vv", "vfsgnj.vf",
+        "vfsgnjn.vv", "vfsgnjn.vf", "vfsgnjx.vv", "vfsgnjx.vf",
+    };
+    uint32_t output[FP32_OPS][4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 4;
+    uint64_t f1 = 0xffffffffc0000000ull;
+    uint64_t t1;
+    size_t i;
+    size_t j;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, code_start + 0x1300, output, sizeof(output)));
+    for (i = 0; i < FP32_OPS; i++) {
+        for (j = 0; j < 4; j++) {
+            TEST_CHECK_(output[i][j] == expected[i][j],
+                        "%s lane %u: got 0x%08x expected 0x%08x",
+                        names[i], (unsigned)j, output[i][j],
+                        expected[i][j]);
+        }
+    }
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == 0x10);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_float32_minmax_sign_smoke(void)
+{
+    uc_engine *uc;
+    uint8_t code[7 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x06, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x08, 1, 1, 1, 5, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+    };
+    uint32_t src2[] = { 0x3f800000u, 0xc0000000u };
+    uint32_t src1[] = { 0x40000000u, 0xc0400000u };
+    uint32_t expected_max[] = { 0x40000000u, 0xc0000000u };
+    uint32_t expected_sgnj[] = { 0xbf800000u, 0xc0000000u };
+    uint32_t out_max[2] = { 0 };
+    uint32_t out_sgnj[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 2;
+    uint64_t f1 = 0xffffffffbf800000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_max, sizeof(out_max)));
+    OK(uc_mem_read(uc, a3, out_sgnj, sizeof(out_sgnj)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_max[i] == expected_max[i]);
+        TEST_CHECK(out_sgnj[i] == expected_sgnj[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_compare(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 2),
+        riscv_encode_rvv_op(0x18, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_mask_ldst(1, 13, 3),
+        riscv_encode_addi(13, 13, 1),
+        riscv_encode_rvv_op(0x1b, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_mask_ldst(1, 13, 4),
+        riscv_encode_addi(13, 13, 1),
+        riscv_encode_rvv_op(0x1c, 1, 1, 1, 5, 5),
+        riscv_encode_rvv_mask_ldst(1, 13, 5),
+        riscv_encode_addi(13, 13, 1),
+        riscv_encode_rvv_op(0x1f, 1, 1, 1, 5, 6),
+        riscv_encode_rvv_mask_ldst(1, 13, 6),
+        riscv_encode_addi(13, 13, 1),
+        riscv_encode_rvv_op(0x19, 1, 1, 1, 5, 7),
+        riscv_encode_rvv_mask_ldst(1, 13, 7),
+        riscv_encode_addi(13, 13, 1),
+        riscv_encode_rvv_op(0x1b, 0, 1, 1, 5, 8),
+        riscv_encode_rvv_mask_ldst(1, 13, 8),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint8_t code[sizeof(insns)];
+    uint8_t mask[] = { 0x05 };
+    uint32_t src2[] = {
+        0x3f800000u, 0x40000000u, 0x7f800001u, 0xbf800000u,
+    };
+    uint32_t src1[] = {
+        0x3f800000u, 0x40400000u, 0x40800000u, 0xc0000000u,
+    };
+    uint8_t expected[] = { 0xf1, 0xf2, 0xfd, 0xf2, 0xfb, 0xfb };
+    uint8_t output[sizeof(expected)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 4;
+    uint64_t t1;
+    uint64_t f1 = 0xffffffff40000000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, code_start + 0x1300, output, sizeof(output)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK_(output[i] == expected[i],
+                    "compare mask %u: got 0x%02x expected 0x%02x",
+                    (unsigned)i, output[i], expected[i]);
+    }
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == 0x10);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_class_merge(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 1),
+        riscv_encode_rvv_op(0x13, 1, 1, 16, 1, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 2),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 3),
+        riscv_encode_rvv_op(0x17, 0, 3, 1, 5, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 4),
+    };
+    uint8_t code[sizeof(insns)];
+    uint8_t mask[] = { 0x05 };
+    uint32_t class_src[] = {
+        0xff800000u, 0x80000000u, 0x00000001u, 0x7f800001u,
+    };
+    uint32_t merge_src[] = {
+        0x3f800000u, 0xbf800000u, 0x40400000u, 0xc0400000u,
+    };
+    uint32_t expected_class[] = { 0x001u, 0x008u, 0x020u, 0x100u };
+    uint32_t expected_merge[] = {
+        0x40000000u, 0xbf800000u, 0x40000000u, 0xc0400000u,
+    };
+    uint32_t out_class[4] = { 0 };
+    uint32_t out_merge[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t t0 = 4;
+    uint64_t f1 = 0xffffffff40000000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, class_src, sizeof(class_src)));
+    OK(uc_mem_write(uc, a3, merge_src, sizeof(merge_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_class, sizeof(out_class)));
+    OK(uc_mem_read(uc, a4, out_merge, sizeof(out_merge)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_class[i] == expected_class[i]);
+        TEST_CHECK(out_merge[i] == expected_merge[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_moves(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_op(0x17, 1, 0, 1, 5, 1),
+        riscv_encode_rvv_ldst(1, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x10, 1, 0, 2, 5, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 2),
+        riscv_encode_rvv_op(0x10, 1, 2, 0, 1, 3),
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t move_src[] = {
+        0x3f800000u, 0xbf800000u, 0x40400000u, 0xc0400000u,
+    };
+    uint32_t expected_broadcast[] = {
+        0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u,
+    };
+    uint32_t expected_scalar_insert[] = {
+        0x40000000u, 0xbf800000u, 0x40400000u, 0xc0400000u,
+    };
+    uint32_t out_broadcast[4] = { 0 };
+    uint32_t out_scalar_insert[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t t0 = 4;
+    uint64_t f1 = 0x0000000040000000ull;
+    uint64_t f2 = 0xffffffff40000000ull;
+    uint64_t f3 = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a1, move_src, sizeof(move_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F2, &f2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a0, out_broadcast, sizeof(out_broadcast)));
+    OK(uc_mem_read(uc, a2, out_scalar_insert, sizeof(out_scalar_insert)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_F3, &f3));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_broadcast[i] == expected_broadcast[i]);
+        TEST_CHECK(out_scalar_insert[i] == expected_scalar_insert[i]);
+    }
+    TEST_CHECK(f3 == 0xffffffff40000000ull);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float64_arith(void)
+{
+    uc_engine *uc;
+    uint8_t code[19 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd8),
+        riscv_encode_rvv_ldst(0, 7, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 7, 1, 11, 2),
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 7, 1, 12, 3),
+        riscv_encode_rvv_op(0x02, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_rvv_op(0x24, 1, 1, 2, 1, 5),
+        riscv_encode_rvv_ldst(1, 7, 1, 14, 5),
+        riscv_encode_rvv_op(0x20, 1, 1, 2, 1, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 15, 6),
+        riscv_encode_rvv_op(0x00, 1, 1, 1, 5, 7),
+        riscv_encode_rvv_ldst(1, 7, 1, 16, 7),
+        riscv_encode_rvv_op(0x02, 1, 1, 1, 5, 8),
+        riscv_encode_rvv_ldst(1, 7, 1, 17, 8),
+        riscv_encode_rvv_op(0x24, 1, 1, 1, 5, 9),
+        riscv_encode_rvv_ldst(1, 7, 1, 28, 9),
+        riscv_encode_rvv_op(0x20, 1, 1, 1, 5, 10),
+        riscv_encode_rvv_ldst(1, 7, 1, 29, 10),
+    };
+    uint64_t src2[] = {
+        0x4020000000000000ull, 0xc018000000000000ull,
+    };
+    uint64_t src1[] = {
+        0x4000000000000000ull, 0xc008000000000000ull,
+    };
+    uint64_t expected_add_vv[] = {
+        0x4024000000000000ull, 0xc022000000000000ull,
+    };
+    uint64_t expected_sub_vv[] = {
+        0x4018000000000000ull, 0xc008000000000000ull,
+    };
+    uint64_t expected_mul_vv[] = {
+        0x4030000000000000ull, 0x4032000000000000ull,
+    };
+    uint64_t expected_div_vv[] = {
+        0x4010000000000000ull, 0x4000000000000000ull,
+    };
+    uint64_t expected_add_vf[] = {
+        0x4024000000000000ull, 0xc010000000000000ull,
+    };
+    uint64_t expected_sub_vf[] = {
+        0x4018000000000000ull, 0xc020000000000000ull,
+    };
+    uint64_t expected_mul_vf[] = {
+        0x4030000000000000ull, 0xc028000000000000ull,
+    };
+    uint64_t expected_div_vf[] = {
+        0x4010000000000000ull, 0xc008000000000000ull,
+    };
+    uint64_t out_add_vv[2] = { 0 };
+    uint64_t out_sub_vv[2] = { 0 };
+    uint64_t out_mul_vv[2] = { 0 };
+    uint64_t out_div_vv[2] = { 0 };
+    uint64_t out_add_vf[2] = { 0 };
+    uint64_t out_sub_vf[2] = { 0 };
+    uint64_t out_mul_vf[2] = { 0 };
+    uint64_t out_div_vf[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 2;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t f1 = 0x4000000000000000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_add_vv, sizeof(out_add_vv)));
+    OK(uc_mem_read(uc, a3, out_sub_vv, sizeof(out_sub_vv)));
+    OK(uc_mem_read(uc, a4, out_mul_vv, sizeof(out_mul_vv)));
+    OK(uc_mem_read(uc, a5, out_div_vv, sizeof(out_div_vv)));
+    OK(uc_mem_read(uc, a6, out_add_vf, sizeof(out_add_vf)));
+    OK(uc_mem_read(uc, a7, out_sub_vf, sizeof(out_sub_vf)));
+    OK(uc_mem_read(uc, t3, out_mul_vf, sizeof(out_mul_vf)));
+    OK(uc_mem_read(uc, t4, out_div_vf, sizeof(out_div_vf)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_add_vv[i] == expected_add_vv[i]);
+        TEST_CHECK(out_sub_vv[i] == expected_sub_vv[i]);
+        TEST_CHECK(out_mul_vv[i] == expected_mul_vv[i]);
+        TEST_CHECK(out_div_vv[i] == expected_div_vv[i]);
+        TEST_CHECK(out_add_vf[i] == expected_add_vf[i]);
+        TEST_CHECK(out_sub_vf[i] == expected_sub_vf[i]);
+        TEST_CHECK(out_mul_vf[i] == expected_mul_vf[i]);
+        TEST_CHECK(out_div_vf[i] == expected_div_vf[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_mask_nanbox(void)
+{
+    uc_engine *uc;
+    uint8_t code[5 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 1),
+        riscv_encode_rvv_op(0x00, 0, 1, 1, 5, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 2),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint32_t src[] = {
+        0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+    };
+    uint32_t expected[] = {
+        0x7fc00000u, 0xffffffffu, 0x7fc00000u, 0x00000000u,
+    };
+    uint32_t output[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t t0 = 3;
+    uint64_t f1 = 0x0000000040000000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK_(output[i] == expected[i],
+                    "lane %u: got 0x%08x expected 0x%08x",
+                    (unsigned)i, output[i], expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_sqrt(void)
+{
+    uc_engine *uc;
+    uint8_t code[5 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_op(0x13, 1, 1, 0, 1, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 11, 2),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint32_t src[] = {
+        0x40800000u, 0x41100000u, 0xbf800000u, 0x3e800000u,
+    };
+    uint32_t expected[] = {
+        0x40000000u, 0x40400000u, 0x7fc00000u, 0x3f000000u,
+    };
+    uint32_t output[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t t0 = 4;
+    uint64_t t1;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, output, sizeof(output)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK_(output[i] == expected[i],
+                    "sqrt lane %u: got 0x%08x expected 0x%08x",
+                    (unsigned)i, output[i], expected[i]);
+    }
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == 0x10);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_estimate(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 3),
+        riscv_encode_rvv_op(0x13, 1, 1, 4, 1, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 2),
+        riscv_encode_rvv_op(0x13, 1, 3, 5, 1, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint32_t frsqrt_src[] = {
+        0x40800000u, 0x3e800000u, 0x00000000u, 0xbf800000u,
+    };
+    uint32_t frec_src[] = {
+        0x40800000u, 0xc0000000u, 0x00000000u, 0x7f800001u,
+    };
+    uint32_t expected_frsqrt[] = {
+        0x3eff0000u, 0x3fff0000u, 0x7f800000u, 0x7fc00000u,
+    };
+    uint32_t expected_frec[] = {
+        0x3e7f0000u, 0xbeff0000u, 0x7f800000u, 0x7fc00000u,
+    };
+    uint32_t out_frsqrt[4] = { 0 };
+    uint32_t out_frec[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 4;
+    uint64_t t1;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, frsqrt_src, sizeof(frsqrt_src)));
+    OK(uc_mem_write(uc, a1, frec_src, sizeof(frec_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_frsqrt, sizeof(out_frsqrt)));
+    OK(uc_mem_read(uc, a3, out_frec, sizeof(out_frec)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK_(out_frsqrt[i] == expected_frsqrt[i],
+                    "frsqrt7 lane %u: got 0x%08x expected 0x%08x",
+                    (unsigned)i, out_frsqrt[i], expected_frsqrt[i]);
+        TEST_CHECK_(out_frec[i] == expected_frec[i],
+                    "frec7 lane %u: got 0x%08x expected 0x%08x",
+                    (unsigned)i, out_frec[i], expected_frec[i]);
+    }
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == 0x18);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_convert(void)
+{
+    uc_engine *uc;
+    uint8_t code[18 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 4),
+        riscv_encode_rvv_op(0x12, 1, 1, 0, 1, 5),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 5),
+        riscv_encode_rvv_op(0x12, 1, 2, 1, 1, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 6),
+        riscv_encode_rvv_op(0x12, 1, 1, 6, 1, 7),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 7),
+        riscv_encode_rvv_op(0x12, 1, 2, 7, 1, 8),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 8),
+        riscv_encode_rvv_op(0x12, 1, 3, 2, 1, 9),
+        riscv_encode_rvv_ldst(1, 6, 1, 28, 9),
+        riscv_encode_rvv_op(0x12, 1, 4, 3, 1, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 10),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint32_t fp_unsigned[] = {
+        0x3fc00000u, 0x40200000u, 0x40400000u, 0x40800000u,
+    };
+    uint32_t fp_signed[] = {
+        0x3fc00000u, 0xc0200000u, 0x40400000u, 0xc0800000u,
+    };
+    uint32_t int_unsigned[] = { 1u, 2u, 0x01000003u, 16u };
+    uint32_t int_signed[] = { 1u, 0xffffffffu, 0x01000003u, 16u };
+    uint32_t expected_xu[] = { 2u, 2u, 3u, 4u };
+    uint32_t expected_x[] = { 2u, 0xfffffffeu, 3u, 0xfffffffcu };
+    uint32_t expected_rtz_xu[] = { 1u, 2u, 3u, 4u };
+    uint32_t expected_rtz_x[] = { 1u, 0xfffffffeu, 3u, 0xfffffffcu };
+    uint32_t expected_f_xu[] = {
+        0x3f800000u, 0x40000000u, 0x4b800002u, 0x41800000u,
+    };
+    uint32_t expected_f_x[] = {
+        0x3f800000u, 0xbf800000u, 0x4b800002u, 0x41800000u,
+    };
+    uint32_t out_xu[4] = { 0 };
+    uint32_t out_x[4] = { 0 };
+    uint32_t out_rtz_xu[4] = { 0 };
+    uint32_t out_rtz_x[4] = { 0 };
+    uint32_t out_f_xu[4] = { 0 };
+    uint32_t out_f_x[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t1;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, fp_unsigned, sizeof(fp_unsigned)));
+    OK(uc_mem_write(uc, a1, fp_signed, sizeof(fp_signed)));
+    OK(uc_mem_write(uc, a2, int_unsigned, sizeof(int_unsigned)));
+    OK(uc_mem_write(uc, a3, int_signed, sizeof(int_signed)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a4, out_xu, sizeof(out_xu)));
+    OK(uc_mem_read(uc, a5, out_x, sizeof(out_x)));
+    OK(uc_mem_read(uc, a6, out_rtz_xu, sizeof(out_rtz_xu)));
+    OK(uc_mem_read(uc, a7, out_rtz_x, sizeof(out_rtz_x)));
+    OK(uc_mem_read(uc, t3, out_f_xu, sizeof(out_f_xu)));
+    OK(uc_mem_read(uc, t4, out_f_x, sizeof(out_f_x)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_xu[i] == expected_xu[i]);
+        TEST_CHECK(out_x[i] == expected_x[i]);
+        TEST_CHECK(out_rtz_xu[i] == expected_rtz_xu[i]);
+        TEST_CHECK(out_rtz_x[i] == expected_rtz_x[i]);
+        TEST_CHECK(out_f_xu[i] == expected_f_xu[i]);
+        TEST_CHECK(out_f_x[i] == expected_f_x[i]);
+    }
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == 0x01);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_float_convert(void)
+{
+    uc_engine *uc;
+    uint8_t code[27 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x12, 1, 1, 8, 1, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_rvv_op(0x12, 1, 1, 9, 1, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 14, 6),
+        riscv_encode_rvv_op(0x12, 1, 1, 14, 1, 8),
+        riscv_encode_rvv_ldst(1, 7, 1, 15, 8),
+        riscv_encode_rvv_op(0x12, 1, 1, 15, 1, 10),
+        riscv_encode_rvv_ldst(1, 7, 1, 16, 10),
+        riscv_encode_rvv_op(0x12, 1, 2, 10, 1, 12),
+        riscv_encode_rvv_ldst(1, 7, 1, 17, 12),
+        riscv_encode_rvv_op(0x12, 1, 3, 11, 1, 14),
+        riscv_encode_rvv_ldst(1, 7, 1, 28, 14),
+        riscv_encode_rvv_op(0x12, 1, 1, 12, 1, 16),
+        riscv_encode_rvv_ldst(1, 7, 1, 29, 16),
+        riscv_encode_rvv_vsetvli(0, 5, 0xc0),
+        riscv_encode_rvv_ldst(0, 0, 1, 8, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 9, 2),
+        riscv_encode_rvv_op(0x12, 1, 1, 10, 1, 4),
+        riscv_encode_rvv_op(0x12, 1, 2, 11, 1, 6),
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_ldst(1, 5, 1, 30, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 31, 6),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint32_t fp_src[] = { 0x3fc00000u, 0x40200000u };
+    uint32_t uint_src[] = { 1u, 0x80000000u };
+    uint32_t int_src[] = { 1u, 0xfffffffeu };
+    uint8_t u8_src[] = { 1u, 2u };
+    uint8_t s8_src[] = { 1u, 0xfeu };
+    uint64_t expected_xu[] = { 2ull, 2ull };
+    uint64_t expected_x[] = { 2ull, 2ull };
+    uint64_t expected_rtz_xu[] = { 1ull, 2ull };
+    uint64_t expected_rtz_x[] = { 1ull, 2ull };
+    uint64_t expected_f_xu[] = {
+        0x3ff0000000000000ull, 0x41e0000000000000ull,
+    };
+    uint64_t expected_f_x[] = {
+        0x3ff0000000000000ull, 0xc000000000000000ull,
+    };
+    uint64_t expected_f_f[] = {
+        0x3ff8000000000000ull, 0x4004000000000000ull,
+    };
+    uint16_t expected_f_xu_b[] = { 0x3c00u, 0x4000u };
+    uint16_t expected_f_x_b[] = { 0x3c00u, 0xc000u };
+    uint64_t out_xu[2] = { 0 };
+    uint64_t out_x[2] = { 0 };
+    uint64_t out_rtz_xu[2] = { 0 };
+    uint64_t out_rtz_x[2] = { 0 };
+    uint64_t out_f_xu[2] = { 0 };
+    uint64_t out_f_x[2] = { 0 };
+    uint64_t out_f_f[2] = { 0 };
+    uint16_t out_f_xu_b[2] = { 0 };
+    uint16_t out_f_x_b[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 2;
+    uint64_t t1;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    uint64_t s0 = code_start + 0x1c00;
+    uint64_t s1 = code_start + 0x1d00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, fp_src, sizeof(fp_src)));
+    OK(uc_mem_write(uc, a1, uint_src, sizeof(uint_src)));
+    OK(uc_mem_write(uc, a2, int_src, sizeof(int_src)));
+    OK(uc_mem_write(uc, s0, u8_src, sizeof(u8_src)));
+    OK(uc_mem_write(uc, s1, s8_src, sizeof(s8_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S1, &s1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_xu, sizeof(out_xu)));
+    OK(uc_mem_read(uc, a4, out_x, sizeof(out_x)));
+    OK(uc_mem_read(uc, a5, out_rtz_xu, sizeof(out_rtz_xu)));
+    OK(uc_mem_read(uc, a6, out_rtz_x, sizeof(out_rtz_x)));
+    OK(uc_mem_read(uc, a7, out_f_xu, sizeof(out_f_xu)));
+    OK(uc_mem_read(uc, t3, out_f_x, sizeof(out_f_x)));
+    OK(uc_mem_read(uc, t4, out_f_f, sizeof(out_f_f)));
+    OK(uc_mem_read(uc, t5, out_f_xu_b, sizeof(out_f_xu_b)));
+    OK(uc_mem_read(uc, t6, out_f_x_b, sizeof(out_f_x_b)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_xu[i] == expected_xu[i]);
+        TEST_CHECK(out_x[i] == expected_x[i]);
+        TEST_CHECK(out_rtz_xu[i] == expected_rtz_xu[i]);
+        TEST_CHECK(out_rtz_x[i] == expected_rtz_x[i]);
+        TEST_CHECK(out_f_xu[i] == expected_f_xu[i]);
+        TEST_CHECK(out_f_x[i] == expected_f_x[i]);
+        TEST_CHECK(out_f_f[i] == expected_f_f[i]);
+        TEST_CHECK_(out_f_xu_b[i] == expected_f_xu_b[i],
+                    "vfwcvt.f.xu.b lane %u: got 0x%04x expected 0x%04x",
+                    (unsigned)i, out_f_xu_b[i], expected_f_xu_b[i]);
+        TEST_CHECK_(out_f_x_b[i] == expected_f_x_b[i],
+                    "vfwcvt.f.x.b lane %u: got 0x%04x expected 0x%04x",
+                    (unsigned)i, out_f_x_b[i], expected_f_x_b[i]);
+    }
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == 0x01);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_narrowing_float_convert(void)
+{
+    uc_engine *uc;
+    uint8_t code[20 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 7, 1, 10, 2),
+        riscv_encode_rvv_ldst(0, 7, 1, 11, 4),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 6),
+        riscv_encode_rvv_ldst(0, 7, 1, 13, 8),
+        riscv_encode_rvv_op(0x12, 1, 2, 16, 1, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 10),
+        riscv_encode_rvv_op(0x12, 1, 2, 17, 1, 11),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 11),
+        riscv_encode_rvv_op(0x12, 1, 2, 22, 1, 12),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 12),
+        riscv_encode_rvv_op(0x12, 1, 4, 18, 1, 13),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 13),
+        riscv_encode_rvv_op(0x12, 1, 6, 19, 1, 14),
+        riscv_encode_rvv_ldst(1, 6, 1, 28, 14),
+        riscv_encode_rvv_op(0x12, 1, 8, 20, 1, 15),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 15),
+        riscv_encode_rvv_op(0x12, 1, 8, 21, 1, 16),
+        riscv_encode_rvv_ldst(1, 6, 1, 30, 16),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint64_t fp_src[] = {
+        0x3ff8000000000000ull, 0x4004000000000000ull,
+    };
+    uint64_t uint_src[] = { 1ull, 0x01000001ull };
+    uint64_t int_src[] = { 0xfffffffffffffffeull, 0xffffffff80000000ull };
+    uint64_t rod_src[] = {
+        0x3ff0000010000000ull, 0x3ff8000000000000ull,
+    };
+    uint32_t expected_xu[] = { 2u, 2u };
+    uint32_t expected_x[] = { 2u, 2u };
+    uint32_t expected_rtz_xu[] = { 1u, 2u };
+    uint32_t expected_f_xu[] = { 0x3f800000u, 0x4b800000u };
+    uint32_t expected_f_x[] = { 0xc0000000u, 0xcf000000u };
+    uint32_t expected_f_f[] = { 0x3f800000u, 0x3fc00000u };
+    uint32_t expected_rod_f_f[] = { 0x3f800001u, 0x3fc00000u };
+    uint32_t out_xu[2] = { 0 };
+    uint32_t out_x[2] = { 0 };
+    uint32_t out_rtz_xu[2] = { 0 };
+    uint32_t out_f_xu[2] = { 0 };
+    uint32_t out_f_x[2] = { 0 };
+    uint32_t out_f_f[2] = { 0 };
+    uint32_t out_rod_f_f[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 2;
+    uint64_t t1;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, fp_src, sizeof(fp_src)));
+    OK(uc_mem_write(uc, a1, uint_src, sizeof(uint_src)));
+    OK(uc_mem_write(uc, a2, int_src, sizeof(int_src)));
+    OK(uc_mem_write(uc, a3, rod_src, sizeof(rod_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a4, out_xu, sizeof(out_xu)));
+    OK(uc_mem_read(uc, a5, out_x, sizeof(out_x)));
+    OK(uc_mem_read(uc, a6, out_rtz_xu, sizeof(out_rtz_xu)));
+    OK(uc_mem_read(uc, a7, out_f_xu, sizeof(out_f_xu)));
+    OK(uc_mem_read(uc, t3, out_f_x, sizeof(out_f_x)));
+    OK(uc_mem_read(uc, t4, out_f_f, sizeof(out_f_f)));
+    OK(uc_mem_read(uc, t5, out_rod_f_f, sizeof(out_rod_f_f)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_xu[i] == expected_xu[i]);
+        TEST_CHECK(out_x[i] == expected_x[i]);
+        TEST_CHECK(out_rtz_xu[i] == expected_rtz_xu[i]);
+        TEST_CHECK(out_f_xu[i] == expected_f_xu[i]);
+        TEST_CHECK(out_f_x[i] == expected_f_x[i]);
+        TEST_CHECK(out_f_f[i] == expected_f_f[i]);
+        TEST_CHECK(out_rod_f_f[i] == expected_rod_f_f[i]);
+    }
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    TEST_CHECK(t1 == 0x01);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_slide(void)
+{
+    uc_engine *uc;
+    uint8_t code[7 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 1),
+        riscv_encode_rvv_op(0x0e, 0, 1, 1, 5, 5),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 5),
+        riscv_encode_rvv_op(0x0f, 1, 1, 2, 5, 1),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 1),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint32_t src[] = {
+        0x40000000u, 0x40400000u, 0x40800000u, 0x40a00000u,
+    };
+    uint32_t expected_up[] = {
+        0x3f800000u, 0xffffffffu, 0x40400000u, 0xffffffffu,
+    };
+    uint32_t expected_down[] = {
+        0x40400000u, 0x40800000u, 0x40a00000u, 0x40c00000u,
+    };
+    uint32_t out_up[4] = { 0 };
+    uint32_t out_down[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 4;
+    uint64_t f1 = 0xffffffff3f800000ull;
+    uint64_t f2 = 0xffffffff40c00000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F2, &f2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_up, sizeof(out_up)));
+    OK(uc_mem_read(uc, a3, out_down, sizeof(out_down)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK_(out_up[i] == expected_up[i],
+                    "up lane %u: got 0x%08x expected 0x%08x",
+                    (unsigned)i, out_up[i], expected_up[i]);
+        TEST_CHECK_(out_down[i] == expected_down[i],
+                    "down lane %u: got 0x%08x expected 0x%08x",
+                    (unsigned)i, out_down[i], expected_down[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float32_fma(void)
+{
+    enum {
+        FMA_VFMACC_VV,
+        FMA_VFNMACC_VV,
+        FMA_VFMSAC_VV,
+        FMA_VFNMSAC_VV,
+        FMA_VFMADD_VV,
+        FMA_VFNMADD_VV,
+        FMA_VFMSUB_VV,
+        FMA_VFNMSUB_VV,
+        FMA_VFMACC_VF,
+        FMA_VFNMACC_VF,
+        FMA_VFMSAC_VF,
+        FMA_VFNMSAC_VF,
+        FMA_VFMADD_VF,
+        FMA_VFNMADD_VF,
+        FMA_VFMSUB_VF,
+        FMA_VFNMSUB_VF,
+        FMA_OPS,
+    };
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2c, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2d, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2e, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2f, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x28, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x29, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2a, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2b, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2c, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2d, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2e, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2f, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x28, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x29, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2a, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_addi(13, 13, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x2b, 1, 1, 5, 5, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t src2[] = { 0x40000000u, 0xc0400000u };
+    uint32_t src1[] = { 0x40a00000u, 0x40e00000u };
+    uint32_t acc[] = { 0x41300000u, 0xc1500000u };
+    uint32_t expected[FMA_OPS][2] = {
+        { 0x41a80000u, 0xc2080000u },
+        { 0xc1a80000u, 0x42080000u },
+        { 0xbf800000u, 0xc1000000u },
+        { 0x3f800000u, 0x41000000u },
+        { 0x42640000u, 0xc2bc0000u },
+        { 0xc2640000u, 0x42bc0000u },
+        { 0x42540000u, 0xc2b00000u },
+        { 0xc2540000u, 0x42b00000u },
+        { 0x41980000u, 0xc1c80000u },
+        { 0xc1980000u, 0x41c80000u },
+        { 0xc0400000u, 0x3f800000u },
+        { 0x40400000u, 0xbf800000u },
+        { 0x42380000u, 0xc25c0000u },
+        { 0xc2380000u, 0x425c0000u },
+        { 0x42280000u, 0xc2440000u },
+        { 0xc2280000u, 0x42440000u },
+    };
+    uint32_t output[FMA_OPS][2] = { { 0 } };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 2;
+    uint64_t f5 = 0xffffffff40800000ull;
+    size_t i;
+    size_t lane;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_mem_write(uc, a2, acc, sizeof(acc)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F5, &f5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, output, sizeof(output)));
+    for (i = 0; i < FMA_OPS; i++) {
+        for (lane = 0; lane < 2; lane++) {
+            TEST_CHECK_(output[i][lane] == expected[i][lane],
+                        "op %u lane %u: got 0x%08x expected 0x%08x",
+                        (unsigned)i, (unsigned)lane, output[i][lane],
+                        expected[i][lane]);
+        }
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_float32_smoke(void)
+{
+    uc_engine *uc;
+    uint8_t code[5 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 3),
+    };
+    uint32_t src2[] = { 0x3f800000u, 0x40000000u };
+    uint32_t src1[] = { 0x40400000u, 0x40800000u };
+    uint32_t expected[] = { 0x40800000u, 0x40c00000u };
+    uint32_t output[2] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    uint32_t t0 = 2;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, output, sizeof(output)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_float_fma(void)
+{
+    enum {
+        FWMA_VFWMACC_VV,
+        FWMA_VFWNMACC_VV,
+        FWMA_VFWMSAC_VV,
+        FWMA_VFWNMSAC_VV,
+        FWMA_VFWMACC_VF,
+        FWMA_VFWNMACC_VF,
+        FWMA_VFWMSAC_VF,
+        FWMA_VFWNMSAC_VF,
+        FWMA_OPS,
+    };
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3c, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3d, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3e, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3f, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3c, 1, 1, 5, 5, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3d, 1, 1, 5, 5, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3e, 1, 1, 5, 5, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_addi(13, 13, 16),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x3f, 1, 1, 5, 5, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t src2[] = { 0x40000000u, 0xc0400000u };
+    uint32_t src1[] = { 0x40a00000u, 0x40e00000u };
+    uint64_t acc[] = {
+        0x4026000000000000ull, 0xc02a000000000000ull,
+    };
+    uint64_t expected[FWMA_OPS][2] = {
+        { 0x4035000000000000ull, 0xc041000000000000ull },
+        { 0xc035000000000000ull, 0x4041000000000000ull },
+        { 0xbff0000000000000ull, 0xc020000000000000ull },
+        { 0x3ff0000000000000ull, 0x4020000000000000ull },
+        { 0x4033000000000000ull, 0xc039000000000000ull },
+        { 0xc033000000000000ull, 0x4039000000000000ull },
+        { 0xc008000000000000ull, 0x3ff0000000000000ull },
+        { 0x4008000000000000ull, 0xbff0000000000000ull },
+    };
+    uint64_t output[FWMA_OPS][2] = { { 0 } };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 2;
+    uint64_t f5 = 0xffffffff40800000ull;
+    size_t i;
+    size_t lane;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_mem_write(uc, a2, acc, sizeof(acc)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F5, &f5));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, output, sizeof(output)));
+    for (i = 0; i < FWMA_OPS; i++) {
+        for (lane = 0; lane < 2; lane++) {
+            TEST_CHECK_(output[i][lane] == expected[i][lane],
+                        "op %u lane %u: got 0x%016llx expected 0x%016llx",
+                        (unsigned)i, (unsigned)lane,
+                        (unsigned long long)output[i][lane],
+                        (unsigned long long)expected[i][lane]);
+        }
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_float_arith(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x30, 1, 1, 2, 1, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 6),
+        riscv_encode_rvv_op(0x32, 1, 1, 2, 1, 8),
+        riscv_encode_rvv_ldst(1, 7, 1, 14, 8),
+        riscv_encode_rvv_op(0x38, 1, 1, 2, 1, 10),
+        riscv_encode_rvv_ldst(1, 7, 1, 15, 10),
+        riscv_encode_rvv_op(0x30, 1, 1, 1, 5, 12),
+        riscv_encode_rvv_ldst(1, 7, 1, 16, 12),
+        riscv_encode_rvv_op(0x32, 1, 1, 1, 5, 14),
+        riscv_encode_rvv_ldst(1, 7, 1, 17, 14),
+        riscv_encode_rvv_op(0x38, 1, 1, 1, 5, 16),
+        riscv_encode_rvv_ldst(1, 7, 1, 28, 16),
+        riscv_encode_rvv_op(0x34, 1, 4, 2, 1, 18),
+        riscv_encode_rvv_ldst(1, 7, 1, 29, 18),
+        riscv_encode_rvv_op(0x36, 1, 4, 2, 1, 20),
+        riscv_encode_rvv_ldst(1, 7, 1, 30, 20),
+        riscv_encode_rvv_op(0x34, 1, 4, 1, 5, 22),
+        riscv_encode_rvv_ldst(1, 7, 1, 31, 22),
+        riscv_encode_rvv_op(0x36, 1, 4, 1, 5, 24),
+        riscv_encode_rvv_ldst(1, 7, 1, 8, 24),
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t src2[] = { 0x3f800000u, 0xc0000000u };
+    uint32_t src1[] = { 0x40400000u, 0x40800000u };
+    uint64_t wide_src[] = {
+        0x4024000000000000ull, 0xc024000000000000ull,
+    };
+    uint64_t expected_add_vv[] = {
+        0x4010000000000000ull, 0x4000000000000000ull,
+    };
+    uint64_t expected_sub_vv[] = {
+        0xc000000000000000ull, 0xc018000000000000ull,
+    };
+    uint64_t expected_mul_vv[] = {
+        0x4008000000000000ull, 0xc020000000000000ull,
+    };
+    uint64_t expected_add_vf[] = {
+        0x4008000000000000ull, 0x0000000000000000ull,
+    };
+    uint64_t expected_sub_vf[] = {
+        0xbff0000000000000ull, 0xc010000000000000ull,
+    };
+    uint64_t expected_mul_vf[] = {
+        0x4000000000000000ull, 0xc010000000000000ull,
+    };
+    uint64_t expected_add_wv[] = {
+        0x402a000000000000ull, 0xc018000000000000ull,
+    };
+    uint64_t expected_sub_wv[] = {
+        0x401c000000000000ull, 0xc02c000000000000ull,
+    };
+    uint64_t expected_add_wf[] = {
+        0x4028000000000000ull, 0xc020000000000000ull,
+    };
+    uint64_t expected_sub_wf[] = {
+        0x4020000000000000ull, 0xc028000000000000ull,
+    };
+    uint64_t out_add_vv[2] = { 0 };
+    uint64_t out_sub_vv[2] = { 0 };
+    uint64_t out_mul_vv[2] = { 0 };
+    uint64_t out_add_vf[2] = { 0 };
+    uint64_t out_sub_vf[2] = { 0 };
+    uint64_t out_mul_vf[2] = { 0 };
+    uint64_t out_add_wv[2] = { 0 };
+    uint64_t out_sub_wv[2] = { 0 };
+    uint64_t out_add_wf[2] = { 0 };
+    uint64_t out_sub_wf[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 2;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    uint64_t s0 = code_start + 0x1c00;
+    uint64_t f1 = 0xffffffff40000000ull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_mem_write(uc, a2, wide_src, sizeof(wide_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S0, &s0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_F1, &f1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_add_vv, sizeof(out_add_vv)));
+    OK(uc_mem_read(uc, a4, out_sub_vv, sizeof(out_sub_vv)));
+    OK(uc_mem_read(uc, a5, out_mul_vv, sizeof(out_mul_vv)));
+    OK(uc_mem_read(uc, a6, out_add_vf, sizeof(out_add_vf)));
+    OK(uc_mem_read(uc, a7, out_sub_vf, sizeof(out_sub_vf)));
+    OK(uc_mem_read(uc, t3, out_mul_vf, sizeof(out_mul_vf)));
+    OK(uc_mem_read(uc, t4, out_add_wv, sizeof(out_add_wv)));
+    OK(uc_mem_read(uc, t5, out_sub_wv, sizeof(out_sub_wv)));
+    OK(uc_mem_read(uc, t6, out_add_wf, sizeof(out_add_wf)));
+    OK(uc_mem_read(uc, s0, out_sub_wf, sizeof(out_sub_wf)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_add_vv[i] == expected_add_vv[i]);
+        TEST_CHECK(out_sub_vv[i] == expected_sub_vv[i]);
+        TEST_CHECK(out_mul_vv[i] == expected_mul_vv[i]);
+        TEST_CHECK(out_add_vf[i] == expected_add_vf[i]);
+        TEST_CHECK(out_sub_vf[i] == expected_sub_vf[i]);
+        TEST_CHECK(out_mul_vf[i] == expected_mul_vf[i]);
+        TEST_CHECK(out_add_wv[i] == expected_add_wv[i]);
+        TEST_CHECK(out_sub_wv[i] == expected_sub_wv[i]);
+        TEST_CHECK(out_add_wf[i] == expected_add_wf[i]);
+        TEST_CHECK(out_sub_wf[i] == expected_sub_wf[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float_illegal(void)
+{
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 1, 3), 0xd0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 1, 3), 0xc0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x00, 0, 1, 2, 1, 0), 0xd0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x30, 1, 1, 2, 1, 4), 0xd8);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x3c, 1, 1, 2, 1, 4), 0xd8);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x3c, 1, 2, 1, 1, 2), 0xd0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x0e, 1, 3, 1, 5, 3), 0xd0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x13, 1, 1, 0, 1, 2), 0xc0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x13, 0, 1, 0, 1, 0), 0xd0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 1, 1, 0, 1, 2), 0xc0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 1, 1, 8, 1, 4), 0xd8);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 1, 2, 8, 1, 2), 0xd0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 1, 1, 20, 1, 2), 0xc0);
+}
+
+static void test_riscv32_rvv_carry_borrow(void)
+{
+    uc_engine *uc;
+    uint8_t code[14 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 7, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 12, 2),
+        riscv_encode_rvv_op(0x10, 0, 1, 2, 0, 3),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 3),
+        riscv_encode_rvv_op(0x10, 0, 1, 5, 4, 4),
+        riscv_encode_rvv_ldst(1, 0, 1, 14, 4),
+        riscv_encode_rvv_op(0x10, 0, 1, 31, 3, 5),
+        riscv_encode_rvv_ldst(1, 0, 1, 15, 5),
+        riscv_encode_rvv_op(0x12, 0, 1, 2, 0, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 16, 6),
+        riscv_encode_rvv_op(0x12, 0, 1, 6, 4, 7),
+        riscv_encode_rvv_ldst(1, 0, 1, 17, 7),
+    };
+    uint8_t carry_mask[] = { 0xad };
+    uint8_t src2[] = { 0xff, 0x10, 0x7f, 0x00, 0x01, 0x80, 0x55, 0x01 };
+    uint8_t src1[] = { 0x01, 0x20, 0x01, 0xff, 0xff, 0x80, 0xaa, 0xfe };
+    uint8_t expected_vvm[] = {
+        0x01, 0x30, 0x81, 0x00, 0x00, 0x01, 0xff, 0x00,
+    };
+    uint8_t expected_vxm[] = {
+        0xfe, 0x0e, 0x7e, 0xff, 0xff, 0x7f, 0x53, 0x00,
+    };
+    uint8_t expected_vim[] = {
+        0xff, 0x0f, 0x7f, 0x00, 0x00, 0x80, 0x54, 0x01,
+    };
+    uint8_t expected_vsbc_vvm[] = {
+        0xfd, 0xf0, 0x7d, 0x00, 0x02, 0xff, 0xab, 0x02,
+    };
+    uint8_t expected_vsbc_vxm[] = {
+        0xfc, 0x0e, 0x7c, 0xfd, 0xff, 0x7d, 0x53, 0xfe,
+    };
+    uint8_t out_vvm[8] = { 0 };
+    uint8_t out_vxm[8] = { 0 };
+    uint8_t out_vim[8] = { 0 };
+    uint8_t out_vsbc_vvm[8] = { 0 };
+    uint8_t out_vsbc_vxm[8] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    uint32_t a3 = code_start + 0x1300;
+    uint32_t a4 = code_start + 0x1400;
+    uint32_t a5 = code_start + 0x1500;
+    uint32_t a6 = code_start + 0x1600;
+    uint32_t a7 = code_start + 0x1700;
+    uint32_t t0 = 0xfe;
+    uint32_t t1 = 2;
+    uint32_t t2 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, carry_mask, sizeof(carry_mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vvm, sizeof(out_vvm)));
+    OK(uc_mem_read(uc, a4, out_vxm, sizeof(out_vxm)));
+    OK(uc_mem_read(uc, a5, out_vim, sizeof(out_vim)));
+    OK(uc_mem_read(uc, a6, out_vsbc_vvm, sizeof(out_vsbc_vvm)));
+    OK(uc_mem_read(uc, a7, out_vsbc_vxm, sizeof(out_vsbc_vxm)));
+    for (i = 0; i < sizeof(expected_vvm); i++) {
+        TEST_CHECK(out_vvm[i] == expected_vvm[i]);
+        TEST_CHECK(out_vxm[i] == expected_vxm[i]);
+        TEST_CHECK(out_vim[i] == expected_vim[i]);
+        TEST_CHECK(out_vsbc_vvm[i] == expected_vsbc_vvm[i]);
+        TEST_CHECK(out_vsbc_vxm[i] == expected_vsbc_vxm[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_carry_borrow_masks(void)
+{
+    uc_engine *uc;
+    uint8_t code[14 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 7, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 12, 2),
+        riscv_encode_rvv_op(0x11, 0, 1, 2, 0, 3),
+        riscv_encode_rvv_mask_ldst(1, 13, 3),
+        riscv_encode_rvv_op(0x11, 1, 1, 5, 4, 4),
+        riscv_encode_rvv_mask_ldst(1, 14, 4),
+        riscv_encode_rvv_op(0x11, 0, 1, 31, 3, 5),
+        riscv_encode_rvv_mask_ldst(1, 15, 5),
+        riscv_encode_rvv_op(0x13, 0, 1, 2, 0, 6),
+        riscv_encode_rvv_mask_ldst(1, 16, 6),
+        riscv_encode_rvv_op(0x13, 1, 1, 6, 4, 7),
+        riscv_encode_rvv_mask_ldst(1, 17, 7),
+    };
+    uint8_t carry_mask[] = { 0xad };
+    uint8_t src2[] = { 0xff, 0x10, 0x7f, 0x00, 0x01, 0x80, 0x55, 0x01 };
+    uint8_t src1[] = { 0x01, 0x20, 0x01, 0xff, 0xff, 0x80, 0xaa, 0xfe };
+    uint8_t expected_vmadc_vvm[] = { 0xb9 };
+    uint8_t expected_vmadc_vxm[] = { 0x01 };
+    uint8_t expected_vmadc_vim[] = { 0xff };
+    uint8_t expected_vmsbc_vvm[] = { 0xfa };
+    uint8_t expected_vmsbc_vxm[] = { 0x98 };
+    uint8_t out_vmadc_vvm[sizeof(expected_vmadc_vvm)] = { 0 };
+    uint8_t out_vmadc_vxm[sizeof(expected_vmadc_vxm)] = { 0 };
+    uint8_t out_vmadc_vim[sizeof(expected_vmadc_vim)] = { 0 };
+    uint8_t out_vmsbc_vvm[sizeof(expected_vmsbc_vvm)] = { 0 };
+    uint8_t out_vmsbc_vxm[sizeof(expected_vmsbc_vxm)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 1;
+    uint64_t t1 = 2;
+    uint64_t t2 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, carry_mask, sizeof(carry_mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vmadc_vvm, sizeof(out_vmadc_vvm)));
+    OK(uc_mem_read(uc, a4, out_vmadc_vxm, sizeof(out_vmadc_vxm)));
+    OK(uc_mem_read(uc, a5, out_vmadc_vim, sizeof(out_vmadc_vim)));
+    OK(uc_mem_read(uc, a6, out_vmsbc_vvm, sizeof(out_vmsbc_vvm)));
+    OK(uc_mem_read(uc, a7, out_vmsbc_vxm, sizeof(out_vmsbc_vxm)));
+    for (i = 0; i < sizeof(expected_vmadc_vvm); i++) {
+        TEST_CHECK(out_vmadc_vvm[i] == expected_vmadc_vvm[i]);
+        TEST_CHECK(out_vmadc_vxm[i] == expected_vmadc_vxm[i]);
+        TEST_CHECK(out_vmadc_vim[i] == expected_vmadc_vim[i]);
+        TEST_CHECK(out_vmsbc_vvm[i] == expected_vmsbc_vvm[i]);
+        TEST_CHECK(out_vmsbc_vxm[i] == expected_vmsbc_vxm[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_carry_borrow_illegal(void)
+{
+    run_riscv64_rvv_illegal(riscv_encode_rvv_op(0x10, 1, 1, 2, 0, 3));
+    run_riscv64_rvv_illegal(riscv_encode_rvv_op(0x10, 0, 1, 2, 0, 0));
+    run_riscv64_rvv_illegal(riscv_encode_rvv_op(0x13, 0, 1, 31, 3, 3));
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x11, 0, 0, 0, 0, 1), 0xc1);
+}
+
+static void test_riscv64_rvv_narrow_shift(void)
+{
+    uc_engine *uc;
+    uint8_t code[18 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 7, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 2),
+        riscv_encode_rvv_op(0x2c, 1, 2, 1, 0, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 4),
+        riscv_encode_rvv_op(0x2d, 1, 2, 1, 0, 5),
+        riscv_encode_rvv_ldst(1, 5, 1, 14, 5),
+        riscv_encode_rvv_op(0x2c, 1, 2, 5, 4, 6),
+        riscv_encode_rvv_ldst(1, 5, 1, 15, 6),
+        riscv_encode_rvv_op(0x2d, 1, 2, 6, 4, 7),
+        riscv_encode_rvv_ldst(1, 5, 1, 16, 7),
+        riscv_encode_rvv_op(0x2c, 1, 2, 4, 3, 8),
+        riscv_encode_rvv_ldst(1, 5, 1, 17, 8),
+        riscv_encode_rvv_op(0x2d, 1, 2, 31, 3, 9),
+        riscv_encode_rvv_ldst(1, 5, 1, 10, 9),
+        riscv_encode_rvv_op(0x2c, 0, 2, 1, 0, 10),
+        riscv_encode_rvv_ldst(1, 5, 1, 28, 10),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint16_t shifts[] = { 20, 1, 8, 31 };
+    uint32_t src[] = {
+        0x8000f000u, 0xffffffffu, 0x0000ff00u, 0x7fffffffu,
+    };
+    uint16_t expected_vnsrl_wv[] = { 0x0800, 0xffff, 0x00ff, 0x0000 };
+    uint16_t expected_vnsra_wv[] = { 0xf800, 0xffff, 0x00ff, 0x0000 };
+    uint16_t expected_vnsrl_wx[] = { 0x0f00, 0xffff, 0x0ff0, 0xffff };
+    uint16_t expected_vnsra_wx[] = { 0xf800, 0xffff, 0x0000, 0x07ff };
+    uint16_t expected_vnsrl_wi[] = { 0x0f00, 0xffff, 0x0ff0, 0xffff };
+    uint16_t expected_vnsra_wi[] = { 0xffff, 0xffff, 0x0000, 0x0000 };
+    uint16_t expected_masked[] = { 0x0800, 0xffff, 0x00ff, 0xffff };
+    uint16_t out_vnsrl_wv[4] = { 0 };
+    uint16_t out_vnsra_wv[4] = { 0 };
+    uint16_t out_vnsrl_wx[4] = { 0 };
+    uint16_t out_vnsra_wx[4] = { 0 };
+    uint16_t out_vnsrl_wi[4] = { 0 };
+    uint16_t out_vnsra_wi[4] = { 0 };
+    uint16_t out_masked[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 36;
+    uint64_t t1 = 20;
+    uint64_t t2 = 4;
+    uint64_t t3 = code_start + 0x1800;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, shifts, sizeof(shifts)));
+    OK(uc_mem_write(uc, a2, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vnsrl_wv, sizeof(out_vnsrl_wv)));
+    OK(uc_mem_read(uc, a4, out_vnsra_wv, sizeof(out_vnsra_wv)));
+    OK(uc_mem_read(uc, a5, out_vnsrl_wx, sizeof(out_vnsrl_wx)));
+    OK(uc_mem_read(uc, a6, out_vnsra_wx, sizeof(out_vnsra_wx)));
+    OK(uc_mem_read(uc, a7, out_vnsrl_wi, sizeof(out_vnsrl_wi)));
+    OK(uc_mem_read(uc, a0, out_vnsra_wi, sizeof(out_vnsra_wi)));
+    OK(uc_mem_read(uc, t3, out_masked, sizeof(out_masked)));
+    for (i = 0; i < sizeof(expected_vnsrl_wv) /
+                    sizeof(expected_vnsrl_wv[0]); i++) {
+        TEST_CHECK(out_vnsrl_wv[i] == expected_vnsrl_wv[i]);
+        TEST_CHECK(out_vnsra_wv[i] == expected_vnsra_wv[i]);
+        TEST_CHECK(out_vnsrl_wx[i] == expected_vnsrl_wx[i]);
+        TEST_CHECK(out_vnsra_wx[i] == expected_vnsra_wx[i]);
+        TEST_CHECK(out_vnsrl_wi[i] == expected_vnsrl_wi[i]);
+        TEST_CHECK(out_vnsra_wi[i] == expected_vnsra_wi[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_narrow_shift_illegal(void)
+{
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2c, 1, 2, 1, 0, 4), 0xd8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2c, 1, 0, 1, 0, 8), 0xcb);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2c, 0, 2, 1, 0, 0), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2c, 1, 0, 4, 0, 2), 0xc9);
+}
+
+static void test_riscv64_rvv_integer_extension(void)
+{
+    uc_engine *uc;
+    uint8_t code[20 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_op(0x12, 1, 1, 4, 2, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 3),
+        riscv_encode_rvv_op(0x12, 1, 1, 5, 2, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 4),
+        riscv_encode_rvv_op(0x12, 1, 2, 6, 2, 5),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 5),
+        riscv_encode_rvv_op(0x12, 1, 2, 7, 2, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 6),
+        riscv_encode_rvv_op(0x12, 0, 1, 4, 2, 7),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 7),
+        riscv_encode_rvv_vsetvli(0, 6, 0xd8),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_op(0x12, 1, 1, 2, 2, 8),
+        riscv_encode_rvv_ldst(1, 7, 1, 17, 8),
+        riscv_encode_rvv_op(0x12, 1, 1, 3, 2, 9),
+        riscv_encode_rvv_ldst(1, 7, 1, 28, 9),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint8_t bytes[] = { 0x01, 0x80, 0xff, 0x7f };
+    uint16_t halves[] = { 0x0001, 0x8001, 0xffff, 0x7fff };
+    uint32_t expected_zext_vf4[] = { 1, 0x80, 0xff, 0x7f };
+    uint32_t expected_sext_vf4[] = {
+        1, 0xffffff80u, 0xffffffffu, 0x7f,
+    };
+    uint32_t expected_zext_vf2[] = { 1, 0x8001, 0xffff, 0x7fff };
+    uint32_t expected_sext_vf2[] = {
+        1, 0xffff8001u, 0xffffffffu, 0x7fff,
+    };
+    uint32_t expected_masked[] = { 1, 0xffffffffu, 0xff, 0xffffffffu };
+    uint64_t expected_zext_vf8[] = { 1, 0x80 };
+    uint64_t expected_sext_vf8[] = { 1, 0xffffffffffffff80ull };
+    uint32_t out_zext_vf4[4] = { 0 };
+    uint32_t out_sext_vf4[4] = { 0 };
+    uint32_t out_zext_vf2[4] = { 0 };
+    uint32_t out_sext_vf2[4] = { 0 };
+    uint32_t out_masked[4] = { 0 };
+    uint64_t out_zext_vf8[2] = { 0 };
+    uint64_t out_sext_vf8[2] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t1 = 2;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, bytes, sizeof(bytes)));
+    OK(uc_mem_write(uc, a2, halves, sizeof(halves)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_zext_vf4, sizeof(out_zext_vf4)));
+    OK(uc_mem_read(uc, a4, out_sext_vf4, sizeof(out_sext_vf4)));
+    OK(uc_mem_read(uc, a5, out_zext_vf2, sizeof(out_zext_vf2)));
+    OK(uc_mem_read(uc, a6, out_sext_vf2, sizeof(out_sext_vf2)));
+    OK(uc_mem_read(uc, a7, out_zext_vf8, sizeof(out_zext_vf8)));
+    OK(uc_mem_read(uc, t3, out_sext_vf8, sizeof(out_sext_vf8)));
+    OK(uc_mem_read(uc, t4, out_masked, sizeof(out_masked)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_zext_vf4[i] == expected_zext_vf4[i]);
+        TEST_CHECK(out_sext_vf4[i] == expected_sext_vf4[i]);
+        TEST_CHECK(out_zext_vf2[i] == expected_zext_vf2[i]);
+        TEST_CHECK(out_sext_vf2[i] == expected_sext_vf2[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_zext_vf8[i] == expected_zext_vf8[i]);
+        TEST_CHECK(out_sext_vf8[i] == expected_sext_vf8[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_integer_extension_illegal(void)
+{
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 1, 1, 6, 2, 3), 0xc0);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 1, 1, 4, 2, 3), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 1, 1, 6, 2, 1), 0xd0);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x12, 0, 1, 4, 2, 0), 0xd0);
+}
+
+static void test_riscv64_rvv_widening_add_sub_vv_vx(void)
+{
+    uc_engine *uc;
+    uint8_t code[22 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_op(0x30, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_rvv_op(0x32, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 6),
+        riscv_encode_rvv_op(0x31, 1, 1, 2, 2, 8),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 8),
+        riscv_encode_rvv_op(0x33, 1, 1, 2, 2, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 10),
+        riscv_encode_rvv_op(0x30, 1, 1, 7, 6, 12),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 12),
+        riscv_encode_rvv_op(0x31, 1, 1, 7, 6, 14),
+        riscv_encode_rvv_ldst(1, 6, 1, 28, 14),
+        riscv_encode_rvv_op(0x32, 1, 1, 7, 6, 16),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 16),
+        riscv_encode_rvv_op(0x33, 1, 1, 7, 6, 18),
+        riscv_encode_rvv_ldst(1, 6, 1, 30, 18),
+        riscv_encode_rvv_op(0x30, 0, 1, 2, 2, 20),
+        riscv_encode_rvv_ldst(1, 6, 1, 31, 20),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint16_t src2[] = { 0xffff, 0x8000, 0x0001, 0x7fff };
+    uint16_t src1[] = { 0x0001, 0xffff, 0x8000, 0x0002 };
+    uint32_t expected_vwaddu_vv[] = {
+        0x00010000u, 0x00017fffu, 0x00008001u, 0x00008001u,
+    };
+    uint32_t expected_vwsubu_vv[] = {
+        0x0000fffeu, 0xffff8001u, 0xffff8001u, 0x00007ffdu,
+    };
+    uint32_t expected_vwadd_vv[] = {
+        0, 0xffff7fffu, 0xffff8001u, 0x00008001u,
+    };
+    uint32_t expected_vwsub_vv[] = {
+        0xfffffffeu, 0xffff8001u, 0x00008001u, 0x00007ffdu,
+    };
+    uint32_t expected_vwaddu_vx[] = {
+        0x0001fffdu, 0x00017ffeu, 0x0000ffffu, 0x00017ffdu,
+    };
+    uint32_t expected_vwadd_vx[] = {
+        0xfffffffdu, 0xffff7ffeu, 0xffffffffu, 0x00007ffdu,
+    };
+    uint32_t expected_vwsubu_vx[] = {
+        1, 0xffff8002u, 0xffff0003u, 0xffff8001u,
+    };
+    uint32_t expected_vwsub_vx[] = {
+        1, 0xffff8002u, 3, 0x00008001u,
+    };
+    uint32_t expected_masked[] = {
+        0x00010000u, 0xffffffffu, 0x00008001u, 0xffffffffu,
+    };
+    uint32_t out_vwaddu_vv[4] = { 0 };
+    uint32_t out_vwsubu_vv[4] = { 0 };
+    uint32_t out_vwadd_vv[4] = { 0 };
+    uint32_t out_vwsub_vv[4] = { 0 };
+    uint32_t out_vwaddu_vx[4] = { 0 };
+    uint32_t out_vwadd_vx[4] = { 0 };
+    uint32_t out_vwsubu_vx[4] = { 0 };
+    uint32_t out_vwsub_vx[4] = { 0 };
+    uint32_t out_masked[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t2 = 0xfffe;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vwaddu_vv, sizeof(out_vwaddu_vv)));
+    OK(uc_mem_read(uc, a4, out_vwsubu_vv, sizeof(out_vwsubu_vv)));
+    OK(uc_mem_read(uc, a5, out_vwadd_vv, sizeof(out_vwadd_vv)));
+    OK(uc_mem_read(uc, a6, out_vwsub_vv, sizeof(out_vwsub_vv)));
+    OK(uc_mem_read(uc, a7, out_vwaddu_vx, sizeof(out_vwaddu_vx)));
+    OK(uc_mem_read(uc, t3, out_vwadd_vx, sizeof(out_vwadd_vx)));
+    OK(uc_mem_read(uc, t4, out_vwsubu_vx, sizeof(out_vwsubu_vx)));
+    OK(uc_mem_read(uc, t5, out_vwsub_vx, sizeof(out_vwsub_vx)));
+    OK(uc_mem_read(uc, t6, out_masked, sizeof(out_masked)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vwaddu_vv[i] == expected_vwaddu_vv[i]);
+        TEST_CHECK(out_vwsubu_vv[i] == expected_vwsubu_vv[i]);
+        TEST_CHECK(out_vwadd_vv[i] == expected_vwadd_vv[i]);
+        TEST_CHECK(out_vwsub_vv[i] == expected_vwsub_vv[i]);
+        TEST_CHECK(out_vwaddu_vx[i] == expected_vwaddu_vx[i]);
+        TEST_CHECK(out_vwadd_vx[i] == expected_vwadd_vx[i]);
+        TEST_CHECK(out_vwsubu_vx[i] == expected_vwsubu_vx[i]);
+        TEST_CHECK(out_vwsub_vx[i] == expected_vwsub_vx[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_add_sub_wv_wx(void)
+{
+    uc_engine *uc;
+    uint8_t code[21 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 2),
+        riscv_encode_rvv_op(0x34, 1, 2, 1, 2, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_rvv_op(0x36, 1, 2, 1, 2, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 6),
+        riscv_encode_rvv_op(0x35, 1, 2, 1, 2, 8),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 8),
+        riscv_encode_rvv_op(0x37, 1, 2, 1, 2, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 10),
+        riscv_encode_rvv_op(0x34, 1, 2, 7, 6, 12),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 12),
+        riscv_encode_rvv_op(0x35, 1, 2, 7, 6, 14),
+        riscv_encode_rvv_ldst(1, 6, 1, 28, 14),
+        riscv_encode_rvv_op(0x36, 1, 2, 7, 6, 16),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 16),
+        riscv_encode_rvv_op(0x37, 1, 2, 7, 6, 18),
+        riscv_encode_rvv_ldst(1, 6, 1, 30, 18),
+        riscv_encode_rvv_op(0x34, 1, 2, 1, 2, 2),
+        riscv_encode_rvv_ldst(1, 6, 1, 31, 2),
+    };
+    uint16_t narrow[] = { 0x0001, 0xffff, 0x8000, 0x0002 };
+    uint32_t wide[] = {
+        0xffffffffu, 0x00008000u, 0xffff8000u, 0x00000001u,
+    };
+    uint32_t expected_vwaddu_wv[] = {
+        0, 0x00017fffu, 0, 3,
+    };
+    uint32_t expected_vwsubu_wv[] = {
+        0xfffffffeu, 0xffff8001u, 0xffff0000u, 0xffffffffu,
+    };
+    uint32_t expected_vwadd_wv[] = {
+        0, 0x00007fffu, 0xffff0000u, 3,
+    };
+    uint32_t expected_vwsub_wv[] = {
+        0xfffffffeu, 0x00008001u, 0, 0xffffffffu,
+    };
+    uint32_t expected_vwaddu_wx[] = {
+        0x0000fffdu, 0x00017ffeu, 0x00007ffeu, 0x0000ffffu,
+    };
+    uint32_t expected_vwadd_wx[] = {
+        0xfffffffdu, 0x00007ffeu, 0xffff7ffeu, 0xffffffffu,
+    };
+    uint32_t expected_vwsubu_wx[] = {
+        0xffff0001u, 0xffff8002u, 0xfffe8002u, 0xffff0003u,
+    };
+    uint32_t expected_vwsub_wx[] = {
+        1, 0x00008002u, 0xffff8002u, 3,
+    };
+    uint32_t out_vwaddu_wv[4] = { 0 };
+    uint32_t out_vwsubu_wv[4] = { 0 };
+    uint32_t out_vwadd_wv[4] = { 0 };
+    uint32_t out_vwsub_wv[4] = { 0 };
+    uint32_t out_vwaddu_wx[4] = { 0 };
+    uint32_t out_vwadd_wx[4] = { 0 };
+    uint32_t out_vwsubu_wx[4] = { 0 };
+    uint32_t out_vwsub_wx[4] = { 0 };
+    uint32_t out_inplace[4] = { 0 };
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t2 = 0xfffe;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a1, narrow, sizeof(narrow)));
+    OK(uc_mem_write(uc, a2, wide, sizeof(wide)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vwaddu_wv, sizeof(out_vwaddu_wv)));
+    OK(uc_mem_read(uc, a4, out_vwsubu_wv, sizeof(out_vwsubu_wv)));
+    OK(uc_mem_read(uc, a5, out_vwadd_wv, sizeof(out_vwadd_wv)));
+    OK(uc_mem_read(uc, a6, out_vwsub_wv, sizeof(out_vwsub_wv)));
+    OK(uc_mem_read(uc, a7, out_vwaddu_wx, sizeof(out_vwaddu_wx)));
+    OK(uc_mem_read(uc, t3, out_vwadd_wx, sizeof(out_vwadd_wx)));
+    OK(uc_mem_read(uc, t4, out_vwsubu_wx, sizeof(out_vwsubu_wx)));
+    OK(uc_mem_read(uc, t5, out_vwsub_wx, sizeof(out_vwsub_wx)));
+    OK(uc_mem_read(uc, t6, out_inplace, sizeof(out_inplace)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vwaddu_wv[i] == expected_vwaddu_wv[i]);
+        TEST_CHECK(out_vwsubu_wv[i] == expected_vwsubu_wv[i]);
+        TEST_CHECK(out_vwadd_wv[i] == expected_vwadd_wv[i]);
+        TEST_CHECK(out_vwsub_wv[i] == expected_vwsub_wv[i]);
+        TEST_CHECK(out_vwaddu_wx[i] == expected_vwaddu_wx[i]);
+        TEST_CHECK(out_vwadd_wx[i] == expected_vwadd_wx[i]);
+        TEST_CHECK(out_vwsubu_wx[i] == expected_vwsubu_wx[i]);
+        TEST_CHECK(out_vwsub_wx[i] == expected_vwsub_wx[i]);
+        TEST_CHECK(out_inplace[i] == expected_vwaddu_wv[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_add_sub_illegal(void)
+{
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x30, 1, 1, 2, 2, 4), 0xd8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x30, 1, 1, 2, 2, 8), 0xcb);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x30, 0, 1, 2, 2, 0), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x30, 1, 2, 1, 2, 2), 0xc8);
+}
+
+static void test_riscv64_rvv_widening_multiply(void)
+{
+    uc_engine *uc;
+    uint8_t code[18 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_op(0x3b, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_rvv_op(0x38, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 6),
+        riscv_encode_rvv_op(0x3a, 1, 1, 2, 2, 8),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 8),
+        riscv_encode_rvv_op(0x3b, 1, 1, 7, 6, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 10),
+        riscv_encode_rvv_op(0x38, 1, 1, 7, 6, 12),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 12),
+        riscv_encode_rvv_op(0x3a, 1, 1, 7, 6, 14),
+        riscv_encode_rvv_ldst(1, 6, 1, 28, 14),
+        riscv_encode_rvv_op(0x3b, 0, 1, 2, 2, 16),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 16),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint16_t src2[] = { 0xffff, 0x8000, 0x1234, 0xfffe };
+    uint16_t src1[] = { 0x0002, 0xffff, 0x0100, 0x8001 };
+    uint32_t expected_vwmul_vv[] = {
+        0xfffffffeu, 0x00008000u, 0x00123400u, 0x0000fffeu,
+    };
+    uint32_t expected_vwmulu_vv[] = {
+        0x0001fffeu, 0x7fff8000u, 0x00123400u, 0x7ffffffeu,
+    };
+    uint32_t expected_vwmulsu_vv[] = {
+        0xfffffffeu, 0x80008000u, 0x00123400u, 0xfffefffeu,
+    };
+    uint32_t expected_vwmul_vx[] = {
+        3, 0x00018000u, 0xffffc964u, 6,
+    };
+    uint32_t expected_vwmulu_vx[] = {
+        0xfffc0003u, 0x7ffe8000u, 0x1233c964u, 0xfffb0006u,
+    };
+    uint32_t expected_vwmulsu_vx[] = {
+        0xffff0003u, 0x80018000u, 0x1233c964u, 0xfffe0006u,
+    };
+    uint32_t expected_masked[] = {
+        0xfffffffeu, 0xffffffffu, 0x00123400u, 0xffffffffu,
+    };
+    uint32_t out_vwmul_vv[4] = { 0 };
+    uint32_t out_vwmulu_vv[4] = { 0 };
+    uint32_t out_vwmulsu_vv[4] = { 0 };
+    uint32_t out_vwmul_vx[4] = { 0 };
+    uint32_t out_vwmulu_vx[4] = { 0 };
+    uint32_t out_vwmulsu_vx[4] = { 0 };
+    uint32_t out_masked[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t2 = 0xfffd;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vwmul_vv, sizeof(out_vwmul_vv)));
+    OK(uc_mem_read(uc, a4, out_vwmulu_vv, sizeof(out_vwmulu_vv)));
+    OK(uc_mem_read(uc, a5, out_vwmulsu_vv, sizeof(out_vwmulsu_vv)));
+    OK(uc_mem_read(uc, a6, out_vwmul_vx, sizeof(out_vwmul_vx)));
+    OK(uc_mem_read(uc, a7, out_vwmulu_vx, sizeof(out_vwmulu_vx)));
+    OK(uc_mem_read(uc, t3, out_vwmulsu_vx, sizeof(out_vwmulsu_vx)));
+    OK(uc_mem_read(uc, t4, out_masked, sizeof(out_masked)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vwmul_vv[i] == expected_vwmul_vv[i]);
+        TEST_CHECK(out_vwmulu_vv[i] == expected_vwmulu_vv[i]);
+        TEST_CHECK(out_vwmulsu_vv[i] == expected_vwmulsu_vv[i]);
+        TEST_CHECK(out_vwmul_vx[i] == expected_vwmul_vx[i]);
+        TEST_CHECK(out_vwmulu_vx[i] == expected_vwmulu_vx[i]);
+        TEST_CHECK(out_vwmulsu_vx[i] == expected_vwmulsu_vx[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_multiply_32(void)
+{
+    uc_engine *uc;
+    uint8_t code[9 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 2),
+        riscv_encode_rvv_op(0x3b, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 4),
+        riscv_encode_rvv_op(0x38, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 14, 6),
+        riscv_encode_rvv_op(0x3a, 1, 1, 7, 6, 8),
+        riscv_encode_rvv_ldst(1, 7, 1, 15, 8),
+    };
+    uint32_t src2[] = { 0xffffffffu, 0x80000000u };
+    uint32_t src1[] = { 3, 0xffffffffu };
+    uint64_t expected_vwmul_vv[] = {
+        0xfffffffffffffffdull, 0x0000000080000000ull,
+    };
+    uint64_t expected_vwmulu_vv[] = {
+        0x00000002fffffffdull, 0x7fffffff80000000ull,
+    };
+    uint64_t expected_vwmulsu_vx[] = {
+        0xffffffff00000003ull, 0x8000000180000000ull,
+    };
+    uint64_t out_vwmul_vv[2] = { 0 };
+    uint64_t out_vwmulu_vv[2] = { 0 };
+    uint64_t out_vwmulsu_vx[2] = { 0 };
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t t0 = 2;
+    uint64_t t2 = 0xfffffffd;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vwmul_vv, sizeof(out_vwmul_vv)));
+    OK(uc_mem_read(uc, a4, out_vwmulu_vv, sizeof(out_vwmulu_vv)));
+    OK(uc_mem_read(uc, a5, out_vwmulsu_vx, sizeof(out_vwmulsu_vx)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_vwmul_vv[i] == expected_vwmul_vv[i]);
+        TEST_CHECK(out_vwmulu_vv[i] == expected_vwmulu_vv[i]);
+        TEST_CHECK(out_vwmulsu_vx[i] == expected_vwmulsu_vx[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_multiply_illegal(void)
+{
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3b, 1, 1, 2, 2, 4), 0xd8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3b, 1, 1, 2, 2, 8), 0xcb);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3b, 0, 1, 2, 2, 0), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3b, 1, 2, 1, 2, 2), 0xc8);
+}
+
+static void test_riscv64_rvv_multiply_add(void)
+{
+    uc_engine *uc;
+    uint8_t code[31 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 3),
+        riscv_encode_rvv_op(0x2d, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 5, 1, 14, 3),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 4),
+        riscv_encode_rvv_op(0x2f, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 15, 4),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 5),
+        riscv_encode_rvv_op(0x29, 1, 1, 2, 2, 5),
+        riscv_encode_rvv_ldst(1, 5, 1, 16, 5),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 6),
+        riscv_encode_rvv_op(0x2b, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 5, 1, 17, 6),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 7),
+        riscv_encode_rvv_op(0x2d, 1, 1, 7, 6, 7),
+        riscv_encode_rvv_ldst(1, 5, 1, 28, 7),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 8),
+        riscv_encode_rvv_op(0x2f, 1, 1, 7, 6, 8),
+        riscv_encode_rvv_ldst(1, 5, 1, 29, 8),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 9),
+        riscv_encode_rvv_op(0x29, 1, 1, 7, 6, 9),
+        riscv_encode_rvv_ldst(1, 5, 1, 30, 9),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 10),
+        riscv_encode_rvv_op(0x2b, 1, 1, 7, 6, 10),
+        riscv_encode_rvv_ldst(1, 5, 1, 31, 10),
+        riscv_encode_rvv_ldst(0, 5, 1, 13, 11),
+        riscv_encode_rvv_op(0x2d, 0, 1, 2, 2, 11),
+        riscv_encode_rvv_ldst(1, 5, 1, 8, 11),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint16_t src2[] = { 0x0002, 0xfffc, 0x8000, 0x0007 };
+    uint16_t src1[] = { 0x0003, 0xfffe, 0x0002, 0x8001 };
+    uint16_t acc[] = { 0x000a, 0x0014, 0xfffb, 0x8000 };
+    uint16_t expected_vmacc_vv[] = { 0x0010, 0x001c, 0xfffb, 0x0007 };
+    uint16_t expected_vnmsac_vv[] = { 0x0004, 0x000c, 0xfffb, 0xfff9 };
+    uint16_t expected_vmadd_vv[] = { 0x0020, 0xffd4, 0x7ff6, 0x8007 };
+    uint16_t expected_vnmsub_vv[] = { 0xffe4, 0x0024, 0x800a, 0x8007 };
+    uint16_t expected_vmacc_vx[] = { 0x0004, 0x0020, 0x7ffb, 0x7feb };
+    uint16_t expected_vnmsac_vx[] = { 0x0010, 0x0008, 0x7ffb, 0x8015 };
+    uint16_t expected_vmadd_vx[] = { 0xffe4, 0xffc0, 0x800f, 0x8007 };
+    uint16_t expected_vnmsub_vx[] = { 0x0020, 0x0038, 0x7ff1, 0x8007 };
+    uint16_t expected_masked[] = { 0x0010, 0xffff, 0xfffb, 0xffff };
+    uint16_t out_vmacc_vv[4] = { 0 };
+    uint16_t out_vnmsac_vv[4] = { 0 };
+    uint16_t out_vmadd_vv[4] = { 0 };
+    uint16_t out_vnmsub_vv[4] = { 0 };
+    uint16_t out_vmacc_vx[4] = { 0 };
+    uint16_t out_vnmsac_vx[4] = { 0 };
+    uint16_t out_vmadd_vx[4] = { 0 };
+    uint16_t out_vnmsub_vx[4] = { 0 };
+    uint16_t out_masked[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t2 = 0xfffd;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    uint64_t s0 = code_start + 0x1c00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_mem_write(uc, a3, acc, sizeof(acc)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_S0, &s0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a4, out_vmacc_vv, sizeof(out_vmacc_vv)));
+    OK(uc_mem_read(uc, a5, out_vnmsac_vv, sizeof(out_vnmsac_vv)));
+    OK(uc_mem_read(uc, a6, out_vmadd_vv, sizeof(out_vmadd_vv)));
+    OK(uc_mem_read(uc, a7, out_vnmsub_vv, sizeof(out_vnmsub_vv)));
+    OK(uc_mem_read(uc, t3, out_vmacc_vx, sizeof(out_vmacc_vx)));
+    OK(uc_mem_read(uc, t4, out_vnmsac_vx, sizeof(out_vnmsac_vx)));
+    OK(uc_mem_read(uc, t5, out_vmadd_vx, sizeof(out_vmadd_vx)));
+    OK(uc_mem_read(uc, t6, out_vnmsub_vx, sizeof(out_vnmsub_vx)));
+    OK(uc_mem_read(uc, s0, out_masked, sizeof(out_masked)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vmacc_vv[i] == expected_vmacc_vv[i]);
+        TEST_CHECK(out_vnmsac_vv[i] == expected_vnmsac_vv[i]);
+        TEST_CHECK(out_vmadd_vv[i] == expected_vmadd_vv[i]);
+        TEST_CHECK(out_vnmsub_vv[i] == expected_vnmsub_vv[i]);
+        TEST_CHECK(out_vmacc_vx[i] == expected_vmacc_vx[i]);
+        TEST_CHECK(out_vnmsac_vx[i] == expected_vnmsac_vx[i]);
+        TEST_CHECK(out_vmadd_vx[i] == expected_vmadd_vx[i]);
+        TEST_CHECK(out_vnmsub_vx[i] == expected_vnmsub_vx[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_multiply_add(void)
+{
+    uc_engine *uc;
+    uint8_t code[28 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 4),
+        riscv_encode_rvv_op(0x3c, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 4),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 6),
+        riscv_encode_rvv_op(0x3d, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 6),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 8),
+        riscv_encode_rvv_op(0x3f, 1, 1, 2, 2, 8),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 8),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 10),
+        riscv_encode_rvv_op(0x3c, 1, 1, 7, 6, 10),
+        riscv_encode_rvv_ldst(1, 6, 1, 17, 10),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 12),
+        riscv_encode_rvv_op(0x3d, 1, 1, 7, 6, 12),
+        riscv_encode_rvv_ldst(1, 6, 1, 28, 12),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 14),
+        riscv_encode_rvv_op(0x3f, 1, 1, 7, 6, 14),
+        riscv_encode_rvv_ldst(1, 6, 1, 29, 14),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 16),
+        riscv_encode_rvv_op(0x3e, 1, 1, 7, 6, 16),
+        riscv_encode_rvv_ldst(1, 6, 1, 30, 16),
+        riscv_encode_rvv_ldst(0, 6, 1, 13, 18),
+        riscv_encode_rvv_op(0x3d, 0, 1, 2, 2, 18),
+        riscv_encode_rvv_ldst(1, 6, 1, 31, 18),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint16_t src2[] = { 0xffff, 0x8000, 0x0005, 0xfffe };
+    uint16_t src1[] = { 0x0002, 0xffff, 0xfffd, 0x0004 };
+    uint32_t acc[] = { 10, 0xfffffff0u, 1000, 0x80000000u };
+    uint32_t expected_vwmaccu_vv[] = {
+        0x00020008u, 0x7fff7ff0u, 0x000503d9u, 0x8003fff8u,
+    };
+    uint32_t expected_vwmacc_vv[] = {
+        0x00000008u, 0x00007ff0u, 0x000003d9u, 0x7ffffff8u,
+    };
+    uint32_t expected_vwmaccsu_vv[] = {
+        0x00020008u, 0xffff7ff0u, 0x000003d9u, 0x8003fff8u,
+    };
+    uint32_t expected_vwmaccu_vx[] = {
+        0xfffc000du, 0x7ffe7ff0u, 0x000503d9u, 0x7ffb0006u,
+    };
+    uint32_t expected_vwmacc_vx[] = {
+        0x0000000du, 0x00017ff0u, 0x000003d9u, 0x80000006u,
+    };
+    uint32_t expected_vwmaccsu_vx[] = {
+        0xfffd000du, 0xfffe7ff0u, 0x000003d9u, 0x7ffd0006u,
+    };
+    uint32_t expected_vwmaccus_vx[] = {
+        0xffff000du, 0x80017ff0u, 0x000503d9u, 0x7ffe0006u,
+    };
+    uint32_t expected_masked[] = {
+        0x00000008u, 0xffffffffu, 0x000003d9u, 0xffffffffu,
+    };
+    uint32_t out_vwmaccu_vv[4] = { 0 };
+    uint32_t out_vwmacc_vv[4] = { 0 };
+    uint32_t out_vwmaccsu_vv[4] = { 0 };
+    uint32_t out_vwmaccu_vx[4] = { 0 };
+    uint32_t out_vwmacc_vx[4] = { 0 };
+    uint32_t out_vwmaccsu_vx[4] = { 0 };
+    uint32_t out_vwmaccus_vx[4] = { 0 };
+    uint32_t out_masked[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t2 = 0xfffd;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_mem_write(uc, a3, acc, sizeof(acc)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a4, out_vwmaccu_vv, sizeof(out_vwmaccu_vv)));
+    OK(uc_mem_read(uc, a5, out_vwmacc_vv, sizeof(out_vwmacc_vv)));
+    OK(uc_mem_read(uc, a6, out_vwmaccsu_vv, sizeof(out_vwmaccsu_vv)));
+    OK(uc_mem_read(uc, a7, out_vwmaccu_vx, sizeof(out_vwmaccu_vx)));
+    OK(uc_mem_read(uc, t3, out_vwmacc_vx, sizeof(out_vwmacc_vx)));
+    OK(uc_mem_read(uc, t4, out_vwmaccsu_vx, sizeof(out_vwmaccsu_vx)));
+    OK(uc_mem_read(uc, t5, out_vwmaccus_vx, sizeof(out_vwmaccus_vx)));
+    OK(uc_mem_read(uc, t6, out_masked, sizeof(out_masked)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vwmaccu_vv[i] == expected_vwmaccu_vv[i]);
+        TEST_CHECK(out_vwmacc_vv[i] == expected_vwmacc_vv[i]);
+        TEST_CHECK(out_vwmaccsu_vv[i] == expected_vwmaccsu_vv[i]);
+        TEST_CHECK(out_vwmaccu_vx[i] == expected_vwmaccu_vx[i]);
+        TEST_CHECK(out_vwmacc_vx[i] == expected_vwmacc_vx[i]);
+        TEST_CHECK(out_vwmaccsu_vx[i] == expected_vwmaccsu_vx[i]);
+        TEST_CHECK(out_vwmaccus_vx[i] == expected_vwmaccus_vx[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_multiply_add_32(void)
+{
+    uc_engine *uc;
+    uint8_t code[15 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 12, 2),
+        riscv_encode_rvv_ldst(0, 7, 1, 13, 4),
+        riscv_encode_rvv_op(0x3d, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 14, 4),
+        riscv_encode_rvv_ldst(0, 7, 1, 13, 6),
+        riscv_encode_rvv_op(0x3c, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 15, 6),
+        riscv_encode_rvv_ldst(0, 7, 1, 13, 8),
+        riscv_encode_rvv_op(0x3f, 1, 1, 7, 6, 8),
+        riscv_encode_rvv_ldst(1, 7, 1, 16, 8),
+        riscv_encode_rvv_ldst(0, 7, 1, 13, 10),
+        riscv_encode_rvv_op(0x3e, 1, 1, 7, 6, 10),
+        riscv_encode_rvv_ldst(1, 7, 1, 17, 10),
+    };
+    uint32_t src2[] = { 0xffffffffu, 0x80000000u };
+    uint32_t src1[] = { 3, 0xffffffffu };
+    uint64_t acc[] = { 10, 0xfffffffffffffff0ull };
+    uint64_t expected_vwmacc_vv[] = {
+        0x0000000000000007ull, 0x000000007ffffff0ull,
+    };
+    uint64_t expected_vwmaccu_vv[] = {
+        0x0000000300000007ull, 0x7fffffff7ffffff0ull,
+    };
+    uint64_t expected_vwmaccsu_vx[] = {
+        0xfffffffd0000000dull, 0xfffffffe7ffffff0ull,
+    };
+    uint64_t expected_vwmaccus_vx[] = {
+        0xffffffff0000000dull, 0x800000017ffffff0ull,
+    };
+    uint64_t out_vwmacc_vv[2] = { 0 };
+    uint64_t out_vwmaccu_vv[2] = { 0 };
+    uint64_t out_vwmaccsu_vx[2] = { 0 };
+    uint64_t out_vwmaccus_vx[2] = { 0 };
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 2;
+    uint64_t t2 = 0xfffffffd;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_mem_write(uc, a3, acc, sizeof(acc)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a4, out_vwmacc_vv, sizeof(out_vwmacc_vv)));
+    OK(uc_mem_read(uc, a5, out_vwmaccu_vv, sizeof(out_vwmaccu_vv)));
+    OK(uc_mem_read(uc, a6, out_vwmaccsu_vx, sizeof(out_vwmaccsu_vx)));
+    OK(uc_mem_read(uc, a7, out_vwmaccus_vx, sizeof(out_vwmaccus_vx)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_vwmacc_vv[i] == expected_vwmacc_vv[i]);
+        TEST_CHECK(out_vwmaccu_vv[i] == expected_vwmaccu_vv[i]);
+        TEST_CHECK(out_vwmaccsu_vx[i] == expected_vwmaccsu_vx[i]);
+        TEST_CHECK(out_vwmaccus_vx[i] == expected_vwmaccus_vx[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_multiply_add_illegal(void)
+{
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2d, 0, 1, 2, 2, 0), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3d, 1, 1, 2, 2, 4), 0xd8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3d, 1, 1, 2, 2, 8), 0xcb);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3d, 0, 1, 2, 2, 0), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3d, 1, 2, 1, 2, 2), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x3e, 0, 1, 7, 6, 0), 0xc8);
+}
+
+static void test_riscv64_rvv_mask_logical(void)
+{
+    uc_engine *uc;
+    uint8_t code[19 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 1),
+        riscv_encode_rvv_mask_ldst(0, 11, 2),
+        riscv_encode_rvv_op(0x19, 0, 1, 2, 2, 3),
+        riscv_encode_rvv_op(0x1d, 0, 1, 2, 2, 4),
+        riscv_encode_rvv_op(0x18, 0, 1, 2, 2, 5),
+        riscv_encode_rvv_op(0x1b, 0, 1, 2, 2, 6),
+        riscv_encode_rvv_op(0x1a, 0, 1, 2, 2, 7),
+        riscv_encode_rvv_op(0x1e, 0, 1, 2, 2, 8),
+        riscv_encode_rvv_op(0x1c, 0, 1, 2, 2, 9),
+        riscv_encode_rvv_op(0x1f, 0, 1, 2, 2, 10),
+        riscv_encode_rvv_mask_ldst(1, 12, 3),
+        riscv_encode_rvv_mask_ldst(1, 13, 4),
+        riscv_encode_rvv_mask_ldst(1, 14, 5),
+        riscv_encode_rvv_mask_ldst(1, 15, 6),
+        riscv_encode_rvv_mask_ldst(1, 17, 7),
+        riscv_encode_rvv_mask_ldst(1, 7, 8),
+        riscv_encode_rvv_mask_ldst(1, 28, 9),
+        riscv_encode_rvv_mask_ldst(1, 29, 10),
+    };
+    uint8_t lhs[] = { 0xb3 };
+    uint8_t rhs[] = { 0x6d };
+    uint8_t expected[] = {
+        0x21, 0xde, 0x92, 0xde, 0xff, 0x00, 0xb3, 0x21,
+    };
+    uint8_t output[sizeof(expected)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 8;
+    uint64_t a7 = code_start + 0x1600;
+    uint64_t t2 = code_start + 0x1700;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, lhs, sizeof(lhs)));
+    OK(uc_mem_write(uc, a1, rhs, sizeof(rhs)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, &output[0], 1));
+    OK(uc_mem_read(uc, a3, &output[1], 1));
+    OK(uc_mem_read(uc, a4, &output[2], 1));
+    OK(uc_mem_read(uc, a5, &output[3], 1));
+    OK(uc_mem_read(uc, a7, &output[4], 1));
+    OK(uc_mem_read(uc, t2, &output[5], 1));
+    OK(uc_mem_read(uc, t3, &output[6], 1));
+    OK(uc_mem_read(uc, t4, &output[7], 1));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_mask_scalar(void)
+{
+    uc_engine *uc;
+    uint8_t code[10 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_mask_ldst(0, 11, 1),
+        riscv_encode_rvv_op(0x10, 1, 1, 16, 2, 12),
+        riscv_encode_rvv_op(0x10, 1, 1, 17, 2, 13),
+        riscv_encode_rvv_op(0x10, 0, 1, 16, 2, 14),
+        riscv_encode_rvv_op(0x10, 0, 1, 17, 2, 15),
+        riscv_encode_rvv_vsetvli(0, 5, 0xc0),
+        riscv_encode_rvv_op(0x10, 1, 1, 16, 2, 16),
+        riscv_encode_rvv_op(0x10, 1, 1, 17, 2, 17),
+    };
+    uint8_t mask[] = { 0x6d };
+    uint8_t source[] = { 0xb2 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = 0;
+    uint64_t a3 = 0;
+    uint64_t a4 = 0;
+    uint64_t a5 = 0;
+    uint64_t a6 = 8;
+    uint64_t a7 = 0;
+    uint64_t t0 = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, source, sizeof(source)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A7, &a7));
+
+    TEST_CHECK(a2 == 4);
+    TEST_CHECK(a3 == 1);
+    TEST_CHECK(a4 == 1);
+    TEST_CHECK(a5 == 5);
+    TEST_CHECK(a6 == 0);
+    TEST_CHECK(a7 == 0xffffffffffffffffull);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_mask_set(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_rvv_mask_ldst(0, 10, 1),
+        riscv_encode_rvv_op(0x14, 1, 1, 1, 2, 2),
+        riscv_encode_rvv_op(0x14, 1, 1, 3, 2, 3),
+        riscv_encode_rvv_op(0x14, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_mask_ldst(1, 11, 2),
+        riscv_encode_rvv_mask_ldst(1, 12, 3),
+        riscv_encode_rvv_mask_ldst(1, 13, 4),
+    };
+    uint8_t source[] = { 0x28 };
+    uint8_t expected[] = { 0x07, 0x0f, 0x08 };
+    uint8_t output[sizeof(expected)] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a6 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, source, sizeof(source)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a1, &output[0], 1));
+    OK(uc_mem_read(uc, a2, &output[1], 1));
+    OK(uc_mem_read(uc, a3, &output[2], 1));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_viota_vid(void)
+{
+    uc_engine *uc;
+    uint8_t code[8 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_mask_ldst(0, 11, 1),
+        riscv_encode_rvv_op(0x14, 1, 1, 16, 2, 2),
+        riscv_encode_rvv_op(0x14, 1, 0, 17, 2, 3),
+        riscv_encode_rvv_op(0x14, 0, 0, 17, 2, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 3),
+    };
+    uint32_t store_vid_masked = riscv_encode_rvv_ldst(1, 5, 1, 14, 4);
+    uint8_t full_code[9 * 4];
+    uint8_t mask[] = { 0x5b };
+    uint8_t source[] = { 0xb2 };
+    uint16_t expected_viota[] = { 0, 0, 1, 1, 1, 2, 3, 3 };
+    uint16_t expected_vid[] = { 0, 1, 2, 3, 4, 5, 6, 7 };
+    uint16_t expected_vid_masked[] = {
+        0, 1, 0xffff, 3, 4, 0xffff, 6, 0xffff,
+    };
+    uint16_t out_viota[8] = { 0 };
+    uint16_t out_vid[8] = { 0 };
+    uint16_t out_vid_masked[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a6 = 8;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&full_code[i * 4], insns[i]);
+    }
+    riscv_insn_to_code(&full_code[8 * 4], store_vid_masked);
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)full_code, sizeof(full_code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, source, sizeof(source)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(full_code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_viota, sizeof(out_viota)));
+    OK(uc_mem_read(uc, a3, out_vid, sizeof(out_vid)));
+    OK(uc_mem_read(uc, a4, out_vid_masked, sizeof(out_vid_masked)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(out_viota[i] == expected_viota[i]);
+        TEST_CHECK(out_vid[i] == expected_vid[i]);
+        TEST_CHECK(out_vid_masked[i] == expected_vid_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_mask_utilities_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x14, 0, 1, 1, 2, 0));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x14, 1, 1, 1, 2, 1));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x14, 1, 1, 16, 2, 1));
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x14, 0, 0, 17, 2, 0));
+}
+
+static void test_riscv64_rvv_fixed_point_saturating(void)
+{
+    uc_engine *uc;
+    uint8_t code[16 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_csr(RISCV_CSR_VXSAT, 0, 1, 0),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 12, 2),
+        riscv_encode_rvv_op(0x20, 1, 1, 2, 0, 3),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 3),
+        riscv_encode_rvv_op(0x21, 1, 1, 15, 3, 4),
+        riscv_encode_rvv_ldst(1, 0, 1, 14, 4),
+        riscv_encode_rvv_op(0x22, 1, 1, 7, 4, 5),
+        riscv_encode_rvv_ldst(1, 0, 1, 15, 5),
+        riscv_encode_rvv_op(0x23, 1, 1, 2, 0, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 29, 6),
+        riscv_encode_csr(RISCV_CSR_VXSAT, 0, 2, 17),
+        riscv_encode_rvv_op(0x20, 0, 1, 2, 0, 7),
+        riscv_encode_rvv_ldst(1, 0, 1, 28, 7),
+    };
+    uint8_t mask[] = { 0x05 };
+    uint8_t src2[] = { 250, 10, 0, 128, 100, 127, 0x80, 5 };
+    uint8_t src1[] = { 10, 250, 1, 128, 100, 1, 0xff, 0xfb };
+    uint8_t expected_vsaddu[] = {
+        0xff, 0xff, 0x01, 0xff, 0xc8, 0x80, 0xff, 0xff,
+    };
+    uint8_t expected_vsadd_vi[] = {
+        0x09, 0x19, 0x0f, 0x8f, 0x73, 0x7f, 0x8f, 0x14,
+    };
+    uint8_t expected_vssubu_vx[] = {
+        0xf3, 0x03, 0x00, 0x79, 0x5d, 0x78, 0x79, 0x00,
+    };
+    uint8_t expected_vssub[] = {
+        0xf0, 0x10, 0xff, 0x00, 0x00, 0x7e, 0x81, 0x0a,
+    };
+    uint8_t expected_masked[] = {
+        0xff, 0xff, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    uint8_t out_vsaddu[8] = { 0 };
+    uint8_t out_vsadd_vi[8] = { 0 };
+    uint8_t out_vssubu_vx[8] = { 0 };
+    uint8_t out_vssub[8] = { 0 };
+    uint8_t out_masked[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 8;
+    uint64_t a7 = 0;
+    uint64_t t2 = 7;
+    uint64_t t3 = code_start + 0x1600;
+    uint64_t t4 = code_start + 0x1700;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vsaddu, sizeof(out_vsaddu)));
+    OK(uc_mem_read(uc, a4, out_vsadd_vi, sizeof(out_vsadd_vi)));
+    OK(uc_mem_read(uc, a5, out_vssubu_vx, sizeof(out_vssubu_vx)));
+    OK(uc_mem_read(uc, t4, out_vssub, sizeof(out_vssub)));
+    OK(uc_mem_read(uc, t3, out_masked, sizeof(out_masked)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A7, &a7));
+    TEST_CHECK(a7 == 1);
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(out_vsaddu[i] == expected_vsaddu[i]);
+        TEST_CHECK(out_vsadd_vi[i] == expected_vsadd_vi[i]);
+        TEST_CHECK(out_vssubu_vx[i] == expected_vssubu_vx[i]);
+        TEST_CHECK(out_vssub[i] == expected_vssub[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_fixed_point_rounding(void)
+{
+    uc_engine *uc;
+    uint8_t code[22 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc0),
+        riscv_encode_csr(RISCV_CSR_VXRM, 5, 1, 0),
+        riscv_encode_csr(RISCV_CSR_VXSAT, 0, 1, 0),
+        riscv_encode_rvv_ldst(0, 0, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 0, 1, 11, 2),
+        riscv_encode_rvv_op(0x08, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 0, 1, 12, 3),
+        riscv_encode_rvv_op(0x09, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 0, 1, 13, 4),
+        riscv_encode_rvv_op(0x0b, 1, 1, 2, 2, 5),
+        riscv_encode_rvv_ldst(1, 0, 1, 14, 5),
+        riscv_encode_rvv_op(0x0a, 1, 1, 7, 6, 6),
+        riscv_encode_rvv_ldst(1, 0, 1, 15, 6),
+        riscv_encode_rvv_op(0x27, 1, 1, 2, 0, 7),
+        riscv_encode_rvv_ldst(1, 0, 1, 17, 7),
+        riscv_encode_rvv_op(0x2a, 1, 1, 2, 3, 8),
+        riscv_encode_rvv_ldst(1, 0, 1, 28, 8),
+        riscv_encode_rvv_op(0x2b, 1, 1, 2, 3, 9),
+        riscv_encode_rvv_ldst(1, 0, 1, 29, 9),
+        riscv_encode_csr(RISCV_CSR_VXRM, 6, 1, 0),
+        riscv_encode_rvv_op(0x2a, 1, 1, 2, 3, 10),
+        riscv_encode_rvv_ldst(1, 0, 1, 30, 10),
+    };
+    uint8_t src2[] = { 10, 11, 5, 250, 0x80, 0x7f, 0x40, 0xc0 };
+    uint8_t src1[] = { 3, 4, 5, 10, 0x80, 1, 0x40, 0xc0 };
+    uint8_t expected_vaaddu[] = {
+        0x07, 0x08, 0x05, 0x82, 0x80, 0x40, 0x40, 0xc0,
+    };
+    uint8_t expected_vaadd[] = {
+        0x07, 0x08, 0x05, 0x02, 0x80, 0x40, 0x40, 0xc0,
+    };
+    uint8_t expected_vasub[] = {
+        0x04, 0x04, 0x00, 0xf8, 0x00, 0x3f, 0x00, 0x00,
+    };
+    uint8_t expected_vasubu_vx[] = {
+        0x04, 0x04, 0x01, 0x7c, 0x3f, 0x3e, 0x1f, 0x5f,
+    };
+    uint8_t expected_vsmul[] = {
+        0x00, 0x00, 0x00, 0x00, 0x7f, 0x01, 0x20, 0x20,
+    };
+    uint8_t expected_vssrl_rnu[] = {
+        0x03, 0x03, 0x01, 0x3f, 0x20, 0x20, 0x10, 0x30,
+    };
+    uint8_t expected_vssra_rnu[] = {
+        0x03, 0x03, 0x01, 0xff, 0xe0, 0x20, 0x10, 0xf0,
+    };
+    uint8_t expected_vssrl_rne[] = {
+        0x02, 0x03, 0x01, 0x3e, 0x20, 0x20, 0x10, 0x30,
+    };
+    uint8_t out_vaaddu[8] = { 0 };
+    uint8_t out_vaadd[8] = { 0 };
+    uint8_t out_vasub[8] = { 0 };
+    uint8_t out_vasubu_vx[8] = { 0 };
+    uint8_t out_vsmul[8] = { 0 };
+    uint8_t out_vssrl_rnu[8] = { 0 };
+    uint8_t out_vssra_rnu[8] = { 0 };
+    uint8_t out_vssrl_rne[8] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 8;
+    uint64_t a7 = code_start + 0x1600;
+    uint64_t t2 = 3;
+    uint64_t t3 = code_start + 0x1700;
+    uint64_t t4 = code_start + 0x1800;
+    uint64_t t5 = code_start + 0x1900;
+    uint64_t t0 = 0;
+    uint64_t t1 = 1;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a1, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T1, &t1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_vaaddu, sizeof(out_vaaddu)));
+    OK(uc_mem_read(uc, a3, out_vaadd, sizeof(out_vaadd)));
+    OK(uc_mem_read(uc, a4, out_vasub, sizeof(out_vasub)));
+    OK(uc_mem_read(uc, a5, out_vasubu_vx, sizeof(out_vasubu_vx)));
+    OK(uc_mem_read(uc, a7, out_vsmul, sizeof(out_vsmul)));
+    OK(uc_mem_read(uc, t3, out_vssrl_rnu, sizeof(out_vssrl_rnu)));
+    OK(uc_mem_read(uc, t4, out_vssra_rnu, sizeof(out_vssra_rnu)));
+    OK(uc_mem_read(uc, t5, out_vssrl_rne, sizeof(out_vssrl_rne)));
+    for (i = 0; i < 8; i++) {
+        TEST_CHECK(out_vaaddu[i] == expected_vaaddu[i]);
+        TEST_CHECK(out_vaadd[i] == expected_vaadd[i]);
+        TEST_CHECK(out_vasub[i] == expected_vasub[i]);
+        TEST_CHECK(out_vasubu_vx[i] == expected_vasubu_vx[i]);
+        TEST_CHECK(out_vsmul[i] == expected_vsmul[i]);
+        TEST_CHECK(out_vssrl_rnu[i] == expected_vssrl_rnu[i]);
+        TEST_CHECK(out_vssra_rnu[i] == expected_vssra_rnu[i]);
+        TEST_CHECK(out_vssrl_rne[i] == expected_vssrl_rne[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_fixed_point_clip(void)
+{
+    uc_engine *uc;
+    uint8_t code[18 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_csr(RISCV_CSR_VXRM, 5, 1, 0),
+        riscv_encode_csr(RISCV_CSR_VXSAT, 0, 1, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x2e, 1, 2, 1, 0, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 4),
+        riscv_encode_rvv_op(0x2f, 1, 2, 1, 0, 5),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 5),
+        riscv_encode_rvv_op(0x2e, 1, 2, 7, 4, 6),
+        riscv_encode_rvv_ldst(1, 5, 1, 14, 6),
+        riscv_encode_rvv_op(0x2f, 1, 2, 7, 4, 7),
+        riscv_encode_rvv_ldst(1, 5, 1, 15, 7),
+        riscv_encode_rvv_op(0x2e, 1, 2, 4, 3, 8),
+        riscv_encode_rvv_ldst(1, 5, 1, 17, 8),
+        riscv_encode_rvv_op(0x2f, 1, 2, 4, 3, 9),
+        riscv_encode_rvv_ldst(1, 5, 1, 28, 9),
+        riscv_encode_csr(RISCV_CSR_VXSAT, 0, 2, 29),
+    };
+    uint16_t shifts[] = { 8, 8, 0, 16 };
+    uint32_t source[] = {
+        0x0001ff00u, 0xffff0000u, 0x00008000u, 0x80000000u,
+    };
+    uint16_t expected_vnclipu_wv[] = {
+        0x01ff, 0xffff, 0x8000, 0x8000,
+    };
+    uint16_t expected_vnclip_wv[] = {
+        0x01ff, 0xff00, 0x7fff, 0x8000,
+    };
+    uint16_t expected_vnclipu_wx[] = {
+        0x01ff, 0xffff, 0x0080, 0xffff,
+    };
+    uint16_t expected_vnclip_wx[] = {
+        0x01ff, 0xff00, 0x0080, 0x8000,
+    };
+    uint16_t expected_vnclipu_wi[] = {
+        0x1ff0, 0xffff, 0x0800, 0xffff,
+    };
+    uint16_t expected_vnclip_wi[] = {
+        0x1ff0, 0xf000, 0x0800, 0x8000,
+    };
+    uint16_t out_vnclipu_wv[4] = { 0 };
+    uint16_t out_vnclip_wv[4] = { 0 };
+    uint16_t out_vnclipu_wx[4] = { 0 };
+    uint16_t out_vnclip_wx[4] = { 0 };
+    uint16_t out_vnclipu_wi[4] = { 0 };
+    uint16_t out_vnclip_wi[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 4;
+    uint64_t a7 = code_start + 0x1600;
+    uint64_t t0 = 0;
+    uint64_t t2 = 8;
+    uint64_t t3 = code_start + 0x1700;
+    uint64_t t4 = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, shifts, sizeof(shifts)));
+    OK(uc_mem_write(uc, a1, source, sizeof(source)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_vnclipu_wv, sizeof(out_vnclipu_wv)));
+    OK(uc_mem_read(uc, a3, out_vnclip_wv, sizeof(out_vnclip_wv)));
+    OK(uc_mem_read(uc, a4, out_vnclipu_wx, sizeof(out_vnclipu_wx)));
+    OK(uc_mem_read(uc, a5, out_vnclip_wx, sizeof(out_vnclip_wx)));
+    OK(uc_mem_read(uc, a7, out_vnclipu_wi, sizeof(out_vnclipu_wi)));
+    OK(uc_mem_read(uc, t3, out_vnclip_wi, sizeof(out_vnclip_wi)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T4, &t4));
+    TEST_CHECK(t4 == 1);
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vnclipu_wv[i] == expected_vnclipu_wv[i]);
+        TEST_CHECK(out_vnclip_wv[i] == expected_vnclip_wv[i]);
+        TEST_CHECK(out_vnclipu_wx[i] == expected_vnclipu_wx[i]);
+        TEST_CHECK(out_vnclip_wx[i] == expected_vnclip_wx[i]);
+        TEST_CHECK(out_vnclipu_wi[i] == expected_vnclipu_wi[i]);
+        TEST_CHECK(out_vnclip_wi[i] == expected_vnclip_wi[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_fixed_point_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x20, 0, 1, 2, 0, 0));
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2e, 1, 2, 1, 0, 4), 0xd8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2e, 0, 2, 1, 0, 0), 0xc8);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x2e, 1, 0, 4, 0, 2), 0xc9);
+}
+
+static void test_riscv64_rvv_integer_reduction_sum_logic(void)
+{
+    uc_engine *uc;
+    uint8_t code[11 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_ldst(0, 5, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 2),
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 3),
+        riscv_encode_rvv_op(0x01, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 4),
+        riscv_encode_rvv_op(0x02, 1, 1, 2, 2, 5),
+        riscv_encode_rvv_ldst(1, 5, 1, 14, 5),
+        riscv_encode_rvv_op(0x03, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 5, 1, 15, 6),
+    };
+    uint16_t source[] = { 0x0003, 0x0005, 0x000c, 0x00f0 };
+    uint16_t seed[] = { 0x00ff, 0, 0, 0 };
+    uint16_t expected_sum[] = { 0x0203, 0xffff, 0xffff, 0xffff };
+    uint16_t expected_and[] = { 0, 0xffff, 0xffff, 0xffff };
+    uint16_t expected_or[] = { 0x00ff, 0xffff, 0xffff, 0xffff };
+    uint16_t expected_xor[] = { 0x0005, 0xffff, 0xffff, 0xffff };
+    uint16_t out_sum[4] = { 0 };
+    uint16_t out_and[4] = { 0 };
+    uint16_t out_or[4] = { 0 };
+    uint16_t out_xor[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, source, sizeof(source)));
+    OK(uc_mem_write(uc, a1, seed, sizeof(seed)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_sum, sizeof(out_sum)));
+    OK(uc_mem_read(uc, a3, out_and, sizeof(out_and)));
+    OK(uc_mem_read(uc, a4, out_or, sizeof(out_or)));
+    OK(uc_mem_read(uc, a5, out_xor, sizeof(out_xor)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_sum[i] == expected_sum[i]);
+        TEST_CHECK(out_and[i] == expected_and[i]);
+        TEST_CHECK(out_or[i] == expected_or[i]);
+        TEST_CHECK(out_xor[i] == expected_xor[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_integer_reduction_minmax(void)
+{
+    uc_engine *uc;
+    uint8_t code[11 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x04, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x05, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_rvv_op(0x06, 1, 1, 2, 2, 5),
+        riscv_encode_rvv_ldst(1, 6, 1, 14, 5),
+        riscv_encode_rvv_op(0x07, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 6),
+    };
+    uint32_t source[] = {
+        0x80000000u, 5, 0xfffffffeu, 0x7fffffffu,
+    };
+    uint32_t seed[] = { 0x10, 0, 0, 0 };
+    uint32_t out_minu[4] = { 0 };
+    uint32_t out_min[4] = { 0 };
+    uint32_t out_maxu[4] = { 0 };
+    uint32_t out_max[4] = { 0 };
+    uint32_t a0 = code_start + 0x1000;
+    uint32_t a1 = code_start + 0x1100;
+    uint32_t a2 = code_start + 0x1200;
+    uint32_t a3 = code_start + 0x1300;
+    uint32_t a4 = code_start + 0x1400;
+    uint32_t a5 = code_start + 0x1500;
+    uint32_t a6 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32,
+                    (const char *)code, sizeof(code));
+    riscv32_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, source, sizeof(source)));
+    OK(uc_mem_write(uc, a1, seed, sizeof(seed)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_minu, sizeof(out_minu)));
+    OK(uc_mem_read(uc, a3, out_min, sizeof(out_min)));
+    OK(uc_mem_read(uc, a4, out_maxu, sizeof(out_maxu)));
+    OK(uc_mem_read(uc, a5, out_max, sizeof(out_max)));
+    TEST_CHECK(out_minu[0] == 5);
+    TEST_CHECK(out_min[0] == 0x80000000u);
+    TEST_CHECK(out_maxu[0] == 0xfffffffeu);
+    TEST_CHECK(out_max[0] == 0x7fffffffu);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_integer_reduction_masked(void)
+{
+    uc_engine *uc;
+    uint8_t code[6 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_op(0x00, 0, 1, 2, 2, 0),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 0),
+    };
+    uint8_t mask[] = { 0x0b };
+    uint16_t source[] = { 1, 10, 100, 1000 };
+    uint16_t seed[] = { 5, 0, 0, 0 };
+    uint16_t expected[] = { 1016, 0xffff, 0xffff, 0xffff };
+    uint16_t output[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a6 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, source, sizeof(source)));
+    OK(uc_mem_write(uc, a2, seed, sizeof(seed)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, output, sizeof(output)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(output[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_reduction_sum(void)
+{
+    uc_engine *uc;
+    uint8_t code[7 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_rvv_ldst(0, 5, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x30, 1, 1, 2, 0, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 4),
+        riscv_encode_rvv_op(0x31, 1, 1, 2, 0, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 6),
+    };
+    uint16_t source[] = { 0xffff, 1, 2, 0xfffe };
+    uint32_t seed[] = { 10, 0, 0, 0 };
+    uint32_t expected_unsigned[] = {
+        0x0002000a, 0xffffffffu, 0xffffffffu, 0xffffffffu,
+    };
+    uint32_t expected_signed[] = {
+        10, 0xffffffffu, 0xffffffffu, 0xffffffffu,
+    };
+    uint32_t out_unsigned[4] = { 0 };
+    uint32_t out_signed[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a6 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, source, sizeof(source)));
+    OK(uc_mem_write(uc, a1, seed, sizeof(seed)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_unsigned, sizeof(out_unsigned)));
+    OK(uc_mem_read(uc, a3, out_signed, sizeof(out_signed)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_unsigned[i] == expected_unsigned[i]);
+        TEST_CHECK(out_signed[i] == expected_signed[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_reduction_32_to_64(void)
+{
+    uc_engine *uc;
+    uint8_t code[7 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 7, 1, 11, 2),
+        riscv_encode_rvv_op(0x30, 1, 1, 2, 0, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 12, 4),
+        riscv_encode_rvv_op(0x31, 1, 1, 2, 0, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 6),
+    };
+    uint32_t source[] = { 0xffffffffu, 1, 0x80000000u };
+    uint64_t seed[] = { 5, 0, 0 };
+    uint64_t expected_unsigned[] = {
+        0x0000000180000005ull,
+        0xffffffffffffffffull,
+        0xffffffffffffffffull,
+    };
+    uint64_t expected_signed[] = {
+        0xffffffff80000005ull,
+        0xffffffffffffffffull,
+        0xffffffffffffffffull,
+    };
+    uint64_t out_unsigned[3] = { 0 };
+    uint64_t out_signed[3] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a6 = 3;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, source, sizeof(source)));
+    OK(uc_mem_write(uc, a1, seed, sizeof(seed)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_unsigned, sizeof(out_unsigned)));
+    OK(uc_mem_read(uc, a3, out_signed, sizeof(out_signed)));
+    for (i = 0; i < 3; i++) {
+        TEST_CHECK(out_unsigned[i] == expected_unsigned[i]);
+        TEST_CHECK(out_signed[i] == expected_signed[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_reduction_illegal(void)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 16, 0xc8),
+        riscv_encode_csr(RISCV_CSR_VSTART, 5, 1, 0),
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 2, 3),
+    };
+    uint64_t a6 = 0;
+    uint64_t t0 = 1;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x00, 1, 1, 2, 2, 3), 0xc9);
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x31, 1, 2, 6, 0, 4), 0xd8);
+}
+
+static void test_riscv64_rvv_float32_reduction(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 6, 1, 11, 2),
+        riscv_encode_rvv_op(0x01, 1, 1, 2, 1, 3),
+        riscv_encode_rvv_ldst(1, 6, 1, 12, 3),
+        riscv_encode_rvv_op(0x03, 1, 1, 2, 1, 4),
+        riscv_encode_rvv_ldst(1, 6, 1, 13, 4),
+        riscv_encode_rvv_ldst(0, 6, 1, 14, 5),
+        riscv_encode_rvv_op(0x05, 1, 5, 2, 1, 6),
+        riscv_encode_rvv_ldst(1, 6, 1, 15, 6),
+        riscv_encode_rvv_op(0x07, 1, 5, 2, 1, 7),
+        riscv_encode_rvv_ldst(1, 6, 1, 16, 7),
+        riscv_encode_csr(RISCV_CSR_FFLAGS, 0, 2, 6),
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t sum_src[] = {
+        0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+    };
+    uint32_t minmax_src[] = {
+        0x3f800000u, 0x7f800001u, 0xc0000000u, 0x40800000u,
+    };
+    uint32_t seed[] = { 0x3f000000u, 0, 0, 0 };
+    uint32_t expected_sum[] = {
+        0x41280000u, 0xffffffffu, 0xffffffffu, 0xffffffffu,
+    };
+    uint32_t expected_min[] = {
+        0xc0000000u, 0xffffffffu, 0xffffffffu, 0xffffffffu,
+    };
+    uint32_t expected_max[] = {
+        0x40800000u, 0xffffffffu, 0xffffffffu, 0xffffffffu,
+    };
+    uint32_t out_usum[4] = { 0 };
+    uint32_t out_osum[4] = { 0 };
+    uint32_t out_min[4] = { 0 };
+    uint32_t out_max[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t t0 = 4;
+    uint64_t t1 = 0;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, sum_src, sizeof(sum_src)));
+    OK(uc_mem_write(uc, a1, seed, sizeof(seed)));
+    OK(uc_mem_write(uc, a4, minmax_src, sizeof(minmax_src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_usum, sizeof(out_usum)));
+    OK(uc_mem_read(uc, a3, out_osum, sizeof(out_osum)));
+    OK(uc_mem_read(uc, a5, out_min, sizeof(out_min)));
+    OK(uc_mem_read(uc, a6, out_max, sizeof(out_max)));
+    OK(uc_reg_read(uc, UC_RISCV_REG_T1, &t1));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_usum[i] == expected_sum[i]);
+        TEST_CHECK(out_osum[i] == expected_sum[i]);
+        TEST_CHECK(out_min[i] == expected_min[i]);
+        TEST_CHECK(out_max[i] == expected_max[i]);
+    }
+    TEST_CHECK(t1 == 0x10);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_widening_float_reduction(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_rvv_ldst(0, 6, 1, 10, 1),
+        riscv_encode_rvv_ldst(0, 7, 1, 11, 4),
+        riscv_encode_rvv_op(0x31, 1, 1, 4, 1, 8),
+        riscv_encode_rvv_ldst(1, 7, 1, 12, 8),
+        riscv_encode_rvv_op(0x33, 1, 1, 4, 1, 10),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 10),
+    };
+    uint8_t code[sizeof(insns)];
+    uint32_t source[] = {
+        0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u,
+    };
+    uint64_t seed[] = { 0x3fe0000000000000ull, 0, 0, 0 };
+    uint64_t expected[] = {
+        0x4025000000000000ull,
+        0xffffffffffffffffull,
+        0xffffffffffffffffull,
+        0xffffffffffffffffull,
+    };
+    uint64_t out_usum[4] = { 0 };
+    uint64_t out_osum[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t t0 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_mem_write(uc, a0, source, sizeof(source)));
+    OK(uc_mem_write(uc, a1, seed, sizeof(seed)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_usum, sizeof(out_usum)));
+    OK(uc_mem_read(uc, a3, out_osum, sizeof(out_osum)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_usum[i] == expected[i]);
+        TEST_CHECK(out_osum[i] == expected[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_float_reduction_illegal(void)
+{
+    uc_engine *uc;
+    uint8_t code[3 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd0),
+        riscv_encode_csr(RISCV_CSR_VSTART, 5, 1, 0),
+        riscv_encode_rvv_op(0x01, 1, 1, 2, 1, 3),
+    };
+    uint64_t t0 = 1;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_fp_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code),
+                               0, 0));
+    OK(uc_close(uc));
+
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x01, 1, 1, 2, 1, 3), 0xc0);
+    run_riscv64_rvv_fp_illegal_vtype(
+        riscv_encode_rvv_op(0x31, 1, 1, 4, 1, 8), 0xd8);
+}
+
+static void test_riscv64_rvv_divide_remainder(void)
+{
+    uc_engine *uc;
+    uint8_t code[20 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_op(0x20, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 3),
+        riscv_encode_rvv_op(0x21, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 14, 4),
+        riscv_encode_rvv_op(0x22, 1, 1, 2, 2, 5),
+        riscv_encode_rvv_ldst(1, 5, 1, 15, 5),
+        riscv_encode_rvv_op(0x23, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 5, 1, 16, 6),
+        riscv_encode_rvv_op(0x20, 1, 1, 7, 6, 7),
+        riscv_encode_rvv_ldst(1, 5, 1, 17, 7),
+        riscv_encode_rvv_op(0x21, 1, 1, 7, 6, 8),
+        riscv_encode_rvv_ldst(1, 5, 1, 28, 8),
+        riscv_encode_rvv_op(0x22, 1, 1, 7, 6, 9),
+        riscv_encode_rvv_ldst(1, 5, 1, 29, 9),
+        riscv_encode_rvv_op(0x23, 1, 1, 7, 6, 10),
+        riscv_encode_rvv_ldst(1, 5, 1, 30, 10),
+    };
+    uint32_t masked_insns[] = {
+        riscv_encode_rvv_op(0x20, 0, 1, 2, 2, 11),
+        riscv_encode_rvv_ldst(1, 5, 1, 31, 11),
+    };
+    uint8_t full_code[(20 + 2) * 4];
+    uint8_t mask[] = { 0x05 };
+    uint16_t src2[] = { 100, 0x8000, 7, 0xfffe };
+    uint16_t src1[] = { 3, 0xffff, 0, 0xfffd };
+    uint16_t expected_vdivu_vv[] = { 33, 0, 0xffff, 1 };
+    uint16_t expected_vdiv_vv[] = { 33, 0x8000, 0xffff, 0 };
+    uint16_t expected_vremu_vv[] = { 1, 0x8000, 7, 1 };
+    uint16_t expected_vrem_vv[] = { 1, 0, 7, 0xfffe };
+    uint16_t expected_vdivu_vx[] = { 0, 0, 0, 1 };
+    uint16_t expected_vdiv_vx[] = { 0xffce, 0x4000, 0xfffd, 1 };
+    uint16_t expected_vremu_vx[] = { 100, 0x8000, 7, 0 };
+    uint16_t expected_vrem_vx[] = { 0, 0, 1, 0 };
+    uint16_t expected_masked[] = { 33, 0xffff, 0xffff, 0xffff };
+    uint16_t out_vdivu_vv[4] = { 0 };
+    uint16_t out_vdiv_vv[4] = { 0 };
+    uint16_t out_vremu_vv[4] = { 0 };
+    uint16_t out_vrem_vv[4] = { 0 };
+    uint16_t out_vdivu_vx[4] = { 0 };
+    uint16_t out_vdiv_vx[4] = { 0 };
+    uint16_t out_vremu_vx[4] = { 0 };
+    uint16_t out_vrem_vx[4] = { 0 };
+    uint16_t out_masked[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t2 = 0xfffe;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&full_code[i * 4], insns[i]);
+    }
+    for (i = 0; i < sizeof(masked_insns) / sizeof(masked_insns[0]); i++) {
+        riscv_insn_to_code(&full_code[(20 + i) * 4], masked_insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)full_code, sizeof(full_code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(full_code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vdivu_vv, sizeof(out_vdivu_vv)));
+    OK(uc_mem_read(uc, a4, out_vdiv_vv, sizeof(out_vdiv_vv)));
+    OK(uc_mem_read(uc, a5, out_vremu_vv, sizeof(out_vremu_vv)));
+    OK(uc_mem_read(uc, a6, out_vrem_vv, sizeof(out_vrem_vv)));
+    OK(uc_mem_read(uc, a7, out_vdivu_vx, sizeof(out_vdivu_vx)));
+    OK(uc_mem_read(uc, t3, out_vdiv_vx, sizeof(out_vdiv_vx)));
+    OK(uc_mem_read(uc, t4, out_vremu_vx, sizeof(out_vremu_vx)));
+    OK(uc_mem_read(uc, t5, out_vrem_vx, sizeof(out_vrem_vx)));
+    OK(uc_mem_read(uc, t6, out_masked, sizeof(out_masked)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vdivu_vv[i] == expected_vdivu_vv[i]);
+        TEST_CHECK(out_vdiv_vv[i] == expected_vdiv_vv[i]);
+        TEST_CHECK(out_vremu_vv[i] == expected_vremu_vv[i]);
+        TEST_CHECK(out_vrem_vv[i] == expected_vrem_vv[i]);
+        TEST_CHECK(out_vdivu_vx[i] == expected_vdivu_vx[i]);
+        TEST_CHECK(out_vdiv_vx[i] == expected_vdiv_vx[i]);
+        TEST_CHECK(out_vremu_vx[i] == expected_vremu_vx[i]);
+        TEST_CHECK(out_vrem_vx[i] == expected_vrem_vx[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_divide_remainder_zero_vx(void)
+{
+    uc_engine *uc;
+    uint8_t code[10 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_op(0x20, 1, 1, 0, 6, 2),
+        riscv_encode_rvv_ldst(1, 5, 1, 12, 2),
+        riscv_encode_rvv_op(0x21, 1, 1, 0, 6, 3),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 3),
+        riscv_encode_rvv_op(0x22, 1, 1, 0, 6, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 14, 4),
+        riscv_encode_rvv_op(0x23, 1, 1, 0, 6, 5),
+        riscv_encode_rvv_ldst(1, 5, 1, 15, 5),
+    };
+    uint16_t src[] = { 100, 0x8000, 7, 0xfffe };
+    uint16_t expected_div[] = { 0xffff, 0xffff, 0xffff, 0xffff };
+    uint16_t expected_rem[] = { 100, 0x8000, 7, 0xfffe };
+    uint16_t out_vdivu[4] = { 0 };
+    uint16_t out_vdiv[4] = { 0 };
+    uint16_t out_vremu[4] = { 0 };
+    uint16_t out_vrem[4] = { 0 };
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t t0 = 4;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a1, src, sizeof(src)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a2, out_vdivu, sizeof(out_vdivu)));
+    OK(uc_mem_read(uc, a3, out_vdiv, sizeof(out_vdiv)));
+    OK(uc_mem_read(uc, a4, out_vremu, sizeof(out_vremu)));
+    OK(uc_mem_read(uc, a5, out_vrem, sizeof(out_vrem)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_vdivu[i] == expected_div[i]);
+        TEST_CHECK(out_vdiv[i] == expected_div[i]);
+        TEST_CHECK(out_vremu[i] == expected_rem[i]);
+        TEST_CHECK(out_vrem[i] == expected_rem[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_divide_remainder_64(void)
+{
+    uc_engine *uc;
+    uint8_t code[11 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd8),
+        riscv_encode_rvv_ldst(0, 7, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 2),
+        riscv_encode_rvv_op(0x21, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 3),
+        riscv_encode_rvv_op(0x23, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 14, 4),
+        riscv_encode_rvv_op(0x21, 1, 1, 7, 6, 5),
+        riscv_encode_rvv_ldst(1, 7, 1, 15, 5),
+        riscv_encode_rvv_op(0x23, 1, 1, 7, 6, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 16, 6),
+    };
+    uint64_t src2[] = { 0x8000000000000000ull, 0xfffffffffffffffbull };
+    uint64_t src1[] = { 0xffffffffffffffffull, 2 };
+    uint64_t expected_vdiv_vv[] = {
+        0x8000000000000000ull, 0xfffffffffffffffeull,
+    };
+    uint64_t expected_vrem_vv[] = { 0, 0xffffffffffffffffull };
+    uint64_t expected_vdiv_vx[] = { 0x8000000000000000ull, 5 };
+    uint64_t expected_vrem_vx[] = { 0, 0 };
+    uint64_t out_vdiv_vv[2] = { 0 };
+    uint64_t out_vrem_vv[2] = { 0 };
+    uint64_t out_vdiv_vx[2] = { 0 };
+    uint64_t out_vrem_vx[2] = { 0 };
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t t0 = 2;
+    uint64_t t2 = 0xffffffffffffffffull;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_vdiv_vv, sizeof(out_vdiv_vv)));
+    OK(uc_mem_read(uc, a4, out_vrem_vv, sizeof(out_vrem_vv)));
+    OK(uc_mem_read(uc, a5, out_vdiv_vx, sizeof(out_vdiv_vx)));
+    OK(uc_mem_read(uc, a6, out_vrem_vx, sizeof(out_vrem_vx)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_vdiv_vv[i] == expected_vdiv_vv[i]);
+        TEST_CHECK(out_vrem_vv[i] == expected_vrem_vv[i]);
+        TEST_CHECK(out_vdiv_vx[i] == expected_vdiv_vx[i]);
+        TEST_CHECK(out_vrem_vx[i] == expected_vrem_vx[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_divide_remainder_illegal(void)
+{
+    run_riscv64_rvv_illegal(
+        riscv_encode_rvv_op(0x20, 0, 1, 2, 2, 0));
+}
+
+static void test_riscv64_rvv_multiply(void)
+{
+    uc_engine *uc;
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xc8),
+        riscv_encode_rvv_mask_ldst(0, 10, 0),
+        riscv_encode_rvv_ldst(0, 5, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 5, 1, 12, 2),
+        riscv_encode_rvv_op(0x25, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 5, 1, 13, 3),
+        riscv_encode_rvv_op(0x27, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 5, 1, 14, 4),
+        riscv_encode_rvv_op(0x24, 1, 1, 2, 2, 5),
+        riscv_encode_rvv_ldst(1, 5, 1, 15, 5),
+        riscv_encode_rvv_op(0x26, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 5, 1, 16, 6),
+        riscv_encode_rvv_op(0x25, 1, 1, 7, 6, 7),
+        riscv_encode_rvv_ldst(1, 5, 1, 17, 7),
+        riscv_encode_rvv_op(0x27, 1, 1, 7, 6, 8),
+        riscv_encode_rvv_ldst(1, 5, 1, 28, 8),
+        riscv_encode_rvv_op(0x24, 1, 1, 7, 6, 9),
+        riscv_encode_rvv_ldst(1, 5, 1, 29, 9),
+    };
+    uint32_t tail_insns[] = {
+        riscv_encode_rvv_op(0x26, 1, 1, 7, 6, 10),
+        riscv_encode_rvv_ldst(1, 5, 1, 30, 10),
+        riscv_encode_rvv_op(0x25, 0, 1, 2, 2, 11),
+        riscv_encode_rvv_ldst(1, 5, 1, 31, 11),
+    };
+    uint8_t full_code[(18 + 4) * 4];
+    uint8_t mask[] = { 0x05 };
+    uint16_t src2[] = { 0xff00, 0x8000, 0x1234, 0xffff };
+    uint16_t src1[] = { 0x0002, 0xffff, 0x0100, 0x8001 };
+    uint16_t expected_mul[] = { 0xfe00, 0x8000, 0x3400, 0x7fff };
+    uint16_t expected_mulh[] = { 0xffff, 0x0000, 0x0012, 0x0000 };
+    uint16_t expected_mulhu[] = { 0x0001, 0x7fff, 0x0012, 0x8000 };
+    uint16_t expected_mulhsu[] = { 0xffff, 0x8000, 0x0012, 0xffff };
+    uint16_t expected_mul_vx[] = { 0x0300, 0x8000, 0xc964, 0x0003 };
+    uint16_t expected_mulh_vx[] = { 0x0000, 0x0001, 0xffff, 0x0000 };
+    uint16_t expected_mulhu_vx[] = { 0xfefd, 0x7ffe, 0x1233, 0xfffc };
+    uint16_t expected_mulhsu_vx[] = { 0xff00, 0x8001, 0x1233, 0xffff };
+    uint16_t expected_masked[] = { 0xfe00, 0xffff, 0x3400, 0xffff };
+    uint16_t out_mul[4] = { 0 };
+    uint16_t out_mulh[4] = { 0 };
+    uint16_t out_mulhu[4] = { 0 };
+    uint16_t out_mulhsu[4] = { 0 };
+    uint16_t out_mul_vx[4] = { 0 };
+    uint16_t out_mulh_vx[4] = { 0 };
+    uint16_t out_mulhu_vx[4] = { 0 };
+    uint16_t out_mulhsu_vx[4] = { 0 };
+    uint16_t out_masked[4] = { 0 };
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 4;
+    uint64_t t2 = 0xfffd;
+    uint64_t t3 = code_start + 0x1800;
+    uint64_t t4 = code_start + 0x1900;
+    uint64_t t5 = code_start + 0x1a00;
+    uint64_t t6 = code_start + 0x1b00;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&full_code[i * 4], insns[i]);
+    }
+    for (i = 0; i < sizeof(tail_insns) / sizeof(tail_insns[0]); i++) {
+        riscv_insn_to_code(&full_code[(18 + i) * 4], tail_insns[i]);
+    }
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)full_code, sizeof(full_code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a0, mask, sizeof(mask)));
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T3, &t3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T4, &t4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T5, &t5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T6, &t6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(full_code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_mul, sizeof(out_mul)));
+    OK(uc_mem_read(uc, a4, out_mulh, sizeof(out_mulh)));
+    OK(uc_mem_read(uc, a5, out_mulhu, sizeof(out_mulhu)));
+    OK(uc_mem_read(uc, a6, out_mulhsu, sizeof(out_mulhsu)));
+    OK(uc_mem_read(uc, a7, out_mul_vx, sizeof(out_mul_vx)));
+    OK(uc_mem_read(uc, t3, out_mulh_vx, sizeof(out_mulh_vx)));
+    OK(uc_mem_read(uc, t4, out_mulhu_vx, sizeof(out_mulhu_vx)));
+    OK(uc_mem_read(uc, t5, out_mulhsu_vx, sizeof(out_mulhsu_vx)));
+    OK(uc_mem_read(uc, t6, out_masked, sizeof(out_masked)));
+    for (i = 0; i < 4; i++) {
+        TEST_CHECK(out_mul[i] == expected_mul[i]);
+        TEST_CHECK(out_mulh[i] == expected_mulh[i]);
+        TEST_CHECK(out_mulhu[i] == expected_mulhu[i]);
+        TEST_CHECK(out_mulhsu[i] == expected_mulhsu[i]);
+        TEST_CHECK(out_mul_vx[i] == expected_mul_vx[i]);
+        TEST_CHECK(out_mulh_vx[i] == expected_mulh_vx[i]);
+        TEST_CHECK(out_mulhu_vx[i] == expected_mulhu_vx[i]);
+        TEST_CHECK(out_mulhsu_vx[i] == expected_mulhsu_vx[i]);
+        TEST_CHECK(out_masked[i] == expected_masked[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_multiply_64(void)
+{
+    uc_engine *uc;
+    uint8_t code[13 * 4];
+    uint32_t insns[] = {
+        riscv_encode_rvv_vsetvli(0, 5, 0xd8),
+        riscv_encode_rvv_ldst(0, 7, 1, 11, 1),
+        riscv_encode_rvv_ldst(0, 7, 1, 12, 2),
+        riscv_encode_rvv_op(0x25, 1, 1, 2, 2, 3),
+        riscv_encode_rvv_ldst(1, 7, 1, 13, 3),
+        riscv_encode_rvv_op(0x27, 1, 1, 2, 2, 4),
+        riscv_encode_rvv_ldst(1, 7, 1, 14, 4),
+        riscv_encode_rvv_op(0x24, 1, 1, 2, 2, 5),
+        riscv_encode_rvv_ldst(1, 7, 1, 15, 5),
+        riscv_encode_rvv_op(0x26, 1, 1, 2, 2, 6),
+        riscv_encode_rvv_ldst(1, 7, 1, 16, 6),
+        riscv_encode_rvv_op(0x24, 1, 1, 7, 6, 7),
+    };
+    uint32_t store_vmulhu_vx = riscv_encode_rvv_ldst(1, 7, 1, 17, 7);
+    uint64_t src2[] = { 0x8000000000000000ull, 0xffffffffffffffffull };
+    uint64_t src1[] = { 2, 0x8000000000000001ull };
+    uint64_t expected_mul[] = { 0, 0x7fffffffffffffffull };
+    uint64_t expected_mulh[] = { 0xffffffffffffffffull, 0 };
+    uint64_t expected_mulhu[] = { 1, 0x8000000000000000ull };
+    uint64_t expected_mulhsu[] = {
+        0xffffffffffffffffull, 0xffffffffffffffffull,
+    };
+    uint64_t expected_mulhu_vx[] = { 1, 1 };
+    uint64_t out_mul[2] = { 0 };
+    uint64_t out_mulh[2] = { 0 };
+    uint64_t out_mulhu[2] = { 0 };
+    uint64_t out_mulhsu[2] = { 0 };
+    uint64_t out_mulhu_vx[2] = { 0 };
+    uint64_t a1 = code_start + 0x1100;
+    uint64_t a2 = code_start + 0x1200;
+    uint64_t a3 = code_start + 0x1300;
+    uint64_t a4 = code_start + 0x1400;
+    uint64_t a5 = code_start + 0x1500;
+    uint64_t a6 = code_start + 0x1600;
+    uint64_t a7 = code_start + 0x1700;
+    uint64_t t0 = 2;
+    uint64_t t2 = 2;
+    size_t i;
+
+    for (i = 0; i < sizeof(insns) / sizeof(insns[0]); i++) {
+        riscv_insn_to_code(&code[i * 4], insns[i]);
+    }
+    riscv_insn_to_code(&code[12 * 4], store_vmulhu_vx);
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+    OK(uc_mem_write(uc, a1, src2, sizeof(src2)));
+    OK(uc_mem_write(uc, a2, src1, sizeof(src1)));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A5, &a5));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A6, &a6));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A7, &a7));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T0, &t0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_T2, &t2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a3, out_mul, sizeof(out_mul)));
+    OK(uc_mem_read(uc, a4, out_mulh, sizeof(out_mulh)));
+    OK(uc_mem_read(uc, a5, out_mulhu, sizeof(out_mulhu)));
+    OK(uc_mem_read(uc, a6, out_mulhsu, sizeof(out_mulhsu)));
+    OK(uc_mem_read(uc, a7, out_mulhu_vx, sizeof(out_mulhu_vx)));
+    for (i = 0; i < 2; i++) {
+        TEST_CHECK(out_mul[i] == expected_mul[i]);
+        TEST_CHECK(out_mulh[i] == expected_mulh[i]);
+        TEST_CHECK(out_mulhu[i] == expected_mulhu[i]);
+        TEST_CHECK(out_mulhsu[i] == expected_mulhsu[i]);
+        TEST_CHECK(out_mulhu_vx[i] == expected_mulhu_vx[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_multiply_illegal(void)
+{
+    run_riscv64_rvv_illegal_vtype(
+        riscv_encode_rvv_op(0x25, 0, 1, 2, 2, 0), 0xc8);
+}
+
+static void test_riscv_fp_rejects_non_fp_models(void)
+{
+    uint32_t flw = riscv_encode_i(0, 10, 2, 1, 0x07);
+    uint32_t flh = 0x00051087;
+    uint32_t fadd_h = 0x04208253;
+
+    run_riscv_non_fp_model_insn_illegal(UC_MODE_RISCV32,
+                                        UC_CPU_RISCV32_SIFIVE_E31, flw);
+    run_riscv_non_fp_model_insn_illegal(UC_MODE_RISCV64,
+                                        UC_CPU_RISCV64_SIFIVE_E51, flw);
+    run_riscv_non_fp_model_insn_illegal(UC_MODE_RISCV32,
+                                        UC_CPU_RISCV32_SIFIVE_E31, flh);
+    run_riscv_non_fp_model_insn_illegal(UC_MODE_RISCV64,
+                                        UC_CPU_RISCV64_SIFIVE_E51, flh);
+    run_riscv_non_fp_model_insn_illegal(UC_MODE_RISCV32,
+                                        UC_CPU_RISCV32_SIFIVE_E31, fadd_h);
+    run_riscv_non_fp_model_insn_illegal(UC_MODE_RISCV64,
+                                        UC_CPU_RISCV64_SIFIVE_E51, fadd_h);
+}
+
+static void test_riscv_rvv_rejects_non_vector_models(void)
+{
+    run_riscv_rvv_non_vector_model_illegal(UC_MODE_RISCV32,
+                                           UC_CPU_RISCV32_SIFIVE_E31);
+    run_riscv_rvv_non_vector_model_illegal(UC_MODE_RISCV64,
+                                           UC_CPU_RISCV64_SIFIVE_E51);
+    run_riscv_rvv_non_vector_model_data_illegal(UC_MODE_RISCV32,
+                                                UC_CPU_RISCV32_SIFIVE_E31);
+    run_riscv_rvv_non_vector_model_data_illegal(UC_MODE_RISCV64,
+                                                UC_CPU_RISCV64_SIFIVE_E51);
+}
+
+static void test_riscv64_rvv_invalid_vtype(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf5\x05\x40"
+        "\x73\x26\x00\xc2"
+        "\xf3\x26\x10\xc2"
+        "\x73\x27\x80\x00";
+    uint64_t a0 = 1;
+    uint64_t a1 = 5;
+    uint64_t a2 = 1;
+    uint64_t a3 = 1;
+    uint64_t a4 = 1;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A4, &a4));
+
+    TEST_CHECK(a0 == 0);
+    TEST_CHECK(a2 == 0);
+    TEST_CHECK(a3 == RISCV64_VTYPE_VILL);
+    TEST_CHECK(a4 == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_vill_blocks_data_path(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x57\xf0\x05\x40"
+        "\x87\x00\x05\x02";
+    uint64_t a0 = code_start + 0x1000;
+    uint64_t a1 = 5;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                               0, 0));
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_vector_csrs(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x13\x03\xf0\x0f"
+        "\x73\x10\x83\x00"
+        "\x73\x25\x80\x00"
+        "\x13\x03\x70\x00"
+        "\x73\x10\xf3\x00"
+        "\x73\x26\xf0\x00"
+        "\xf3\x26\xa0\x00"
+        "\x73\x27\x90\x00"
+        "\xf3\x27\x00\x30";
+    uint64_t a0 = 0;
+    uint64_t a2 = 0;
+    uint64_t a3 = 0;
+    uint64_t a4 = 0;
+    uint64_t a5 = 0;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A2, &a2));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A3, &a3));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A4, &a4));
+    OK(uc_reg_read(uc, UC_RISCV_REG_A5, &a5));
+
+    TEST_CHECK(a0 == 127);
+    TEST_CHECK(a2 == 7);
+    TEST_CHECK(a3 == 3);
+    TEST_CHECK(a4 == 1);
+    TEST_CHECK(a5 == (RISCV64_MSTATUS_SD | RISCV_MSTATUS_VS_DIRTY));
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_public_scalar_regs(void)
+{
+    uc_engine *uc;
+    char code[] = "\x13\x00\x00\x00";
+    uint64_t value;
+    uint64_t out;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+
+    value = 9;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VSTART, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VSTART, &out));
+    TEST_CHECK(out == 9);
+
+    value = 7;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VXRM, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VXRM, &out));
+    TEST_CHECK(out == 3);
+
+    value = 3;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VXSAT, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VXSAT, &out));
+    TEST_CHECK(out == 1);
+
+    value = (2 << 1) | 1;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VCSR, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VCSR, &out));
+    TEST_CHECK(out == 5);
+
+    value = 1000;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VL, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VL, &out));
+    TEST_CHECK(out == 16);
+
+    value = RISCV64_VTYPE_VILL | 7;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VTYPE, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VTYPE, &out));
+    TEST_CHECK(out == RISCV64_VTYPE_VILL);
+    OK(uc_reg_read(uc, UC_RISCV_REG_VL, &out));
+    TEST_CHECK(out == 0);
+
+    value = 0xc0;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VTYPE, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VTYPE, &out));
+    TEST_CHECK(out == 0xc0);
+
+    value = 5;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VL, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VL, &out));
+    TEST_CHECK(out == 5);
+
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VLENB, &out));
+    TEST_CHECK(out == 16);
+    value = 32;
+    uc_assert_err(UC_ERR_ARG,
+                  uc_reg_write(uc, UC_RISCV_REG_VLENB, &value));
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv32_rvv_public_scalar_regs(void)
+{
+    uc_engine *uc;
+    char code[] = "\x13\x00\x00\x00";
+    uint32_t value;
+    uint32_t out;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV32, code,
+                    sizeof(code) - 1);
+    riscv32_enable_vector_state(uc);
+
+    value = 7;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VCSR, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VXRM, &out));
+    TEST_CHECK(out == 3);
+    OK(uc_reg_read(uc, UC_RISCV_REG_VXSAT, &out));
+    TEST_CHECK(out == 1);
+
+    value = 0xc0;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VTYPE, &value));
+    value = 20;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VL, &value));
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VL, &out));
+    TEST_CHECK(out == 16);
+
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VLENB, &out));
+    TEST_CHECK(out == 16);
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_public_vector_reg(void)
+{
+    uc_engine *uc;
+    uint32_t code[] = {
+        riscv_encode_rvv_vsetvli(0, 11, 0xc0),
+        riscv_encode_rvv_ldst(1, 0, 1, 10, 1),
+    };
+    uc_riscv_vreg v1 = { { 0 } };
+    uc_riscv_vreg actual = { { 0 } };
+    uint8_t output[16] = { 0 };
+    uint64_t a0 = code_start + 0x1200;
+    uint64_t a1 = sizeof(output);
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64,
+                    (const char *)code, sizeof(code));
+    riscv64_enable_vector_state(uc);
+
+    for (i = 0; i < sizeof(v1.bytes); i++) {
+        v1.bytes[i] = (uint8_t)(0x80 + i);
+    }
+
+    OK(uc_reg_write(uc, UC_RISCV_REG_V1, &v1));
+    OK(uc_reg_read(uc, UC_RISCV_REG_V1, &actual));
+    for (i = 0; i < sizeof(output); i++) {
+        TEST_CHECK(actual.bytes[i] == v1.bytes[i]);
+    }
+    for (i = sizeof(output); i < sizeof(actual.bytes); i++) {
+        TEST_CHECK(actual.bytes[i] == 0);
+    }
+
+    OK(uc_reg_write(uc, UC_RISCV_REG_A0, &a0));
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_mem_read(uc, a0, output, sizeof(output)));
+    for (i = 0; i < sizeof(output); i++) {
+        TEST_CHECK(output[i] == v1.bytes[i]);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_context_save_restore_public_regs(void)
+{
+    uc_engine *uc;
+    uc_context *context;
+    char code[] = "\x13\x00\x00\x00";
+    uc_riscv_vreg saved_v1 = { { 0 } };
+    uc_riscv_vreg zero_v1 = { { 0 } };
+    uc_riscv_vreg actual_v1 = { { 0 } };
+    uint64_t value;
+    uint64_t out;
+    size_t i;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    riscv64_enable_vector_state(uc);
+
+    value = 0xc0;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VTYPE, &value));
+    value = 5;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VL, &value));
+    value = 5;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VCSR, &value));
+    for (i = 0; i < 16; i++) {
+        saved_v1.bytes[i] = (uint8_t)(0x30 + i);
+    }
+    OK(uc_reg_write(uc, UC_RISCV_REG_V1, &saved_v1));
+
+    OK(uc_context_alloc(uc, &context));
+    OK(uc_context_save(uc, context));
+
+    value = RISCV64_VTYPE_VILL;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VTYPE, &value));
+    value = 0;
+    OK(uc_reg_write(uc, UC_RISCV_REG_VCSR, &value));
+    OK(uc_reg_write(uc, UC_RISCV_REG_V1, &zero_v1));
+
+    OK(uc_context_restore(uc, context));
+
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VTYPE, &out));
+    TEST_CHECK(out == 0xc0);
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VL, &out));
+    TEST_CHECK(out == 5);
+    out = 0;
+    OK(uc_reg_read(uc, UC_RISCV_REG_VCSR, &out));
+    TEST_CHECK(out == 5);
+    OK(uc_reg_read(uc, UC_RISCV_REG_V1, &actual_v1));
+    for (i = 0; i < 16; i++) {
+        TEST_CHECK(actual_v1.bytes[i] == saved_v1.bytes[i]);
+    }
+    for (i = 16; i < sizeof(actual_v1.bytes); i++) {
+        TEST_CHECK(actual_v1.bytes[i] == 0);
+    }
+
+    OK(uc_context_free(context));
+    OK(uc_close(uc));
+}
+
+static void test_riscv64_rvv_requires_vs(void)
+{
+    uc_engine *uc;
+    char code[] = "\x57\xf5\x05\x0c";
+    uint64_t a1 = 5;
+
+    uc_common_setup(&uc, UC_ARCH_RISCV, UC_MODE_RISCV64, code,
+                    sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_RISCV_REG_A1, &a1));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + sizeof(code) - 1,
+                               0, 0));
+
+    OK(uc_close(uc));
+}
+
 TEST_LIST = {
     {"test_riscv32_nop", test_riscv32_nop},
     {"test_riscv64_nop", test_riscv64_nop},
+    {"test_riscv64_sstc_stimecmp", test_riscv64_sstc_stimecmp},
+    {"test_riscv32_zihintpause", test_riscv32_zihintpause},
+    {"test_riscv64_zihintpause", test_riscv64_zihintpause},
     {"test_riscv32_3steps_pc_update", test_riscv32_3steps_pc_update},
     {"test_riscv64_3steps_pc_update", test_riscv64_3steps_pc_update},
     {"test_riscv64_until_at_page_end", test_riscv64_until_at_page_end},
@@ -926,6 +11254,13 @@ TEST_LIST = {
     {"test_riscv64_fclass_s_produced_nanbox",
      test_riscv64_fclass_s_produced_nanbox},
     {"test_riscv64_fmv_w_x_nanbox", test_riscv64_fmv_w_x_nanbox},
+    {"test_riscv64_fmin_fmax_snan", test_riscv64_fmin_fmax_snan},
+    {"test_riscv32_zfh_load_store_move", test_riscv32_zfh_load_store_move},
+    {"test_riscv64_zfh_load_store_move", test_riscv64_zfh_load_store_move},
+    {"test_riscv64_zfh_arith_compare_class",
+     test_riscv64_zfh_arith_compare_class},
+    {"test_riscv32_zfh_conversions", test_riscv32_zfh_conversions},
+    {"test_riscv64_zfh_conversions", test_riscv64_zfh_conversions},
     {"test_riscv64_ecall", test_riscv64_ecall},
     {"test_riscv32_mmio_map", test_riscv32_mmio_map},
     {"test_riscv64_mmio_map", test_riscv64_mmio_map},
@@ -938,4 +11273,288 @@ TEST_LIST = {
      test_riscv_correct_address_in_long_jump_hook},
     {"test_riscv_mmu", test_riscv_mmu},
     {"test_riscv_priv", test_riscv_priv},
+    {"test_riscv64_pmp_na4_load", test_riscv64_pmp_na4_load},
+    {"test_riscv64_pmp_na4_rejects_outside",
+     test_riscv64_pmp_na4_rejects_outside},
+    {"test_riscv64_pmp_na4_rejects_store",
+     test_riscv64_pmp_na4_rejects_store},
+    {"test_riscv32_svinval", test_riscv32_svinval},
+    {"test_riscv64_svinval", test_riscv64_svinval},
+    {"test_riscv_svinval_hinval_requires_rvh",
+     test_riscv_svinval_hinval_requires_rvh},
+    {"test_riscv_svinval_requires_s", test_riscv_svinval_requires_s},
+    {"test_riscv32_rvh_hlv_hsv", test_riscv32_rvh_hlv_hsv},
+    {"test_riscv64_rvh_hlv_hsv", test_riscv64_rvh_hlv_hsv},
+    {"test_riscv64_rvh_hlvx_vs_stage_xonly_mxr",
+     test_riscv64_rvh_hlvx_vs_stage_xonly_mxr},
+    {"test_riscv_rvh_requires_h", test_riscv_rvh_requires_h},
+    {"test_riscv_rvh_requires_hlsx", test_riscv_rvh_requires_hlsx},
+    {"test_riscv_rvh_hstatus_layout", test_riscv_rvh_hstatus_layout},
+    {"test_riscv_rvh_hedeleg_mask", test_riscv_rvh_hedeleg_mask},
+    {"test_riscv_rvh_hu_allows_u_mode", test_riscv_rvh_hu_allows_u_mode},
+    {"test_riscv_rvh_hu_tb_flags", test_riscv_rvh_hu_tb_flags},
+    {"test_riscv_rvh_virtual_instruction_fault",
+     test_riscv_rvh_virtual_instruction_fault},
+    {"test_riscv64_rvh_indirect_g_stage_fault",
+     test_riscv64_rvh_indirect_g_stage_fault},
+    {"test_riscv_virtual_wfi_fault", test_riscv_virtual_wfi_fault},
+    {"test_riscv_rvh_hfence", test_riscv_rvh_hfence},
+    {"test_riscv_rvh_hfence_requires_h",
+     test_riscv_rvh_hfence_requires_h},
+    {"test_riscv64_rv128_opcodes_rejected",
+     test_riscv64_rv128_opcodes_rejected},
+    {"test_riscv32_xventanacondops", test_riscv32_xventanacondops},
+    {"test_riscv64_xventanacondops", test_riscv64_xventanacondops},
+    {"test_riscv32_zba", test_riscv32_zba},
+    {"test_riscv64_zba", test_riscv64_zba},
+    {"test_riscv32_zbc", test_riscv32_zbc},
+    {"test_riscv64_zbc", test_riscv64_zbc},
+    {"test_riscv32_zbkb", test_riscv32_zbkb},
+    {"test_riscv64_zbkb", test_riscv64_zbkb},
+    {"test_riscv32_zbkx", test_riscv32_zbkx},
+    {"test_riscv64_zbkx", test_riscv64_zbkx},
+    {"test_riscv32_zknh_sha256", test_riscv32_zknh_sha256},
+    {"test_riscv64_zknh_sha256", test_riscv64_zknh_sha256},
+    {"test_riscv32_zknh_sha512", test_riscv32_zknh_sha512},
+    {"test_riscv64_zknh_sha512", test_riscv64_zknh_sha512},
+    {"test_riscv_zksh_sm3", test_riscv_zksh_sm3},
+    {"test_riscv32_zkne_aes", test_riscv32_zkne_aes},
+    {"test_riscv32_zknd_aes", test_riscv32_zknd_aes},
+    {"test_riscv64_zkne_zknd_aes", test_riscv64_zkne_zknd_aes},
+    {"test_riscv_zksed_sm4", test_riscv_zksed_sm4},
+    {"test_riscv_zkr_seed", test_riscv_zkr_seed},
+    {"test_riscv32_zbb_unary", test_riscv32_zbb_unary},
+    {"test_riscv64_zbb_unary", test_riscv64_zbb_unary},
+    {"test_riscv32_zbb_binary", test_riscv32_zbb_binary},
+    {"test_riscv64_zbb_binary", test_riscv64_zbb_binary},
+    {"test_riscv32_zbb_rotate", test_riscv32_zbb_rotate},
+    {"test_riscv64_zbb_rotate", test_riscv64_zbb_rotate},
+    {"test_riscv64_zbb_word", test_riscv64_zbb_word},
+    {"test_riscv_zbb_illegal_encodings", test_riscv_zbb_illegal_encodings},
+    {"test_riscv32_zbs_register", test_riscv32_zbs_register},
+    {"test_riscv64_zbs_register", test_riscv64_zbs_register},
+    {"test_riscv32_zbs_immediate", test_riscv32_zbs_immediate},
+    {"test_riscv64_zbs_immediate", test_riscv64_zbs_immediate},
+    {"test_riscv32_rvv_vsetvli_csrs", test_riscv32_rvv_vsetvli_csrs},
+    {"test_riscv64_rvv_vsetvli_csrs", test_riscv64_rvv_vsetvli_csrs},
+    {"test_riscv64_rvv_vsetvl_clamp", test_riscv64_rvv_vsetvl_clamp},
+    {"test_riscv32_rvv_vsetivli", test_riscv32_rvv_vsetivli},
+    {"test_riscv64_rvv_vle8_vadd_vv_vse8",
+     test_riscv64_rvv_vle8_vadd_vv_vse8},
+    {"test_riscv32_rvv_vle8_vadd_vv_vse8",
+     test_riscv32_rvv_vle8_vadd_vv_vse8},
+    {"test_riscv64_rvv_vmv_vi_vadd_vi_vse32",
+     test_riscv64_rvv_vmv_vi_vadd_vi_vse32},
+    {"test_riscv64_rvv_tail_agnostic_vmv",
+     test_riscv64_rvv_tail_agnostic_vmv},
+    {"test_riscv32_rvv_vle16_vsub_vse16",
+     test_riscv32_rvv_vle16_vsub_vse16},
+    {"test_riscv64_rvv_vle64_moves_logic_compare",
+     test_riscv64_rvv_vle64_moves_logic_compare},
+    {"test_riscv64_rvv_scalar_moves_compare",
+     test_riscv64_rvv_scalar_moves_compare},
+    {"test_riscv32_rvv_minmax", test_riscv32_rvv_minmax},
+    {"test_riscv64_rvv_reverse_subtract",
+     test_riscv64_rvv_reverse_subtract},
+    {"test_riscv32_rvv_reverse_subtract",
+     test_riscv32_rvv_reverse_subtract},
+    {"test_riscv64_rvv_slide", test_riscv64_rvv_slide},
+    {"test_riscv32_rvv_slide", test_riscv32_rvv_slide},
+    {"test_riscv64_rvv_slide_illegal",
+     test_riscv64_rvv_slide_illegal},
+    {"test_riscv64_rvv_gather_compress",
+     test_riscv64_rvv_gather_compress},
+    {"test_riscv32_rvv_gather_compress",
+     test_riscv32_rvv_gather_compress},
+    {"test_riscv64_rvv_whole_register_move",
+     test_riscv64_rvv_whole_register_move},
+    {"test_riscv64_rvv_gather_compress_move_illegal",
+     test_riscv64_rvv_gather_compress_move_illegal},
+    {"test_riscv64_rvv_vmerge", test_riscv64_rvv_vmerge},
+    {"test_riscv64_rvv_shift_vv", test_riscv64_rvv_shift_vv},
+    {"test_riscv32_rvv_shift_vx_vi", test_riscv32_rvv_shift_vx_vi},
+    {"test_riscv32_rvv_mask_load_store",
+     test_riscv32_rvv_mask_load_store},
+    {"test_riscv64_rvv_masked_unit_stride",
+     test_riscv64_rvv_masked_unit_stride},
+    {"test_riscv64_rvv_unit_stride_fault_vstart",
+     test_riscv64_rvv_unit_stride_fault_vstart},
+    {"test_riscv64_rvv_strided_fault_vstart",
+     test_riscv64_rvv_strided_fault_vstart},
+    {"test_riscv64_rvv_indexed_fault_vstart",
+     test_riscv64_rvv_indexed_fault_vstart},
+    {"test_riscv64_rvv_unit_stride_segment_memory",
+     test_riscv64_rvv_unit_stride_segment_memory},
+    {"test_riscv64_rvv_masked_unit_stride_segment",
+     test_riscv64_rvv_masked_unit_stride_segment},
+    {"test_riscv64_rvv_unit_stride_segment_illegal",
+     test_riscv64_rvv_unit_stride_segment_illegal},
+    {"test_riscv64_rvv_strided_load_store",
+     test_riscv64_rvv_strided_load_store},
+    {"test_riscv64_rvv_negative_stride_load",
+     test_riscv64_rvv_negative_stride_load},
+    {"test_riscv64_rvv_masked_strided_memory",
+     test_riscv64_rvv_masked_strided_memory},
+    {"test_riscv64_rvv_strided_segment_memory",
+     test_riscv64_rvv_strided_segment_memory},
+    {"test_riscv64_rvv_strided_memory_illegal",
+     test_riscv64_rvv_strided_memory_illegal},
+    {"test_riscv64_rvv_indexed_load_store",
+     test_riscv64_rvv_indexed_load_store},
+    {"test_riscv64_rvv_indexed_mixed_widths",
+     test_riscv64_rvv_indexed_mixed_widths},
+    {"test_riscv64_rvv_masked_indexed_memory",
+     test_riscv64_rvv_masked_indexed_memory},
+    {"test_riscv64_rvv_indexed_segment_memory",
+     test_riscv64_rvv_indexed_segment_memory},
+    {"test_riscv64_rvv_indexed_memory_illegal",
+     test_riscv64_rvv_indexed_memory_illegal},
+    {"test_riscv64_rvv_fault_only_first_segment",
+     test_riscv64_rvv_fault_only_first_segment},
+    {"test_riscv64_rvv_masked_fault_only_first",
+     test_riscv64_rvv_masked_fault_only_first},
+    {"test_riscv64_rvv_fault_only_first_partial_fault",
+     test_riscv64_rvv_fault_only_first_partial_fault},
+    {"test_riscv64_rvv_fault_only_first_first_fault",
+     test_riscv64_rvv_fault_only_first_first_fault},
+    {"test_riscv64_rvv_fault_only_first_illegal",
+     test_riscv64_rvv_fault_only_first_illegal},
+    {"test_riscv64_rvv_whole_register_single",
+     test_riscv64_rvv_whole_register_single},
+    {"test_riscv64_rvv_whole_register_groups",
+     test_riscv64_rvv_whole_register_groups},
+    {"test_riscv64_rvv_whole_register_ignores_vl_vill",
+     test_riscv64_rvv_whole_register_ignores_vl_vill},
+    {"test_riscv64_rvv_whole_register_vstart",
+     test_riscv64_rvv_whole_register_vstart},
+    {"test_riscv64_rvv_whole_register_illegal",
+     test_riscv64_rvv_whole_register_illegal},
+    {"test_riscv32_rvv_whole_register_smoke",
+     test_riscv32_rvv_whole_register_smoke},
+    {"test_riscv64_rvv_float32_arith", test_riscv64_rvv_float32_arith},
+    {"test_riscv64_rvv_float32_minmax_sign",
+     test_riscv64_rvv_float32_minmax_sign},
+    {"test_riscv32_rvv_float32_minmax_sign_smoke",
+     test_riscv32_rvv_float32_minmax_sign_smoke},
+    {"test_riscv64_rvv_float32_compare", test_riscv64_rvv_float32_compare},
+    {"test_riscv64_rvv_float32_class_merge",
+     test_riscv64_rvv_float32_class_merge},
+    {"test_riscv64_rvv_float32_moves", test_riscv64_rvv_float32_moves},
+    {"test_riscv64_rvv_float64_arith", test_riscv64_rvv_float64_arith},
+    {"test_riscv64_rvv_float32_mask_nanbox",
+     test_riscv64_rvv_float32_mask_nanbox},
+    {"test_riscv64_rvv_float32_sqrt", test_riscv64_rvv_float32_sqrt},
+    {"test_riscv64_rvv_float32_estimate",
+     test_riscv64_rvv_float32_estimate},
+    {"test_riscv64_rvv_float32_convert",
+     test_riscv64_rvv_float32_convert},
+    {"test_riscv64_rvv_widening_float_convert",
+     test_riscv64_rvv_widening_float_convert},
+    {"test_riscv64_rvv_narrowing_float_convert",
+     test_riscv64_rvv_narrowing_float_convert},
+    {"test_riscv64_rvv_float32_slide", test_riscv64_rvv_float32_slide},
+    {"test_riscv64_rvv_float32_fma", test_riscv64_rvv_float32_fma},
+    {"test_riscv32_rvv_float32_smoke", test_riscv32_rvv_float32_smoke},
+    {"test_riscv64_rvv_widening_float_fma",
+     test_riscv64_rvv_widening_float_fma},
+    {"test_riscv64_rvv_widening_float_arith",
+     test_riscv64_rvv_widening_float_arith},
+    {"test_riscv64_rvv_float_illegal", test_riscv64_rvv_float_illegal},
+    {"test_riscv32_rvv_carry_borrow", test_riscv32_rvv_carry_borrow},
+    {"test_riscv64_rvv_carry_borrow_masks",
+     test_riscv64_rvv_carry_borrow_masks},
+    {"test_riscv64_rvv_carry_borrow_illegal",
+     test_riscv64_rvv_carry_borrow_illegal},
+    {"test_riscv64_rvv_narrow_shift", test_riscv64_rvv_narrow_shift},
+    {"test_riscv64_rvv_narrow_shift_illegal",
+     test_riscv64_rvv_narrow_shift_illegal},
+    {"test_riscv64_rvv_integer_extension",
+     test_riscv64_rvv_integer_extension},
+    {"test_riscv64_rvv_integer_extension_illegal",
+     test_riscv64_rvv_integer_extension_illegal},
+    {"test_riscv64_rvv_widening_add_sub_vv_vx",
+     test_riscv64_rvv_widening_add_sub_vv_vx},
+    {"test_riscv64_rvv_widening_add_sub_wv_wx",
+     test_riscv64_rvv_widening_add_sub_wv_wx},
+    {"test_riscv64_rvv_widening_add_sub_illegal",
+     test_riscv64_rvv_widening_add_sub_illegal},
+    {"test_riscv64_rvv_widening_multiply",
+     test_riscv64_rvv_widening_multiply},
+    {"test_riscv64_rvv_widening_multiply_32",
+     test_riscv64_rvv_widening_multiply_32},
+    {"test_riscv64_rvv_widening_multiply_illegal",
+     test_riscv64_rvv_widening_multiply_illegal},
+    {"test_riscv64_rvv_multiply_add",
+     test_riscv64_rvv_multiply_add},
+    {"test_riscv64_rvv_widening_multiply_add",
+     test_riscv64_rvv_widening_multiply_add},
+    {"test_riscv64_rvv_widening_multiply_add_32",
+     test_riscv64_rvv_widening_multiply_add_32},
+    {"test_riscv64_rvv_multiply_add_illegal",
+     test_riscv64_rvv_multiply_add_illegal},
+    {"test_riscv64_rvv_mask_logical",
+     test_riscv64_rvv_mask_logical},
+    {"test_riscv64_rvv_mask_scalar",
+     test_riscv64_rvv_mask_scalar},
+    {"test_riscv64_rvv_mask_set",
+     test_riscv64_rvv_mask_set},
+    {"test_riscv64_rvv_viota_vid",
+     test_riscv64_rvv_viota_vid},
+    {"test_riscv64_rvv_mask_utilities_illegal",
+     test_riscv64_rvv_mask_utilities_illegal},
+    {"test_riscv64_rvv_fixed_point_saturating",
+     test_riscv64_rvv_fixed_point_saturating},
+    {"test_riscv64_rvv_fixed_point_rounding",
+     test_riscv64_rvv_fixed_point_rounding},
+    {"test_riscv64_rvv_fixed_point_clip",
+     test_riscv64_rvv_fixed_point_clip},
+    {"test_riscv64_rvv_fixed_point_illegal",
+     test_riscv64_rvv_fixed_point_illegal},
+    {"test_riscv64_rvv_integer_reduction_sum_logic",
+     test_riscv64_rvv_integer_reduction_sum_logic},
+    {"test_riscv32_rvv_integer_reduction_minmax",
+     test_riscv32_rvv_integer_reduction_minmax},
+    {"test_riscv64_rvv_integer_reduction_masked",
+     test_riscv64_rvv_integer_reduction_masked},
+    {"test_riscv64_rvv_widening_reduction_sum",
+     test_riscv64_rvv_widening_reduction_sum},
+    {"test_riscv64_rvv_widening_reduction_32_to_64",
+     test_riscv64_rvv_widening_reduction_32_to_64},
+    {"test_riscv64_rvv_reduction_illegal",
+     test_riscv64_rvv_reduction_illegal},
+    {"test_riscv64_rvv_float32_reduction",
+     test_riscv64_rvv_float32_reduction},
+    {"test_riscv64_rvv_widening_float_reduction",
+     test_riscv64_rvv_widening_float_reduction},
+    {"test_riscv64_rvv_float_reduction_illegal",
+     test_riscv64_rvv_float_reduction_illegal},
+    {"test_riscv64_rvv_divide_remainder",
+     test_riscv64_rvv_divide_remainder},
+    {"test_riscv64_rvv_divide_remainder_zero_vx",
+     test_riscv64_rvv_divide_remainder_zero_vx},
+    {"test_riscv64_rvv_divide_remainder_64",
+     test_riscv64_rvv_divide_remainder_64},
+    {"test_riscv64_rvv_divide_remainder_illegal",
+     test_riscv64_rvv_divide_remainder_illegal},
+    {"test_riscv64_rvv_multiply", test_riscv64_rvv_multiply},
+    {"test_riscv64_rvv_multiply_64", test_riscv64_rvv_multiply_64},
+    {"test_riscv64_rvv_multiply_illegal",
+     test_riscv64_rvv_multiply_illegal},
+    {"test_riscv_fp_rejects_non_fp_models",
+     test_riscv_fp_rejects_non_fp_models},
+    {"test_riscv_rvv_rejects_non_vector_models",
+     test_riscv_rvv_rejects_non_vector_models},
+    {"test_riscv64_rvv_invalid_vtype", test_riscv64_rvv_invalid_vtype},
+    {"test_riscv64_rvv_vill_blocks_data_path",
+     test_riscv64_rvv_vill_blocks_data_path},
+    {"test_riscv64_rvv_vector_csrs", test_riscv64_rvv_vector_csrs},
+    {"test_riscv64_rvv_public_scalar_regs",
+     test_riscv64_rvv_public_scalar_regs},
+    {"test_riscv32_rvv_public_scalar_regs",
+     test_riscv32_rvv_public_scalar_regs},
+    {"test_riscv64_rvv_public_vector_reg",
+     test_riscv64_rvv_public_vector_reg},
+    {"test_riscv64_rvv_context_save_restore_public_regs",
+     test_riscv64_rvv_context_save_restore_public_regs},
+    {"test_riscv64_rvv_requires_vs", test_riscv64_rvv_requires_vs},
     {NULL, NULL}};
diff --git a/tests/unit/test_s390x.c b/tests/unit/test_s390x.c
index 7677bf90af..f6bed82b2c 100644
--- a/tests/unit/test_s390x.c
+++ b/tests/unit/test_s390x.c
@@ -2,6 +2,8 @@
 
 const uint64_t code_start = 0x1000;
 const uint64_t code_len = 0x4000;
+const uint64_t s390x_data_start = 0x2000;
+const uint64_t s390x_cr0_vector_afp = 0x0000000000060000ull;
 
 static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
                             const char *code, uint64_t size)
@@ -11,6 +13,9 @@ static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
     OK(uc_mem_write(*uc, code_start, code, size));
 }
 
+static void s390x_check_bytes(const char *name, const uint8_t *actual,
+                              const uint8_t *expected, size_t size);
+
 static void test_s390x_lr(void)
 {
     char code[] = "\x18\x23"; // lr %r2, %r3
@@ -33,4 +38,1512 @@ static void test_s390x_lr(void)
     OK(uc_close(uc));
 }
 
-TEST_LIST = {{"test_s390x_lr", test_s390x_lr}, {NULL, NULL}};
+static void run_logic_case(const uint8_t code[4], uint64_t r1,
+                           uint64_t r2, uint64_t r3,
+                           uint64_t expected_r1, uint32_t expected_cc)
+{
+    uc_engine *uc;
+    uint64_t pswm;
+
+    OK(uc_open(UC_ARCH_S390X, UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_S390X_GEN15A));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, 4));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r1));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R3, &r3));
+
+    OK(uc_emu_start(uc, code_start, code_start + 4, 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r1));
+    OK(uc_reg_read(uc, UC_S390X_REG_PSWM, &pswm));
+    TEST_CHECK(r1 == expected_r1);
+    TEST_CHECK(((pswm >> 44) & 3) == expected_cc);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_mie3_logic(void)
+{
+    const uint8_t ncrk[] = "\xb9\xf5\x30\x42";
+    const uint8_t nnrk[] = "\xb9\x74\x30\x42";
+    const uint8_t nork[] = "\xb9\x76\x30\x42";
+    const uint8_t nxrk[] = "\xb9\x77\x30\x42";
+    const uint8_t ocrk[] = "\xb9\x75\x30\x42";
+    const uint8_t ncgrk[] = "\xb9\xe5\x30\x42";
+    const uint8_t nngrk[] = "\xb9\x64\x30\x42";
+    const uint8_t nogrk[] = "\xb9\x66\x30\x42";
+    const uint8_t nxgrk[] = "\xb9\x67\x30\x42";
+    const uint8_t ocgrk[] = "\xb9\x65\x30\x42";
+    const uint64_t high = 0xaaaaaaaa00000000ull;
+
+    run_logic_case(ncrk, high, 0xf0, 0x0f, high | 0xf0, 1);
+    run_logic_case(nnrk, high, 0xf0f0, 0x0ff0, high | 0xffffff0f, 1);
+    run_logic_case(nork, high, 0xffffffff, 0, high, 0);
+    run_logic_case(nxrk, high, 0x12345678, 0x12345678,
+                   high | 0xffffffff, 1);
+    run_logic_case(ocrk, high, 0x00f0, 0x0ff0, high | 0xfffff0ff, 1);
+    run_logic_case(ncgrk, 0, 0xff00ff00ff00ff00ull,
+                   0x00ff00ff00ff00ffull, 0xff00ff00ff00ff00ull, 1);
+    run_logic_case(nngrk, 0, 0xffff0000ffff0000ull,
+                   0x00ff00ff00ff00ffull, 0xff00ffffff00ffffull, 1);
+    run_logic_case(nogrk, 0, 0xffffffffffffffffull, 0, 0, 0);
+    run_logic_case(nxgrk, 0, 0x123456789abcdef0ull,
+                   0x123456789abcdef0ull, 0xffffffffffffffffull, 1);
+    run_logic_case(ocgrk, 0, 0x00000000000000f0ull,
+                   0x0000000000000ff0ull, 0xfffffffffffff0ffull, 1);
+}
+
+static void run_select_case(const uint8_t code[6], uint64_t r1,
+                            uint64_t r2, uint64_t r3,
+                            uint64_t expected_r1)
+{
+    uc_engine *uc;
+
+    OK(uc_open(UC_ARCH_S390X, UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_S390X_GEN15A));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, 6));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r1));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R3, &r3));
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r1));
+    TEST_CHECK(r1 == expected_r1);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_mie3_select(void)
+{
+    const uint8_t selr_true[] = "\x12\x00\xb9\xf0\x38\x42";
+    const uint8_t selr_false[] = "\x12\x00\xb9\xf0\x34\x42";
+    const uint8_t selgr_true[] = "\x12\x00\xb9\xe3\x38\x42";
+    const uint8_t selgr_false[] = "\x12\x00\xb9\xe3\x34\x42";
+    const uint8_t selfhr_true[] = "\x12\x00\xb9\xc0\x38\x42";
+    const uint8_t selfhr_false[] = "\x12\x00\xb9\xc0\x34\x42";
+    const uint64_t r1 = 0xaaaabbbbccccddddull;
+    const uint64_t r2 = 0x1111222233334444ull;
+    const uint64_t r3 = 0x5555666677778888ull;
+
+    run_select_case(selr_true, r1, r2, r3, 0xaaaabbbb33334444ull);
+    run_select_case(selr_false, r1, r2, r3, 0xaaaabbbb77778888ull);
+    run_select_case(selgr_true, r1, r2, r3, r2);
+    run_select_case(selgr_false, r1, r2, r3, r3);
+    run_select_case(selfhr_true, r1, r2, r3, 0x11112222ccccddddull);
+    run_select_case(selfhr_false, r1, r2, r3, 0x55556666ccccddddull);
+}
+
+static void test_s390x_locfhr(void)
+{
+    const uint8_t locfhr_true[] = "\x12\x00\xb9\xe0\x80\x42";
+    const uint8_t locfhr_false[] = "\x12\x00\xb9\xe0\x40\x42";
+    const uint64_t r1 = 0x1111111122222222ull;
+    const uint64_t r2 = 0x3333333344444444ull;
+
+    run_select_case(locfhr_true, r1, r2, 0, 0x3333333322222222ull);
+    run_select_case(locfhr_false, r1, r2, 0, r1);
+}
+
+static uint64_t s390x_pack_bytes(const uint8_t *bytes)
+{
+    uint64_t value = 0;
+    int i;
+
+    for (i = 0; i < 8; i++) {
+        value = (value << 8) | bytes[i];
+    }
+    return value;
+}
+
+static void s390x_unpack_bytes(uint64_t value, uint8_t *bytes)
+{
+    int i;
+
+    for (i = 7; i >= 0; i--) {
+        bytes[i] = value;
+        value >>= 8;
+    }
+}
+
+static uint32_t s390x_read_cc(uc_engine *uc)
+{
+    uint64_t pswm;
+
+    OK(uc_reg_read(uc, UC_S390X_REG_PSWM, &pswm));
+    return (pswm >> 44) & 3;
+}
+
+static void run_s390x_chrl_case(const uint8_t code[6],
+                                const uint8_t data[8], uint64_t r4,
+                                uint32_t expected_cc)
+{
+    uc_engine *uc;
+
+    OK(uc_open(UC_ARCH_S390X, UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_S390X_GEN15A));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, 6));
+    OK(uc_mem_write(uc, s390x_data_start, data, 8));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    TEST_CHECK(s390x_read_cc(uc) == expected_cc);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_compare_halfword_relative_long(void)
+{
+    const uint8_t chrl[] = { 0xc6, 0x45, 0x00, 0x00, 0x08, 0x00 };
+    const uint8_t cghrl[] = { 0xc6, 0x44, 0x00, 0x00, 0x08, 0x00 };
+    const uint8_t wide_positive[] = {
+        0x7f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t signed_negative[] = {
+        0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    uint64_t r4;
+
+    r4 = 100000;
+    run_s390x_chrl_case(chrl, wide_positive, r4, 2);
+    run_s390x_chrl_case(cghrl, wide_positive, r4, 2);
+
+    r4 = 0;
+    run_s390x_chrl_case(chrl, signed_negative, r4, 2);
+    run_s390x_chrl_case(cghrl, signed_negative, r4, 2);
+}
+
+static void run_s390x_clgit_case(const uint8_t code[6])
+{
+    uc_engine *uc;
+    uint64_t r4 = 0x8000;
+
+    OK(uc_open(UC_ARCH_S390X, UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_S390X_GEN15A));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, 6));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_compare_logical_immediate_trap(void)
+{
+    const uint8_t clfit[] = { 0xec, 0x40, 0x80, 0x00, 0x40, 0x73 };
+    const uint8_t clgit[] = { 0xec, 0x40, 0x80, 0x00, 0x40, 0x71 };
+
+    run_s390x_clgit_case(clfit);
+    run_s390x_clgit_case(clgit);
+}
+
+static void s390x_setup_scalar_case(uc_engine **uc, const uint8_t *code,
+                                    size_t code_size)
+{
+    OK(uc_open(UC_ARCH_S390X, UC_MODE_BIG_ENDIAN, uc));
+    OK(uc_ctl_set_cpu_model(*uc, UC_CPU_S390X_GEN15A));
+    OK(uc_mem_map(*uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(*uc, code_start, code, code_size));
+}
+
+static void test_s390x_mie2_add_sub_halfword(void)
+{
+    const uint8_t code[] = {
+        0xe3, 0x40, 0x20, 0x00, 0x00, 0x38,
+        0xe3, 0x40, 0x20, 0x02, 0x00, 0x39,
+    };
+    const uint8_t data[] = { 0xff, 0xfe, 0x00, 0x04 };
+    uc_engine *uc;
+    uint64_t r2 = s390x_data_start;
+    uint64_t r4 = 5;
+
+    s390x_setup_scalar_case(&uc, code, sizeof(code));
+    OK(uc_mem_write(uc, s390x_data_start, data, sizeof(data)));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r4));
+    TEST_CHECK(r4 == UINT64_MAX);
+    TEST_CHECK(s390x_read_cc(uc) == 1);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_mie2_branch_indirect(void)
+{
+    const uint8_t code[] = {
+        0xe3, 0x80, 0x20, 0x00, 0x00, 0x47,
+        0xa7, 0x49, 0x00, 0x01,
+    };
+    uint8_t target[8];
+    uc_engine *uc;
+    uint64_t r2 = s390x_data_start;
+    uint64_t r4 = 0x1234;
+
+    s390x_unpack_bytes(code_start + sizeof(code), target);
+    s390x_setup_scalar_case(&uc, code, sizeof(code));
+    OK(uc_mem_write(uc, s390x_data_start, target, sizeof(target)));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r4));
+    TEST_CHECK(r4 == 0x1234);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_lzrf(void)
+{
+    const uint8_t lzrf[] = { 0xe3, 0x40, 0x20, 0x00, 0x00, 0x3b };
+    const uint8_t value[] = { 0x12, 0x34, 0x56, 0x78 };
+    uc_engine *uc;
+    uint64_t r2 = s390x_data_start;
+    uint64_t r4 = 0xaaaaaaaa55555555ull;
+
+    s390x_setup_scalar_case(&uc, lzrf, sizeof(lzrf));
+    OK(uc_mem_write(uc, s390x_data_start, value, sizeof(value)));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(lzrf), 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r4));
+    TEST_CHECK(r4 == 0xaaaaaaaa12345600ull);
+    OK(uc_close(uc));
+}
+
+static void test_s390x_monitor_call(void)
+{
+    const uint8_t mc_disabled[] = {
+        0xaf, 0x00, 0x20, 0x00,
+        0xa7, 0x49, 0x00, 0x7b,
+    };
+    const uint8_t mc_invalid_class[] = { 0xaf, 0x10, 0x20, 0x00 };
+    const uint8_t mc_enabled[] = { 0xaf, 0x00, 0x20, 0x00 };
+    const uint8_t expected_class[] = { 0x00, 0x00 };
+    const uint8_t expected_code[] = {
+        0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x20, 0x00,
+    };
+    uint8_t actual_class[sizeof(expected_class)];
+    uint8_t actual_code[sizeof(expected_code)];
+    uc_engine *uc;
+    uint64_t r2 = s390x_data_start;
+    uint64_t r4 = 0;
+    uint64_t cr8 = 0x8000;
+
+    s390x_setup_scalar_case(&uc, mc_disabled, sizeof(mc_disabled));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(mc_disabled),
+                    0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r4));
+    TEST_CHECK(r4 == 123);
+    OK(uc_close(uc));
+
+    s390x_setup_scalar_case(&uc, mc_invalid_class,
+                            sizeof(mc_invalid_class));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start,
+                               code_start + sizeof(mc_invalid_class), 0, 0));
+    OK(uc_close(uc));
+
+    s390x_setup_scalar_case(&uc, mc_enabled, sizeof(mc_enabled));
+    OK(uc_mem_map(uc, 0, 0x1000, UC_PROT_ALL));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_CR8, &cr8));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start,
+                               code_start + sizeof(mc_enabled), 0, 0));
+    OK(uc_mem_read(uc, 0x94, actual_class, sizeof(actual_class)));
+    OK(uc_mem_read(uc, 0xb0, actual_code, sizeof(actual_code)));
+    s390x_check_bytes("mc_class", actual_class, expected_class,
+                      sizeof(actual_class));
+    s390x_check_bytes("mc_code", actual_code, expected_code,
+                      sizeof(actual_code));
+    OK(uc_close(uc));
+}
+
+static void test_s390x_mie3_move_right_to_left(void)
+{
+    const uint8_t mvcrl[] = { 0xe5, 0x0a, 0x20, 0x01, 0x30, 0x00 };
+    const uint8_t input[] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+    };
+    const uint8_t expected[] = {
+        0x10, 0x10, 0x11, 0x12, 0x13, 0x14, 0x16, 0x17,
+    };
+    uint8_t actual[sizeof(input)];
+    uc_engine *uc;
+    size_t i;
+    uint64_t r0 = 4;
+    uint64_t r2 = s390x_data_start;
+    uint64_t r3 = s390x_data_start;
+
+    s390x_setup_scalar_case(&uc, mvcrl, sizeof(mvcrl));
+    OK(uc_mem_write(uc, s390x_data_start, input, sizeof(input)));
+    OK(uc_reg_write(uc, UC_S390X_REG_R0, &r0));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R3, &r3));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(mvcrl), 0, 0));
+
+    OK(uc_mem_read(uc, s390x_data_start, actual, sizeof(actual)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(actual[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+}
+
+static void run_s390x_mie2_mul_pair_case(const uint8_t *code,
+                                         size_t code_size, uint64_t r2,
+                                         uint64_t r3, uint64_t r4, uint64_t r5,
+                                         const uint8_t *data,
+                                         size_t data_size,
+                                         uint64_t expected_high,
+                                         uint64_t expected_low)
+{
+    uc_engine *uc;
+    uint64_t data_addr = s390x_data_start;
+    uint64_t r2_value = data_size != 0 ? data_addr : r2;
+
+    s390x_setup_scalar_case(&uc, code, code_size);
+    if (data_size != 0) {
+        OK(uc_mem_write(uc, s390x_data_start, data, data_size));
+    }
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2_value));
+    OK(uc_reg_write(uc, UC_S390X_REG_R3, &r3));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+    OK(uc_reg_write(uc, UC_S390X_REG_R5, &r5));
+
+    OK(uc_emu_start(uc, code_start, code_start + code_size, 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r4));
+    OK(uc_reg_read(uc, UC_S390X_REG_R5, &r5));
+    TEST_CHECK(r4 == expected_high);
+    TEST_CHECK(r5 == expected_low);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_mie2_multiply_128(void)
+{
+    const uint8_t mgrk[] = { 0xb9, 0xec, 0x30, 0x42 };
+    const uint8_t mg[] = { 0xe3, 0x40, 0x20, 0x00, 0x00, 0x84 };
+    const uint8_t four[] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+    };
+
+    run_s390x_mie2_mul_pair_case(mgrk, sizeof(mgrk),
+                                 (uint64_t)(int64_t)-3, 7, 0, 0,
+                                 NULL, 0, UINT64_MAX,
+                                 (uint64_t)(int64_t)-21);
+    run_s390x_mie2_mul_pair_case(mg, sizeof(mg), 0, 0, 0,
+                                 (uint64_t)(int64_t)-6, four,
+                                 sizeof(four), UINT64_MAX,
+                                 (uint64_t)(int64_t)-24);
+}
+
+static void run_s390x_mie2_multiply_cc_case(const uint8_t *code,
+                                            size_t code_size, uint64_t r2,
+                                            uint64_t r3, uint64_t r4,
+                                            const uint8_t *data,
+                                            size_t data_size,
+                                            uint64_t expected_r4,
+                                            uint32_t expected_cc)
+{
+    uc_engine *uc;
+    uint64_t data_addr = s390x_data_start;
+
+    s390x_setup_scalar_case(&uc, code, code_size);
+    if (data_size != 0) {
+        OK(uc_mem_write(uc, s390x_data_start, data, data_size));
+        OK(uc_reg_write(uc, UC_S390X_REG_R2, &data_addr));
+    } else {
+        OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    }
+    OK(uc_reg_write(uc, UC_S390X_REG_R3, &r3));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + code_size, 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r4));
+    TEST_CHECK(r4 == expected_r4);
+    TEST_CHECK(s390x_read_cc(uc) == expected_cc);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_mie2_multiply_single_cc(void)
+{
+    const uint8_t mgh[] = { 0xe3, 0x40, 0x20, 0x00, 0x00, 0x3c };
+    const uint8_t msrkc[] = { 0xb9, 0xfd, 0x30, 0x42 };
+    const uint8_t msc[] = { 0xe3, 0x40, 0x20, 0x00, 0x00, 0x53 };
+    const uint8_t msgrkc[] = { 0xb9, 0xed, 0x30, 0x42 };
+    const uint8_t msgc[] = { 0xe3, 0x40, 0x20, 0x00, 0x00, 0x83 };
+    const uint8_t neg_three_h[] = { 0xff, 0xfd };
+    const uint8_t four_w[] = { 0x00, 0x00, 0x00, 0x04 };
+    const uint8_t three_g[] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03,
+    };
+    uc_engine *uc;
+    uint64_t r2 = s390x_data_start;
+    uint64_t r4 = (uint64_t)(int64_t)-7;
+
+    s390x_setup_scalar_case(&uc, mgh, sizeof(mgh));
+    OK(uc_mem_write(uc, s390x_data_start, neg_three_h,
+                    sizeof(neg_three_h)));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(mgh), 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_R4, &r4));
+    TEST_CHECK(r4 == 21);
+    OK(uc_close(uc));
+
+    run_s390x_mie2_multiply_cc_case(msrkc, sizeof(msrkc),
+                                    0x7fffffffull, 2,
+                                    0xaaaaaaaa00000000ull, NULL, 0,
+                                    0xaaaaaaaafffffffeull, 3);
+    run_s390x_mie2_multiply_cc_case(msc, sizeof(msc), 0, 0,
+                                    0xaaaaaaaa00000003ull, four_w,
+                                    sizeof(four_w),
+                                    0xaaaaaaaa0000000cull, 2);
+    run_s390x_mie2_multiply_cc_case(msgrkc, sizeof(msgrkc),
+                                    0x7fffffffffffffffull, 2, 0, NULL, 0,
+                                    UINT64_MAX - 1, 3);
+    run_s390x_mie2_multiply_cc_case(msgc, sizeof(msgc), 0, 0,
+                                    (uint64_t)(int64_t)-7, three_g,
+                                    sizeof(three_g),
+                                    (uint64_t)(int64_t)-21, 1);
+}
+
+static void test_s390x_kma_query(void)
+{
+    const uint8_t kma_query[] = { 0xb9, 0x29, 0x60, 0x24 };
+    const uint8_t kma_bad_pair[] = { 0xb9, 0x29, 0x20, 0x24 };
+    const uint8_t expected[] = {
+        0x80, 0x00, 0x38, 0x38, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    uint8_t actual[sizeof(expected)];
+    uc_engine *uc;
+    uint64_t r0 = 0;
+    uint64_t r1 = s390x_data_start;
+    uint64_t r2 = s390x_data_start + 0x100;
+    uint64_t r4 = s390x_data_start + 0x200;
+    uint64_t r6 = s390x_data_start + 0x300;
+    size_t i;
+
+    s390x_setup_scalar_case(&uc, kma_query, sizeof(kma_query));
+    OK(uc_reg_write(uc, UC_S390X_REG_R0, &r0));
+    OK(uc_reg_write(uc, UC_S390X_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+    OK(uc_reg_write(uc, UC_S390X_REG_R6, &r6));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(kma_query), 0, 0));
+
+    OK(uc_mem_read(uc, s390x_data_start, actual, sizeof(actual)));
+    for (i = 0; i < sizeof(expected); i++) {
+        TEST_CHECK(actual[i] == expected[i]);
+    }
+    OK(uc_close(uc));
+
+    s390x_setup_scalar_case(&uc, kma_bad_pair, sizeof(kma_bad_pair));
+    OK(uc_reg_write(uc, UC_S390X_REG_R0, &r0));
+    OK(uc_reg_write(uc, UC_S390X_REG_R1, &r1));
+    OK(uc_reg_write(uc, UC_S390X_REG_R2, &r2));
+    OK(uc_reg_write(uc, UC_S390X_REG_R4, &r4));
+
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start,
+                               code_start + sizeof(kma_bad_pair), 0, 0));
+    OK(uc_close(uc));
+}
+
+static void s390x_write_vec(uc_engine *uc, unsigned int reg,
+                            const uint8_t bytes[16])
+{
+    uint64_t low = s390x_pack_bytes(bytes);
+    uint64_t high = s390x_pack_bytes(bytes + 8);
+
+    OK(uc_reg_write(uc, UC_S390X_REG_F0 + reg, &low));
+    OK(uc_reg_write(uc, UC_S390X_REG_F0_HI + reg, &high));
+}
+
+static void s390x_read_vec(uc_engine *uc, unsigned int reg, uint8_t bytes[16])
+{
+    uint64_t low;
+    uint64_t high;
+
+    OK(uc_reg_read(uc, UC_S390X_REG_F0 + reg, &low));
+    OK(uc_reg_read(uc, UC_S390X_REG_F0_HI + reg, &high));
+    s390x_unpack_bytes(low, bytes);
+    s390x_unpack_bytes(high, bytes + 8);
+}
+
+static void s390x_check_bytes(const char *name, const uint8_t *actual,
+                              const uint8_t *expected, size_t len)
+{
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        if (actual[i] != expected[i]) {
+            break;
+        }
+    }
+
+    if (!TEST_CHECK(i == len)) {
+        TEST_MSG("%s[%zu]: expected 0x%02x, got 0x%02x",
+                 name, i, expected[i], actual[i]);
+    }
+}
+
+static void s390x_setup_vector_case(uc_engine **uc, const uint8_t *code,
+                                    size_t code_size)
+{
+    uint64_t cr0;
+    uint64_t r2 = s390x_data_start;
+
+    OK(uc_open(UC_ARCH_S390X, UC_MODE_BIG_ENDIAN, uc));
+    OK(uc_ctl_set_cpu_model(*uc, UC_CPU_S390X_GEN15A));
+    OK(uc_mem_map(*uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(*uc, code_start, code, code_size));
+    OK(uc_reg_read(*uc, UC_S390X_REG_CR0, &cr0));
+    cr0 |= s390x_cr0_vector_afp;
+    OK(uc_reg_write(*uc, UC_S390X_REG_CR0, &cr0));
+    OK(uc_reg_write(*uc, UC_S390X_REG_R2, &r2));
+}
+
+static void run_s390x_ve2_load_case(const uint8_t code[6],
+                                    const uint8_t input[16],
+                                    const uint8_t initial[16],
+                                    const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    OK(uc_mem_write(uc, s390x_data_start, input, 16));
+    s390x_write_vec(uc, 0, initial);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes("vector", actual, expected, 16);
+    OK(uc_close(uc));
+}
+
+static void run_s390x_ve2_store_case(const uint8_t code[6],
+                                     const uint8_t input[16],
+                                     const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+    uint8_t memory[16];
+    int i;
+
+    for (i = 0; i < 16; i++) {
+        memory[i] = 0xa5;
+    }
+
+    s390x_setup_vector_case(&uc, code, 6);
+    OK(uc_mem_write(uc, s390x_data_start, memory, sizeof(memory)));
+    s390x_write_vec(uc, 0, input);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    OK(uc_mem_read(uc, s390x_data_start, actual, sizeof(actual)));
+    s390x_check_bytes("memory", actual, expected, 16);
+    OK(uc_close(uc));
+}
+
+static void run_s390x_ve2_shift_case(const char *name, const uint8_t code[6],
+                                     const uint8_t v1[16],
+                                     const uint8_t v2[16],
+                                     const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 1, v1);
+    s390x_write_vec(uc, 2, v2);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes(name, actual, expected, 16);
+    OK(uc_close(uc));
+}
+
+static void run_s390x_ve2_invalid_case(const uint8_t code[6])
+{
+    uc_engine *uc;
+
+    s390x_setup_vector_case(&uc, code, 6);
+    uc_assert_err(UC_ERR_EXCEPTION,
+                  uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+    OK(uc_close(uc));
+}
+
+static void run_vstrs_case(const uint8_t *code, const uint8_t searched[16],
+                           const uint8_t substr[16], uint8_t substr_len,
+                           uint64_t expected_offset, uint32_t expected_cc)
+{
+    uc_engine *uc;
+    uint8_t length[16] = { 0 };
+    uint64_t cr0;
+    uint64_t offset;
+    uint64_t high;
+    uint64_t pswm;
+
+    OK(uc_open(UC_ARCH_S390X, UC_MODE_BIG_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_S390X_GEN15A));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, 6));
+
+    length[7] = substr_len;
+    OK(uc_reg_read(uc, UC_S390X_REG_CR0, &cr0));
+    cr0 |= s390x_cr0_vector_afp;
+    OK(uc_reg_write(uc, UC_S390X_REG_CR0, &cr0));
+    s390x_write_vec(uc, 1, searched);
+    s390x_write_vec(uc, 2, substr);
+    s390x_write_vec(uc, 3, length);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    OK(uc_reg_read(uc, UC_S390X_REG_F0, &offset));
+    OK(uc_reg_read(uc, UC_S390X_REG_F0_HI, &high));
+    OK(uc_reg_read(uc, UC_S390X_REG_PSWM, &pswm));
+    TEST_CHECK(offset == expected_offset);
+    TEST_CHECK(high == 0);
+    TEST_CHECK(((pswm >> 44) & 3) == expected_cc);
+
+    OK(uc_close(uc));
+}
+
+static void test_s390x_vstrs(void)
+{
+    uint8_t vstrs[] = "\xe7\x01\x20\x00\x30\x8b";
+    uint8_t vstrs_zs[] = "\xe7\x01\x20\x20\x30\x8b";
+    uint8_t vstrs16[] = "\xe7\x01\x21\x00\x30\x8b";
+    uint8_t vstrs32[] = "\xe7\x01\x22\x00\x30\x8b";
+    uint8_t vstrs_bad_m5[] = "\xe7\x01\x23\x00\x30\x8b";
+    uint8_t vstrs_bad_m6[] = "\xe7\x01\x20\x10\x30\x8b";
+    const uint8_t searched_full[16] = {
+        'a', 'b', 'c', 'n', 'e', 'e', 'd', 'l',
+        'e', 'x', 'y', 'z', '0', '1', '2', '3',
+    };
+    const uint8_t substr_nee[16] = {
+        'n', 'e', 'e', 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t searched_partial[16] = {
+        'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+        'x', 'x', 'x', 'x', 'x', 'x', 'n', 'e',
+    };
+    const uint8_t searched_zs[16] = {
+        'a', 'b', 0, 'n', 'e', 'e', 'd', 'l',
+        'e', 'x', 'y', 'z', '0', '1', '2', '3',
+    };
+    const uint8_t substr_needle[16] = {
+        'n', 'e', 'e', 'd', 'l', 'e', 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t searched16[16] = {
+        0x11, 0x11, 0x22, 0x22, 0xab, 0xcd, 0x34, 0x56,
+        0xab, 0xcd, 0x34, 0x56, 0x77, 0x77, 0x88, 0x88,
+    };
+    const uint8_t substr16[16] = {
+        0xab, 0xcd, 0x34, 0x56, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t searched32[16] = {
+        0x01, 0x02, 0x03, 0x04, 0x0a, 0x0b, 0x0c, 0x0d,
+        0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18,
+    };
+    const uint8_t substr32[16] = {
+        0x0a, 0x0b, 0x0c, 0x0d, 0x11, 0x12, 0x13, 0x14,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    run_vstrs_case(vstrs, searched_full, substr_nee, 3, 3, 2);
+    run_vstrs_case(vstrs, searched_partial, substr_nee, 3, 14, 3);
+    run_vstrs_case(vstrs_zs, searched_zs, substr_needle, 6, 16, 1);
+    run_vstrs_case(vstrs16, searched16, substr16, 4, 4, 2);
+    run_vstrs_case(vstrs32, searched32, substr32, 8, 4, 2);
+    run_s390x_ve2_invalid_case(vstrs_bad_m5);
+    run_s390x_ve2_invalid_case(vstrs_bad_m6);
+}
+
+static void run_s390x_vbperm_case(const uint8_t v1[16], const uint8_t v2[16],
+                                  const uint8_t expected[16])
+{
+    const uint8_t vbperm[] = { 0xe7, 0x01, 0x20, 0x00, 0x00, 0x85 };
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, vbperm, sizeof(vbperm));
+    s390x_write_vec(uc, 1, v1);
+    s390x_write_vec(uc, 2, v2);
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(vbperm), 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes("vbperm", actual, expected, sizeof(actual));
+    OK(uc_close(uc));
+}
+
+static void test_s390x_vbperm(void)
+{
+    const uint8_t selected_bits[16] = {
+        0xa5, 0x5a, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t selectors[16] = {
+        0, 1, 2, 3, 4, 5, 6, 7,
+        8, 9, 10, 11, 12, 13, 14, 15,
+    };
+    const uint8_t expected_selected[16] = {
+        0, 0, 0, 0, 0, 0, 0xa5, 0x5a,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+    const uint8_t all_ones[16] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t selectors_with_overflow[16] = {
+        0, 1, 2, 3, 4, 5, 6, 7,
+        128, 129, 130, 131, 132, 133, 134, 135,
+    };
+    const uint8_t expected_overflow[16] = {
+        0, 0, 0, 0, 0, 0, 0xff, 0,
+        0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    run_s390x_vbperm_case(selected_bits, selectors, expected_selected);
+    run_s390x_vbperm_case(all_ones, selectors_with_overflow,
+                          expected_overflow);
+}
+
+static void run_s390x_vmsl_case(const uint8_t code[6],
+                                const uint8_t expected[16])
+{
+    const uint8_t v2[] = {
+        0, 0, 0, 0, 0, 0, 0, 2,
+        0, 0, 0, 0, 0, 0, 0, 3,
+    };
+    const uint8_t v3[] = {
+        0, 0, 0, 0, 0, 0, 0, 4,
+        0, 0, 0, 0, 0, 0, 0, 5,
+    };
+    const uint8_t v4[] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 7,
+    };
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 2, v2);
+    s390x_write_vec(uc, 3, v3);
+    s390x_write_vec(uc, 4, v4);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes("vmsl", actual, expected, sizeof(actual));
+    OK(uc_close(uc));
+}
+
+static void test_s390x_vmsl(void)
+{
+    const uint8_t vmsl[] = { 0xe7, 0x02, 0x33, 0x00, 0x40, 0xb8 };
+    const uint8_t vmsl_shifted[] = {
+        0xe7, 0x02, 0x33, 0xc0, 0x40, 0xb8,
+    };
+    const uint8_t vmsl_invalid_m5[] = {
+        0xe7, 0x02, 0x32, 0x00, 0x40, 0xb8,
+    };
+    const uint8_t expected[] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 30,
+    };
+    const uint8_t expected_shifted[] = {
+        0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 53,
+    };
+
+    run_s390x_vmsl_case(vmsl, expected);
+    run_s390x_vmsl_case(vmsl_shifted, expected_shifted);
+    run_s390x_ve2_invalid_case(vmsl_invalid_m5);
+}
+
+static void run_s390x_vfp3_case(const char *name, const uint8_t code[6],
+                                const uint8_t v2[16],
+                                const uint8_t v3[16],
+                                const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 1, v2);
+    s390x_write_vec(uc, 2, v3);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes(name, actual, expected, sizeof(actual));
+    OK(uc_close(uc));
+}
+
+static void run_s390x_vfp4_case(const char *name, const uint8_t code[6],
+                                const uint8_t v2[16],
+                                const uint8_t v3[16],
+                                const uint8_t v4[16],
+                                const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 1, v2);
+    s390x_write_vec(uc, 2, v3);
+    s390x_write_vec(uc, 3, v4);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes(name, actual, expected, sizeof(actual));
+    OK(uc_close(uc));
+}
+
+static void run_s390x_vfp2_case(const char *name, const uint8_t code[6],
+                                const uint8_t v2[16],
+                                const uint8_t expected[16])
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 1, v2);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes(name, actual, expected, sizeof(actual));
+    OK(uc_close(uc));
+}
+
+static void run_s390x_vfp2_cc_case(const char *name, const uint8_t code[6],
+                                   const uint8_t v2[16],
+                                   const uint8_t expected[16],
+                                   uint32_t expected_cc)
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 1, v2);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes(name, actual, expected, sizeof(actual));
+    TEST_CHECK(s390x_read_cc(uc) == expected_cc);
+    OK(uc_close(uc));
+}
+
+static void run_s390x_wfc_case(const uint8_t code[6],
+                               const uint8_t v1[16],
+                               const uint8_t v2[16],
+                               uint32_t expected_cc)
+{
+    uc_engine *uc;
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 0, v1);
+    s390x_write_vec(uc, 1, v2);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    TEST_CHECK(s390x_read_cc(uc) == expected_cc);
+    OK(uc_close(uc));
+}
+
+static void run_s390x_vfp3_cc_case(const char *name, const uint8_t code[6],
+                                   const uint8_t v2[16],
+                                   const uint8_t v3[16],
+                                   const uint8_t expected[16],
+                                   uint32_t expected_cc)
+{
+    uc_engine *uc;
+    uint8_t actual[16];
+
+    s390x_setup_vector_case(&uc, code, 6);
+    s390x_write_vec(uc, 1, v2);
+    s390x_write_vec(uc, 2, v3);
+
+    OK(uc_emu_start(uc, code_start, code_start + 6, 0, 0));
+
+    s390x_read_vec(uc, 0, actual);
+    s390x_check_bytes(name, actual, expected, sizeof(actual));
+    TEST_CHECK(s390x_read_cc(uc) == expected_cc);
+    OK(uc_close(uc));
+}
+
+static void test_s390x_vfp_minmax(void)
+{
+    const uint8_t vfmax64[] = { 0xe7, 0x01, 0x20, 0x00, 0x30, 0xef };
+    const uint8_t vfmin64[] = { 0xe7, 0x01, 0x20, 0x00, 0x30, 0xee };
+    const uint8_t vfmin64_abs[] = { 0xe7, 0x01, 0x20, 0x80, 0x30, 0xee };
+    const uint8_t vfmax128[] = { 0xe7, 0x01, 0x20, 0x00, 0x40, 0xef };
+    const uint8_t vfmax_bad_m5[] = { 0xe7, 0x01, 0x20, 0x01, 0x30, 0xef };
+    const uint8_t vfmax_bad_m6[] = { 0xe7, 0x01, 0x20, 0x50, 0x30, 0xef };
+    const uint8_t values_a[] = {
+        0xc0, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t values_b[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_max[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_min[] = {
+        0xc0, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_abs_min[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_one[] = {
+        0x3f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_two[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+
+    run_s390x_vfp3_case("vfmax64", vfmax64, values_a, values_b,
+                        expected_max);
+    run_s390x_vfp3_case("vfmin64", vfmin64, values_a, values_b,
+                        expected_min);
+    run_s390x_vfp3_case("vfmin64_abs", vfmin64_abs, values_a, values_b,
+                        expected_abs_min);
+    run_s390x_vfp3_case("vfmax128", vfmax128, quad_one, quad_two,
+                        quad_two);
+    run_s390x_ve2_invalid_case(vfmax_bad_m5);
+    run_s390x_ve2_invalid_case(vfmax_bad_m6);
+}
+
+static void test_s390x_vfp_negated_fma(void)
+{
+    const uint8_t vfma64[] = { 0xe7, 0x01, 0x23, 0x00, 0x30, 0x8f };
+    const uint8_t vfms64[] = { 0xe7, 0x01, 0x23, 0x00, 0x30, 0x8e };
+    const uint8_t vfnma64[] = { 0xe7, 0x01, 0x23, 0x00, 0x30, 0x9f };
+    const uint8_t vfnms64[] = { 0xe7, 0x01, 0x23, 0x00, 0x30, 0x9e };
+    const uint8_t vfnma64_single[] = {
+        0xe7, 0x01, 0x23, 0x08, 0x30, 0x9f,
+    };
+    const uint8_t vfnma32[] = { 0xe7, 0x01, 0x22, 0x00, 0x30, 0x9f };
+    const uint8_t vfnma_bad_m5[] = { 0xe7, 0x01, 0x23, 0x01, 0x30, 0x9f };
+    const uint8_t vfnma_bad_fpf[] = { 0xe7, 0x01, 0x25, 0x00, 0x30, 0x9f };
+    const uint8_t mul_a64[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t mul_b64[] = {
+        0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x3f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t add_c64[] = {
+        0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_vfma64[] = {
+        0x40, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_vfms64[] = {
+        0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_vfnma64[] = {
+        0xc0, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xbf, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_vfnms64[] = {
+        0xc0, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x40, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_single[] = {
+        0xc0, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t mul_a32[] = {
+        0x40, 0x00, 0x00, 0x00, 0xc0, 0x80, 0x00, 0x00,
+        0x3f, 0x80, 0x00, 0x00, 0xbf, 0x00, 0x00, 0x00,
+    };
+    const uint8_t mul_b32[] = {
+        0x40, 0x40, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00,
+        0xc0, 0x00, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00,
+    };
+    const uint8_t add_c32[] = {
+        0x3f, 0x80, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+        0x3f, 0x00, 0x00, 0x00, 0xbf, 0x80, 0x00, 0x00,
+    };
+    const uint8_t expected_vfnma32[] = {
+        0xc0, 0xe0, 0x00, 0x00, 0xbf, 0x80, 0x00, 0x00,
+        0x3f, 0xc0, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+    };
+
+    run_s390x_vfp4_case("vfma64", vfma64, mul_a64, mul_b64, add_c64,
+                        expected_vfma64);
+    run_s390x_vfp4_case("vfms64", vfms64, mul_a64, mul_b64, add_c64,
+                        expected_vfms64);
+    run_s390x_vfp4_case("vfnma64", vfnma64, mul_a64, mul_b64, add_c64,
+                        expected_vfnma64);
+    run_s390x_vfp4_case("vfnms64", vfnms64, mul_a64, mul_b64, add_c64,
+                        expected_vfnms64);
+    run_s390x_vfp4_case("vfnma64_single", vfnma64_single, mul_a64,
+                        mul_b64, add_c64, expected_single);
+    run_s390x_vfp4_case("vfnma32", vfnma32, mul_a32, mul_b32, add_c32,
+                        expected_vfnma32);
+    run_s390x_ve2_invalid_case(vfnma_bad_m5);
+    run_s390x_ve2_invalid_case(vfnma_bad_fpf);
+}
+
+static void test_s390x_vfp_arith_32_128(void)
+{
+    const uint8_t vfa32[] = { 0xe7, 0x01, 0x20, 0x00, 0x20, 0xe3 };
+    const uint8_t vfd32[] = { 0xe7, 0x01, 0x20, 0x00, 0x20, 0xe5 };
+    const uint8_t vfm32[] = { 0xe7, 0x01, 0x20, 0x00, 0x20, 0xe7 };
+    const uint8_t vfs32[] = { 0xe7, 0x01, 0x20, 0x00, 0x20, 0xe2 };
+    const uint8_t vfa128[] = { 0xe7, 0x01, 0x20, 0x00, 0x40, 0xe3 };
+    const uint8_t vfd128[] = { 0xe7, 0x01, 0x20, 0x00, 0x40, 0xe5 };
+    const uint8_t vfm128[] = { 0xe7, 0x01, 0x20, 0x00, 0x40, 0xe7 };
+    const uint8_t vfs128[] = { 0xe7, 0x01, 0x20, 0x00, 0x40, 0xe2 };
+    const uint8_t values_a32[] = {
+        0x3f, 0x80, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00,
+        0x40, 0x40, 0x00, 0x00, 0xc0, 0x80, 0x00, 0x00,
+    };
+    const uint8_t values_b32[] = {
+        0x40, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00,
+        0xbf, 0x80, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_add32[] = {
+        0x40, 0x40, 0x00, 0x00, 0xbf, 0xc0, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0xc0, 0xc0, 0x00, 0x00,
+    };
+    const uint8_t expected_div32[] = {
+        0x3f, 0x00, 0x00, 0x00, 0xc0, 0x80, 0x00, 0x00,
+        0xc0, 0x40, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_mul32[] = {
+        0x40, 0x00, 0x00, 0x00, 0xbf, 0x80, 0x00, 0x00,
+        0xc0, 0x40, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_sub32[] = {
+        0xbf, 0x80, 0x00, 0x00, 0xc0, 0x20, 0x00, 0x00,
+        0x40, 0x80, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_one[] = {
+        0x3f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_two[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_three[] = {
+        0x40, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_four[] = {
+        0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+
+    run_s390x_vfp3_case("vfa32", vfa32, values_a32, values_b32,
+                        expected_add32);
+    run_s390x_vfp3_case("vfd32", vfd32, values_a32, values_b32,
+                        expected_div32);
+    run_s390x_vfp3_case("vfm32", vfm32, values_a32, values_b32,
+                        expected_mul32);
+    run_s390x_vfp3_case("vfs32", vfs32, values_a32, values_b32,
+                        expected_sub32);
+    run_s390x_vfp3_case("vfa128", vfa128, quad_one, quad_two, quad_three);
+    run_s390x_vfp3_case("vfd128", vfd128, quad_four, quad_two, quad_two);
+    run_s390x_vfp3_case("vfm128", vfm128, quad_two, quad_two, quad_four);
+    run_s390x_vfp3_case("vfs128", vfs128, quad_four, quad_two, quad_two);
+}
+
+static void test_s390x_vfp_compare_32_128(void)
+{
+    const uint8_t wfc32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xcb };
+    const uint8_t wfk128[] = { 0xe7, 0x01, 0x00, 0x00, 0x40, 0xca };
+    const uint8_t vfce32_cc[] = {
+        0xe7, 0x01, 0x20, 0x10, 0x20, 0xe8,
+    };
+    const uint8_t vfche32[] = { 0xe7, 0x01, 0x20, 0x00, 0x20, 0xea };
+    const uint8_t vfch128_cc[] = {
+        0xe7, 0x01, 0x20, 0x10, 0x40, 0xeb,
+    };
+    const uint8_t f32_a[] = {
+        0x3f, 0x80, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x40, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00,
+    };
+    const uint8_t f32_b[] = {
+        0x3f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x40, 0x40, 0x00, 0x00, 0x40, 0xa0, 0x00, 0x00,
+    };
+    const uint8_t f32_c[] = {
+        0x3f, 0x80, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00,
+    };
+    const uint8_t vfce_expected[] = {
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t vfche_expected[] = {
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+    const uint8_t quad_one[] = {
+        0x3f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_two[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_all_ones[] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+
+    run_s390x_wfc_case(wfc32, quad_one, quad_two, 1);
+    run_s390x_wfc_case(wfk128, quad_one, quad_one, 0);
+    run_s390x_vfp3_cc_case("vfce32", vfce32_cc, f32_a, f32_b,
+                           vfce_expected, 1);
+    run_s390x_vfp3_case("vfche32", vfche32, f32_a, f32_c,
+                        vfche_expected);
+    run_s390x_vfp3_cc_case("vfch128", vfch128_cc, quad_two, quad_one,
+                           quad_all_ones, 0);
+}
+
+static void test_s390x_vfp_convert_round(void)
+{
+    const uint8_t vcdg32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xc3 };
+    const uint8_t vcdlg32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xc1 };
+    const uint8_t vcgd32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xc2 };
+    const uint8_t vclgd32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xc0 };
+    const uint8_t vfi32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xc7 };
+    const uint8_t vfi128[] = { 0xe7, 0x01, 0x00, 0x00, 0x40, 0xc7 };
+    const uint8_t vfll64[] = { 0xe7, 0x01, 0x00, 0x00, 0x30, 0xc4 };
+    const uint8_t vflr128[] = { 0xe7, 0x01, 0x00, 0x00, 0x40, 0xc5 };
+    const uint8_t int_values[] = {
+        0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xfe,
+        0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xfc,
+    };
+    const uint8_t uint_values[] = {
+        0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02,
+        0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04,
+    };
+    const uint8_t float_signed[] = {
+        0x3f, 0x80, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00,
+        0x40, 0x40, 0x00, 0x00, 0xc0, 0x80, 0x00, 0x00,
+    };
+    const uint8_t float_unsigned[] = {
+        0x3f, 0x80, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x40, 0x40, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00,
+    };
+    const uint8_t vfi_input32[] = {
+        0x3f, 0xa0, 0x00, 0x00, 0xc0, 0x30, 0x00, 0x00,
+        0x40, 0x50, 0x00, 0x00, 0xc0, 0x98, 0x00, 0x00,
+    };
+    const uint8_t vfi_expected32[] = {
+        0x3f, 0x80, 0x00, 0x00, 0xc0, 0x40, 0x00, 0x00,
+        0x40, 0x40, 0x00, 0x00, 0xc0, 0xa0, 0x00, 0x00,
+    };
+    const uint8_t double_one_two[] = {
+        0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_one[] = {
+        0x3f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_one_half[] = {
+        0x3f, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_two[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t double_one_from_quad[] = {
+        0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+
+    run_s390x_vfp2_case("vcdg32", vcdg32, int_values, float_signed);
+    run_s390x_vfp2_case("vcdlg32", vcdlg32, uint_values, float_unsigned);
+    run_s390x_vfp2_case("vcgd32", vcgd32, float_signed, int_values);
+    run_s390x_vfp2_case("vclgd32", vclgd32, float_unsigned, uint_values);
+    run_s390x_vfp2_case("vfi32", vfi32, vfi_input32, vfi_expected32);
+    run_s390x_vfp2_case("vfi128", vfi128, quad_one_half, quad_two);
+    run_s390x_vfp2_case("vfll64", vfll64, double_one_two, quad_one);
+    run_s390x_vfp2_case("vflr128", vflr128, quad_one,
+                        double_one_from_quad);
+}
+
+static void test_s390x_vfp_sign_sqrt_class(void)
+{
+    const uint8_t vfpso32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xcc };
+    const uint8_t vfpso128_abs[] = {
+        0xe7, 0x01, 0x00, 0x20, 0x40, 0xcc,
+    };
+    const uint8_t vfsq32[] = { 0xe7, 0x01, 0x00, 0x00, 0x20, 0xce };
+    const uint8_t vfsq128[] = { 0xe7, 0x01, 0x00, 0x00, 0x40, 0xce };
+    const uint8_t vftci32[] = { 0xe7, 0x01, 0x30, 0x00, 0x20, 0x4a };
+    const uint8_t vftci128[] = { 0xe7, 0x01, 0x30, 0x00, 0x40, 0x4a };
+    const uint8_t sign_input32[] = {
+        0x3f, 0x80, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
+    };
+    const uint8_t sign_expected32[] = {
+        0xbf, 0x80, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_neg_one[] = {
+        0xbf, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_one[] = {
+        0x3f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t sqrt_input32[] = {
+        0x40, 0x80, 0x00, 0x00, 0x41, 0x10, 0x00, 0x00,
+        0x41, 0x80, 0x00, 0x00, 0x41, 0xc8, 0x00, 0x00,
+    };
+    const uint8_t sqrt_expected32[] = {
+        0x40, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
+        0x40, 0x80, 0x00, 0x00, 0x40, 0xa0, 0x00, 0x00,
+    };
+    const uint8_t quad_two[] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_four[] = {
+        0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t class_input32[] = {
+        0x3f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0xc0, 0x00, 0x00, 0x00, 0x7f, 0x80, 0x00, 0x00,
+    };
+    const uint8_t class_expected32[] = {
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+        0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t quad_all_ones[] = {
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    };
+
+    run_s390x_vfp2_case("vfpso32", vfpso32, sign_input32,
+                        sign_expected32);
+    run_s390x_vfp2_case("vfpso128", vfpso128_abs, quad_neg_one, quad_one);
+    run_s390x_vfp2_case("vfsq32", vfsq32, sqrt_input32, sqrt_expected32);
+    run_s390x_vfp2_case("vfsq128", vfsq128, quad_four, quad_two);
+    run_s390x_vfp2_cc_case("vftci32", vftci32, class_input32,
+                           class_expected32, 1);
+    run_s390x_vfp2_cc_case("vftci128", vftci128, quad_one,
+                           quad_all_ones, 0);
+}
+
+static void test_s390x_ve2_byte_reverse_load(void)
+{
+    const uint8_t data[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t initial[16] = {
+        0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+        0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+    };
+    const uint8_t vlebrh[] = { 0xe6, 0x00, 0x20, 0x00, 0x00, 0x01 };
+    const uint8_t vlbrrep_h[] = { 0xe6, 0x00, 0x20, 0x00, 0x10, 0x05 };
+    const uint8_t vllebrz_f_left[] = {
+        0xe6, 0x00, 0x20, 0x00, 0x60, 0x04,
+    };
+    const uint8_t vlbr_h[] = { 0xe6, 0x00, 0x20, 0x00, 0x10, 0x06 };
+    const uint8_t vler_f[] = { 0xe6, 0x00, 0x20, 0x00, 0x20, 0x07 };
+    const uint8_t expected_vlebrh[16] = {
+        0x01, 0x00, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+        0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+    };
+    const uint8_t expected_vlbrrep_h[16] = {
+        0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
+        0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00,
+    };
+    const uint8_t expected_vllebrz_f_left[16] = {
+        0x03, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_vlbr_h[16] = {
+        0x01, 0x00, 0x03, 0x02, 0x05, 0x04, 0x07, 0x06,
+        0x09, 0x08, 0x0b, 0x0a, 0x0d, 0x0c, 0x0f, 0x0e,
+    };
+    const uint8_t expected_vler_f[16] = {
+        0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b,
+        0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03,
+    };
+
+    run_s390x_ve2_load_case(vlebrh, data, initial, expected_vlebrh);
+    run_s390x_ve2_load_case(vlbrrep_h, data, initial, expected_vlbrrep_h);
+    run_s390x_ve2_load_case(vllebrz_f_left, data, initial,
+                            expected_vllebrz_f_left);
+    run_s390x_ve2_load_case(vlbr_h, data, initial, expected_vlbr_h);
+    run_s390x_ve2_load_case(vler_f, data, initial, expected_vler_f);
+}
+
+static void test_s390x_ve2_byte_reverse_store(void)
+{
+    const uint8_t input[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t vstebrh[] = { 0xe6, 0x00, 0x20, 0x00, 0x00, 0x09 };
+    const uint8_t vstbr_f[] = { 0xe6, 0x00, 0x20, 0x00, 0x20, 0x0e };
+    const uint8_t vster_h[] = { 0xe6, 0x00, 0x20, 0x00, 0x10, 0x0f };
+    const uint8_t expected_vstebrh[16] = {
+        0x01, 0x00, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5,
+        0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5,
+    };
+    const uint8_t expected_vstbr_f[16] = {
+        0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04,
+        0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c,
+    };
+    const uint8_t expected_vster_h[16] = {
+        0x0e, 0x0f, 0x0c, 0x0d, 0x0a, 0x0b, 0x08, 0x09,
+        0x06, 0x07, 0x04, 0x05, 0x02, 0x03, 0x00, 0x01,
+    };
+
+    run_s390x_ve2_store_case(vstebrh, input, expected_vstebrh);
+    run_s390x_ve2_store_case(vstbr_f, input, expected_vstbr_f);
+    run_s390x_ve2_store_case(vster_h, input, expected_vster_h);
+}
+
+static void test_s390x_ve2_double_shift(void)
+{
+    const uint8_t vsld[] = { 0xe7, 0x01, 0x20, 0x01, 0x00, 0x86 };
+    const uint8_t vsrd[] = { 0xe7, 0x01, 0x20, 0x01, 0x00, 0x87 };
+    const uint8_t vsldb[] = { 0xe7, 0x01, 0x20, 0x01, 0x00, 0x77 };
+    const uint8_t vsld_invalid[] = {
+        0xe7, 0x01, 0x20, 0x08, 0x00, 0x86,
+    };
+    const uint8_t vsrd_invalid[] = {
+        0xe7, 0x01, 0x20, 0x08, 0x00, 0x87,
+    };
+    const uint8_t v1_vsld[16] = {
+        0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t v2_vsld[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t v1_vsrd[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+    };
+    const uint8_t v2_vsrd[16] = {
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t v1_vsldb[16] = {
+        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+    };
+    const uint8_t v2_vsldb[16] = {
+        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+    };
+    const uint8_t expected_shift[16] = {
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+    const uint8_t expected_vsldb[16] = {
+        0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+        0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+    };
+
+    run_s390x_ve2_shift_case("vsld", vsld, v1_vsld, v2_vsld,
+                             expected_shift);
+    run_s390x_ve2_shift_case("vsrd", vsrd, v1_vsrd, v2_vsrd,
+                             expected_shift);
+    run_s390x_ve2_shift_case("vsldb", vsldb, v1_vsldb, v2_vsldb,
+                             expected_vsldb);
+    run_s390x_ve2_invalid_case(vsld_invalid);
+    run_s390x_ve2_invalid_case(vsrd_invalid);
+}
+
+TEST_LIST = {
+    {"test_s390x_lr", test_s390x_lr},
+    {"test_s390x_compare_halfword_relative_long",
+     test_s390x_compare_halfword_relative_long},
+    {"test_s390x_compare_logical_immediate_trap",
+     test_s390x_compare_logical_immediate_trap},
+    {"test_s390x_mie2_add_sub_halfword",
+     test_s390x_mie2_add_sub_halfword},
+    {"test_s390x_mie2_branch_indirect", test_s390x_mie2_branch_indirect},
+    {"test_s390x_mie2_multiply_128", test_s390x_mie2_multiply_128},
+    {"test_s390x_mie2_multiply_single_cc",
+     test_s390x_mie2_multiply_single_cc},
+    {"test_s390x_lzrf", test_s390x_lzrf},
+    {"test_s390x_kma_query", test_s390x_kma_query},
+    {"test_s390x_monitor_call", test_s390x_monitor_call},
+    {"test_s390x_mie3_move_right_to_left",
+     test_s390x_mie3_move_right_to_left},
+    {"test_s390x_mie3_logic", test_s390x_mie3_logic},
+    {"test_s390x_mie3_select", test_s390x_mie3_select},
+    {"test_s390x_locfhr", test_s390x_locfhr},
+    {"test_s390x_vstrs", test_s390x_vstrs},
+    {"test_s390x_vbperm", test_s390x_vbperm},
+    {"test_s390x_vmsl", test_s390x_vmsl},
+    {"test_s390x_vfp_minmax", test_s390x_vfp_minmax},
+    {"test_s390x_vfp_negated_fma", test_s390x_vfp_negated_fma},
+    {"test_s390x_vfp_arith_32_128", test_s390x_vfp_arith_32_128},
+    {"test_s390x_vfp_compare_32_128", test_s390x_vfp_compare_32_128},
+    {"test_s390x_vfp_convert_round", test_s390x_vfp_convert_round},
+    {"test_s390x_vfp_sign_sqrt_class", test_s390x_vfp_sign_sqrt_class},
+    {"test_s390x_ve2_byte_reverse_load", test_s390x_ve2_byte_reverse_load},
+    {"test_s390x_ve2_byte_reverse_store", test_s390x_ve2_byte_reverse_store},
+    {"test_s390x_ve2_double_shift", test_s390x_ve2_double_shift},
+    {NULL, NULL},
+};
diff --git a/tests/unit/test_sparc.c b/tests/unit/test_sparc.c
index 2f7aa5a21f..a91189b407 100644
--- a/tests/unit/test_sparc.c
+++ b/tests/unit/test_sparc.c
@@ -14,9 +14,109 @@ static void test_virtual_read(void)
     uc_assert_err(UC_ERR_ARG, uc_vmem_read(uc, code_start, UC_PROT_READ, &u8, sizeof(u8)));
     OK(uc_ctl_tlb_mode(uc, UC_TLB_VIRTUAL));
     OK(uc_vmem_read(uc, code_start, UC_PROT_READ, &u8, sizeof(u8)));
+
+    OK(uc_close(uc));
+}
+
+static void test_sparc32_public_registers(void)
+{
+    uc_engine *uc;
+    uint32_t f0 = 0x11223344;
+    uint32_t f1 = 0x55667788;
+    uint32_t f31 = 0xaabbccdd;
+    uint32_t fcc0 = 3;
+    uint32_t icc = 0xb;
+    uint32_t y = 0xcafebabe;
+    uint32_t psr;
+    uint32_t got;
+
+    OK(uc_open(UC_ARCH_SPARC, UC_MODE_SPARC32 | UC_MODE_BIG_ENDIAN, &uc));
+
+    OK(uc_reg_write(uc, UC_SPARC_REG_F0, &f0));
+    OK(uc_reg_write(uc, UC_SPARC_REG_F1, &f1));
+    OK(uc_reg_write(uc, UC_SPARC_REG_F31, &f31));
+    OK(uc_reg_write(uc, UC_SPARC_REG_FCC0, &fcc0));
+    OK(uc_reg_write(uc, UC_SPARC_REG_ICC, &icc));
+    OK(uc_reg_write(uc, UC_SPARC_REG_Y, &y));
+
+    OK(uc_reg_read(uc, UC_SPARC_REG_F0, &got));
+    TEST_CHECK(got == f0);
+    OK(uc_reg_read(uc, UC_SPARC_REG_F1, &got));
+    TEST_CHECK(got == f1);
+    OK(uc_reg_read(uc, UC_SPARC_REG_F31, &got));
+    TEST_CHECK(got == f31);
+    OK(uc_reg_read(uc, UC_SPARC_REG_FCC0, &got));
+    TEST_CHECK(got == fcc0);
+    OK(uc_reg_read(uc, UC_SPARC_REG_ICC, &got));
+    TEST_CHECK(got == icc);
+    OK(uc_reg_read(uc, UC_SPARC_REG_Y, &got));
+    TEST_CHECK(got == y);
+    OK(uc_reg_read(uc, UC_SPARC_REG_PSR, &psr));
+    TEST_CHECK(((psr >> 20) & 0xf) == icc);
+
+    OK(uc_close(uc));
+}
+
+static void test_sparc64_public_registers(void)
+{
+    uc_engine *uc;
+    uint64_t f32 = 0x1122334455667788ull;
+    uint64_t f62 = 0x8877665544332211ull;
+    uint64_t y = 0x123456789abcdef0ull;
+    uint64_t got64;
+    uint32_t f0 = 0xa1b2c3d4;
+    uint32_t f31 = 0x0badf00d;
+    uint32_t fcc0 = 1;
+    uint32_t fcc1 = 2;
+    uint32_t fcc2 = 3;
+    uint32_t fcc3 = 0;
+    uint32_t icc = 5;
+    uint32_t xcc = 0xa;
+    uint32_t got32;
+
+    OK(uc_open(UC_ARCH_SPARC, UC_MODE_SPARC64 | UC_MODE_BIG_ENDIAN, &uc));
+
+    OK(uc_reg_write(uc, UC_SPARC_REG_F0, &f0));
+    OK(uc_reg_write(uc, UC_SPARC_REG_F31, &f31));
+    OK(uc_reg_write(uc, UC_SPARC_REG_F32, &f32));
+    OK(uc_reg_write(uc, UC_SPARC_REG_F62, &f62));
+    OK(uc_reg_write(uc, UC_SPARC_REG_FCC0, &fcc0));
+    OK(uc_reg_write(uc, UC_SPARC_REG_FCC1, &fcc1));
+    OK(uc_reg_write(uc, UC_SPARC_REG_FCC2, &fcc2));
+    OK(uc_reg_write(uc, UC_SPARC_REG_FCC3, &fcc3));
+    OK(uc_reg_write(uc, UC_SPARC_REG_ICC, &icc));
+    OK(uc_reg_write(uc, UC_SPARC_REG_XCC, &xcc));
+    OK(uc_reg_write(uc, UC_SPARC_REG_Y, &y));
+
+    OK(uc_reg_read(uc, UC_SPARC_REG_F0, &got32));
+    TEST_CHECK(got32 == f0);
+    OK(uc_reg_read(uc, UC_SPARC_REG_F31, &got32));
+    TEST_CHECK(got32 == f31);
+    OK(uc_reg_read(uc, UC_SPARC_REG_F32, &got64));
+    TEST_CHECK(got64 == f32);
+    OK(uc_reg_read(uc, UC_SPARC_REG_F62, &got64));
+    TEST_CHECK(got64 == f62);
+    OK(uc_reg_read(uc, UC_SPARC_REG_FCC0, &got32));
+    TEST_CHECK(got32 == fcc0);
+    OK(uc_reg_read(uc, UC_SPARC_REG_FCC1, &got32));
+    TEST_CHECK(got32 == fcc1);
+    OK(uc_reg_read(uc, UC_SPARC_REG_FCC2, &got32));
+    TEST_CHECK(got32 == fcc2);
+    OK(uc_reg_read(uc, UC_SPARC_REG_FCC3, &got32));
+    TEST_CHECK(got32 == fcc3);
+    OK(uc_reg_read(uc, UC_SPARC_REG_ICC, &got32));
+    TEST_CHECK(got32 == icc);
+    OK(uc_reg_read(uc, UC_SPARC_REG_XCC, &got32));
+    TEST_CHECK(got32 == xcc);
+    OK(uc_reg_read(uc, UC_SPARC_REG_Y, &got64));
+    TEST_CHECK(got64 == y);
+
+    OK(uc_close(uc));
 }
 
 TEST_LIST = {
         {"test_virtual_read", test_virtual_read},
+        {"test_sparc32_public_registers", test_sparc32_public_registers},
+        {"test_sparc64_public_registers", test_sparc64_public_registers},
         {NULL, NULL}
 };
diff --git a/tests/unit/test_tricore.c b/tests/unit/test_tricore.c
index f699ff9fe2..5669d54fdc 100644
--- a/tests/unit/test_tricore.c
+++ b/tests/unit/test_tricore.c
@@ -1,6 +1,95 @@
 #include "unicorn_test.h"
 
-const uint64_t code_start = 0x1000;
+const uint64_t code_start = 0x10000;
 const uint64_t code_len = 0x4000;
 
-TEST_LIST = {{NULL, NULL}};
+static void uc_map_code(uc_engine *uc, const uint8_t *code, size_t size)
+{
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, size));
+}
+
+static void uc_common_setup(uc_engine **uc, const uint8_t *code, size_t size)
+{
+    OK(uc_open(UC_ARCH_TRICORE, UC_MODE_LITTLE_ENDIAN, uc));
+    uc_map_code(*uc, code, size);
+}
+
+static void test_tricore_mov_dreg(void)
+{
+    const uint8_t code[] = {
+        0x82, 0x11,
+        0xbb, 0x00, 0x00, 0x08,
+    };
+    uc_engine *uc;
+    uint32_t d0 = 0;
+    uint32_t d1 = 0;
+    uint32_t pc = 0;
+
+    uc_common_setup(&uc, code, sizeof(code));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_reg_read(uc, UC_TRICORE_REG_D0, &d0));
+    OK(uc_reg_read(uc, UC_TRICORE_REG_D1, &d1));
+    OK(uc_reg_read(uc, UC_TRICORE_REG_PC, &pc));
+    TEST_CHECK_(d0 == 0x8000, "d0 = 0x%08x", d0);
+    TEST_CHECK_(d1 == 1, "d1 = 0x%08x", d1);
+    TEST_CHECK(pc == code_start + sizeof(code));
+
+    OK(uc_close(uc));
+}
+
+static void test_tricore_reg_roundtrip_one(uc_engine *uc, int reg,
+                                           uint32_t value)
+{
+    uint32_t out = 0;
+
+    OK(uc_reg_write(uc, reg, &value));
+    OK(uc_reg_read(uc, reg, &out));
+    TEST_CHECK(out == value);
+}
+
+static void test_tricore_csfr_reg_roundtrip(void)
+{
+    uc_engine *uc;
+    uint32_t code = 0;
+
+    uc_common_setup(&uc, (const uint8_t *)&code, sizeof(code));
+
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_DPR0_U, 0x11112222);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_DPR3_L, 0x33334444);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_CPR0_U, 0x55556666);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_CPR3_L, 0x77778888);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_DPM2, 0x10203040);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_CPM3, 0x50607080);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_MMU_CON, 0xaabbccdd);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_MMU_TFA, 0x01020304);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_BMACON, 0x11223344);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_CCPIER, 0x55667788);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_DBGSR, 0x00000001);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_TR1EVT, 0x13579bdf);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_CCTRL, 0x2468ace0);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_M3CNT, 0xdeadbeef);
+    test_tricore_reg_roundtrip_one(uc, UC_TRICORE_REG_PSW, 0xf8000123);
+
+    OK(uc_close(uc));
+}
+
+static void test_tricore_cpu_model(void)
+{
+    uc_engine *uc;
+
+    OK(uc_open(UC_ARCH_TRICORE, UC_MODE_LITTLE_ENDIAN, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_TRICORE_TC1796));
+    TEST_CHECK(uc_ctl_set_cpu_model(uc, UC_CPU_TRICORE_ENDING) ==
+               UC_ERR_ARG);
+    OK(uc_close(uc));
+}
+
+TEST_LIST = {
+    {"test_tricore_mov_dreg", test_tricore_mov_dreg},
+    {"test_tricore_csfr_reg_roundtrip", test_tricore_csfr_reg_roundtrip},
+    {"test_tricore_cpu_model", test_tricore_cpu_model},
+    {NULL, NULL},
+};
diff --git a/tests/unit/test_x86.c b/tests/unit/test_x86.c
index 1a99c8b9e5..59eded91b9 100644
--- a/tests/unit/test_x86.c
+++ b/tests/unit/test_x86.c
@@ -7,6 +7,23 @@ const uint64_t code_len = 0x4000;
 #define MEM_SIZE 1024 * 1024
 #define MEM_STACK MEM_BASE + (MEM_SIZE / 2)
 #define MEM_TEXT MEM_STACK + 4096
+#define TEST_MSR_IA32_XFD 0x000001c4
+#define TEST_MSR_IA32_XFD_ERR 0x000001c5
+#define TEST_MSR_IA32_PKRS 0x000006e1
+#define TEST_MSR_ARCH_LBR_CTL 0x000014ce
+#define TEST_MSR_ARCH_LBR_DEPTH 0x000014cf
+#define TEST_MSR_ARCH_LBR_FROM_0 0x00001500
+#define TEST_MSR_ARCH_LBR_TO_0 0x00001600
+#define TEST_MSR_ARCH_LBR_INFO_0 0x00001200
+#define TEST_MSR_IA32_XSS 0x00000da0
+#define TEST_X86_CPUID_7_0_EBX_AVX2 (1U << 5)
+#define TEST_X86_CPUID_7_0_EBX_AVX512F (1U << 16)
+#define TEST_X86_CPUID_7_0_EBX_AVX512DQ (1U << 17)
+#define TEST_X86_CPUID_7_0_EBX_AVX512CD (1U << 28)
+#define TEST_X86_CPUID_7_0_EBX_AVX512BW (1U << 30)
+#define TEST_X86_CPUID_7_0_EBX_AVX512VL (1U << 31)
+#define TEST_X86_CPUID_7_0_ECX_VAES (1U << 9)
+#define TEST_X86_CPUID_7_0_ECX_VPCLMULQDQ (1U << 10)
 
 static void uc_common_setup(uc_engine **uc, uc_arch arch, uc_mode mode,
                             const char *code, uint64_t size)
@@ -280,6 +297,803 @@ static void test_x86_inc_dec_pxor(void)
     OK(uc_close(uc));
 }
 
+static void test_x86_avx_vpxor_ymm(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc5\xfd\xef\xc1";
+    uint64_t ymm0[4] = {0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL,
+                        0x8899aabbccddeeffULL, 0x0011223344556677ULL};
+    uint64_t ymm1[4] = {0x8090a0b0c0d0e0f0ULL, 0x0010203040506070ULL,
+                        0x1020304050607080ULL, 0xfedcba9876543210ULL};
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_32, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+
+    TEST_CHECK(ymm0[0] == 0x8899aabbccddeeffULL);
+    TEST_CHECK(ymm0[1] == 0x0011223344556677ULL);
+    TEST_CHECK(ymm0[2] == 0x98b99afb9cbd9e7fULL);
+    TEST_CHECK(ymm0[3] == 0xfecd98ab32015467ULL);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx_vex128_zero_upper_one(int cpu_model)
+{
+    uc_engine *uc;
+    char code[] = "\xc5\xf1\xef\xc2";
+    uint64_t ymm0[4] = {0xffffffffffffffffULL, 0xeeeeeeeeeeeeeeeeULL,
+                        0xddddddddddddddddULL, 0xccccccccccccccccULL};
+    uint64_t ymm1[4] = {0x0011223344556677ULL, 0x8899aabbccddeeffULL,
+                        0x1020304050607080ULL, 0xfedcba9876543210ULL};
+    uint64_t ymm2[4] = {0xff00ff00aa55aa55ULL, 0x123456789abcdef0ULL,
+                        0x0f1e2d3c4b5a6978ULL, 0x8877665544332211ULL};
+    uint64_t expected[4] = {0xff11dd33ee00cc22ULL, 0x9aadfcc35661300fULL,
+                            0, 0};
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, cpu_model));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM2, &ymm2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+
+    TEST_CHECK(memcmp(ymm0, expected, sizeof(ymm0)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx_vex128_zero_upper(void)
+{
+    test_x86_avx_vex128_zero_upper_one(UC_CPU_X86_HASWELL);
+    test_x86_avx_vex128_zero_upper_one(UC_CPU_X86_ICELAKE_CLIENT);
+}
+
+static void test_x86_avx_scalar_zero_upper_ss_one(int cpu_model)
+{
+    uc_engine *uc;
+    char code[] = "\xc5\xf2\x58\xc2";
+    uint32_t ymm0[8] = {
+        0xffffffff, 0xeeeeeeee, 0xdddddddd, 0xcccccccc,
+        0xbbbbbbbb, 0xaaaaaaaa, 0x99999999, 0x88888888,
+    };
+    uint32_t ymm1[8] = {
+        0x3fc00000, 0x11223344, 0x55667788, 0x99aabbcc,
+        0x12345678, 0x23456789, 0x3456789a, 0x456789ab,
+    };
+    uint32_t ymm2[8] = {
+        0x40100000, 0x80818283, 0x84858687, 0x88898a8b,
+        0x8c8d8e8f, 0x90919293, 0x94959697, 0x98999a9b,
+    };
+    uint32_t expected[8] = {
+        0x40700000, 0x11223344, 0x55667788, 0x99aabbcc,
+        0, 0, 0, 0,
+    };
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, cpu_model));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM2, &ymm2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+
+    TEST_CHECK(memcmp(ymm0, expected, sizeof(ymm0)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx_scalar_zero_upper_sd_one(int cpu_model)
+{
+    uc_engine *uc;
+    char code[] = "\xc5\xf3\x58\xc2";
+    uint64_t ymm0[4] = {0xffffffffffffffffULL, 0xeeeeeeeeeeeeeeeeULL,
+                        0xddddddddddddddddULL, 0xccccccccccccccccULL};
+    uint64_t ymm1[4] = {0x3ff8000000000000ULL, 0x1122334455667788ULL,
+                        0x123456789abcdef0ULL, 0x0f1e2d3c4b5a6978ULL};
+    uint64_t ymm2[4] = {0x4002000000000000ULL, 0x8899aabbccddeeffULL,
+                        0x1020304050607080ULL, 0xfedcba9876543210ULL};
+    uint64_t expected[4] = {0x400e000000000000ULL, 0x1122334455667788ULL,
+                            0, 0};
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, cpu_model));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM2, &ymm2));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+
+    TEST_CHECK(memcmp(ymm0, expected, sizeof(ymm0)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx_scalar_zero_upper(void)
+{
+    test_x86_avx_scalar_zero_upper_ss_one(UC_CPU_X86_HASWELL);
+    test_x86_avx_scalar_zero_upper_sd_one(UC_CPU_X86_HASWELL);
+    test_x86_avx_scalar_zero_upper_ss_one(UC_CPU_X86_ICELAKE_CLIENT);
+    test_x86_avx_scalar_zero_upper_sd_one(UC_CPU_X86_ICELAKE_CLIENT);
+}
+
+static void test_x86_avx_fma_ps(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe2\x79\x98\xc1";
+    uint32_t xmm0[4] = {
+        0x40000000, 0x40400000, 0x40800000, 0x40a00000,
+    };
+    uint32_t xmm1[4] = {
+        0x41200000, 0x41a00000, 0x41f00000, 0x42200000,
+    };
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_XMM0, &xmm0));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM1, &xmm1));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_XMM0, &xmm0));
+
+    TEST_CHECK(xmm0[0] == 0x41b00000);
+    TEST_CHECK(xmm0[1] == 0x427c0000);
+    TEST_CHECK(xmm0[2] == 0x42f80000);
+    TEST_CHECK(xmm0[3] == 0x434d0000);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_fma_scalar_variants(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\xc4\xe2\x71\x99\xc2"
+        "\xc4\xe2\xd9\xbf\xdd"
+        "\xc4\xc2\x45\xa6\xf0";
+    uint32_t xmm0[4] = {
+        0x40000000, 0x41300000, 0x41400000, 0x41500000,
+    };
+    uint32_t xmm1[4] = {
+        0x41200000, 0, 0, 0,
+    };
+    uint32_t xmm2[4] = {
+        0x40400000, 0, 0, 0,
+    };
+    uint64_t xmm3[2] = {
+        0x4014000000000000ULL, 0,
+    };
+    uint64_t xmm4[2] = {
+        0x4000000000000000ULL, 0,
+    };
+    uint64_t xmm5[2] = {
+        0x4008000000000000ULL, 0,
+    };
+    uint32_t ymm6[8] = {
+        0x3f800000, 0x40000000, 0x40400000, 0x40800000,
+        0x40a00000, 0x40c00000, 0x40e00000, 0x41000000,
+    };
+    uint32_t ymm7[8] = {
+        0x40000000, 0x40000000, 0x40000000, 0x40000000,
+        0x40000000, 0x40000000, 0x40000000, 0x40000000,
+    };
+    uint32_t ymm8[8] = {
+        0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,
+        0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,
+    };
+    uint32_t expected6[8] = {
+        0x3f800000, 0x40a00000, 0x40a00000, 0x41100000,
+        0x41100000, 0x41500000, 0x41500000, 0x41880000,
+    };
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_XMM0, &xmm0));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM1, &xmm1));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM2, &xmm2));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM3, &xmm3));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM4, &xmm4));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM5, &xmm5));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM6, &ymm6));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM7, &ymm7));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM8, &ymm8));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_XMM0, &xmm0));
+    OK(uc_reg_read(uc, UC_X86_REG_XMM3, &xmm3));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM6, &ymm6));
+
+    TEST_CHECK(xmm0[0] == 0x41800000);
+    TEST_CHECK(xmm3[0] == 0xc026000000000000ULL);
+    TEST_CHECK(memcmp(ymm6, expected6, sizeof(ymm6)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx2_broadcast_permute(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\xc4\xe2\x7d\x58\x00"
+        "\xc4\xe2\x4d\x36\xef"
+        "\xc4\xe2\x7d\x5a\x50\x20"
+        "\xc4\xe3\x75\x46\xda\x21";
+    uint64_t rax = code_start + 0x100;
+    uint32_t value = 0x11223344;
+    uint32_t block[4] = {
+        0xa0a1a2a3, 0xb0b1b2b3, 0xc0c1c2c3, 0xd0d1d2d3,
+    };
+    uint32_t ymm0[8];
+    uint32_t ymm1[8] = { 1, 2, 3, 4, 5, 6, 7, 8 };
+    uint32_t ymm2[8];
+    uint32_t ymm3[8];
+    uint32_t ymm5[8];
+    uint32_t ymm6[8] = { 7, 0, 6, 1, 5, 2, 4, 3 };
+    uint32_t ymm7[8] = { 10, 20, 30, 40, 50, 60, 70, 80 };
+    uint32_t expected3[8] = {
+        5, 6, 7, 8,
+        0xa0a1a2a3, 0xb0b1b2b3, 0xc0c1c2c3, 0xd0d1d2d3,
+    };
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, rax, &value, sizeof(value)));
+    OK(uc_mem_write(uc, rax + 0x20, block, sizeof(block)));
+
+    OK(uc_reg_write(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM6, &ymm6));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM7, &ymm7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM2, &ymm2));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM3, &ymm3));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM5, &ymm5));
+
+    for (size_t i = 0; i < 8; i++) {
+        TEST_CHECK(ymm0[i] == value);
+    }
+    TEST_CHECK(ymm5[0] == 80);
+    TEST_CHECK(ymm5[1] == 10);
+    TEST_CHECK(ymm5[2] == 70);
+    TEST_CHECK(ymm5[3] == 20);
+    TEST_CHECK(ymm5[4] == 60);
+    TEST_CHECK(ymm5[5] == 30);
+    TEST_CHECK(ymm5[6] == 50);
+    TEST_CHECK(ymm5[7] == 40);
+    TEST_CHECK(memcmp(ymm2, block, sizeof(block)) == 0);
+    TEST_CHECK(memcmp(&ymm2[4], block, sizeof(block)) == 0);
+    TEST_CHECK(memcmp(ymm3, expected3, sizeof(ymm3)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx2_variable_shifts(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\xc4\xe2\x75\x47\xc2"
+        "\xc4\xe2\x5d\x45\xdd"
+        "\xc4\xc2\x45\x46\xf0"
+        "\xc4\x42\xad\x47\xcb"
+        "\xc4\x42\x95\x45\xe6";
+    uint32_t ymm1[8] = {
+        1, 2, 0x80000000, 0xffffffff,
+        0x12345678, 0x7fffffff, 0x89abcdef, 0x00010000,
+    };
+    uint32_t ymm2[8] = { 0, 1, 4, 31, 32, 33, 8, 16 };
+    uint32_t ymm4[8] = {
+        0xffffffff, 0x80000000, 0x7fffffff, 0x12345678,
+        1, 0x80000001, 0xf0000000, 0x00ff00ff,
+    };
+    uint32_t ymm5[8] = { 0, 1, 4, 31, 32, 33, 8, 16 };
+    uint32_t ymm7[8] = {
+        0xffffffff, 0x80000000, 0x7fffffff, 0x80000000,
+        1, 0x80000000, 0xf0000000, 0x00ff00ff,
+    };
+    uint32_t ymm8[8] = { 0, 1, 4, 31, 32, 33, 8, 16 };
+    uint64_t ymm10[4] = {
+        1, 0x8000000000000000ULL,
+        0x0123456789abcdefULL, 0xffffffffffffffffULL,
+    };
+    uint64_t ymm11[4] = { 0, 1, 64, 8 };
+    uint64_t ymm13[4] = {
+        0xffffffffffffffffULL, 0x8000000000000000ULL,
+        0x0123456789abcdefULL, 0x00ff00ff00ff00ffULL,
+    };
+    uint64_t ymm14[4] = { 0, 1, 64, 8 };
+    uint32_t expected0[8] = {
+        1, 4, 0, 0x80000000, 0, 0, 0xabcdef00, 0,
+    };
+    uint32_t expected3[8] = {
+        0xffffffff, 0x40000000, 0x07ffffff, 0, 0, 0,
+        0x00f00000, 0x000000ff,
+    };
+    uint32_t expected6[8] = {
+        0xffffffff, 0xc0000000, 0x07ffffff, 0xffffffff,
+        0, 0xffffffff, 0xfff00000, 0x000000ff,
+    };
+    uint64_t expected9[4] = {
+        1, 0, 0, 0xffffffffffffff00ULL,
+    };
+    uint64_t expected12[4] = {
+        0xffffffffffffffffULL, 0x4000000000000000ULL,
+        0, 0x0000ff00ff00ff00ULL,
+    };
+    uint32_t ymm0[8];
+    uint32_t ymm3[8];
+    uint32_t ymm6[8];
+    uint64_t ymm9[4];
+    uint64_t ymm12[4];
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM2, &ymm2));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM4, &ymm4));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM5, &ymm5));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM7, &ymm7));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM8, &ymm8));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM10, &ymm10));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM11, &ymm11));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM13, &ymm13));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM14, &ymm14));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM3, &ymm3));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM6, &ymm6));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM9, &ymm9));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM12, &ymm12));
+
+    TEST_CHECK(memcmp(ymm0, expected0, sizeof(ymm0)) == 0);
+    TEST_CHECK(memcmp(ymm3, expected3, sizeof(ymm3)) == 0);
+    TEST_CHECK(memcmp(ymm6, expected6, sizeof(ymm6)) == 0);
+    TEST_CHECK(memcmp(ymm9, expected9, sizeof(ymm9)) == 0);
+    TEST_CHECK(memcmp(ymm12, expected12, sizeof(ymm12)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx2_mask_gather(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\xc4\xe2\x45\x8c\x30"
+        "\xc4\xe2\x6d\x90\x04\x88";
+    uint64_t rax = code_start + 0x100;
+    uint32_t data[8] = {
+        0x10001000, 0x20002000, 0x30003000, 0x40004000,
+        0x50005000, 0x60006000, 0x70007000, 0x80008000,
+    };
+    uint32_t ymm0[8] = {
+        0x11111111, 0x22222222, 0x33333333, 0x44444444,
+        0x55555555, 0x66666666, 0x77777777, 0x88888888,
+    };
+    uint32_t ymm1[8] = { 7, 0, 5, 2, 1, 4, 3, 6 };
+    uint32_t ymm2[8] = {
+        0x80000000, 0, 0xffffffff, 0,
+        0x80000000, 0x7fffffff, 0, 0xffffffff,
+    };
+    uint32_t ymm7[8] = {
+        0x80000000, 0, 0xffffffff, 0x7fffffff,
+        0x80000000, 0, 0xffffffff, 0,
+    };
+    uint32_t expected0[8] = {
+        0x80008000, 0x22222222, 0x60006000, 0x44444444,
+        0x20002000, 0x66666666, 0x77777777, 0x70007000,
+    };
+    uint32_t expected2[8] = { 0 };
+    uint32_t expected6[8] = {
+        0x10001000, 0, 0x30003000, 0,
+        0x50005000, 0, 0x70007000, 0,
+    };
+    uint32_t ymm6[8] = { 0 };
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_mem_write(uc, rax, data, sizeof(data)));
+
+    OK(uc_reg_write(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM2, &ymm2));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM7, &ymm7));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM2, &ymm2));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM6, &ymm6));
+
+    TEST_CHECK(memcmp(ymm0, expected0, sizeof(ymm0)) == 0);
+    TEST_CHECK(memcmp(ymm2, expected2, sizeof(ymm2)) == 0);
+    TEST_CHECK(memcmp(ymm6, expected6, sizeof(ymm6)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_avx_vzeroall(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc5\xfc\x77";
+    uint64_t ymm0[4] = {
+        0x1111111111111111ULL, 0x2222222222222222ULL,
+        0x3333333333333333ULL, 0x4444444444444444ULL,
+    };
+    uint64_t ymm15[4] = {
+        0x5555555555555555ULL, 0x6666666666666666ULL,
+        0x7777777777777777ULL, 0x8888888888888888ULL,
+    };
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM15, &ymm15));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM15, &ymm15));
+
+    for (size_t i = 0; i < 4; i++) {
+        TEST_CHECK(ymm0[i] == 0);
+        TEST_CHECK(ymm15[i] == 0);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_aes_pclmul(void)
+{
+    uc_engine *uc;
+    char code[] =
+        "\x66\x0f\x38\xdc\xc1"
+        "\x66\x0f\x38\xde\xd3"
+        "\x66\x0f\x38\xdb\xe5"
+        "\x66\x0f\x3a\xdf\xf7\x1b"
+        "\x66\x45\x0f\x3a\x44\xc1\x11";
+    uint8_t xmm0[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    uint8_t xmm1[16] = {
+        0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
+        0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+    };
+    uint8_t xmm2[16] = {
+        0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30,
+        0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a,
+    };
+    uint8_t xmm3[16] = {
+        0x13, 0x11, 0x1d, 0x7f, 0xe3, 0x94, 0x4a, 0x17,
+        0xf3, 0x07, 0xa7, 0x8b, 0x4d, 0x2b, 0x30, 0xc5,
+    };
+    uint8_t xmm5[16] = {
+        0xac, 0x19, 0x28, 0x57, 0x77, 0xfa, 0xd1, 0x5c,
+        0x66, 0xdc, 0x29, 0x00, 0xf3, 0x21, 0x41, 0x6a,
+    };
+    uint8_t xmm7[16] = {
+        0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6,
+        0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
+    };
+    uint8_t xmm8[16] = {
+        0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe,
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+    };
+    uint8_t xmm9[16] = {
+        0x55, 0xaa, 0x00, 0xff, 0x11, 0xee, 0x22, 0xdd,
+        0x33, 0xcc, 0x44, 0xbb, 0x55, 0xaa, 0x66, 0x99,
+    };
+    const uint8_t expected_xmm0[16] = {
+        0x6c, 0x77, 0xeb, 0xd5, 0xff, 0x6d, 0xf2, 0x7e,
+        0xaa, 0x00, 0x39, 0xf0, 0xd1, 0xe9, 0x8b, 0xa3,
+    };
+    const uint8_t expected_xmm2[16] = {
+        0xd4, 0x4f, 0x0a, 0xfb, 0xa3, 0x23, 0x94, 0xd3,
+        0x52, 0x84, 0x00, 0xc6, 0x83, 0x41, 0x84, 0x98,
+    };
+    const uint8_t expected_xmm4[16] = {
+        0x3b, 0x98, 0x30, 0x59, 0xd8, 0x02, 0x7e, 0xa4,
+        0x49, 0x17, 0x3b, 0xf6, 0xc2, 0xa6, 0x99, 0x04,
+    };
+    const uint8_t expected_xmm6[16] = {
+        0x34, 0xe4, 0xb5, 0x24, 0xff, 0xb5, 0x24, 0x34,
+        0x01, 0x8a, 0x84, 0xeb, 0x91, 0x84, 0xeb, 0x01,
+    };
+    const uint8_t expected_xmm8[16] = {
+        0x33, 0xf9, 0xa9, 0x26, 0x51, 0x89, 0xaf, 0x7e,
+        0x13, 0xd9, 0xa9, 0x26, 0x71, 0xa9, 0xaf, 0x7e,
+    };
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_XMM0, &xmm0));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM1, &xmm1));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM2, &xmm2));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM3, &xmm3));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM5, &xmm5));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM7, &xmm7));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM8, &xmm8));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM9, &xmm9));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_XMM0, &xmm0));
+    OK(uc_reg_read(uc, UC_X86_REG_XMM2, &xmm2));
+    OK(uc_reg_read(uc, UC_X86_REG_XMM4, &xmm5));
+    OK(uc_reg_read(uc, UC_X86_REG_XMM6, &xmm7));
+    OK(uc_reg_read(uc, UC_X86_REG_XMM8, &xmm8));
+
+    TEST_CHECK(memcmp(xmm0, expected_xmm0, sizeof(xmm0)) == 0);
+    TEST_CHECK(memcmp(xmm2, expected_xmm2, sizeof(xmm2)) == 0);
+    TEST_CHECK(memcmp(xmm5, expected_xmm4, sizeof(xmm5)) == 0);
+    TEST_CHECK(memcmp(xmm7, expected_xmm6, sizeof(xmm7)) == 0);
+    TEST_CHECK(memcmp(xmm8, expected_xmm8, sizeof(xmm8)) == 0);
+
+    OK(uc_close(uc));
+}
+
+static uint32_t test_x86_cpuid_7_0_ecx(uc_cpu_x86 cpu_model)
+{
+    uc_engine *uc;
+    char code[] = "\x0f\xa2";
+    uint32_t eax = 7;
+    uint32_t ecx = 0;
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, cpu_model));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_X86_REG_EAX, &eax));
+    OK(uc_reg_write(uc, UC_X86_REG_ECX, &ecx));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_ECX, &ecx));
+
+    OK(uc_close(uc));
+    return ecx;
+}
+
+static uint32_t test_x86_cpuid_7_0_ebx(uc_cpu_x86 cpu_model)
+{
+    uc_engine *uc;
+    char code[] = "\x0f\xa2";
+    uint32_t eax = 7;
+    uint32_t ebx;
+    uint32_t ecx = 0;
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, cpu_model));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+    OK(uc_reg_write(uc, UC_X86_REG_EAX, &eax));
+    OK(uc_reg_write(uc, UC_X86_REG_ECX, &ecx));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_EBX, &ebx));
+
+    OK(uc_close(uc));
+    return ebx;
+}
+
+static void test_x86_avx512_tcg_mask(void)
+{
+    const uint32_t avx512_mask = TEST_X86_CPUID_7_0_EBX_AVX512F |
+                                 TEST_X86_CPUID_7_0_EBX_AVX512DQ |
+                                 TEST_X86_CPUID_7_0_EBX_AVX512CD |
+                                 TEST_X86_CPUID_7_0_EBX_AVX512BW |
+                                 TEST_X86_CPUID_7_0_EBX_AVX512VL;
+    uint32_t ebx;
+
+    ebx = test_x86_cpuid_7_0_ebx(UC_CPU_X86_SKYLAKE_SERVER);
+    TEST_CHECK((ebx & TEST_X86_CPUID_7_0_EBX_AVX2) != 0);
+    TEST_CHECK((ebx & avx512_mask) == 0);
+
+    ebx = test_x86_cpuid_7_0_ebx(UC_CPU_X86_ICELAKE_SERVER);
+    TEST_CHECK((ebx & TEST_X86_CPUID_7_0_EBX_AVX2) != 0);
+    TEST_CHECK((ebx & avx512_mask) == 0);
+}
+
+static void test_x86_vaes_vex_gating(void)
+{
+    uc_engine *uc;
+    char vaesenc_xmm[] = "\xc4\xe2\x79\xdc\xc1";
+    char vaesenc_ymm[] = "\xc4\xe2\x7d\xdc\xc1";
+    char vaes_ymm[] =
+        "\xc4\xe2\x7d\xdc\xc1"
+        "\xc4\xe2\x6d\xde\xd3";
+    uint8_t xmm0[16] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+    };
+    uint8_t xmm1[16] = {
+        0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
+        0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+    };
+    uint8_t ymm0[32] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+        0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe,
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+    };
+    uint8_t ymm1[32] = {
+        0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
+        0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+        0x55, 0xaa, 0x00, 0xff, 0x11, 0xee, 0x22, 0xdd,
+        0x33, 0xcc, 0x44, 0xbb, 0x55, 0xaa, 0x66, 0x99,
+    };
+    uint8_t ymm2[32] = {
+        0x69, 0xc4, 0xe0, 0xd8, 0x6a, 0x7b, 0x04, 0x30,
+        0xd8, 0xcd, 0xb7, 0x80, 0x70, 0xb4, 0xc5, 0x5a,
+        0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe,
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+    };
+    uint8_t ymm3[32] = {
+        0x13, 0x11, 0x1d, 0x7f, 0xe3, 0x94, 0x4a, 0x17,
+        0xf3, 0x07, 0xa7, 0x8b, 0x4d, 0x2b, 0x30, 0xc5,
+        0x55, 0xaa, 0x00, 0xff, 0x11, 0xee, 0x22, 0xdd,
+        0x33, 0xcc, 0x44, 0xbb, 0x55, 0xaa, 0x66, 0x99,
+    };
+    const uint8_t expected_xmm0[16] = {
+        0x6c, 0x77, 0xeb, 0xd5, 0xff, 0x6d, 0xf2, 0x7e,
+        0xaa, 0x00, 0x39, 0xf0, 0xd1, 0xe9, 0x8b, 0xa3,
+    };
+    const uint8_t expected_ymm0[32] = {
+        0x6c, 0x77, 0xeb, 0xd5, 0xff, 0x6d, 0xf2, 0x7e,
+        0xaa, 0x00, 0x39, 0xf0, 0xd1, 0xe9, 0x8b, 0xa3,
+        0x6c, 0xfe, 0x98, 0x85, 0x72, 0x00, 0x6b, 0xfc,
+        0xf6, 0xaf, 0xcc, 0x10, 0x66, 0x5f, 0x61, 0xdf,
+    };
+    const uint8_t expected_ymm2[32] = {
+        0xd4, 0x4f, 0x0a, 0xfb, 0xa3, 0x23, 0x94, 0xd3,
+        0x52, 0x84, 0x00, 0xc6, 0x83, 0x41, 0x84, 0x98,
+        0x3b, 0xc6, 0x56, 0xbd, 0x3d, 0x4c, 0xe5, 0x5d,
+        0x85, 0x0f, 0xac, 0x73, 0x29, 0xbf, 0xc3, 0x09,
+    };
+
+    TEST_CHECK((test_x86_cpuid_7_0_ecx(UC_CPU_X86_HASWELL) &
+                TEST_X86_CPUID_7_0_ECX_VAES) == 0);
+    TEST_CHECK((test_x86_cpuid_7_0_ecx(UC_CPU_X86_ICELAKE_CLIENT) &
+                TEST_X86_CPUID_7_0_ECX_VAES) != 0);
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, vaesenc_xmm, sizeof(vaesenc_xmm) - 1));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM0, &xmm0));
+    OK(uc_reg_write(uc, UC_X86_REG_XMM1, &xmm1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(vaesenc_xmm) - 1,
+                    0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_XMM0, &xmm0));
+    TEST_CHECK(memcmp(xmm0, expected_xmm0, sizeof(xmm0)) == 0);
+    OK(uc_close(uc));
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, vaesenc_ymm, sizeof(vaesenc_ymm) - 1));
+    uc_assert_err(UC_ERR_INSN_INVALID,
+                  uc_emu_start(uc, code_start,
+                               code_start + sizeof(vaesenc_ymm) - 1, 0, 0));
+    OK(uc_close(uc));
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_ICELAKE_CLIENT));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, vaes_ymm, sizeof(vaes_ymm) - 1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM2, &ymm2));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM3, &ymm3));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(vaes_ymm) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM2, &ymm2));
+    TEST_CHECK(memcmp(ymm0, expected_ymm0, sizeof(ymm0)) == 0);
+    TEST_CHECK(memcmp(ymm2, expected_ymm2, sizeof(ymm2)) == 0);
+    OK(uc_close(uc));
+}
+
+static void test_x86_vpclmulqdq_tcg_mask(void)
+{
+    uc_engine *uc;
+    char pclmul_xmm[] = "\xc4\xe3\x79\x44\xc1\x11";
+    char pclmul_ymm[] = "\xc4\xe3\x7d\x44\xc1\x11";
+    uint8_t ymm0[32] = {
+        0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+        0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+        0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe,
+        0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+    };
+    uint8_t ymm1[32] = {
+        0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
+        0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+        0x55, 0xaa, 0x00, 0xff, 0x11, 0xee, 0x22, 0xdd,
+        0x33, 0xcc, 0x44, 0xbb, 0x55, 0xaa, 0x66, 0x99,
+    };
+    const uint8_t expected_ymm0[32] = {
+        0xb8, 0xfc, 0xa8, 0x02, 0x00, 0xff, 0x10, 0x01,
+        0xa8, 0xfd, 0xb8, 0x03, 0x10, 0xfe, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    };
+
+    TEST_CHECK((test_x86_cpuid_7_0_ecx(UC_CPU_X86_HASWELL) &
+                TEST_X86_CPUID_7_0_ECX_VPCLMULQDQ) == 0);
+    TEST_CHECK((test_x86_cpuid_7_0_ecx(UC_CPU_X86_ICELAKE_CLIENT) &
+                TEST_X86_CPUID_7_0_ECX_VPCLMULQDQ) == 0);
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, pclmul_xmm, sizeof(pclmul_xmm) - 1));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM0, &ymm0));
+    OK(uc_reg_write(uc, UC_X86_REG_YMM1, &ymm1));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(pclmul_xmm) - 1,
+                    0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_YMM0, &ymm0));
+    TEST_CHECK(memcmp(ymm0, expected_ymm0, sizeof(ymm0)) == 0);
+    OK(uc_close(uc));
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_ICELAKE_CLIENT));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, pclmul_ymm, sizeof(pclmul_ymm) - 1));
+    uc_assert_err(UC_ERR_INSN_INVALID,
+                  uc_emu_start(uc, code_start,
+                               code_start + sizeof(pclmul_ymm) - 1, 0, 0));
+
+    OK(uc_close(uc));
+}
+
 static void test_x86_relative_jump(void)
 {
     uc_engine *uc;
@@ -828,6 +1642,147 @@ static void test_x86_486_cpuid(void)
     OK(uc_close(uc));
 }
 
+static void test_x86_qemu72_xsave_cpuid(void)
+{
+    uc_engine *uc;
+    char code[] = "\x0f\xa2";
+    uint32_t eax = 0xd;
+    uint32_t ebx;
+    uint32_t ecx = 1;
+    uint32_t edx;
+    uint32_t eip = code_start;
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_32, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
+    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
+    OK(uc_mem_write(uc, code_start, code, sizeof(code) - 1));
+
+    OK(uc_reg_write(uc, UC_X86_REG_EAX, &eax));
+    OK(uc_reg_write(uc, UC_X86_REG_ECX, &ecx));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_EAX, &eax));
+    OK(uc_reg_read(uc, UC_X86_REG_EBX, &ebx));
+    OK(uc_reg_read(uc, UC_X86_REG_ECX, &ecx));
+    OK(uc_reg_read(uc, UC_X86_REG_EDX, &edx));
+
+    TEST_CHECK(eax != 0);
+    TEST_CHECK(ebx >= 512);
+    TEST_CHECK((ecx & ~(1U << 15)) == 0);
+    TEST_CHECK(edx == 0);
+
+    eax = 0xd;
+    ecx = 0;
+    OK(uc_reg_write(uc, UC_X86_REG_EAX, &eax));
+    OK(uc_reg_write(uc, UC_X86_REG_ECX, &ecx));
+    OK(uc_reg_write(uc, UC_X86_REG_EIP, &eip));
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+
+    OK(uc_reg_read(uc, UC_X86_REG_EAX, &eax));
+    OK(uc_reg_read(uc, UC_X86_REG_EBX, &ebx));
+    OK(uc_reg_read(uc, UC_X86_REG_ECX, &ecx));
+    OK(uc_reg_read(uc, UC_X86_REG_EDX, &edx));
+
+    TEST_CHECK((eax & 0x7) == 0x7);
+    TEST_CHECK(ebx >= 512);
+    TEST_CHECK(ecx >= ebx);
+    TEST_CHECK(edx == 0);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_opmask_registers(void)
+{
+    uc_engine *uc;
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+
+    for (int i = 0; i < 8; i++) {
+        uint64_t in = 0x1122334455667700ULL + i;
+        uint64_t out = 0;
+        int reg = UC_X86_REG_K0 + i;
+
+        OK(uc_reg_write(uc, reg, &in));
+        OK(uc_reg_read(uc, reg, &out));
+        TEST_CHECK(out == in);
+    }
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_qemu72_msr_state(void)
+{
+    uc_engine *uc;
+    uc_x86_msr msr;
+
+    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+
+    msr.rid = TEST_MSR_IA32_XFD;
+    msr.value = 0x12345678abcdef00ULL;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 0x12345678abcdef00ULL);
+
+    msr.rid = TEST_MSR_IA32_XFD_ERR;
+    msr.value = 0xfedcba9876543210ULL;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 0xfedcba9876543210ULL);
+
+    msr.rid = TEST_MSR_IA32_PKRS;
+    msr.value = 0xa5a55a5aULL;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 0xa5a55a5aULL);
+
+    msr.rid = TEST_MSR_ARCH_LBR_CTL;
+    msr.value = 0x19;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 0x19);
+
+    msr.rid = TEST_MSR_ARCH_LBR_DEPTH;
+    msr.value = 32;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 32);
+
+    msr.rid = TEST_MSR_ARCH_LBR_FROM_0 + 3;
+    msr.value = 0x1111222233334444ULL;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 0x1111222233334444ULL);
+
+    msr.rid = TEST_MSR_ARCH_LBR_TO_0 + 3;
+    msr.value = 0x5555666677778888ULL;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 0x5555666677778888ULL);
+
+    msr.rid = TEST_MSR_ARCH_LBR_INFO_0 + 3;
+    msr.value = 0x9999aaaabbbbccccULL;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = 0;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK(msr.value == 0x9999aaaabbbbccccULL);
+
+    msr.rid = TEST_MSR_IA32_XSS;
+    msr.value = UINT64_MAX;
+    OK(uc_reg_write(uc, UC_X86_REG_MSR, &msr));
+    msr.value = UINT64_MAX;
+    OK(uc_reg_read(uc, UC_X86_REG_MSR, &msr));
+    TEST_CHECK((msr.value & ~(1ULL << 15)) == 0);
+
+    OK(uc_close(uc));
+}
+
 // This is a regression bug.
 static void test_x86_clear_tb_cache(void)
 {
@@ -1189,6 +2144,27 @@ static void test_x86_pdep32_zero_extend(void)
     OK(uc_close(uc));
 }
 
+static void test_x86_pext32_zero_extend(void)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe2\x62\xf5\xc1"; /* pext eax, ebx, ecx */
+    uint64_t rax = 0xffffffffffffffffULL;
+    uint64_t rbx = 0xffffffffabcdef00ULL;
+    uint64_t rcx = 0xffffffff0000ff00ULL;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &rbx));
+    OK(uc_reg_write(uc, UC_X86_REG_RCX, &rcx));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 1));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+
+    TEST_CHECK(rax == 0xef);
+
+    OK(uc_close(uc));
+}
+
 static void test_x86_nested_emu_start_cb(uc_engine *uc, uint64_t addr,
                                          size_t size, void *data)
 {
@@ -1329,6 +2305,65 @@ static void test_x86_blsi_cf(void)
     test_x86_blsi_cf_case(0, 0, false, true);
 }
 
+static void test_x86_blsr_flags_case(uint64_t src, uint64_t expected_dst,
+                                     bool expected_cf, bool expected_zf)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe2\xf8\xf3\xcb"; /* blsr rax, rbx */
+    uint64_t rax;
+    uint64_t rflags;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &src));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_read(uc, UC_X86_REG_RFLAGS, &rflags));
+
+    TEST_CHECK(rax == expected_dst);
+    TEST_CHECK((bool)(rflags & 1) == expected_cf);
+    TEST_CHECK((bool)(rflags & 0x40) == expected_zf);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_blsr_flags(void)
+{
+    test_x86_blsr_flags_case(0x28, 0x20, false, false);
+    test_x86_blsr_flags_case(1, 0, false, true);
+    test_x86_blsr_flags_case(0, 0, true, true);
+}
+
+static void test_x86_blsmsk_flags_case(uint64_t src, uint64_t expected_dst,
+                                       bool expected_cf, bool expected_zf,
+                                       bool expected_sf)
+{
+    uc_engine *uc;
+    char code[] = "\xc4\xe2\xf8\xf3\xd3"; /* blsmsk rax, rbx */
+    uint64_t rax;
+    uint64_t rflags;
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code) - 1);
+    OK(uc_reg_write(uc, UC_X86_REG_RBX, &src));
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc, UC_X86_REG_RAX, &rax));
+    OK(uc_reg_read(uc, UC_X86_REG_RFLAGS, &rflags));
+
+    TEST_CHECK(rax == expected_dst);
+    TEST_CHECK((bool)(rflags & 1) == expected_cf);
+    TEST_CHECK((bool)(rflags & 0x40) == expected_zf);
+    TEST_CHECK((bool)(rflags & 0x80) == expected_sf);
+
+    OK(uc_close(uc));
+}
+
+static void test_x86_blsmsk_flags(void)
+{
+    test_x86_blsmsk_flags_case(0x28, 0x0f, false, false, false);
+    test_x86_blsmsk_flags_case(0, UINT64_MAX, true, false, true);
+}
+
 static void test_x86_bzhi_index_case(uint64_t index, uint64_t expected_dst,
                                      bool expected_cf, bool expected_sf)
 {
@@ -1487,11 +2522,12 @@ static void test_x86_invalid_vex_l(void)
 {
     uc_engine *uc;
 
-    /* vmovdqu ymm1, [rcx] */
-    char code[] = {'\xC5', '\xFE', '\x6F', '\x09'};
+    /* andn eax, eax, eax with reserved VEX.L set */
+    char code[] = {'\xC4', '\xE2', '\x7F', '\xF2', '\xC0'};
 
     /* initialize memory and run emulation  */
     OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));
+    OK(uc_ctl_set_cpu_model(uc, UC_CPU_X86_HASWELL));
     OK(uc_mem_map(uc, 0, 2 * 1024 * 1024, UC_PROT_ALL));
 
     OK(uc_mem_write(uc, 0, code, sizeof(code) / sizeof(code[0])));
@@ -2332,7 +3368,7 @@ static void test_x86_hook_insn_rdtscp(void)
     OK(uc_close(uc));
 }
 
-static void test_x86_dr7()
+static void test_x86_dr7(void)
 {
     uc_engine *uc;
     char code[] =
@@ -2355,7 +3391,7 @@ static void test_x86_hook_block_cb(uc_engine *uc, uint64_t address,
     *((uint64_t *)user_data) += 1;
 }
 
-static void test_x86_hook_block()
+static void test_x86_hook_block(void)
 {
     uc_engine *uc;
     char code[] = "\xeb\x02\x90\x90\x90\x90\x90\x90"; // jmp 4; nop; nop; nop;
@@ -2410,6 +3446,19 @@ TEST_LIST = {
     {"test_x86_out", test_x86_out},
     {"test_x86_mem_hook_all", test_x86_mem_hook_all},
     {"test_x86_inc_dec_pxor", test_x86_inc_dec_pxor},
+    {"test_x86_avx_vpxor_ymm", test_x86_avx_vpxor_ymm},
+    {"test_x86_avx_vex128_zero_upper", test_x86_avx_vex128_zero_upper},
+    {"test_x86_avx_scalar_zero_upper", test_x86_avx_scalar_zero_upper},
+    {"test_x86_avx_fma_ps", test_x86_avx_fma_ps},
+    {"test_x86_fma_scalar_variants", test_x86_fma_scalar_variants},
+    {"test_x86_avx2_broadcast_permute", test_x86_avx2_broadcast_permute},
+    {"test_x86_avx2_variable_shifts", test_x86_avx2_variable_shifts},
+    {"test_x86_avx2_mask_gather", test_x86_avx2_mask_gather},
+    {"test_x86_avx_vzeroall", test_x86_avx_vzeroall},
+    {"test_x86_aes_pclmul", test_x86_aes_pclmul},
+    {"test_x86_avx512_tcg_mask", test_x86_avx512_tcg_mask},
+    {"test_x86_vaes_vex_gating", test_x86_vaes_vex_gating},
+    {"test_x86_vpclmulqdq_tcg_mask", test_x86_vpclmulqdq_tcg_mask},
     {"test_x86_relative_jump", test_x86_relative_jump},
     {"test_x86_loop", test_x86_loop},
     {"test_x86_invalid_mem_read", test_x86_invalid_mem_read},
@@ -2430,6 +3479,9 @@ TEST_LIST = {
     {"test_x86_sysenter", test_x86_sysenter},
     {"test_x86_hook_cpuid", test_x86_hook_cpuid},
     {"test_x86_486_cpuid", test_x86_486_cpuid},
+    {"test_x86_qemu72_xsave_cpuid", test_x86_qemu72_xsave_cpuid},
+    {"test_x86_opmask_registers", test_x86_opmask_registers},
+    {"test_x86_qemu72_msr_state", test_x86_qemu72_msr_state},
     {"test_x86_clear_tb_cache", test_x86_clear_tb_cache},
     {"test_x86_clear_empty_tb", test_x86_clear_empty_tb},
     {"test_x86_hook_tcg_op", test_x86_hook_tcg_op},
@@ -2440,11 +3492,14 @@ TEST_LIST = {
     {"test_x86_shld_rip_relative_imm", test_x86_shld_rip_relative_imm},
     {"test_x86_shrd_rip_relative_imm", test_x86_shrd_rip_relative_imm},
     {"test_x86_pdep32_zero_extend", test_x86_pdep32_zero_extend},
+    {"test_x86_pext32_zero_extend", test_x86_pext32_zero_extend},
     {"test_x86_nested_emu_start", test_x86_nested_emu_start},
     {"test_x86_nested_emu_stop", test_x86_nested_emu_stop},
     {"test_x86_64_nested_emu_start_error", test_x86_64_nested_emu_start_error},
     {"test_x86_eflags_reserved_bit", test_x86_eflags_reserved_bit},
     {"test_x86_blsi_cf", test_x86_blsi_cf},
+    {"test_x86_blsr_flags", test_x86_blsr_flags},
+    {"test_x86_blsmsk_flags", test_x86_blsmsk_flags},
     {"test_x86_bzhi_index_boundary", test_x86_bzhi_index_boundary},
     {"test_x86_nested_uc_emu_start_exits", test_x86_nested_uc_emu_start_exits},
     {"test_x86_clear_count_cache", test_x86_clear_count_cache},
diff --git a/uc.c b/uc.c
index 5cd49fd8a2..df2d9625f7 100644
--- a/uc.c
+++ b/uc.c
@@ -35,6 +35,8 @@ static void clear_deleted_hooks(uc_engine *uc);
 static uc_err uc_snapshot(uc_engine *uc);
 static uc_err uc_restore_latest_snapshot(uc_engine *uc);
 
+#define UC_MTE_TAG_STORAGE_GRANULE 32
+
 #if defined(__APPLE__) && defined(HAVE_PTHREAD_JIT_PROTECT) &&                 \
     (defined(__arm__) || defined(__aarch64__))
 static void save_jit_state(uc_engine *uc)
@@ -392,6 +394,12 @@ uc_err uc_open(uc_arch arch, uc_mode mode, uc_engine **result)
                 free(uc);
                 return UC_ERR_MODE;
             }
+            if (((mode & UC_MODE_MICRO) && !(mode & UC_MODE_MIPS32)) ||
+                ((mode & UC_MODE_MIPS3) && !(mode & UC_MODE_MIPS64)) ||
+                ((mode & UC_MODE_MIPS32R6) && !(mode & UC_MODE_MIPS32))) {
+                free(uc);
+                return UC_ERR_MODE;
+            }
             if (mode & UC_MODE_BIG_ENDIAN) {
 #ifdef UNICORN_HAS_MIPS
                 if (mode & UC_MODE_MIPS32) {
@@ -1475,6 +1483,68 @@ static uint8_t *copy_region(struct uc_struct *uc, MemoryRegion *mr)
     return block;
 }
 
+static bool copy_mte_tags(RAMBlock *block, uint8_t **tags,
+                          ram_addr_t *tag_size)
+{
+    *tags = NULL;
+    *tag_size = 0;
+    if (block->mte_tags == NULL || block->mte_tags_size == 0) {
+        return true;
+    }
+
+    *tags = g_malloc0(block->mte_tags_size);
+    if (*tags == NULL) {
+        return false;
+    }
+
+    memcpy(*tags, block->mte_tags, block->mte_tags_size);
+    *tag_size = block->mte_tags_size;
+    return true;
+}
+
+static bool restore_mte_tags(struct uc_struct *uc, uint64_t address,
+                             uint64_t size, uint64_t source_offset,
+                             const uint8_t *tags, ram_addr_t tag_size)
+{
+    MemoryRegion *mr;
+    RAMBlock *block;
+    ram_addr_t source_tag_offset;
+    ram_addr_t copy_size;
+
+    if (tags == NULL || size == 0) {
+        return true;
+    }
+
+    source_tag_offset = source_offset / UC_MTE_TAG_STORAGE_GRANULE;
+    if (source_tag_offset >= tag_size) {
+        return true;
+    }
+
+    mr = uc->memory_mapping(uc, address);
+    if (mr == NULL || !mr->ram || mr->ram_block == NULL) {
+        return false;
+    }
+
+    block = mr->ram_block;
+    if (block->mte_tags == NULL) {
+        block->mte_tags_size =
+            (block->max_length + UC_MTE_TAG_STORAGE_GRANULE - 1) /
+            UC_MTE_TAG_STORAGE_GRANULE;
+        block->mte_tags = g_malloc0(block->mte_tags_size);
+        if (block->mte_tags == NULL) {
+            block->mte_tags_size = 0;
+            return false;
+        }
+    }
+
+    copy_size = (size + UC_MTE_TAG_STORAGE_GRANULE - 1) /
+                UC_MTE_TAG_STORAGE_GRANULE;
+    copy_size = MIN(copy_size, tag_size - source_tag_offset);
+    copy_size = MIN(copy_size, block->mte_tags_size);
+    memcpy(block->mte_tags, tags + source_tag_offset, copy_size);
+    return true;
+}
+
 /*
     This function is similar to split_region, but for MMIO memory.
 
@@ -1578,6 +1648,8 @@ static bool split_region(struct uc_struct *uc, MemoryRegion *mr,
     uint64_t l_size, m_size, r_size;
     RAMBlock *block = NULL;
     bool prealloc = false;
+    uint8_t *tag_backup = NULL;
+    ram_addr_t tag_backup_size = 0;
 
     chunk_end = address + size;
 
@@ -1617,6 +1689,9 @@ static bool split_region(struct uc_struct *uc, MemoryRegion *mr,
             return false;
         }
     }
+    if (!copy_mte_tags(block, &tag_backup, &tag_backup_size)) {
+        goto error;
+    }
 
     // save the essential information required for the split before mr gets
     // deleted
@@ -1667,6 +1742,10 @@ static bool split_region(struct uc_struct *uc, MemoryRegion *mr,
                 goto error;
             }
         }
+        if (!restore_mte_tags(uc, begin, l_size, 0, tag_backup,
+                              tag_backup_size)) {
+            goto error;
+        }
     }
 
     if (m_size > 0 && !do_delete) {
@@ -1684,6 +1763,10 @@ static bool split_region(struct uc_struct *uc, MemoryRegion *mr,
                 goto error;
             }
         }
+        if (!restore_mte_tags(uc, address, m_size, l_size, tag_backup,
+                              tag_backup_size)) {
+            goto error;
+        }
     }
 
     if (r_size > 0) {
@@ -1701,14 +1784,20 @@ static bool split_region(struct uc_struct *uc, MemoryRegion *mr,
                 goto error;
             }
         }
+        if (!restore_mte_tags(uc, chunk_end, r_size, l_size + m_size,
+                              tag_backup, tag_backup_size)) {
+            goto error;
+        }
     }
 
+    g_free(tag_backup);
     if (!prealloc) {
         free(backup);
     }
     return true;
 
 error:
+    g_free(tag_backup);
     if (!prealloc) {
         free(backup);
     }
@@ -2881,6 +2970,11 @@ uc_err uc_ctl(uc_engine *uc, uc_control_type control, ...)
                     err = UC_ERR_ARG;
                     break;
                 }
+            } else if (uc->arch == UC_ARCH_TRICORE) {
+                if (model >= UC_CPU_TRICORE_ENDING) {
+                    err = UC_ERR_ARG;
+                    break;
+                }
             } else {
                 err = UC_ERR_ARG;
                 break;

From ed08a4f387422acdbcccf0bcc229d401c1a768b5 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 09:51:54 +0300
Subject: [PATCH 12/32] mingw build fix

---
 qemu/target/ppc/translate/fp-impl.inc.c  |  2 ++
 qemu/target/ppc/translate/vsx-impl.inc.c | 10 ++++++++++
 2 files changed, 12 insertions(+)

diff --git a/qemu/target/ppc/translate/fp-impl.inc.c b/qemu/target/ppc/translate/fp-impl.inc.c
index 2e247d4dbe..18b59bb7af 100644
--- a/qemu/target/ppc/translate/fp-impl.inc.c
+++ b/qemu/target/ppc/translate/fp-impl.inc.c
@@ -1289,6 +1289,7 @@ GEN_STFS(stfd, st64_i64, 0x16, PPC_FLOAT);
 /* stfs stfsu stfsux stfsx */
 GEN_STFS(stfs, st32fs, 0x14, PPC_FLOAT);
 
+#if defined(TARGET_PPC64)
 static void gen_plfs(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -1376,6 +1377,7 @@ static void gen_pstfd(DisasContext *ctx)
     tcg_temp_free(tcg_ctx, ea);
     tcg_temp_free_i64(tcg_ctx, t0);
 }
+#endif
 
 /* stfdepx (external PID lfdx) */
 static void gen_stfdepx(DisasContext *ctx)
diff --git a/qemu/target/ppc/translate/vsx-impl.inc.c b/qemu/target/ppc/translate/vsx-impl.inc.c
index 54cf755a71..288e0252a2 100644
--- a/qemu/target/ppc/translate/vsx-impl.inc.c
+++ b/qemu/target/ppc/translate/vsx-impl.inc.c
@@ -345,10 +345,12 @@ static void gen_##name(DisasContext *ctx)                   \
 VSX_VECTOR_LOAD(lxv, ld_i64, 0)
 VSX_VECTOR_LOAD(lxvx, ld_i64, 1)
 
+#if defined(TARGET_PPC64)
 static int prefixed_8ls_xt(DisasContext *ctx)
 {
     return ((opc1(ctx->opcode) & 1) << 5) | rD(ctx->opcode);
 }
+#endif
 
 static int vsx_tsxp_rt(DisasContext *ctx)
 {
@@ -478,6 +480,7 @@ static void gen_lxvpx(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
 
+#if defined(TARGET_PPC64)
 static void gen_plxv(DisasContext *ctx, bool paired)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -506,6 +509,7 @@ static void gen_plxv(DisasContext *ctx, bool paired)
     tcg_temp_free_i64(tcg_ctx, xth);
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
+#endif
 
 #define VSX_VECTOR_STORE(name, op, indexed)                 \
 static void gen_##name(DisasContext *ctx)                   \
@@ -609,6 +613,7 @@ static void gen_stxvpx(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
 
+#if defined(TARGET_PPC64)
 static void gen_pstxv(DisasContext *ctx, bool paired)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -637,6 +642,7 @@ static void gen_pstxv(DisasContext *ctx, bool paired)
     tcg_temp_free_i64(tcg_ctx, xth);
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
+#endif
 
 static void gen_lxvrx(DisasContext *ctx, MemOp mop)
 {
@@ -782,6 +788,7 @@ static void gen_##name(DisasContext *ctx)                         \
 VSX_LOAD_SCALAR_DS(lxsd, ld64_i64)
 VSX_LOAD_SCALAR_DS(lxssp, ld32fs)
 
+#if defined(TARGET_PPC64)
 static void gen_plxsd(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -837,6 +844,7 @@ static void gen_plxssp(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, xth);
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
+#endif
 
 #define VSX_STORE_SCALAR(name, operation)                     \
 static void gen_##name(DisasContext *ctx)                     \
@@ -1017,6 +1025,7 @@ static void gen_##name(DisasContext *ctx)                         \
 VSX_STORE_SCALAR_DS(stxsd, st64_i64)
 VSX_STORE_SCALAR_DS(stxssp, st32fs)
 
+#if defined(TARGET_PPC64)
 static void gen_pstxsd(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -1060,6 +1069,7 @@ static void gen_pstxssp(DisasContext *ctx)
     tcg_temp_free(tcg_ctx, ea);
     tcg_temp_free_i64(tcg_ctx, xth);
 }
+#endif
 
 static void gen_mfvsrwz(DisasContext *ctx)
 {

From 117cea33d9de57e9fa8eb0d9517984ff4d502e16 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 09:51:54 +0300
Subject: [PATCH 13/32] mingw build fix

---
 CMakeLists.txt                           | 17 +++++++++++++++++
 qemu/target/ppc/translate/fp-impl.inc.c  |  2 ++
 qemu/target/ppc/translate/vsx-impl.inc.c | 10 ++++++++++
 tests/unit/test_arm.c                    |  6 ++++++
 tests/unit/test_ppc.c                    |  6 +++---
 5 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2bf906beb3..9f89810692 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1536,6 +1536,23 @@ if(UNICORN_BUILD_TESTS)
         target_link_libraries(${TEST_FILE} PRIVATE
             ${SAMPLES_LIB}
         )
+        if(TEST_FILE STREQUAL "test_arm")
+            if(MSVC)
+                target_compile_options(${TEST_FILE} PRIVATE
+                    -DNEED_CPU_H
+                    /FIarm.h
+                    /I${CMAKE_CURRENT_SOURCE_DIR}/msvc/arm-softmmu
+                    /I${CMAKE_CURRENT_SOURCE_DIR}/qemu/target/arm
+                )
+            else()
+                target_compile_options(${TEST_FILE} PRIVATE
+                    -DNEED_CPU_H
+                    -include arm.h
+                    -I${CMAKE_BINARY_DIR}/arm-softmmu
+                    -I${CMAKE_CURRENT_SOURCE_DIR}/qemu/target/arm
+                )
+            endif()
+        endif()
         add_test(${TEST_FILE} ${TEST_FILE})
         if(ANDROID_ABI)
             file(APPEND ${CMAKE_BINARY_DIR}/adb.sh "adb push ${TEST_FILE} /data/local/tmp/build/\n")
diff --git a/qemu/target/ppc/translate/fp-impl.inc.c b/qemu/target/ppc/translate/fp-impl.inc.c
index 2e247d4dbe..18b59bb7af 100644
--- a/qemu/target/ppc/translate/fp-impl.inc.c
+++ b/qemu/target/ppc/translate/fp-impl.inc.c
@@ -1289,6 +1289,7 @@ GEN_STFS(stfd, st64_i64, 0x16, PPC_FLOAT);
 /* stfs stfsu stfsux stfsx */
 GEN_STFS(stfs, st32fs, 0x14, PPC_FLOAT);
 
+#if defined(TARGET_PPC64)
 static void gen_plfs(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -1376,6 +1377,7 @@ static void gen_pstfd(DisasContext *ctx)
     tcg_temp_free(tcg_ctx, ea);
     tcg_temp_free_i64(tcg_ctx, t0);
 }
+#endif
 
 /* stfdepx (external PID lfdx) */
 static void gen_stfdepx(DisasContext *ctx)
diff --git a/qemu/target/ppc/translate/vsx-impl.inc.c b/qemu/target/ppc/translate/vsx-impl.inc.c
index 54cf755a71..288e0252a2 100644
--- a/qemu/target/ppc/translate/vsx-impl.inc.c
+++ b/qemu/target/ppc/translate/vsx-impl.inc.c
@@ -345,10 +345,12 @@ static void gen_##name(DisasContext *ctx)                   \
 VSX_VECTOR_LOAD(lxv, ld_i64, 0)
 VSX_VECTOR_LOAD(lxvx, ld_i64, 1)
 
+#if defined(TARGET_PPC64)
 static int prefixed_8ls_xt(DisasContext *ctx)
 {
     return ((opc1(ctx->opcode) & 1) << 5) | rD(ctx->opcode);
 }
+#endif
 
 static int vsx_tsxp_rt(DisasContext *ctx)
 {
@@ -478,6 +480,7 @@ static void gen_lxvpx(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
 
+#if defined(TARGET_PPC64)
 static void gen_plxv(DisasContext *ctx, bool paired)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -506,6 +509,7 @@ static void gen_plxv(DisasContext *ctx, bool paired)
     tcg_temp_free_i64(tcg_ctx, xth);
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
+#endif
 
 #define VSX_VECTOR_STORE(name, op, indexed)                 \
 static void gen_##name(DisasContext *ctx)                   \
@@ -609,6 +613,7 @@ static void gen_stxvpx(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
 
+#if defined(TARGET_PPC64)
 static void gen_pstxv(DisasContext *ctx, bool paired)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -637,6 +642,7 @@ static void gen_pstxv(DisasContext *ctx, bool paired)
     tcg_temp_free_i64(tcg_ctx, xth);
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
+#endif
 
 static void gen_lxvrx(DisasContext *ctx, MemOp mop)
 {
@@ -782,6 +788,7 @@ static void gen_##name(DisasContext *ctx)                         \
 VSX_LOAD_SCALAR_DS(lxsd, ld64_i64)
 VSX_LOAD_SCALAR_DS(lxssp, ld32fs)
 
+#if defined(TARGET_PPC64)
 static void gen_plxsd(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -837,6 +844,7 @@ static void gen_plxssp(DisasContext *ctx)
     tcg_temp_free_i64(tcg_ctx, xth);
     tcg_temp_free_i64(tcg_ctx, xtl);
 }
+#endif
 
 #define VSX_STORE_SCALAR(name, operation)                     \
 static void gen_##name(DisasContext *ctx)                     \
@@ -1017,6 +1025,7 @@ static void gen_##name(DisasContext *ctx)                         \
 VSX_STORE_SCALAR_DS(stxsd, st64_i64)
 VSX_STORE_SCALAR_DS(stxssp, st32fs)
 
+#if defined(TARGET_PPC64)
 static void gen_pstxsd(DisasContext *ctx)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
@@ -1060,6 +1069,7 @@ static void gen_pstxssp(DisasContext *ctx)
     tcg_temp_free(tcg_ctx, ea);
     tcg_temp_free_i64(tcg_ctx, xth);
 }
+#endif
 
 static void gen_mfvsrwz(DisasContext *ctx)
 {
diff --git a/tests/unit/test_arm.c b/tests/unit/test_arm.c
index c74a97494f..2b2c8ceb94 100644
--- a/tests/unit/test_arm.c
+++ b/tests/unit/test_arm.c
@@ -1,8 +1,14 @@
 #include "unicorn_test.h"
 #include "uc_priv.h"
+#ifndef NEED_CPU_H
 #define NEED_CPU_H
+#define TEST_ARM_LOCAL_NEED_CPU_H
+#endif
 #include "target/arm/cpu.h"
+#ifdef TEST_ARM_LOCAL_NEED_CPU_H
 #undef NEED_CPU_H
+#undef TEST_ARM_LOCAL_NEED_CPU_H
+#endif
 #include <string.h>
 
 const uint64_t code_start = 0x1000;
diff --git a/tests/unit/test_ppc.c b/tests/unit/test_ppc.c
index 950e8349c6..b684d32d1a 100644
--- a/tests/unit/test_ppc.c
+++ b/tests/unit/test_ppc.c
@@ -490,7 +490,7 @@ static void test_ppc64_power10_vmx_mask_materialize_extract(void)
     uc_engine *uc;
     uint64_t b_addr = code_start + 0x1000;
     uint64_t expand_addr = code_start + 0x1100;
-    uint64_t h_addr = code_start + 0x1200;
+    uint64_t half_addr = code_start + 0x1200;
     uint64_t w_addr = code_start + 0x1300;
     uint64_t d_addr = code_start + 0x1400;
     uint64_t q_addr = code_start + 0x1500;
@@ -570,7 +570,7 @@ static void test_ppc64_power10_vmx_mask_materialize_extract(void)
     OK(uc_reg_write(uc, UC_PPC_REG_16, &q_addr));
     OK(uc_reg_write(uc, UC_PPC_REG_27, &b_addr));
     OK(uc_reg_write(uc, UC_PPC_REG_28, &expand_addr));
-    OK(uc_reg_write(uc, UC_PPC_REG_29, &h_addr));
+    OK(uc_reg_write(uc, UC_PPC_REG_29, &half_addr));
     OK(uc_reg_write(uc, UC_PPC_REG_30, &w_addr));
     OK(uc_reg_write(uc, UC_PPC_REG_31, &d_addr));
 
@@ -580,7 +580,7 @@ static void test_ppc64_power10_vmx_mask_materialize_extract(void)
     TEST_CHECK(memcmp(dst, expected_b, sizeof(dst)) == 0);
     OK(uc_mem_read(uc, expand_addr, dst, sizeof(dst)));
     TEST_CHECK(memcmp(dst, expected_b, sizeof(dst)) == 0);
-    OK(uc_mem_read(uc, h_addr, dst, sizeof(dst)));
+    OK(uc_mem_read(uc, half_addr, dst, sizeof(dst)));
     TEST_CHECK(memcmp(dst, expected_h, sizeof(dst)) == 0);
     OK(uc_mem_read(uc, w_addr, dst, sizeof(dst)));
     TEST_CHECK(memcmp(dst, expected_w, sizeof(dst)) == 0);

From 97ce4b755c4029c696372509afcc6dbda67f19c0 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 10:55:39 +0300
Subject: [PATCH 14/32] msvc 32bit shared build fix

---
 CMakeLists.txt                       |  17 -
 qemu/target/s390x/translate_vx.inc.c |  30 +-
 tests/unit/test_arm.c                | 536 +--------------------------
 3 files changed, 22 insertions(+), 561 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9f89810692..2bf906beb3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1536,23 +1536,6 @@ if(UNICORN_BUILD_TESTS)
         target_link_libraries(${TEST_FILE} PRIVATE
             ${SAMPLES_LIB}
         )
-        if(TEST_FILE STREQUAL "test_arm")
-            if(MSVC)
-                target_compile_options(${TEST_FILE} PRIVATE
-                    -DNEED_CPU_H
-                    /FIarm.h
-                    /I${CMAKE_CURRENT_SOURCE_DIR}/msvc/arm-softmmu
-                    /I${CMAKE_CURRENT_SOURCE_DIR}/qemu/target/arm
-                )
-            else()
-                target_compile_options(${TEST_FILE} PRIVATE
-                    -DNEED_CPU_H
-                    -include arm.h
-                    -I${CMAKE_BINARY_DIR}/arm-softmmu
-                    -I${CMAKE_CURRENT_SOURCE_DIR}/qemu/target/arm
-                )
-            endif()
-        endif()
         add_test(${TEST_FILE} ${TEST_FILE})
         if(ANDROID_ABI)
             file(APPEND ${CMAKE_BINARY_DIR}/adb.sh "adb push ${TEST_FILE} /data/local/tmp/build/\n")
diff --git a/qemu/target/s390x/translate_vx.inc.c b/qemu/target/s390x/translate_vx.inc.c
index 465715eb6d..1be66df2df 100644
--- a/qemu/target/s390x/translate_vx.inc.c
+++ b/qemu/target/s390x/translate_vx.inc.c
@@ -3540,17 +3540,27 @@ static DisasJumpType op_vfpso(DisasContext *s, DisasOps *o)
     switch (fpf) {
     case FPF_SHORT:
         if (!se) {
-            switch (m5) {
-            case 0:
-                gen_gvec_fn_2i(tcg_ctx, xori, ES_32, v1, v2, 1ull << 31);
-                break;
-            case 1:
-                gen_gvec_fn_2i(tcg_ctx, ori, ES_32, v1, v2, 1ull << 31);
-                break;
-            case 2:
-                gen_gvec_fn_2i(tcg_ctx, andi, ES_32, v1, v2, (1ull << 31) - 1);
-                break;
+            TCGv_i32 tmp32 = tcg_temp_new_i32(tcg_ctx);
+            int i;
+
+            for (i = 0; i < 4; i++) {
+                read_vec_element_i32(tcg_ctx, tmp32, v2, i, ES_32);
+                switch (m5) {
+                case 0:
+                    tcg_gen_xori_i32(tcg_ctx, tmp32, tmp32,
+                                      (int32_t)0x80000000u);
+                    break;
+                case 1:
+                    tcg_gen_ori_i32(tcg_ctx, tmp32, tmp32,
+                                    (int32_t)0x80000000u);
+                    break;
+                case 2:
+                    tcg_gen_andi_i32(tcg_ctx, tmp32, tmp32, 0x7fffffff);
+                    break;
+                }
+                write_vec_element_i32(tcg_ctx, tmp32, v1, i, ES_32);
             }
+            tcg_temp_free_i32(tcg_ctx, tmp32);
             return DISAS_NEXT;
         }
         break;
diff --git a/tests/unit/test_arm.c b/tests/unit/test_arm.c
index 2b2c8ceb94..968ab7b27e 100644
--- a/tests/unit/test_arm.c
+++ b/tests/unit/test_arm.c
@@ -1,14 +1,4 @@
 #include "unicorn_test.h"
-#include "uc_priv.h"
-#ifndef NEED_CPU_H
-#define NEED_CPU_H
-#define TEST_ARM_LOCAL_NEED_CPU_H
-#endif
-#include "target/arm/cpu.h"
-#ifdef TEST_ARM_LOCAL_NEED_CPU_H
-#undef NEED_CPU_H
-#undef TEST_ARM_LOCAL_NEED_CPU_H
-#endif
 #include <string.h>
 
 const uint64_t code_start = 0x1000;
@@ -580,124 +570,6 @@ static void test_arm_m_control(void)
     OK(uc_close(uc));
 }
 
-static uint32_t test_arm_m33_sau_tt_query(uc_engine *uc, uint32_t address,
-                                          uint8_t op)
-{
-    uint32_t result = 0;
-    uint8_t code[] = {
-        0x41, 0xe8, 0x00, 0xf0, /* tt-family r0, r1 */
-    };
-
-    code[2] = op << 6;
-    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R0, &result));
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &address));
-    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_R0, &result));
-
-    return result;
-}
-
-static void test_arm_m33_sau_region(CPUARMState *env, uint32_t region,
-                                    uint32_t base, uint32_t limit,
-                                    bool non_secure_callable)
-{
-    env->sau.rbar[region] = base & ~0x1fU;
-    env->sau.rlar[region] = (limit & ~0x1fU) | 1U;
-    if (non_secure_callable) {
-        env->sau.rlar[region] |= 2U;
-    }
-}
-
-static void test_arm_m33_sau_tt(void)
-{
-    uc_engine *uc;
-    ARMCPU *cpu;
-    CPUARMState *env;
-    uint32_t result;
-
-    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
-    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M33));
-    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
-
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    if (!TEST_CHECK(cpu->sau_sregion >= 2)) {
-        OK(uc_close(uc));
-        return;
-    }
-
-    env->sau.ctrl = 0;
-    memset(env->sau.rbar, 0, sizeof(*env->sau.rbar) * cpu->sau_sregion);
-    memset(env->sau.rlar, 0, sizeof(*env->sau.rlar) * cpu->sau_sregion);
-
-    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
-    TEST_CHECK(result & (1U << 22));
-    TEST_CHECK(!(result & (1U << 17)));
-
-    env->sau.ctrl = 2;
-    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
-    TEST_CHECK(!(result & (1U << 22)));
-    TEST_CHECK(!(result & (1U << 17)));
-
-    env->sau.ctrl = 1;
-    test_arm_m33_sau_region(env, 0, 0x2000, 0x2fff, false);
-    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
-    TEST_CHECK(!(result & (1U << 22)));
-    TEST_CHECK(result & (1U << 17));
-    TEST_CHECK(((result >> 8) & 0xff) == 0);
-
-    test_arm_m33_sau_region(env, 1, 0x3000, 0x3fff, true);
-    result = test_arm_m33_sau_tt_query(uc, 0x3000, 0);
-    TEST_CHECK(result & (1U << 22));
-    TEST_CHECK(result & (1U << 17));
-    TEST_CHECK(((result >> 8) & 0xff) == 1);
-
-    test_arm_m33_sau_region(env, 1, 0x2000, 0x2fff, true);
-    result = test_arm_m33_sau_tt_query(uc, 0x2000, 0);
-    TEST_CHECK(result & (1U << 22));
-    TEST_CHECK(!(result & (1U << 17)));
-
-    result = test_arm_m33_sau_tt_query(uc, 0xe000e010, 0);
-    TEST_CHECK(result & (1U << 22));
-
-    result = test_arm_m33_sau_tt_query(uc, 0xe000e010, 2);
-    TEST_CHECK(!(result & (1U << 22)));
-
-    OK(uc_close(uc));
-}
-
-static void test_arm_m55_pmsav8_pxn(void)
-{
-    uc_engine *uc;
-    ARMCPU *cpu;
-    CPUARMState *env;
-    uint32_t r0 = 0;
-    uc_err err;
-    const uint8_t code[] = {
-        0x2a, 0x20, /* movs r0, #42 */
-    };
-
-    OK(uc_open(UC_ARCH_ARM, UC_MODE_THUMB | UC_MODE_MCLASS, &uc));
-    OK(uc_ctl_set_cpu_model(uc, UC_CPU_ARM_CORTEX_M55));
-    OK(uc_mem_map(uc, code_start, code_len, UC_PROT_ALL));
-    OK(uc_mem_write(uc, code_start, code, sizeof(code)));
-
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.mpu_ctrl[M_REG_S] = R_V7M_MPU_CTRL_ENABLE_MASK;
-    env->pmsav8.rbar[M_REG_S][0] = (uint32_t)code_start | (1U << 1);
-    env->pmsav8.rlar[M_REG_S][0] =
-        ((uint32_t)(code_start + code_len - 1) & ~0x1fU) | (1U << 4) | 1U;
-
-    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
-    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
-    OK(uc_reg_read(uc, UC_ARM_REG_R0, &r0));
-    TEST_CHECK(r0 == 0);
-
-    OK(uc_close(uc));
-}
-
 static void test_arm_m55_mve_id(void)
 {
     uc_engine *uc;
@@ -1100,27 +972,9 @@ static void test_arm_store_le64(uint8_t *data, uint64_t value)
 
 static void test_arm_m_profile_activate_fp_context(uc_engine *uc)
 {
-    const uint32_t control_fpca = 1U << 2;
-    const uint32_t control_sfpa = 1U << 3;
-    ARMCPU *cpu = ARM_CPU(uc->cpu);
-    CPUARMState *env = &cpu->env;
-
-    env->v7m.control[M_REG_S] |= control_fpca | control_sfpa;
-    env->v7m.cpacr[M_REG_S] |= 3U << 20;
-    env->v7m.fpccr[M_REG_S] &= ~R_V7M_FPCCR_LSPACT_MASK;
-    env->v7m.fpccr[M_REG_NS] &= ~R_V7M_FPCCR_LSPACT_MASK;
-    FIELD_DP32(env->hflags, TBFLAG_M32, LSPACT, 0, env->hflags);
-    FIELD_DP32(env->hflags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 0,
-               env->hflags);
-    FIELD_DP32(env->hflags, TBFLAG_M32, FPCCR_S_WRONG, 0, env->hflags);
-}
-
-static void test_arm_m_profile_disable_lazy_fp(uc_engine *uc)
-{
-    ARMCPU *cpu = ARM_CPU(uc->cpu);
-    CPUARMState *env = &cpu->env;
+    uint32_t control = (1U << 2) | (1U << 3);
 
-    env->v7m.fpccr[M_REG_S] &= ~R_V7M_FPCCR_LSPEN_MASK;
+    OK(uc_reg_write(uc, UC_ARM_REG_CONTROL, &control));
 }
 
 static void test_arm_m55_sysreg_mem(void)
@@ -1213,388 +1067,6 @@ static void test_arm_m55_sysreg_mem(void)
     OK(uc_close(uc));
 }
 
-static void test_arm_m55_vlldm_vlstm(void)
-{
-    const uint32_t xpsr_t = 1U << 24;
-    const uint32_t eci_a0a1 = 2U << 12;
-    const uint32_t epsr_condexec_mask = 0xfc00 | (3U << 25);
-    const uint64_t data_addr = code_start + 0x1000;
-    uc_engine *uc;
-    uint8_t code[4];
-    uint8_t mem[0x48] = { 0 };
-    uint32_t r1 = (uint32_t)data_addr;
-    uint32_t s0 = 0x11223344;
-    uint32_t s1 = 0x55667788;
-    uint32_t fpscr = 0x90000000;
-    uint32_t vpr = 0x00abcdef;
-    uint32_t epsr;
-    uc_err err;
-
-    test_arm_emit32(code, 0, 0x0a00ec21); /* vlstm r1, t1 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, sizeof(code),
-                    UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    test_arm_m_profile_disable_lazy_fp(uc);
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
-    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
-    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
-    epsr = xpsr_t | eci_a0a1;
-    OK(uc_reg_write(uc, UC_ARM_REG_EPSR, &epsr));
-    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
-    memset(mem, 0, sizeof(mem));
-    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_read(uc, UC_ARM_REG_EPSR, &epsr));
-    TEST_CHECK(test_arm_load_le(mem, 4) == s0);
-    TEST_CHECK(test_arm_load_le(mem + 4, 4) == s1);
-    TEST_CHECK((test_arm_load_le(mem + 0x40, 4) & 0xf0000000) ==
-               (fpscr & 0xf0000000));
-    TEST_CHECK(test_arm_load_le(mem + 0x44, 4) == vpr);
-    TEST_CHECK_((epsr & epsr_condexec_mask) == 0, "epsr=0x%08x", epsr);
-    OK(uc_close(uc));
-
-    memset(mem, 0, sizeof(mem));
-    s0 = 0x89abcdef;
-    s1 = 0x76543210;
-    fpscr = 0x60000000;
-    vpr = 0x00123456;
-    test_arm_emit32(code, 0, 0x0a80ec21); /* vlstm r1, t2 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, sizeof(code),
-                    UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    test_arm_m_profile_disable_lazy_fp(uc);
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
-    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
-    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
-    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
-    memset(mem, 0, sizeof(mem));
-    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
-    TEST_CHECK(test_arm_load_le(mem, 4) == s0);
-    TEST_CHECK(test_arm_load_le(mem + 4, 4) == s1);
-    TEST_CHECK((test_arm_load_le(mem + 0x40, 4) & 0xf0000000) ==
-               (fpscr & 0xf0000000));
-    TEST_CHECK(test_arm_load_le(mem + 0x44, 4) == vpr);
-    OK(uc_close(uc));
-
-    memset(mem, 0, sizeof(mem));
-    test_arm_store_le(mem, 4, 0x0badc0de);
-    test_arm_store_le(mem + 4, 4, 0xf00d1234);
-    test_arm_store_le(mem + 0x40, 4, 0xa0000000);
-    test_arm_store_le(mem + 0x44, 4, 0x00fedcba);
-    s0 = 0;
-    s1 = 0;
-    fpscr = 0;
-    vpr = 0;
-    test_arm_emit32(code, 0, 0x0a80ec31); /* vlldm r1, t2 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, sizeof(code),
-                    UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    test_arm_m_profile_disable_lazy_fp(uc);
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
-    OK(uc_reg_write(uc, UC_ARM_REG_S0, &s0));
-    OK(uc_reg_write(uc, UC_ARM_REG_S1, &s1));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_write(uc, UC_ARM_REG_VPR, &vpr));
-    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_S0, &s0));
-    OK(uc_reg_read(uc, UC_ARM_REG_S1, &s1));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_read(uc, UC_ARM_REG_VPR, &vpr));
-    TEST_CHECK(s0 == 0x0badc0de);
-    TEST_CHECK(s1 == 0xf00d1234);
-    TEST_CHECK((fpscr & 0xf0000000) == 0xa0000000);
-    TEST_CHECK(vpr == 0x00fedcba);
-    OK(uc_close(uc));
-
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, sizeof(code),
-                    UC_CPU_ARM_CORTEX_M33);
-    test_arm_enable_vfp(uc);
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &r1));
-    err = uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0);
-    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
-    OK(uc_close(uc));
-}
-
-static void test_arm_m55_fpcxt_sysreg(void)
-{
-    const uint32_t control_fpca = 1U << 2;
-    const uint32_t control_sfpa = 1U << 3;
-    const uint32_t fpcr_v = 1U << 28;
-    const uint32_t fpcr_c = 1U << 29;
-    const uint32_t fpcr_z = 1U << 30;
-    const uint32_t fpcr_n = 1U << 31;
-    const uint32_t nzcv_mask = fpcr_n | fpcr_z | fpcr_c | fpcr_v;
-    const uint64_t data_addr = code_start + 0x1000;
-    const uint32_t fpdscr_ns = 0x00070000;
-    uc_engine *uc;
-    ARMCPU *cpu;
-    CPUARMState *env;
-    uint8_t code[8];
-    uint8_t mem[16] = { 0 };
-    uint32_t fpscr_initial = 0x0a040000 | fpcr_n | fpcr_c;
-    uint32_t fpcxt_write = 0x86040000;
-    uint32_t expected_write = fpcxt_write & ~nzcv_mask;
-    uint32_t expected_read;
-    uint32_t r1 = 0;
-    uint32_t r2 = fpcxt_write;
-    uint32_t r3 = 0;
-    uint32_t fpscr = fpscr_initial;
-    uint32_t rbase = (uint32_t)data_addr;
-    uc_err err;
-
-    test_arm_emit32(code, 0, 0x3a10eeff); /* vmrs r3,fpcxt_s */
-    test_arm_emit32(code, 4, 0x2a10eeef); /* vmsr fpcxt_s,r2 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, sizeof(code),
-                    UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    expected_read = (fpscr & ~nzcv_mask) | 0x80000000;
-    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_R3, &r3));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_(r3 == expected_read,
-                "r3=0x%08x", r3);
-    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) != 0,
-                "control=0x%08x", env->v7m.control[M_REG_S]);
-    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_write,
-                "fpscr=0x%08x expected=0x%08x", fpscr, expected_write);
-    TEST_CHECK_((fpscr & nzcv_mask) == 0, "fpscr=0x%08x", fpscr);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0x1a10eefe); /* vmrs r1,fpcxt_ns */
-    test_arm_emit32(code, 4, 0x0a10eeee); /* vmsr fpcxt_ns,r0 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, sizeof(code),
-                    UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
-    env->v7m.fpccr[M_REG_NS] |= R_V7M_FPCCR_ASPEN_MASK;
-    env->v7m.control[M_REG_S] &= ~control_fpca;
-    fpscr = 0x01030000;
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    expected_read = fpscr & ~nzcv_mask;
-    OK(uc_reg_write(uc, UC_ARM_REG_R0, &fpcxt_write));
-    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_(r1 == fpdscr_ns, "r1=0x%08x", r1);
-    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_read,
-                "fpscr=0x%08x", fpscr);
-    TEST_CHECK_((env->v7m.control[M_REG_S] & control_fpca) == 0,
-                "control=0x%08x", env->v7m.control[M_REG_S]);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0x1a10eefe); /* vmrs r1,fpcxt_ns */
-    test_arm_emit32(code, 4, 0x0a10eeee); /* vmsr fpcxt_ns,r0 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, sizeof(code),
-                    UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
-    env->v7m.control[M_REG_S] &= ~control_sfpa;
-    fpscr = fpscr_initial;
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    expected_read = fpscr & ~nzcv_mask;
-    OK(uc_reg_write(uc, UC_ARM_REG_R0, &fpcxt_write));
-    OK(uc_emu_start(uc, code_start | 1, code_start + sizeof(code), 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_R1, &r1));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_(r1 == expected_read, "r1=0x%08x", r1);
-    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) != 0,
-                "control=0x%08x", env->v7m.control[M_REG_S]);
-    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_write,
-                "fpscr=0x%08x expected=0x%08x", fpscr, expected_write);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0xef80edc1); /* vstr fpcxt_s,[r1,#0] */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
-    fpscr = fpscr_initial;
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &rbase));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    expected_read = (fpscr & ~nzcv_mask) | 0x80000000;
-    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
-    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_(test_arm_load_le(mem, 4) == expected_read,
-                "stored=0x%08x", test_arm_load_le(mem, 4));
-    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) == 0,
-                "control=0x%08x", env->v7m.control[M_REG_S]);
-    TEST_CHECK_((fpscr & ~nzcv_mask) == fpdscr_ns,
-                "fpscr=0x%08x", fpscr);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0xef80edd1); /* vldr fpcxt_s,[r1,#0] */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    fpscr = 0;
-    memset(mem, 0, sizeof(mem));
-    test_arm_store_le(mem, 4, fpcxt_write);
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &rbase));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_((env->v7m.control[M_REG_S] & control_sfpa) != 0,
-                "control=0x%08x", env->v7m.control[M_REG_S]);
-    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_write,
-                "fpscr=0x%08x expected=0x%08x", fpscr, expected_write);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0xcf80edc1); /* vstr fpcxt_ns,[r1,#0] */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    test_arm_m_profile_activate_fp_context(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
-    env->v7m.control[M_REG_S] &= ~control_sfpa;
-    fpscr = fpscr_initial;
-    memset(mem, 0, sizeof(mem));
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R1, &rbase));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    expected_read = fpscr & ~nzcv_mask;
-    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
-    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_(test_arm_load_le(mem, 4) == expected_read,
-                "stored=0x%08x", test_arm_load_le(mem, 4));
-    TEST_CHECK_((fpscr & ~nzcv_mask) == fpdscr_ns,
-                "fpscr=0x%08x", fpscr);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0xcf81ecf2); /* vldr fpcxt_ns,[r2],#4 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.fpccr[M_REG_NS] |= R_V7M_FPCCR_ASPEN_MASK;
-    env->v7m.control[M_REG_S] &= ~control_fpca;
-    fpscr = 0x04460000;
-    r2 = (uint32_t)data_addr;
-    test_arm_store_le(mem, 4, fpcxt_write);
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
-    TEST_CHECK_(r2 == (uint32_t)data_addr, "initial r2=0x%08x", r2);
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    expected_read = fpscr & ~nzcv_mask;
-    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_(r2 == (uint32_t)data_addr + 4, "r2=0x%08x", r2);
-    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_read,
-                "fpscr=0x%08x", fpscr);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0xcf81ece2); /* vstr fpcxt_ns,[r2],#4 */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
-    test_arm_enable_vfp(uc);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.fpdscr[M_REG_NS] = fpdscr_ns;
-    env->v7m.fpccr[M_REG_NS] |= R_V7M_FPCCR_ASPEN_MASK;
-    env->v7m.control[M_REG_S] &= ~control_fpca;
-    fpscr = 0x04460000;
-    r2 = (uint32_t)data_addr;
-    memset(mem, 0, sizeof(mem));
-    OK(uc_mem_write(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_write(uc, UC_ARM_REG_R2, &r2));
-    OK(uc_reg_write(uc, UC_ARM_REG_FPSCR, &fpscr));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    expected_read = fpscr & ~nzcv_mask;
-    OK(uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0));
-    OK(uc_reg_read(uc, UC_ARM_REG_R2, &r2));
-    OK(uc_mem_read(uc, data_addr, mem, sizeof(mem)));
-    OK(uc_reg_read(uc, UC_ARM_REG_FPSCR, &fpscr));
-    TEST_CHECK_(r2 == (uint32_t)data_addr + 4, "r2=0x%08x", r2);
-    TEST_CHECK_(test_arm_load_le(mem, 4) == fpdscr_ns,
-                "stored=0x%08x", test_arm_load_le(mem, 4));
-    TEST_CHECK_((fpscr & ~nzcv_mask) == expected_read,
-                "fpscr=0x%08x", fpscr);
-    OK(uc_close(uc));
-
-    test_arm_emit32(code, 0, 0x1a10eefe); /* vmrs r1,fpcxt_ns */
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M55);
-    cpu = ARM_CPU(uc->cpu);
-    env = &cpu->env;
-    env->v7m.control[M_REG_S] |= control_fpca;
-    env->v7m.fpccr[M_REG_NS] &= ~R_V7M_FPCCR_ASPEN_MASK;
-    env->v7m.cpacr[M_REG_S] &= ~(3U << 20);
-    env->v7m.cpacr[M_REG_NS] &= ~(3U << 20);
-    FIELD_DP32(env->hflags, TBFLAG_ANY, FPEXC_EL, 1, env->hflags);
-    FIELD_DP32(env->hflags, TBFLAG_M32, LSPACT, 0, env->hflags);
-    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
-    TEST_CHECK_(err == UC_ERR_EXCEPTION, "err=%u", (unsigned)err);
-    OK(uc_close(uc));
-
-    uc_common_setup(&uc, UC_ARCH_ARM,
-                    UC_MODE_THUMB | UC_MODE_MCLASS,
-                    (const char *)code, 4, UC_CPU_ARM_CORTEX_M33);
-    test_arm_enable_vfp(uc);
-    err = uc_emu_start(uc, code_start | 1, code_start + 4, 0, 0);
-    TEST_CHECK_(err == UC_ERR_INSN_INVALID, "err=%u", (unsigned)err);
-    OK(uc_close(uc));
-}
-
 static void test_arm_m55_vscclrm(void)
 {
     const uint32_t xpsr_t = 1U << 24;
@@ -12476,8 +11948,6 @@ TEST_LIST = {{"test_arm_nop", test_arm_nop},
              {"test_arm_i8mm", test_arm_i8mm},
              {"test_arm_bf16", test_arm_bf16},
              {"test_arm_m_control", test_arm_m_control},
-             {"test_arm_m33_sau_tt", test_arm_m33_sau_tt},
-             {"test_arm_m55_pmsav8_pxn", test_arm_m55_pmsav8_pxn},
              {"test_arm_m55_mve_id", test_arm_m55_mve_id},
              {"test_arm_m55_vpr_public_reg", test_arm_m55_vpr_public_reg},
              {"test_arm_m55_vpr_sysreg", test_arm_m55_vpr_sysreg},
@@ -12485,8 +11955,6 @@ TEST_LIST = {{"test_arm_nop", test_arm_nop},
              {"test_arm_m55_fpscr_nzcvqc_sysreg",
               test_arm_m55_fpscr_nzcvqc_sysreg},
              {"test_arm_m55_sysreg_mem", test_arm_m55_sysreg_mem},
-             {"test_arm_m55_vlldm_vlstm", test_arm_m55_vlldm_vlstm},
-             {"test_arm_m55_fpcxt_sysreg", test_arm_m55_fpcxt_sysreg},
              {"test_arm_m55_vscclrm", test_arm_m55_vscclrm},
              {"test_arm_m55_vctp", test_arm_m55_vctp},
              {"test_arm_m55_mve_eci", test_arm_m55_mve_eci},

From 57eecb5587c67f66fa1f74e0f303bf96d64c538e Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 11:39:10 +0300
Subject: [PATCH 15/32] fix aarch64 host tcg branches

---
 qemu/tcg/aarch64/tcg-target.inc.c | 33 +++++++++++++------------------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index 44eceb3a6c..5fd85959a9 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -594,7 +594,7 @@ typedef enum {
     I3617_CMEQ0     = 0x0e209800,
     I3617_CMLT0     = 0x0e20a800,
     I3617_CMGE0     = 0x2e208800,
-    I3617_CMLE0     = 0x2e20a800,
+    I3617_CMLE0     = 0x2e209800,
     I3617_NOT       = 0x2e205800,
     I3617_ABS       = 0x0e20b800,
     I3617_NEG       = 0x2e20b800,
@@ -832,11 +832,7 @@ static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
 {
     unsigned h, l, r, c;
 
-    // Unicorn Hack (wtdcode):
-    // I have no clue about this assert and it seems the logic here is same with QEMU at least 7.2.1
-    // That said, qemu probably suffers the same issue but maybe no one emulates mips on M1?
-    // Disabling this still passes all unit tests so let's go with it.
-    // tcg_debug_assert(is_limm(limm));
+    /* Some legacy lowering paths can reach non-logical immediates here. */
 
     h = clz64(limm);
     l = ctz64(limm);
@@ -1255,7 +1251,7 @@ static inline void tcg_out_shl(TCGContext *s, TCGType ext,
 {
     int bits = ext ? 64 : 32;
     int max = bits - 1;
-    tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
+    tcg_out_ubfm(s, ext, rd, rn, (bits - m) & max, (max - m) & max);
 }
 
 static inline void tcg_out_shr(TCGContext *s, TCGType ext,
@@ -1282,9 +1278,8 @@ static inline void tcg_out_rotr(TCGContext *s, TCGType ext,
 static inline void tcg_out_rotl(TCGContext *s, TCGType ext,
                                 TCGReg rd, TCGReg rn, unsigned int m)
 {
-    int bits = ext ? 64 : 32;
-    int max = bits - 1;
-    tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
+    int max = ext ? 63 : 31;
+    tcg_out_extr(s, ext, rd, rn, rn, -m & max);
 }
 
 static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd,
@@ -1312,21 +1307,21 @@ static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
     }
 }
 
-static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
+static inline void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target)
 {
-    ptrdiff_t offset = target - s->code_ptr;
+    ptrdiff_t offset = tcg_pcrel_diff(s, (void *)target) >> 2;
     tcg_debug_assert(offset == sextract64(offset, 0, 26));
     tcg_out_insn(s, 3206, B, offset);
 }
 
-static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target)
+static inline void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target)
 {
-    ptrdiff_t offset = target - s->code_ptr;
+    ptrdiff_t offset = tcg_pcrel_diff(s, (void *)target) >> 2;
     if (offset == sextract64(offset, 0, 26)) {
-        tcg_out_insn(s, 3206, BL, offset);
+        tcg_out_insn(s, 3206, B, offset);
     } else {
-        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
-        tcg_out_insn(s, 3207, BR, TCG_REG_TMP);
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X9, (intptr_t)target);
+        tcg_out_insn(s, 3207, BR, TCG_REG_X9);
     }
 }
 
@@ -1335,9 +1330,9 @@ static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
     tcg_out_insn(s, 3207, BLR, reg);
 }
 
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
+static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
 {
-    ptrdiff_t offset = target - s->code_ptr;
+    ptrdiff_t offset = tcg_pcrel_diff(s, (void *)target) >> 2;
     if (offset == sextract64(offset, 0, 26)) {
         tcg_out_insn(s, 3206, BL, offset);
     } else {

From 3cd880fef61f9652b2397b90825d4e7b5b241b24 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 11:56:09 +0300
Subject: [PATCH 16/32] fix aarch64 host tcg call prototype

Keep the aarch64 backend call emitter on the reduced tree's tcg_out_call pointer ABI so ubuntu-aarch64 builds compile again.

Use concise CTest failure output in Build UC2 so parallel CI logs expose the failing test case instead of losing it inside verbose interleaved output.
---
 .github/workflows/build-uc2.yml   | 16 ++++++++--------
 qemu/tcg/aarch64/tcg-target.inc.c |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build-uc2.yml b/.github/workflows/build-uc2.yml
index 7d023657a1..2ee8a2eed5 100644
--- a/.github/workflows/build-uc2.yml
+++ b/.github/workflows/build-uc2.yml
@@ -168,7 +168,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip --config ${{ env.BUILD_TYPE }}
-          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
           mv ${{ env.BUILD_TYPE }} instdir
 
       - name: '🛠️ Win MSVC 32 setup'
@@ -195,7 +195,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip --config ${{ env.BUILD_TYPE }}
-          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
           mv ${{ env.BUILD_TYPE }} instdir
 
       - name: '🚧 Win MINGW build'
@@ -224,7 +224,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
 
       - name: '📦 Pack artifact'
         if: always()
@@ -312,7 +312,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
 
       - name: '🚧 Android x86_64 build'
         if: contains(matrix.config.name, 'android')
@@ -474,7 +474,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
 
       - name: '🚧 Linux aarch64 build'
         if: contains(matrix.config.arch, 'aarch64')
@@ -492,7 +492,7 @@ jobs:
             -DCMAKE_INSTALL_PREFIX:PATH=instdir
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
 
       - name: '🚧 Linux ppc64le build'
         if: contains(matrix.config.arch, 'ppc64le')
@@ -518,7 +518,7 @@ jobs:
               -DCMAKE_INSTALL_PREFIX:PATH=/instdir
             cmake --build . --config ${{ env.BUILD_TYPE }}
             cmake --install . --strip
-            ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+            ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
 
       - name: '📦 Pack artifact'
         if: always()
@@ -620,7 +620,7 @@ jobs:
             -DBUILD_SHARED_LIBS=${{ matrix.config.shared }}
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip
-          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} -VV -C ${{ env.BUILD_TYPE }}
+          ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
 
       - name: '📦 Pack artifact'
         if: always()
diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index 5fd85959a9..f591a5a296 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -1330,7 +1330,7 @@ static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
     tcg_out_insn(s, 3207, BLR, reg);
 }
 
-static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
+static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
 {
     ptrdiff_t offset = tcg_pcrel_diff(s, (void *)target) >> 2;
     if (offset == sextract64(offset, 0, 26)) {

From 94e4c9cf7948c1a8f8f78868d78dea1f8c68be22 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 12:51:33 +0300
Subject: [PATCH 17/32] fix qemu72 cross-host ci failures

Align host TCG direct jump patching with the QEMU 7.2 tc_ptr/jmp_rx/jmp_rw ABI, add missing non-goto_ptr host backend definitions, and fix PPC host TCG opcode/table drift.

Move variable target page size state out of release-mode macros that depended on a local uc variable, using arch-postfixed target_page_bits_state instead.

Fix POSIX MIPS CPU alignment and preserve microMIPS entry state, and avoid Apple JIT state asserts when virtualized macOS runners cannot report SPRR permissions.
---
 CMakeLists.txt                        | 141 ++++++++++++++++----------
 qemu/aarch64.h                        |   1 +
 qemu/accel/tcg/cpu-exec.c             |   4 +-
 qemu/arm.h                            |   1 +
 qemu/exec-vary.c                      |  53 ++++++----
 qemu/include/exec/cpu-all.h           |  18 ++--
 qemu/include/exec/cpu_ldst.h          |   3 -
 qemu/include/tcg/tcg-apple-jit.h      |   5 +-
 qemu/m68k.h                           |   1 +
 qemu/mips.h                           |   1 +
 qemu/mips64.h                         |   1 +
 qemu/mips64el.h                       |   1 +
 qemu/mipsel.h                         |   1 +
 qemu/ppc.h                            |   1 +
 qemu/ppc64.h                          |   1 +
 qemu/riscv32.h                        |   1 +
 qemu/riscv64.h                        |   1 +
 qemu/s390x.h                          |   1 +
 qemu/sparc.h                          |   1 +
 qemu/sparc64.h                        |   1 +
 qemu/target/mips/cpu.c                |   4 +-
 qemu/target/mips/translate.c          |   2 +-
 qemu/tcg/aarch64/tcg-target.h         |   2 +-
 qemu/tcg/aarch64/tcg-target.inc.c     |  12 +--
 qemu/tcg/arm/tcg-target.h             |   1 +
 qemu/tcg/i386/tcg-target.h            |   5 +-
 qemu/tcg/loongarch64/tcg-target.h     |   2 +-
 qemu/tcg/loongarch64/tcg-target.inc.c |  12 +--
 qemu/tcg/mips/tcg-target.h            |   1 +
 qemu/tcg/mips/tcg-target.inc.c        |   8 +-
 qemu/tcg/ppc/tcg-target.h             |   1 +
 qemu/tcg/ppc/tcg-target.inc.c         |  28 ++---
 qemu/tcg/ppc/tcg-target.opc.h         |   1 +
 qemu/tcg/riscv/tcg-target.h           |   1 +
 qemu/tcg/s390/tcg-target.h            |   7 +-
 qemu/tcg/sparc/tcg-target.h           |   2 +-
 qemu/tcg/sparc/tcg-target.inc.c       |  14 +--
 qemu/tricore.h                        |   1 +
 qemu/x86_64.h                         |   1 +
 symbols.sh                            |   1 +
 40 files changed, 206 insertions(+), 138 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2bf906beb3..245c8350de 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -102,6 +102,26 @@ else()
     include_directories(
         ${CMAKE_BINARY_DIR}
     )
+    find_program(QEMU_SH_EXECUTABLE
+        NAMES sh
+        PATHS
+            "C:/msys64/usr/bin"
+            "C:/Program Files/Git/usr/bin"
+        NO_DEFAULT_PATH
+    )
+    if(NOT QEMU_SH_EXECUTABLE)
+        find_program(QEMU_SH_EXECUTABLE NAMES sh)
+    endif()
+    if(NOT QEMU_SH_EXECUTABLE)
+        message(FATAL_ERROR "A POSIX shell is required to configure QEMU")
+    endif()
+    get_filename_component(QEMU_SH_DIR "${QEMU_SH_EXECUTABLE}" DIRECTORY)
+    if(WIN32)
+        set(QEMU_PATH_SEPARATOR ";")
+    else()
+        set(QEMU_PATH_SEPARATOR ":")
+    endif()
+    set(ENV{PATH} "${QEMU_SH_DIR}${QEMU_PATH_SEPARATOR}$ENV{PATH}")
 endif()
 
 include_directories(
@@ -374,98 +394,115 @@ else()
     #     ${TARGET_LIST}
     #     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}"
     # )
-    execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/configure
+    function(qemu_create_config CONFIG_INPUT CONFIG_OUTPUT)
+        execute_process(
+            COMMAND ${QEMU_SH_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
+            INPUT_FILE ${CONFIG_INPUT}
+            OUTPUT_FILE ${CONFIG_OUTPUT}
+            RESULT_VARIABLE CREATE_CONFIG_RESULT
+        )
+        if(NOT CREATE_CONFIG_RESULT EQUAL 0)
+            message(FATAL_ERROR "create_config failed for ${CONFIG_INPUT}")
+        endif()
+    endfunction()
+
+    execute_process(COMMAND ${QEMU_SH_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/qemu/configure
         --cc=${CMAKE_C_COMPILER}
         ${EXTRA_CFLAGS}
         ${TARGET_LIST}
         WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+        RESULT_VARIABLE QEMU_CONFIGURE_RESULT
     )
-    execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-        INPUT_FILE ${CMAKE_BINARY_DIR}/config-host.mak
-        OUTPUT_FILE ${CMAKE_BINARY_DIR}/config-host.h
+    if(NOT QEMU_CONFIGURE_RESULT EQUAL 0)
+        message(FATAL_ERROR "qemu/configure failed")
+    endif()
+
+    qemu_create_config(
+        ${CMAKE_BINARY_DIR}/config-host.mak
+        ${CMAKE_BINARY_DIR}/config-host.h
     )
     if(UNICORN_HAS_X86)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/x86_64-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/x86_64-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/x86_64-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/x86_64-softmmu/config-target.h
         )
     endif()
     if(UNICORN_HAS_ARM)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/arm-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/arm-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/arm-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/arm-softmmu/config-target.h
         )
     endif()
     if(UNICORN_HAS_AARCH64)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/aarch64-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/aarch64-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/aarch64-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/aarch64-softmmu/config-target.h
         )
     endif()
     if(UNICORN_HAS_M68K)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/m68k-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/m68k-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/m68k-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/m68k-softmmu/config-target.h
         )
     endif()
     if(UNICORN_HAS_MIPS)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/mips-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/mips-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/mips-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/mips-softmmu/config-target.h
         )
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/mipsel-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/mipsel-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/mipsel-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/mipsel-softmmu/config-target.h
         )
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/mips64-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/mips64-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/mips64-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/mips64-softmmu/config-target.h
         )
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/mips64el-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/mips64el-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/mips64el-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/mips64el-softmmu/config-target.h
         )
     endif()
     if(UNICORN_HAS_SPARC)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/sparc-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/sparc-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/sparc-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/sparc-softmmu/config-target.h
         )
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/sparc64-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/sparc64-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/sparc64-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/sparc64-softmmu/config-target.h
         )
     endif()
     if(UNICORN_HAS_PPC)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/ppc-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/ppc-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/ppc-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/ppc-softmmu/config-target.h
         )
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/ppc64-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/ppc64-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/ppc64-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/ppc64-softmmu/config-target.h
         )
     endif()
     if(UNICORN_HAS_RISCV)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/riscv32-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/riscv32-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/riscv32-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/riscv32-softmmu/config-target.h
         )
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/riscv64-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/riscv64-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/riscv64-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/riscv64-softmmu/config-target.h
         )
     endif()
     if (UNICORN_HAS_S390X)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/s390x-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/s390x-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/s390x-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/s390x-softmmu/config-target.h
         )
     endif()
     if (UNICORN_HAS_TRICORE)
-        execute_process(COMMAND sh ${CMAKE_CURRENT_SOURCE_DIR}/qemu/scripts/create_config
-            INPUT_FILE ${CMAKE_BINARY_DIR}/tricore-softmmu/config-target.mak
-            OUTPUT_FILE ${CMAKE_BINARY_DIR}/tricore-softmmu/config-target.h
+        qemu_create_config(
+            ${CMAKE_BINARY_DIR}/tricore-softmmu/config-target.mak
+            ${CMAKE_BINARY_DIR}/tricore-softmmu/config-target.h
         )
     endif()
     add_compile_options(
diff --git a/qemu/aarch64.h b/qemu/aarch64.h
index 222dbe1156..7b9231922c 100644
--- a/qemu/aarch64.h
+++ b/qemu/aarch64.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_aarch64
 #define qemu_target_page_bits qemu_target_page_bits_aarch64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_aarch64
+#define target_page_bits_state target_page_bits_state_aarch64
 #define target_words_bigendian target_words_bigendian_aarch64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_aarch64
 #define ram_block_discard_range ram_block_discard_range_aarch64
diff --git a/qemu/accel/tcg/cpu-exec.c b/qemu/accel/tcg/cpu-exec.c
index 1343b81571..ab75a5ff10 100644
--- a/qemu/accel/tcg/cpu-exec.c
+++ b/qemu/accel/tcg/cpu-exec.c
@@ -199,7 +199,9 @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
     if (TCG_TARGET_HAS_direct_jump) {
         uintptr_t offset = tb->jmp_target_arg[n];
         uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
-        tb_target_set_jmp_target(tc_ptr, tc_ptr + offset, addr);
+        uintptr_t jmp_rx = tc_ptr + offset;
+        uintptr_t jmp_rw = jmp_rx;
+        tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr);
     } else {
         tb->jmp_target_arg[n] = addr;
     }
diff --git a/qemu/arm.h b/qemu/arm.h
index 63466c452e..97bf686bac 100644
--- a/qemu/arm.h
+++ b/qemu/arm.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_arm
 #define qemu_target_page_bits qemu_target_page_bits_arm
 #define qemu_target_page_bits_min qemu_target_page_bits_min_arm
+#define target_page_bits_state target_page_bits_state_arm
 #define target_words_bigendian target_words_bigendian_arm
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_arm
 #define ram_block_discard_range ram_block_discard_range_arm
diff --git a/qemu/exec-vary.c b/qemu/exec-vary.c
index b2f5a6789d..78f41d1876 100644
--- a/qemu/exec-vary.c
+++ b/qemu/exec-vary.c
@@ -26,6 +26,10 @@
 
 #include <uc_priv.h>
 
+#ifdef TARGET_PAGE_BITS_VARY
+TargetPageBits target_page_bits_state;
+#endif
+
 bool set_preferred_target_page_bits(struct uc_struct *uc, int bits)
 {
     /*
@@ -35,22 +39,22 @@ bool set_preferred_target_page_bits(struct uc_struct *uc, int bits)
      * a particular size.
      */
 #ifdef TARGET_PAGE_BITS_VARY
-    //assert(bits >= TARGET_PAGE_BITS_MIN);
-    if (uc->init_target_page == NULL) {
-        uc->init_target_page = calloc(1, sizeof(TargetPageBits));
-    } else {
-        return false;
-    }
+    TargetPageBits *page = uc->init_target_page;
 
     if (bits < TARGET_PAGE_BITS_MIN) {
         return false;
     }
 
-    if (uc->init_target_page->bits == 0 || uc->init_target_page->bits > bits) {
-        if (uc->init_target_page->decided) {
+    if (page == NULL) {
+        page = g_new0(TargetPageBits, 1);
+        uc->init_target_page = page;
+    }
+
+    if (page->bits == 0 || page->bits > bits) {
+        if (page->decided) {
             return false;
         }
-        uc->init_target_page->bits = bits;
+        page->bits = bits;
     }
 #endif
     return true;
@@ -59,25 +63,30 @@ bool set_preferred_target_page_bits(struct uc_struct *uc, int bits)
 void finalize_target_page_bits(struct uc_struct *uc)
 {
 #ifdef TARGET_PAGE_BITS_VARY
-    if (uc->init_target_page == NULL) {
-        uc->init_target_page = calloc(1, sizeof(TargetPageBits));
-    } else {
-        return;
-    }
+    TargetPageBits *page = uc->init_target_page;
 
-    if (uc->target_bits != 0) {
-        uc->init_target_page->bits = uc->target_bits;
+    if (page == NULL) {
+        page = g_new0(TargetPageBits, 1);
+        uc->init_target_page = page;
     }
 
-    if (uc->init_target_page->bits == 0) {
-        uc->init_target_page->bits = TARGET_PAGE_BITS_MIN;
+    if (!page->decided) {
+        if (uc->target_bits != 0) {
+            page->bits = uc->target_bits;
+        }
+
+        if (page->bits == 0) {
+            page->bits = TARGET_PAGE_BITS_MIN;
+        }
+        page->mask = ((target_ulong)-1) << page->bits;
+        page->decided = true;
     }
-    uc->init_target_page->mask = ((target_ulong)-1) << uc->init_target_page->bits;
-    uc->init_target_page->decided = true;
+
+    target_page_bits_state = *page;
 
     /*
-     * For the benefit of an -flto build, prevent the compiler from
-     * hoisting a read from target_page before we finish initializing.
+     * For the benefit of an -flto build, prevent the compiler from hoisting
+     * a read from target_page_bits_state before we finish initializing.
      */
     barrier();
 #endif
diff --git a/qemu/include/exec/cpu-all.h b/qemu/include/exec/cpu-all.h
index ddac720740..2db754fd99 100644
--- a/qemu/include/exec/cpu-all.h
+++ b/qemu/include/exec/cpu-all.h
@@ -221,20 +221,18 @@ typedef struct TargetPageBits {
     int bits;
     target_long mask;
 } TargetPageBits;
-#if defined(CONFIG_ATTRIBUTE_ALIAS) || !defined(IN_EXEC_VARY)
-extern const TargetPageBits target_page;
-#else
-extern TargetPageBits target_page;
-#endif
+extern TargetPageBits target_page_bits_state;
 
 #ifdef CONFIG_DEBUG_TCG
-#define TARGET_PAGE_BITS   ({ assert(target_page.decided); target_page.bits; })
-#define TARGET_PAGE_MASK   ({ assert(target_page.decided); target_page.mask; })
+#define TARGET_PAGE_BITS \
+    ({ assert(target_page_bits_state.decided); target_page_bits_state.bits; })
+#define TARGET_PAGE_MASK \
+    ({ assert(target_page_bits_state.decided); target_page_bits_state.mask; })
 #else
-#define TARGET_PAGE_BITS   uc->init_target_page->bits
-#define TARGET_PAGE_MASK   uc->init_target_page->mask
+#define TARGET_PAGE_BITS   target_page_bits_state.bits
+#define TARGET_PAGE_MASK   target_page_bits_state.mask
 #endif
-#define TARGET_PAGE_SIZE   (-(int)TARGET_PAGE_MASK) // qq
+#define TARGET_PAGE_SIZE   (-(int)TARGET_PAGE_MASK)
 #else
 #define TARGET_PAGE_BITS_MIN TARGET_PAGE_BITS
 #define TARGET_PAGE_SIZE   (1 << TARGET_PAGE_BITS)
diff --git a/qemu/include/exec/cpu_ldst.h b/qemu/include/exec/cpu_ldst.h
index 3f504bd2b7..145fabbb79 100644
--- a/qemu/include/exec/cpu_ldst.h
+++ b/qemu/include/exec/cpu_ldst.h
@@ -98,9 +98,6 @@ static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry)
 static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx,
                                   target_ulong addr)
 {
-#ifdef TARGET_ARM
-    struct uc_struct *uc = env->uc;
-#endif
     uintptr_t size_mask = env_tlb(env)->f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS;
 
     return (addr >> TARGET_PAGE_BITS) & size_mask;
diff --git a/qemu/include/tcg/tcg-apple-jit.h b/qemu/include/tcg/tcg-apple-jit.h
index 4ace2c7be9..2854f20a1b 100644
--- a/qemu/include/tcg/tcg-apple-jit.h
+++ b/qemu/include/tcg/tcg-apple-jit.h
@@ -77,8 +77,9 @@ static inline void assert_executable(bool executable) {
     uint64_t v = read_sprr_perm();
 
     if (!v) {
-        assert(executable == thread_executable());
+        return;
     }
+    assert(executable == thread_executable());
 }
 
 #else
@@ -152,4 +153,4 @@ static inline void jit_write_protect(int enabled)
 
 #endif
 
-#endif /* define TCG_APPLE_JIT_H */
\ No newline at end of file
+#endif /* define TCG_APPLE_JIT_H */
diff --git a/qemu/m68k.h b/qemu/m68k.h
index 76072a5bfb..814f82782b 100644
--- a/qemu/m68k.h
+++ b/qemu/m68k.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_m68k
 #define qemu_target_page_bits qemu_target_page_bits_m68k
 #define qemu_target_page_bits_min qemu_target_page_bits_min_m68k
+#define target_page_bits_state target_page_bits_state_m68k
 #define target_words_bigendian target_words_bigendian_m68k
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_m68k
 #define ram_block_discard_range ram_block_discard_range_m68k
diff --git a/qemu/mips.h b/qemu/mips.h
index 91440cb504..5179e3f49e 100644
--- a/qemu/mips.h
+++ b/qemu/mips.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_mips
 #define qemu_target_page_bits qemu_target_page_bits_mips
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mips
+#define target_page_bits_state target_page_bits_state_mips
 #define target_words_bigendian target_words_bigendian_mips
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mips
 #define ram_block_discard_range ram_block_discard_range_mips
diff --git a/qemu/mips64.h b/qemu/mips64.h
index 24422eef29..c5622f2d35 100644
--- a/qemu/mips64.h
+++ b/qemu/mips64.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_mips64
 #define qemu_target_page_bits qemu_target_page_bits_mips64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mips64
+#define target_page_bits_state target_page_bits_state_mips64
 #define target_words_bigendian target_words_bigendian_mips64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mips64
 #define ram_block_discard_range ram_block_discard_range_mips64
diff --git a/qemu/mips64el.h b/qemu/mips64el.h
index 0e6e85b557..819d28eb8f 100644
--- a/qemu/mips64el.h
+++ b/qemu/mips64el.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_mips64el
 #define qemu_target_page_bits qemu_target_page_bits_mips64el
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mips64el
+#define target_page_bits_state target_page_bits_state_mips64el
 #define target_words_bigendian target_words_bigendian_mips64el
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mips64el
 #define ram_block_discard_range ram_block_discard_range_mips64el
diff --git a/qemu/mipsel.h b/qemu/mipsel.h
index c7495fed90..8c1700f439 100644
--- a/qemu/mipsel.h
+++ b/qemu/mipsel.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_mipsel
 #define qemu_target_page_bits qemu_target_page_bits_mipsel
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mipsel
+#define target_page_bits_state target_page_bits_state_mipsel
 #define target_words_bigendian target_words_bigendian_mipsel
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mipsel
 #define ram_block_discard_range ram_block_discard_range_mipsel
diff --git a/qemu/ppc.h b/qemu/ppc.h
index bc6683ca62..d6ea3a01f9 100644
--- a/qemu/ppc.h
+++ b/qemu/ppc.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_ppc
 #define qemu_target_page_bits qemu_target_page_bits_ppc
 #define qemu_target_page_bits_min qemu_target_page_bits_min_ppc
+#define target_page_bits_state target_page_bits_state_ppc
 #define target_words_bigendian target_words_bigendian_ppc
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_ppc
 #define ram_block_discard_range ram_block_discard_range_ppc
diff --git a/qemu/ppc64.h b/qemu/ppc64.h
index a42068e85d..dd094542e6 100644
--- a/qemu/ppc64.h
+++ b/qemu/ppc64.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_ppc64
 #define qemu_target_page_bits qemu_target_page_bits_ppc64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_ppc64
+#define target_page_bits_state target_page_bits_state_ppc64
 #define target_words_bigendian target_words_bigendian_ppc64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_ppc64
 #define ram_block_discard_range ram_block_discard_range_ppc64
diff --git a/qemu/riscv32.h b/qemu/riscv32.h
index 975e0893ae..a651e76bdc 100644
--- a/qemu/riscv32.h
+++ b/qemu/riscv32.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_riscv32
 #define qemu_target_page_bits qemu_target_page_bits_riscv32
 #define qemu_target_page_bits_min qemu_target_page_bits_min_riscv32
+#define target_page_bits_state target_page_bits_state_riscv32
 #define target_words_bigendian target_words_bigendian_riscv32
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_riscv32
 #define ram_block_discard_range ram_block_discard_range_riscv32
diff --git a/qemu/riscv64.h b/qemu/riscv64.h
index 90cc133731..9364f21048 100644
--- a/qemu/riscv64.h
+++ b/qemu/riscv64.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_riscv64
 #define qemu_target_page_bits qemu_target_page_bits_riscv64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_riscv64
+#define target_page_bits_state target_page_bits_state_riscv64
 #define target_words_bigendian target_words_bigendian_riscv64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_riscv64
 #define ram_block_discard_range ram_block_discard_range_riscv64
diff --git a/qemu/s390x.h b/qemu/s390x.h
index e5e85370af..c6f6cc2231 100644
--- a/qemu/s390x.h
+++ b/qemu/s390x.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_s390x
 #define qemu_target_page_bits qemu_target_page_bits_s390x
 #define qemu_target_page_bits_min qemu_target_page_bits_min_s390x
+#define target_page_bits_state target_page_bits_state_s390x
 #define target_words_bigendian target_words_bigendian_s390x
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_s390x
 #define ram_block_discard_range ram_block_discard_range_s390x
diff --git a/qemu/sparc.h b/qemu/sparc.h
index 76b9eea506..a48f65659b 100644
--- a/qemu/sparc.h
+++ b/qemu/sparc.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_sparc
 #define qemu_target_page_bits qemu_target_page_bits_sparc
 #define qemu_target_page_bits_min qemu_target_page_bits_min_sparc
+#define target_page_bits_state target_page_bits_state_sparc
 #define target_words_bigendian target_words_bigendian_sparc
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_sparc
 #define ram_block_discard_range ram_block_discard_range_sparc
diff --git a/qemu/sparc64.h b/qemu/sparc64.h
index 4d4b0317aa..bf1350af2e 100644
--- a/qemu/sparc64.h
+++ b/qemu/sparc64.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_sparc64
 #define qemu_target_page_bits qemu_target_page_bits_sparc64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_sparc64
+#define target_page_bits_state target_page_bits_state_sparc64
 #define target_words_bigendian target_words_bigendian_sparc64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_sparc64
 #define ram_block_discard_range ram_block_discard_range_sparc64
diff --git a/qemu/target/mips/cpu.c b/qemu/target/mips/cpu.c
index f1fe63d39e..129e137728 100644
--- a/qemu/target/mips/cpu.c
+++ b/qemu/target/mips/cpu.c
@@ -31,7 +31,7 @@ static void mips_cpu_set_pc(CPUState *cs, vaddr value)
     CPUMIPSState *env = &cpu->env;
 
     env->active_tc.PC = value & ~(target_ulong)1;
-    if (value & 1) {
+    if ((value & 1) || (cs->uc->mode & UC_MODE_MICRO)) {
         env->hflags |= MIPS_HFLAG_M16;
     } else {
         env->hflags &= ~(MIPS_HFLAG_M16);
@@ -157,7 +157,7 @@ MIPSCPU *cpu_mips_init(struct uc_struct *uc)
     CPUClass *cc;
     CPUMIPSState *env;
 
-    cpu = qemu_memalign(8, sizeof(*cpu));
+    cpu = qemu_memalign(16, sizeof(*cpu));
     if (cpu == NULL) {
         return NULL;
     }
diff --git a/qemu/target/mips/translate.c b/qemu/target/mips/translate.c
index 66d785c1b2..7bf04edeba 100644
--- a/qemu/target/mips/translate.c
+++ b/qemu/target/mips/translate.c
@@ -32044,7 +32044,7 @@ void cpu_state_reset(CPUMIPSState *env)
         env->CP0_PWField = 0x02;
     }
 
-    if (env->CP0_Config3 & (1 << CP0C3_ISA) & (1 << (CP0C3_ISA + 1))) {
+    if ((env->CP0_Config3 & (3 << CP0C3_ISA)) == (3 << CP0C3_ISA)) {
         /*  microMIPS on reset when Config3.ISA is 3 */
         env->hflags |= MIPS_HFLAG_M16;
     }
diff --git a/qemu/tcg/aarch64/tcg-target.h b/qemu/tcg/aarch64/tcg-target.h
index 13993a70e5..fd917cfdc3 100644
--- a/qemu/tcg/aarch64/tcg-target.h
+++ b/qemu/tcg/aarch64/tcg-target.h
@@ -163,7 +163,7 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 #endif
 }
 
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
 #ifdef CONFIG_SOFTMMU
 #define TCG_TARGET_NEED_LDST_LABELS
diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index f591a5a296..43944eeed8 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -1341,21 +1341,21 @@ static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
     }
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
-                              uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+                              uintptr_t jmp_rw, uintptr_t addr)
 {
     tcg_insn_unit i1, i2;
     TCGType rt = TCG_TYPE_I64;
     TCGReg  rd = TCG_REG_TMP;
     uint64_t pair;
 
-    ptrdiff_t offset = addr - jmp_addr;
+    ptrdiff_t offset = addr - jmp_rx;
 
     if (offset == sextract64(offset, 0, 26)) {
         i1 = I3206_B | ((offset >> 2) & 0x3ffffff);
         i2 = NOP;
     } else {
-        offset = (addr >> 12) - (jmp_addr >> 12);
+        offset = (addr >> 12) - (jmp_rx >> 12);
 
         /* patch ADRP */
         i1 = I3406_ADRP | (offset & 3) << 29 | (offset & 0x1ffffc) << (5 - 2) | rd;
@@ -1363,8 +1363,8 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
         i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
     }
     pair = (uint64_t)i2 << 32 | i1;
-    atomic_set((uint64_t *)jmp_addr, pair);
-    flush_icache_range(jmp_addr, jmp_addr + 8);
+    atomic_set((uint64_t *)jmp_rw, pair);
+    flush_icache_range(jmp_rx, jmp_rx + 8);
 }
 
 static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
diff --git a/qemu/tcg/arm/tcg-target.h b/qemu/tcg/arm/tcg-target.h
index 486269eff1..238c2d2ab7 100644
--- a/qemu/tcg/arm/tcg-target.h
+++ b/qemu/tcg/arm/tcg-target.h
@@ -156,6 +156,7 @@ extern bool use_neon_instructions;
 /* not defined -- call should be eliminated at compile time */
 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
+#define TCG_TARGET_HAS_goto_ptr         0
 #define TCG_TARGET_NEED_LDST_LABELS
 #define TCG_TARGET_NEED_POOL_LABELS
 
diff --git a/qemu/tcg/i386/tcg-target.h b/qemu/tcg/i386/tcg-target.h
index 24ba5d19be..a9426cbeee 100644
--- a/qemu/tcg/i386/tcg-target.h
+++ b/qemu/tcg/i386/tcg-target.h
@@ -209,10 +209,11 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 }
 
 static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
-                                            uintptr_t jmp_addr, uintptr_t addr)
+                                            uintptr_t jmp_rx,
+                                            uintptr_t jmp_rw, uintptr_t addr)
 {
     /* patch the branch destination */
-    *(int32_t *)jmp_addr = addr - (jmp_addr + 4);
+    *(int32_t *)jmp_rw = addr - (jmp_rx + 4);
     /* no need to flush icache explicitly */
 }
 
diff --git a/qemu/tcg/loongarch64/tcg-target.h b/qemu/tcg/loongarch64/tcg-target.h
index 4578e7cace..6c0e090c3f 100644
--- a/qemu/tcg/loongarch64/tcg-target.h
+++ b/qemu/tcg/loongarch64/tcg-target.h
@@ -220,7 +220,7 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
     __builtin___clear_cache((char *)start, (char *)stop);
 }
 
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
 #define TCG_TARGET_NEED_LDST_LABELS
 
diff --git a/qemu/tcg/loongarch64/tcg-target.inc.c b/qemu/tcg/loongarch64/tcg-target.inc.c
index 0f2aa5304f..b6a53a7207 100644
--- a/qemu/tcg/loongarch64/tcg-target.inc.c
+++ b/qemu/tcg/loongarch64/tcg-target.inc.c
@@ -1386,11 +1386,11 @@ static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
     }
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
-                              uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+                              uintptr_t jmp_rw, uintptr_t addr)
 {
     uintptr_t d_addr = addr;
-    ptrdiff_t d_disp = (ptrdiff_t)(d_addr - jmp_addr) >> 2;
+    ptrdiff_t d_disp = (ptrdiff_t)(d_addr - jmp_rx) >> 2;
     tcg_insn_unit insn;
 
     /* Either directly branch, or load slot address for indirect branch. */
@@ -1398,13 +1398,13 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
         insn = encode_sd10k16_insn(OPC_B, d_disp);
     } else {
         uintptr_t i_addr = addr;
-        intptr_t i_disp = i_addr - jmp_addr;
+        intptr_t i_disp = i_addr - jmp_rx;
         insn = encode_dsj20_insn(OPC_PCADDU2I, TCG_REG_TMP0, i_disp >> 2);
     }
 
-    *(tcg_insn_unit *)jmp_addr =  insn;
+    *(tcg_insn_unit *)jmp_rw =  insn;
     // flush_idcache_range(jmp_rx, jmp_rw, 4);
-    flush_icache_range(jmp_addr, jmp_addr + 8);
+    flush_icache_range(jmp_rx, jmp_rx + 8);
 }
 
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
diff --git a/qemu/tcg/mips/tcg-target.h b/qemu/tcg/mips/tcg-target.h
index 7669213175..cac24e14d8 100644
--- a/qemu/tcg/mips/tcg-target.h
+++ b/qemu/tcg/mips/tcg-target.h
@@ -207,6 +207,7 @@ extern bool use_mips32r2_instructions;
 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t)
     QEMU_ERROR("code path is reachable");
 
+#define TCG_TARGET_HAS_goto_ptr         0
 #define TCG_TARGET_NEED_LDST_LABELS
 
 #endif
diff --git a/qemu/tcg/mips/tcg-target.inc.c b/qemu/tcg/mips/tcg-target.inc.c
index 7eb598685b..18f83e63a6 100644
--- a/qemu/tcg/mips/tcg-target.inc.c
+++ b/qemu/tcg/mips/tcg-target.inc.c
@@ -2662,11 +2662,11 @@ static void tcg_target_init(TCGContext *s)
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP);   /* global pointer */
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
-                              uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+                              uintptr_t jmp_rw, uintptr_t addr)
 {
-    atomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2));
-    flush_icache_range(jmp_addr, jmp_addr + 4);
+    atomic_set((uint32_t *)jmp_rw, deposit32(OPC_J, 0, 26, addr >> 2));
+    flush_icache_range(jmp_rx, jmp_rx + 4);
 }
 
 typedef struct {
diff --git a/qemu/tcg/ppc/tcg-target.h b/qemu/tcg/ppc/tcg-target.h
index b5cd225cfa..9bd91da97b 100644
--- a/qemu/tcg/ppc/tcg-target.h
+++ b/qemu/tcg/ppc/tcg-target.h
@@ -182,6 +182,7 @@ extern bool have_vsx;
 
 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
+#define TCG_TARGET_HAS_goto_ptr         0
 #define TCG_TARGET_DEFAULT_MO (0)
 #define TCG_TARGET_HAS_MEMORY_BSWAP     1
 
diff --git a/qemu/tcg/ppc/tcg-target.inc.c b/qemu/tcg/ppc/tcg-target.inc.c
index 2f5c1c6f32..6428246ead 100644
--- a/qemu/tcg/ppc/tcg-target.inc.c
+++ b/qemu/tcg/ppc/tcg-target.inc.c
@@ -1726,13 +1726,13 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
     tcg_out32(s, insn);
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
-                              uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+                              uintptr_t jmp_rw, uintptr_t addr)
 {
     if (TCG_TARGET_REG_BITS == 64) {
         tcg_insn_unit i1, i2;
         intptr_t tb_diff = addr - tc_ptr;
-        intptr_t br_diff = addr - (jmp_addr + 4);
+        intptr_t br_diff = addr - (jmp_rx + 4);
         uint64_t pair;
 
         /* This does not exercise the range of the branch, but we do
@@ -1756,13 +1756,13 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
 
         /* As per the enclosing if, this is ppc64.  Avoid the _Static_assert
            within atomic_set that would fail to build a ppc32 host.  */
-        atomic_set__nocheck((uint64_t *)jmp_addr, pair);
-        flush_icache_range(jmp_addr, jmp_addr + 8);
+        atomic_set__nocheck((uint64_t *)jmp_rw, pair);
+        flush_icache_range(jmp_rx, jmp_rx + 8);
     } else {
-        intptr_t diff = addr - jmp_addr;
+        intptr_t diff = addr - jmp_rx;
         tcg_debug_assert(in_range_b(diff));
-        atomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
-        flush_icache_range(jmp_addr, jmp_addr + 4);
+        atomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc));
+        flush_icache_range(jmp_rx, jmp_rx + 4);
     }
 }
 
@@ -1816,28 +1816,28 @@ static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
 #endif
 }
 
-static const uint32_t qemu_ldx_opc[16] = {
+static const uint32_t qemu_ldx_opc[(MO_SSIZE + MO_BSWAP) + 1] = {
     [MO_UB] = LBZX,
     [MO_UW] = LHZX,
     [MO_UL] = LWZX,
-    [MO_Q]  = LDX,
+    [MO_UQ] = LDX,
     [MO_SW] = LHAX,
     [MO_SL] = LWAX,
     [MO_BSWAP | MO_UB] = LBZX,
     [MO_BSWAP | MO_UW] = LHBRX,
     [MO_BSWAP | MO_UL] = LWBRX,
-    [MO_BSWAP | MO_Q]  = LDBRX,
+    [MO_BSWAP | MO_UQ] = LDBRX,
 };
 
-static const uint32_t qemu_stx_opc[16] = {
+static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = {
     [MO_UB] = STBX,
     [MO_UW] = STHX,
     [MO_UL] = STWX,
-    [MO_Q]  = STDX,
+    [MO_UQ] = STDX,
     [MO_BSWAP | MO_UB] = STBX,
     [MO_BSWAP | MO_UW] = STHBRX,
     [MO_BSWAP | MO_UL] = STWBRX,
-    [MO_BSWAP | MO_Q]  = STDBRX,
+    [MO_BSWAP | MO_UQ] = STDBRX,
 };
 
 static const uint32_t qemu_exts_opc[4] = {
diff --git a/qemu/tcg/ppc/tcg-target.opc.h b/qemu/tcg/ppc/tcg-target.opc.h
index db514403c3..1373f77e82 100644
--- a/qemu/tcg/ppc/tcg-target.opc.h
+++ b/qemu/tcg/ppc/tcg-target.opc.h
@@ -30,3 +30,4 @@ DEF(ppc_msum_vec, 1, 3, 0, IMPLVEC)
 DEF(ppc_muleu_vec, 1, 2, 0, IMPLVEC)
 DEF(ppc_mulou_vec, 1, 2, 0, IMPLVEC)
 DEF(ppc_pkum_vec, 1, 2, 0, IMPLVEC)
+DEF(ppc_rotl_vec, 1, 2, 0, IMPLVEC)
diff --git a/qemu/tcg/riscv/tcg-target.h b/qemu/tcg/riscv/tcg-target.h
index 11c9b3e4f4..5dcca685e1 100644
--- a/qemu/tcg/riscv/tcg-target.h
+++ b/qemu/tcg/riscv/tcg-target.h
@@ -163,6 +163,7 @@ typedef enum {
 /* not defined -- call should be eliminated at compile time */
 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
+#define TCG_TARGET_HAS_goto_ptr         0
 #define TCG_TARGET_DEFAULT_MO (0)
 
 #define TCG_TARGET_NEED_LDST_LABELS
diff --git a/qemu/tcg/s390/tcg-target.h b/qemu/tcg/s390/tcg-target.h
index 07accabbd1..807e4ed940 100644
--- a/qemu/tcg/s390/tcg-target.h
+++ b/qemu/tcg/s390/tcg-target.h
@@ -150,11 +150,12 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 }
 
 static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
-                                            uintptr_t jmp_addr, uintptr_t addr)
+                                            uintptr_t jmp_rx,
+                                            uintptr_t jmp_rw, uintptr_t addr)
 {
     /* patch the branch destination */
-    intptr_t disp = addr - (jmp_addr - 2);
-    atomic_set((int32_t *)jmp_addr, disp / 2);
+    intptr_t disp = addr - (jmp_rx - 2);
+    atomic_set((int32_t *)jmp_rw, disp / 2);
     /* no need to flush icache explicitly */
 }
 
diff --git a/qemu/tcg/sparc/tcg-target.h b/qemu/tcg/sparc/tcg-target.h
index 633841ebf2..81099906d0 100644
--- a/qemu/tcg/sparc/tcg-target.h
+++ b/qemu/tcg/sparc/tcg-target.h
@@ -176,7 +176,7 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
     }
 }
 
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
 
 #define TCG_TARGET_NEED_POOL_LABELS
 
diff --git a/qemu/tcg/sparc/tcg-target.inc.c b/qemu/tcg/sparc/tcg-target.inc.c
index fb13db72f0..86c24c2dba 100644
--- a/qemu/tcg/sparc/tcg-target.inc.c
+++ b/qemu/tcg/sparc/tcg-target.inc.c
@@ -1829,11 +1829,11 @@ void tcg_register_jit(TCGContext *s, void *buf, size_t buf_size)
     tcg_register_jit_int(s, buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
 
-void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
-                              uintptr_t addr)
+void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
+                              uintptr_t jmp_rw, uintptr_t addr)
 {
     intptr_t tb_disp = addr - tc_ptr;
-    intptr_t br_disp = addr - jmp_addr;
+    intptr_t br_disp = addr - jmp_rx;
     tcg_insn_unit i1, i2;
 
     /* We can reach the entire address space for ILP32.
@@ -1842,8 +1842,8 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
     tcg_debug_assert(br_disp == (int32_t)br_disp);
 
     if (!USE_REG_TB) {
-        atomic_set((uint32_t *)jmp_addr, deposit32(CALL, 0, 30, br_disp >> 2));
-        flush_icache_range(jmp_addr, jmp_addr + 4);
+        atomic_set((uint32_t *)jmp_rw, deposit32(CALL, 0, 30, br_disp >> 2));
+        flush_icache_range(jmp_rx, jmp_rx + 4);
         return;
     }
 
@@ -1866,6 +1866,6 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
               | INSN_IMM13((tb_disp & 0x3ff) | -0x400));
     }
 
-    atomic_set((uint64_t *)jmp_addr, deposit64(i2, 32, 32, i1));
-    flush_icache_range(jmp_addr, jmp_addr + 8);
+    atomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1));
+    flush_icache_range(jmp_rx, jmp_rx + 8);
 }
diff --git a/qemu/tricore.h b/qemu/tricore.h
index 43c5806a1c..98090bd6c9 100644
--- a/qemu/tricore.h
+++ b/qemu/tricore.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_tricore
 #define qemu_target_page_bits qemu_target_page_bits_tricore
 #define qemu_target_page_bits_min qemu_target_page_bits_min_tricore
+#define target_page_bits_state target_page_bits_state_tricore
 #define target_words_bigendian target_words_bigendian_tricore
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_tricore
 #define ram_block_discard_range ram_block_discard_range_tricore
diff --git a/qemu/x86_64.h b/qemu/x86_64.h
index a64dc44c84..4aa82bca1b 100644
--- a/qemu/x86_64.h
+++ b/qemu/x86_64.h
@@ -110,6 +110,7 @@
 #define qemu_target_page_size qemu_target_page_size_x86_64
 #define qemu_target_page_bits qemu_target_page_bits_x86_64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_x86_64
+#define target_page_bits_state target_page_bits_state_x86_64
 #define target_words_bigendian target_words_bigendian_x86_64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_x86_64
 #define ram_block_discard_range ram_block_discard_range_x86_64
diff --git a/symbols.sh b/symbols.sh
index 6ca25d24ae..7da98d10ba 100755
--- a/symbols.sh
+++ b/symbols.sh
@@ -110,6 +110,7 @@ cpu_memory_rw_debug \
 qemu_target_page_size \
 qemu_target_page_bits \
 qemu_target_page_bits_min \
+target_page_bits_state \
 target_words_bigendian \
 cpu_physical_memory_is_io \
 ram_block_discard_range \

From 05fbaf19844fb15acce3e8894e4b556bc2f80a3f Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 13:04:39 +0300
Subject: [PATCH 18/32] fix per-engine target page bits

Keep variable target page size state per Unicorn engine instead of arch-global state, while preserving the improved finalization path.

Add a public control regression that runs two ARM engines with different page sizes in the same process.
---
 qemu/aarch64.h               |  1 -
 qemu/arm.h                   |  1 -
 qemu/exec-vary.c             |  8 +-------
 qemu/include/exec/cpu-all.h  |  9 ++++-----
 qemu/include/exec/cpu_ldst.h |  3 +++
 qemu/m68k.h                  |  1 -
 qemu/mips.h                  |  1 -
 qemu/mips64.h                |  1 -
 qemu/mips64el.h              |  1 -
 qemu/mipsel.h                |  1 -
 qemu/ppc.h                   |  1 -
 qemu/ppc64.h                 |  1 -
 qemu/riscv32.h               |  1 -
 qemu/riscv64.h               |  1 -
 qemu/s390x.h                 |  1 -
 qemu/sparc.h                 |  1 -
 qemu/sparc64.h               |  1 -
 qemu/tricore.h               |  1 -
 qemu/x86_64.h                |  1 -
 symbols.sh                   |  1 -
 tests/unit/test_ctl.c        | 29 +++++++++++++++++++++++++++++
 21 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/qemu/aarch64.h b/qemu/aarch64.h
index 7b9231922c..222dbe1156 100644
--- a/qemu/aarch64.h
+++ b/qemu/aarch64.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_aarch64
 #define qemu_target_page_bits qemu_target_page_bits_aarch64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_aarch64
-#define target_page_bits_state target_page_bits_state_aarch64
 #define target_words_bigendian target_words_bigendian_aarch64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_aarch64
 #define ram_block_discard_range ram_block_discard_range_aarch64
diff --git a/qemu/arm.h b/qemu/arm.h
index 97bf686bac..63466c452e 100644
--- a/qemu/arm.h
+++ b/qemu/arm.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_arm
 #define qemu_target_page_bits qemu_target_page_bits_arm
 #define qemu_target_page_bits_min qemu_target_page_bits_min_arm
-#define target_page_bits_state target_page_bits_state_arm
 #define target_words_bigendian target_words_bigendian_arm
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_arm
 #define ram_block_discard_range ram_block_discard_range_arm
diff --git a/qemu/exec-vary.c b/qemu/exec-vary.c
index 78f41d1876..93a78f4b77 100644
--- a/qemu/exec-vary.c
+++ b/qemu/exec-vary.c
@@ -26,10 +26,6 @@
 
 #include <uc_priv.h>
 
-#ifdef TARGET_PAGE_BITS_VARY
-TargetPageBits target_page_bits_state;
-#endif
-
 bool set_preferred_target_page_bits(struct uc_struct *uc, int bits)
 {
     /*
@@ -82,11 +78,9 @@ void finalize_target_page_bits(struct uc_struct *uc)
         page->decided = true;
     }
 
-    target_page_bits_state = *page;
-
     /*
      * For the benefit of an -flto build, prevent the compiler from hoisting
-     * a read from target_page_bits_state before we finish initializing.
+     * a read from the target page data before we finish initializing.
      */
     barrier();
 #endif
diff --git a/qemu/include/exec/cpu-all.h b/qemu/include/exec/cpu-all.h
index 2db754fd99..d61c4acc59 100644
--- a/qemu/include/exec/cpu-all.h
+++ b/qemu/include/exec/cpu-all.h
@@ -221,16 +221,15 @@ typedef struct TargetPageBits {
     int bits;
     target_long mask;
 } TargetPageBits;
-extern TargetPageBits target_page_bits_state;
 
 #ifdef CONFIG_DEBUG_TCG
 #define TARGET_PAGE_BITS \
-    ({ assert(target_page_bits_state.decided); target_page_bits_state.bits; })
+    ({ assert(uc->init_target_page->decided); uc->init_target_page->bits; })
 #define TARGET_PAGE_MASK \
-    ({ assert(target_page_bits_state.decided); target_page_bits_state.mask; })
+    ({ assert(uc->init_target_page->decided); uc->init_target_page->mask; })
 #else
-#define TARGET_PAGE_BITS   target_page_bits_state.bits
-#define TARGET_PAGE_MASK   target_page_bits_state.mask
+#define TARGET_PAGE_BITS   uc->init_target_page->bits
+#define TARGET_PAGE_MASK   uc->init_target_page->mask
 #endif
 #define TARGET_PAGE_SIZE   (-(int)TARGET_PAGE_MASK)
 #else
diff --git a/qemu/include/exec/cpu_ldst.h b/qemu/include/exec/cpu_ldst.h
index 145fabbb79..3f504bd2b7 100644
--- a/qemu/include/exec/cpu_ldst.h
+++ b/qemu/include/exec/cpu_ldst.h
@@ -98,6 +98,9 @@ static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry)
 static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx,
                                   target_ulong addr)
 {
+#ifdef TARGET_ARM
+    struct uc_struct *uc = env->uc;
+#endif
     uintptr_t size_mask = env_tlb(env)->f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS;
 
     return (addr >> TARGET_PAGE_BITS) & size_mask;
diff --git a/qemu/m68k.h b/qemu/m68k.h
index 814f82782b..76072a5bfb 100644
--- a/qemu/m68k.h
+++ b/qemu/m68k.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_m68k
 #define qemu_target_page_bits qemu_target_page_bits_m68k
 #define qemu_target_page_bits_min qemu_target_page_bits_min_m68k
-#define target_page_bits_state target_page_bits_state_m68k
 #define target_words_bigendian target_words_bigendian_m68k
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_m68k
 #define ram_block_discard_range ram_block_discard_range_m68k
diff --git a/qemu/mips.h b/qemu/mips.h
index 5179e3f49e..91440cb504 100644
--- a/qemu/mips.h
+++ b/qemu/mips.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_mips
 #define qemu_target_page_bits qemu_target_page_bits_mips
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mips
-#define target_page_bits_state target_page_bits_state_mips
 #define target_words_bigendian target_words_bigendian_mips
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mips
 #define ram_block_discard_range ram_block_discard_range_mips
diff --git a/qemu/mips64.h b/qemu/mips64.h
index c5622f2d35..24422eef29 100644
--- a/qemu/mips64.h
+++ b/qemu/mips64.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_mips64
 #define qemu_target_page_bits qemu_target_page_bits_mips64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mips64
-#define target_page_bits_state target_page_bits_state_mips64
 #define target_words_bigendian target_words_bigendian_mips64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mips64
 #define ram_block_discard_range ram_block_discard_range_mips64
diff --git a/qemu/mips64el.h b/qemu/mips64el.h
index 819d28eb8f..0e6e85b557 100644
--- a/qemu/mips64el.h
+++ b/qemu/mips64el.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_mips64el
 #define qemu_target_page_bits qemu_target_page_bits_mips64el
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mips64el
-#define target_page_bits_state target_page_bits_state_mips64el
 #define target_words_bigendian target_words_bigendian_mips64el
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mips64el
 #define ram_block_discard_range ram_block_discard_range_mips64el
diff --git a/qemu/mipsel.h b/qemu/mipsel.h
index 8c1700f439..c7495fed90 100644
--- a/qemu/mipsel.h
+++ b/qemu/mipsel.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_mipsel
 #define qemu_target_page_bits qemu_target_page_bits_mipsel
 #define qemu_target_page_bits_min qemu_target_page_bits_min_mipsel
-#define target_page_bits_state target_page_bits_state_mipsel
 #define target_words_bigendian target_words_bigendian_mipsel
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_mipsel
 #define ram_block_discard_range ram_block_discard_range_mipsel
diff --git a/qemu/ppc.h b/qemu/ppc.h
index d6ea3a01f9..bc6683ca62 100644
--- a/qemu/ppc.h
+++ b/qemu/ppc.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_ppc
 #define qemu_target_page_bits qemu_target_page_bits_ppc
 #define qemu_target_page_bits_min qemu_target_page_bits_min_ppc
-#define target_page_bits_state target_page_bits_state_ppc
 #define target_words_bigendian target_words_bigendian_ppc
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_ppc
 #define ram_block_discard_range ram_block_discard_range_ppc
diff --git a/qemu/ppc64.h b/qemu/ppc64.h
index dd094542e6..a42068e85d 100644
--- a/qemu/ppc64.h
+++ b/qemu/ppc64.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_ppc64
 #define qemu_target_page_bits qemu_target_page_bits_ppc64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_ppc64
-#define target_page_bits_state target_page_bits_state_ppc64
 #define target_words_bigendian target_words_bigendian_ppc64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_ppc64
 #define ram_block_discard_range ram_block_discard_range_ppc64
diff --git a/qemu/riscv32.h b/qemu/riscv32.h
index a651e76bdc..975e0893ae 100644
--- a/qemu/riscv32.h
+++ b/qemu/riscv32.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_riscv32
 #define qemu_target_page_bits qemu_target_page_bits_riscv32
 #define qemu_target_page_bits_min qemu_target_page_bits_min_riscv32
-#define target_page_bits_state target_page_bits_state_riscv32
 #define target_words_bigendian target_words_bigendian_riscv32
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_riscv32
 #define ram_block_discard_range ram_block_discard_range_riscv32
diff --git a/qemu/riscv64.h b/qemu/riscv64.h
index 9364f21048..90cc133731 100644
--- a/qemu/riscv64.h
+++ b/qemu/riscv64.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_riscv64
 #define qemu_target_page_bits qemu_target_page_bits_riscv64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_riscv64
-#define target_page_bits_state target_page_bits_state_riscv64
 #define target_words_bigendian target_words_bigendian_riscv64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_riscv64
 #define ram_block_discard_range ram_block_discard_range_riscv64
diff --git a/qemu/s390x.h b/qemu/s390x.h
index c6f6cc2231..e5e85370af 100644
--- a/qemu/s390x.h
+++ b/qemu/s390x.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_s390x
 #define qemu_target_page_bits qemu_target_page_bits_s390x
 #define qemu_target_page_bits_min qemu_target_page_bits_min_s390x
-#define target_page_bits_state target_page_bits_state_s390x
 #define target_words_bigendian target_words_bigendian_s390x
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_s390x
 #define ram_block_discard_range ram_block_discard_range_s390x
diff --git a/qemu/sparc.h b/qemu/sparc.h
index a48f65659b..76b9eea506 100644
--- a/qemu/sparc.h
+++ b/qemu/sparc.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_sparc
 #define qemu_target_page_bits qemu_target_page_bits_sparc
 #define qemu_target_page_bits_min qemu_target_page_bits_min_sparc
-#define target_page_bits_state target_page_bits_state_sparc
 #define target_words_bigendian target_words_bigendian_sparc
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_sparc
 #define ram_block_discard_range ram_block_discard_range_sparc
diff --git a/qemu/sparc64.h b/qemu/sparc64.h
index bf1350af2e..4d4b0317aa 100644
--- a/qemu/sparc64.h
+++ b/qemu/sparc64.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_sparc64
 #define qemu_target_page_bits qemu_target_page_bits_sparc64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_sparc64
-#define target_page_bits_state target_page_bits_state_sparc64
 #define target_words_bigendian target_words_bigendian_sparc64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_sparc64
 #define ram_block_discard_range ram_block_discard_range_sparc64
diff --git a/qemu/tricore.h b/qemu/tricore.h
index 98090bd6c9..43c5806a1c 100644
--- a/qemu/tricore.h
+++ b/qemu/tricore.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_tricore
 #define qemu_target_page_bits qemu_target_page_bits_tricore
 #define qemu_target_page_bits_min qemu_target_page_bits_min_tricore
-#define target_page_bits_state target_page_bits_state_tricore
 #define target_words_bigendian target_words_bigendian_tricore
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_tricore
 #define ram_block_discard_range ram_block_discard_range_tricore
diff --git a/qemu/x86_64.h b/qemu/x86_64.h
index 4aa82bca1b..a64dc44c84 100644
--- a/qemu/x86_64.h
+++ b/qemu/x86_64.h
@@ -110,7 +110,6 @@
 #define qemu_target_page_size qemu_target_page_size_x86_64
 #define qemu_target_page_bits qemu_target_page_bits_x86_64
 #define qemu_target_page_bits_min qemu_target_page_bits_min_x86_64
-#define target_page_bits_state target_page_bits_state_x86_64
 #define target_words_bigendian target_words_bigendian_x86_64
 #define cpu_physical_memory_is_io cpu_physical_memory_is_io_x86_64
 #define ram_block_discard_range ram_block_discard_range_x86_64
diff --git a/symbols.sh b/symbols.sh
index 7da98d10ba..6ca25d24ae 100755
--- a/symbols.sh
+++ b/symbols.sh
@@ -110,7 +110,6 @@ cpu_memory_rw_debug \
 qemu_target_page_size \
 qemu_target_page_bits \
 qemu_target_page_bits_min \
-target_page_bits_state \
 target_words_bigendian \
 cpu_physical_memory_is_io \
 ram_block_discard_range \
diff --git a/tests/unit/test_ctl.c b/tests/unit/test_ctl.c
index 5fe65e57b0..cae29c9a10 100644
--- a/tests/unit/test_ctl.c
+++ b/tests/unit/test_ctl.c
@@ -196,6 +196,33 @@ static void test_uc_ctl_change_page_size(void)
     OK(uc_close(uc));
     OK(uc_close(uc2));
 }
+
+static void test_uc_ctl_arm_page_size_per_engine(void)
+{
+    uc_engine *uc4k;
+    uc_engine *uc1k;
+    uint32_t page_size;
+    uint32_t r0 = 0;
+    const char code[] = "\x01\x00\xa0\xe3";
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_ARM, &uc4k));
+    OK(uc_ctl_set_page_size(uc4k, 4096));
+    OK(uc_mem_map(uc4k, 0x1000, 0x1000, UC_PROT_ALL));
+    OK(uc_mem_write(uc4k, 0x1000, code, sizeof(code) - 1));
+
+    OK(uc_open(UC_ARCH_ARM, UC_MODE_ARM, &uc1k));
+    OK(uc_ctl_set_page_size(uc1k, 1024));
+    OK(uc_mem_map(uc1k, 0x400, 0x400, UC_PROT_ALL));
+
+    OK(uc_ctl_get_page_size(uc4k, &page_size));
+    TEST_CHECK(page_size == 4096);
+    OK(uc_emu_start(uc4k, 0x1000, 0x1000 + sizeof(code) - 1, 0, 0));
+    OK(uc_reg_read(uc4k, UC_ARM_REG_R0, &r0));
+    TEST_CHECK(r0 == 1);
+
+    OK(uc_close(uc1k));
+    OK(uc_close(uc4k));
+}
 #endif
 
 // Test requires UC_ARCH_ARM64.
@@ -407,6 +434,8 @@ TEST_LIST = {
     {"test_uc_ctl_tb_cache", test_uc_ctl_tb_cache},
 #ifdef UNICORN_HAS_ARM
     {"test_uc_ctl_change_page_size", test_uc_ctl_change_page_size},
+    {"test_uc_ctl_arm_page_size_per_engine",
+     test_uc_ctl_arm_page_size_per_engine},
     {"test_uc_ctl_arm_cpu", test_uc_ctl_arm_cpu},
 #endif
 #ifdef UNICORN_HAS_ARM64

From 6b380e6f547406c29d6ee0a4cf542235619d0195 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 13:28:48 +0300
Subject: [PATCH 19/32] fix host code cache flush

Synchronize generated code writes with instruction fetches when patching TCG direct jumps and finalizing generated code. This keeps the reduced JIT path aligned with QEMU 7.2 host cache semantics on non-coherent host caches.
---
 qemu/include/qemu/cacheflush.h        | 33 +++++++++++++++++++++++++++
 qemu/tcg/aarch64/tcg-target.inc.c     |  2 +-
 qemu/tcg/loongarch64/tcg-target.inc.c |  3 +--
 qemu/tcg/ppc/tcg-target.inc.c         |  4 ++--
 qemu/tcg/sparc/tcg-target.inc.c       |  4 ++--
 qemu/tcg/tcg.c                        |  7 ++++--
 6 files changed, 44 insertions(+), 9 deletions(-)
 create mode 100644 qemu/include/qemu/cacheflush.h

diff --git a/qemu/include/qemu/cacheflush.h b/qemu/include/qemu/cacheflush.h
new file mode 100644
index 0000000000..2a9e790a37
--- /dev/null
+++ b/qemu/include/qemu/cacheflush.h
@@ -0,0 +1,33 @@
+/*
+ * Flush host instruction and data caches for generated code.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_CACHEFLUSH_H
+#define QEMU_CACHEFLUSH_H
+
+#if defined(__APPLE__) && (defined(__aarch64__) || defined(__arm__))
+#include <libkern/OSCacheControl.h>
+#endif
+
+static inline void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
+{
+#if defined(__i386__) || defined(__x86_64__) || defined(__s390__)
+    /* Instruction and data caches are coherent. */
+    (void)rx;
+    (void)rw;
+    (void)len;
+#elif defined(__APPLE__) && (defined(__aarch64__) || defined(__arm__))
+    sys_dcache_flush((void *)rw, len);
+    sys_icache_invalidate((void *)rx, len);
+#else
+    if (rw != rx) {
+        __builtin___clear_cache((char *)rw, (char *)rw + len);
+    }
+    __builtin___clear_cache((char *)rx, (char *)rx + len);
+#endif
+}
+
+#endif /* QEMU_CACHEFLUSH_H */
diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index 43944eeed8..277740e466 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -1364,7 +1364,7 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
     }
     pair = (uint64_t)i2 << 32 | i1;
     atomic_set((uint64_t *)jmp_rw, pair);
-    flush_icache_range(jmp_rx, jmp_rx + 8);
+    flush_idcache_range(jmp_rx, jmp_rw, 8);
 }
 
 static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
diff --git a/qemu/tcg/loongarch64/tcg-target.inc.c b/qemu/tcg/loongarch64/tcg-target.inc.c
index b6a53a7207..6a4c5ed9a2 100644
--- a/qemu/tcg/loongarch64/tcg-target.inc.c
+++ b/qemu/tcg/loongarch64/tcg-target.inc.c
@@ -1403,8 +1403,7 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
     }
 
     *(tcg_insn_unit *)jmp_rw =  insn;
-    // flush_idcache_range(jmp_rx, jmp_rw, 4);
-    flush_icache_range(jmp_rx, jmp_rx + 8);
+    flush_idcache_range(jmp_rx, jmp_rw, 4);
 }
 
 static void tcg_out_op(TCGContext *s, TCGOpcode opc,
diff --git a/qemu/tcg/ppc/tcg-target.inc.c b/qemu/tcg/ppc/tcg-target.inc.c
index 6428246ead..d3d09547d1 100644
--- a/qemu/tcg/ppc/tcg-target.inc.c
+++ b/qemu/tcg/ppc/tcg-target.inc.c
@@ -1757,12 +1757,12 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
         /* As per the enclosing if, this is ppc64.  Avoid the _Static_assert
            within atomic_set that would fail to build a ppc32 host.  */
         atomic_set__nocheck((uint64_t *)jmp_rw, pair);
-        flush_icache_range(jmp_rx, jmp_rx + 8);
+        flush_idcache_range(jmp_rx, jmp_rw, 8);
     } else {
         intptr_t diff = addr - jmp_rx;
         tcg_debug_assert(in_range_b(diff));
         atomic_set((uint32_t *)jmp_rw, B | (diff & 0x3fffffc));
-        flush_icache_range(jmp_rx, jmp_rx + 4);
+        flush_idcache_range(jmp_rx, jmp_rw, 4);
     }
 }
 
diff --git a/qemu/tcg/sparc/tcg-target.inc.c b/qemu/tcg/sparc/tcg-target.inc.c
index 86c24c2dba..a7ce51440e 100644
--- a/qemu/tcg/sparc/tcg-target.inc.c
+++ b/qemu/tcg/sparc/tcg-target.inc.c
@@ -1843,7 +1843,7 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
 
     if (!USE_REG_TB) {
         atomic_set((uint32_t *)jmp_rw, deposit32(CALL, 0, 30, br_disp >> 2));
-        flush_icache_range(jmp_rx, jmp_rx + 4);
+        flush_idcache_range(jmp_rx, jmp_rw, 4);
         return;
     }
 
@@ -1867,5 +1867,5 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
     }
 
     atomic_set((uint64_t *)jmp_rw, deposit64(i2, 32, 32, i1));
-    flush_icache_range(jmp_rx, jmp_rx + 8);
+    flush_idcache_range(jmp_rx, jmp_rw, 8);
 }
diff --git a/qemu/tcg/tcg.c b/qemu/tcg/tcg.c
index 0b542afe3f..17f4de7cc0 100644
--- a/qemu/tcg/tcg.c
+++ b/qemu/tcg/tcg.c
@@ -31,6 +31,7 @@
 #undef DEBUG_JIT
 
 #include "qemu/cutils.h"
+#include "qemu/cacheflush.h"
 #include "qemu/host-utils.h"
 #include "qemu/timer.h"
 
@@ -857,7 +858,8 @@ void tcg_prologue_init(TCGContext *s)
 #endif
 
     buf1 = s->code_ptr;
-    flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
+    flush_idcache_range((uintptr_t)buf0, (uintptr_t)buf0,
+                        (uintptr_t)buf1 - (uintptr_t)buf0);
 
     /* Deduct the prologue from the buffer.  */
     prologue_size = tcg_current_code_size(s);
@@ -3870,7 +3872,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
     }
 
     /* flush instruction cache */
-    flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
+    flush_idcache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_buf,
+                        (uintptr_t)s->code_ptr - (uintptr_t)s->code_buf);
 
     return tcg_current_code_size(s);
 }

From 720120c581592bf8e13ca596e1171212df2e671e Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 14:25:14 +0300
Subject: [PATCH 20/32] fix qemu 7.2 ci regressions

---
 qemu/include/qemu/atomic.h     |  94 ++++++++++++++++++++++++++++++
 qemu/include/qemu/cacheflush.h |  36 ++++++++++++
 qemu/target/mips/translate.c   |  31 ++++++----
 tests/unit/test_ctl.c          | 103 ++++++---------------------------
 4 files changed, 168 insertions(+), 96 deletions(-)

diff --git a/qemu/include/qemu/atomic.h b/qemu/include/qemu/atomic.h
index 849221007b..9844160a12 100644
--- a/qemu/include/qemu/atomic.h
+++ b/qemu/include/qemu/atomic.h
@@ -304,4 +304,98 @@
 
 #endif /* __ATOMIC_RELAXED */
 
+#ifndef qatomic_read__nocheck
+#define qatomic_read__nocheck atomic_read__nocheck
+#endif
+#ifndef qatomic_read
+#define qatomic_read atomic_read
+#endif
+#ifndef qatomic_set__nocheck
+#define qatomic_set__nocheck atomic_set__nocheck
+#endif
+#ifndef qatomic_set
+#define qatomic_set atomic_set
+#endif
+#ifndef qatomic_rcu_read
+#define qatomic_rcu_read atomic_rcu_read
+#endif
+#ifndef qatomic_rcu_set
+#define qatomic_rcu_set atomic_rcu_set
+#endif
+#ifndef qatomic_xchg__nocheck
+#define qatomic_xchg__nocheck atomic_xchg__nocheck
+#endif
+#ifndef qatomic_xchg
+#define qatomic_xchg atomic_xchg
+#endif
+#ifndef qatomic_cmpxchg__nocheck
+#define qatomic_cmpxchg__nocheck atomic_cmpxchg__nocheck
+#endif
+#ifndef qatomic_cmpxchg
+#define qatomic_cmpxchg atomic_cmpxchg
+#endif
+#ifndef qatomic_fetch_inc
+#define qatomic_fetch_inc atomic_fetch_inc
+#endif
+#ifndef qatomic_fetch_dec
+#define qatomic_fetch_dec atomic_fetch_dec
+#endif
+#ifndef qatomic_fetch_add
+#define qatomic_fetch_add atomic_fetch_add
+#endif
+#ifndef qatomic_fetch_sub
+#define qatomic_fetch_sub atomic_fetch_sub
+#endif
+#ifndef qatomic_fetch_and
+#define qatomic_fetch_and atomic_fetch_and
+#endif
+#ifndef qatomic_fetch_or
+#define qatomic_fetch_or atomic_fetch_or
+#endif
+#ifndef qatomic_fetch_xor
+#define qatomic_fetch_xor atomic_fetch_xor
+#endif
+#ifndef qatomic_inc_fetch
+#define qatomic_inc_fetch atomic_inc_fetch
+#endif
+#ifndef qatomic_dec_fetch
+#define qatomic_dec_fetch atomic_dec_fetch
+#endif
+#ifndef qatomic_add_fetch
+#define qatomic_add_fetch atomic_add_fetch
+#endif
+#ifndef qatomic_sub_fetch
+#define qatomic_sub_fetch atomic_sub_fetch
+#endif
+#ifndef qatomic_and_fetch
+#define qatomic_and_fetch atomic_and_fetch
+#endif
+#ifndef qatomic_or_fetch
+#define qatomic_or_fetch atomic_or_fetch
+#endif
+#ifndef qatomic_xor_fetch
+#define qatomic_xor_fetch atomic_xor_fetch
+#endif
+#ifndef qatomic_inc
+#define qatomic_inc atomic_inc
+#endif
+#ifndef qatomic_dec
+#define qatomic_dec atomic_dec
+#endif
+#ifndef qatomic_add
+#define qatomic_add atomic_add
+#endif
+#ifndef qatomic_sub
+#define qatomic_sub atomic_sub
+#endif
+#ifndef qatomic_and
+#define qatomic_and atomic_and
+#endif
+#ifndef qatomic_or
+#define qatomic_or atomic_or
+#endif
+#ifndef qatomic_xor
+#define qatomic_xor atomic_xor
+#endif
+
 #endif /* QEMU_ATOMIC_H */
diff --git a/qemu/include/qemu/cacheflush.h b/qemu/include/qemu/cacheflush.h
index 2a9e790a37..744e5fada5 100644
--- a/qemu/include/qemu/cacheflush.h
+++ b/qemu/include/qemu/cacheflush.h
@@ -12,6 +12,16 @@
 #include <libkern/OSCacheControl.h>
 #endif
 
+#if defined(__aarch64__) && !defined(__APPLE__)
+static inline uint64_t qemu_aarch64_read_ctr_el0(void)
+{
+    uint64_t ctr_el0;
+
+    asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr_el0));
+    return ctr_el0;
+}
+#endif
+
 static inline void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 {
 #if defined(__i386__) || defined(__x86_64__) || defined(__s390__)
@@ -19,6 +29,32 @@ static inline void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
     (void)rx;
     (void)rw;
     (void)len;
+#elif defined(__aarch64__) && !defined(__APPLE__)
+    const unsigned int ctr_idc = 1u << 28;
+    const unsigned int ctr_dic = 1u << 29;
+    const uint64_t ctr_el0 = qemu_aarch64_read_ctr_el0();
+    const uintptr_t dcache_lsize = 4 << ((ctr_el0 >> 16) & 15);
+    const uintptr_t icache_lsize = 4 << (ctr_el0 & 15);
+    uintptr_t p;
+
+    if (!(ctr_el0 & ctr_idc)) {
+        for (p = rw & ~(dcache_lsize - 1);
+             p < rw + len; p += dcache_lsize) {
+            asm volatile("dc\tcvau, %0" : : "r"(p) : "memory");
+        }
+    }
+
+    asm volatile("dsb\tish" : : : "memory");
+
+    if (!(ctr_el0 & ctr_dic)) {
+        for (p = rx & ~(icache_lsize - 1);
+             p < rx + len; p += icache_lsize) {
+            asm volatile("ic\tivau, %0" : : "r"(p) : "memory");
+        }
+        asm volatile("dsb\tish" : : : "memory");
+    }
+
+    asm volatile("isb" : : : "memory");
 #elif defined(__APPLE__) && (defined(__aarch64__) || defined(__arm__))
     sys_dcache_flush((void *)rw, len);
     sys_icache_invalidate((void *)rx, len);
diff --git a/qemu/target/mips/translate.c b/qemu/target/mips/translate.c
index 7bf04edeba..1994b9767d 100644
--- a/qemu/target/mips/translate.c
+++ b/qemu/target/mips/translate.c
@@ -2538,6 +2538,7 @@ typedef struct DisasContext {
     target_ulong page_start;
     uint32_t opcode;
     uint64_t insn_flags;
+    int32_t CP0_Config0;
     int32_t CP0_Config1;
     int32_t CP0_Config2;
     int32_t CP0_Config3;
@@ -2572,6 +2573,11 @@ typedef struct DisasContext {
     struct uc_struct *uc;
 } DisasContext;
 
+static inline bool cpu_is_bigendian(DisasContext *ctx)
+{
+    return extract32(ctx->CP0_Config0, CP0C0_BE, 1);
+}
+
 #define DISAS_STOP       DISAS_TARGET_0
 #define DISAS_EXIT       DISAS_TARGET_1
 
@@ -6034,9 +6040,9 @@ static void gen_loongson_lswc2(DisasContext *ctx, int rt, int rs)
             t1 = tcg_temp_new(tcg_ctx);
             tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
             tcg_gen_andi_tl(tcg_ctx, t1, t0, 3);
-#ifndef TARGET_WORDS_BIGENDIAN
-            tcg_gen_xori_tl(tcg_ctx, t1, t1, 3);
-#endif
+            if (!cpu_is_bigendian(ctx)) {
+                tcg_gen_xori_tl(tcg_ctx, t1, t1, 3);
+            }
             tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
             tcg_gen_andi_tl(tcg_ctx, t0, t0, ~3);
             tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUL);
@@ -6064,9 +6070,9 @@ static void gen_loongson_lswc2(DisasContext *ctx, int rt, int rs)
             t1 = tcg_temp_new(tcg_ctx);
             tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
             tcg_gen_andi_tl(tcg_ctx, t1, t0, 3);
-#ifdef TARGET_WORDS_BIGENDIAN
-            tcg_gen_xori_tl(tcg_ctx, t1, t1, 3);
-#endif
+            if (cpu_is_bigendian(ctx)) {
+                tcg_gen_xori_tl(tcg_ctx, t1, t1, 3);
+            }
             tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
             tcg_gen_andi_tl(tcg_ctx, t0, t0, ~3);
             tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUL);
@@ -6096,9 +6102,9 @@ static void gen_loongson_lswc2(DisasContext *ctx, int rt, int rs)
             t1 = tcg_temp_new(tcg_ctx);
             tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
             tcg_gen_andi_tl(tcg_ctx, t1, t0, 7);
-#ifndef TARGET_WORDS_BIGENDIAN
-            tcg_gen_xori_tl(tcg_ctx, t1, t1, 7);
-#endif
+            if (!cpu_is_bigendian(ctx)) {
+                tcg_gen_xori_tl(tcg_ctx, t1, t1, 7);
+            }
             tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
             tcg_gen_andi_tl(tcg_ctx, t0, t0, ~7);
             tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ);
@@ -6118,9 +6124,9 @@ static void gen_loongson_lswc2(DisasContext *ctx, int rt, int rs)
             t1 = tcg_temp_new(tcg_ctx);
             tcg_gen_qemu_ld_tl(tcg_ctx, t1, t0, ctx->mem_idx, MO_UB);
             tcg_gen_andi_tl(tcg_ctx, t1, t0, 7);
-#ifdef TARGET_WORDS_BIGENDIAN
-            tcg_gen_xori_tl(tcg_ctx, t1, t1, 7);
-#endif
+            if (cpu_is_bigendian(ctx)) {
+                tcg_gen_xori_tl(tcg_ctx, t1, t1, 7);
+            }
             tcg_gen_shli_tl(tcg_ctx, t1, t1, 3);
             tcg_gen_andi_tl(tcg_ctx, t0, t0, ~7);
             tcg_gen_qemu_ld_tl(tcg_ctx, t0, t0, ctx->mem_idx, MO_TEUQ);
@@ -31494,6 +31500,7 @@ static void mips_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs)
     ctx->page_start = ctx->base.pc_first & TARGET_PAGE_MASK;
     ctx->saved_pc = -1;
     ctx->insn_flags = env->insn_flags;
+    ctx->CP0_Config0 = env->CP0_Config0;
     ctx->CP0_Config1 = env->CP0_Config1;
     ctx->CP0_Config2 = env->CP0_Config2;
     ctx->CP0_Config3 = env->CP0_Config3;
diff --git a/tests/unit/test_ctl.c b/tests/unit/test_ctl.c
index cae29c9a10..904fea3fab 100644
--- a/tests/unit/test_ctl.c
+++ b/tests/unit/test_ctl.c
@@ -1,67 +1,6 @@
 #include "unicorn_test.h"
-#include <time.h>
 #include <string.h>
 
-// We have to copy this for Android.
-#ifdef _WIN32
-
-#include "windows.h"
-
-#define NANOSECONDS_PER_SECOND 1000000000LL
-
-static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
-{
-    union {
-        uint64_t ll;
-        struct {
-            uint32_t low, high;
-        } l;
-    } u, res;
-    uint64_t rl, rh;
-
-    u.ll = a;
-    rl = (uint64_t)u.l.low * (uint64_t)b;
-    rh = (uint64_t)u.l.high * (uint64_t)b;
-    rh += (rl >> 32);
-    res.l.high = rh / c;
-    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
-    return res.ll;
-}
-
-static int64_t get_freq(void)
-{
-    LARGE_INTEGER freq;
-    int ret = QueryPerformanceFrequency(&freq);
-    if (ret == 0) {
-        fprintf(stderr, "Could not calibrate ticks\n");
-        exit(1);
-    }
-    return freq.QuadPart;
-}
-
-static inline int64_t get_clock_realtime(void)
-{
-    LARGE_INTEGER ti;
-    QueryPerformanceCounter(&ti);
-    return muldiv64(ti.QuadPart, NANOSECONDS_PER_SECOND, get_freq());
-}
-
-#else
-
-#include <sys/time.h>
-#include "sys/mman.h"
-
-/* get host real time in nanosecond */
-static inline int64_t get_clock_realtime(void)
-{
-    struct timeval tv;
-
-    gettimeofday(&tv, NULL);
-    return tv.tv_sec * 1000000000LL + (tv.tv_usec * 1000);
-}
-
-#endif
-
 const uint64_t code_start = 0x1000;
 const uint64_t code_len = 0x4000;
 
@@ -125,19 +64,6 @@ static void test_uc_ctl_exits(void)
     OK(uc_close(uc));
 }
 
-double time_emulation(uc_engine *uc, uint64_t start, uint64_t end)
-{
-    int64_t t1, t2;
-
-    t1 = get_clock_realtime();
-
-    OK(uc_emu_start(uc, start, end, 0, 0));
-
-    t2 = get_clock_realtime();
-
-    return t2 - t1;
-}
-
 #define TB_COUNT (8)
 #define TCG_MAX_INSNS (512) // from tcg.h
 #define CODE_LEN TB_COUNT *TCG_MAX_INSNS
@@ -146,31 +72,40 @@ static void test_uc_ctl_tb_cache(void)
 {
     uc_engine *uc;
     char code[CODE_LEN + 1];
-    double standard, cached, evicted;
 
     memset(code, 0x90, CODE_LEN);
     code[CODE_LEN] = 0;
 
     uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_32, code, sizeof(code) - 1);
 
-    standard = time_emulation(uc, code_start, code_start + sizeof(code) - 1);
-
     for (int i = 0; i < TB_COUNT; i++) {
-        OK(uc_ctl_request_cache(uc, code_start + i * TCG_MAX_INSNS, NULL));
+        uc_tb tb;
+        uint64_t pc = code_start + i * TCG_MAX_INSNS;
+
+        OK(uc_ctl_request_cache(uc, pc, &tb));
+        TEST_CHECK(tb.pc == pc);
+        TEST_CHECK(tb.size > 0);
+        TEST_CHECK(tb.icount > 0);
     }
 
-    cached = time_emulation(uc, code_start, code_start + sizeof(code) - 1);
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
 
     for (int i = 0; i < TB_COUNT; i++) {
         OK(uc_ctl_remove_cache(uc, code_start + i * TCG_MAX_INSNS,
                                code_start + i * TCG_MAX_INSNS + 1));
     }
-    evicted = time_emulation(uc, code_start, code_start + sizeof(code) - 1);
 
-    // In fact, evicted is also slightly faster than standard but we don't do
-    // this guarantee.
-    TEST_CHECK(cached < standard);
-    TEST_CHECK(evicted > cached);
+    for (int i = 0; i < TB_COUNT; i++) {
+        uc_tb tb;
+        uint64_t pc = code_start + i * TCG_MAX_INSNS;
+
+        OK(uc_ctl_request_cache(uc, pc, &tb));
+        TEST_CHECK(tb.pc == pc);
+        TEST_CHECK(tb.size > 0);
+        TEST_CHECK(tb.icount > 0);
+    }
+
+    OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0));
 
     OK(uc_close(uc));
 }

From 222aaf20ec9f1b22fe701711e7cc7ab8d12a841d Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 15:09:41 +0300
Subject: [PATCH 21/32] fix qemu 7.2 ci followups

Align the reduced TCG generated-code pointers with the QEMU 7.2 RX/RW address model so host backends compute branch offsets, labels, prologue addresses, and jump patches against executable addresses while still writing through writable addresses.

Use runtime MIPS guest endianness for unaligned store helpers and declare the PPC host cache flush helper used by reduced per-target builds.
---
 qemu/accel/tcg/cpu-exec.c         |   2 +-
 qemu/accel/tcg/translate-all.c    |   5 +-
 qemu/include/tcg/tcg.h            |  18 ++++-
 qemu/target/mips/op_helper.c      | 123 +++++++++++++++++-------------
 qemu/tcg/aarch64/tcg-target.inc.c |  36 +++++----
 qemu/tcg/ppc/tcg-target.h         |   1 +
 qemu/tcg/tcg.c                    |  18 +++--
 7 files changed, 124 insertions(+), 79 deletions(-)

diff --git a/qemu/accel/tcg/cpu-exec.c b/qemu/accel/tcg/cpu-exec.c
index ab75a5ff10..f37c83dddf 100644
--- a/qemu/accel/tcg/cpu-exec.c
+++ b/qemu/accel/tcg/cpu-exec.c
@@ -200,7 +200,7 @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
         uintptr_t offset = tb->jmp_target_arg[n];
         uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
         uintptr_t jmp_rx = tc_ptr + offset;
-        uintptr_t jmp_rw = jmp_rx;
+        uintptr_t jmp_rw = (uintptr_t)tcg_splitwx_to_rw((void *)jmp_rx);
         tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr);
     } else {
         tb->jmp_target_arg[n] = addr;
diff --git a/qemu/accel/tcg/translate-all.c b/qemu/accel/tcg/translate-all.c
index 0d65cfe2a1..fec3f7a976 100644
--- a/qemu/accel/tcg/translate-all.c
+++ b/qemu/accel/tcg/translate-all.c
@@ -338,7 +338,8 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit)
      * tcg_init_ctx.code_gen_buffer check_offset will wrap to way
      * above the code_gen_buffer_size
      */
-    check_offset = host_pc - (uintptr_t) uc->tcg_ctx->code_gen_buffer;
+    check_offset = (uintptr_t)tcg_splitwx_to_rw((void *)host_pc) -
+        (uintptr_t)uc->tcg_ctx->code_gen_buffer;
 
     if (check_offset < uc->tcg_ctx->code_gen_buffer_size) {
         tb = tcg_tb_lookup(tcg_ctx, host_pc);
@@ -1742,7 +1743,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
     }
 
     gen_code_buf = tcg_ctx->code_gen_ptr;
-    tb->tc.ptr = gen_code_buf;
+    tb->tc.ptr = (tcg_insn_unit *)tcg_splitwx_to_rx(gen_code_buf);
     tb->pc = pc;
     tb->cs_base = cs_base;
     tb->flags = flags;
diff --git a/qemu/include/tcg/tcg.h b/qemu/include/tcg/tcg.h
index b4dfd34a02..56e09be3ef 100644
--- a/qemu/include/tcg/tcg.h
+++ b/qemu/include/tcg/tcg.h
@@ -821,6 +821,18 @@ struct TCGContext {
     bool use_lsx_instructions;
 };
 
+extern uintptr_t tcg_splitwx_diff;
+
+static inline const void *tcg_splitwx_to_rx(void *rw)
+{
+    return rw ? (void *)((uintptr_t)rw + tcg_splitwx_diff) : NULL;
+}
+
+static inline void *tcg_splitwx_to_rw(const void *rx)
+{
+    return rx ? (void *)((uintptr_t)rx - tcg_splitwx_diff) : NULL;
+}
+
 static inline size_t temp_idx(TCGContext *tcg_ctx, TCGTemp *ts)
 {
     ptrdiff_t n = ts - tcg_ctx->temps;
@@ -1240,7 +1252,7 @@ static inline TCGLabel *arg_label(TCGArg i)
  * correct result.
  */
 
-static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
+static inline ptrdiff_t tcg_ptr_byte_diff(const void *a, const void *b)
 {
     return (char *)a - (char *)b;
 }
@@ -1254,9 +1266,9 @@ static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
  * to the destination address.
  */
 
-static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, void *target)
+static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, const void *target)
 {
-    return tcg_ptr_byte_diff(target, s->code_ptr);
+    return tcg_ptr_byte_diff(target, tcg_splitwx_to_rx(s->code_ptr));
 }
 
 /**
diff --git a/qemu/target/mips/op_helper.c b/qemu/target/mips/op_helper.c
index a3e8156c0b..a9933f8f5b 100644
--- a/qemu/target/mips/op_helper.c
+++ b/qemu/target/mips/op_helper.c
@@ -323,31 +323,45 @@ HELPER_LD_ATOMIC(lld, ldq, 0x7, (target_ulong))
 #endif
 #undef HELPER_LD_ATOMIC
 
-#ifdef TARGET_WORDS_BIGENDIAN
-#define GET_LMASK(v) ((v) & 3)
-#define GET_OFFSET(addr, offset) (addr + (offset))
-#else
-#define GET_LMASK(v) (((v) & 3) ^ 3)
-#define GET_OFFSET(addr, offset) (addr - (offset))
-#endif
+static inline bool cpu_is_bigendian(CPUMIPSState *env)
+{
+    return extract32(env->CP0_Config0, CP0C0_BE, 1);
+}
+
+static inline target_ulong get_lmask(CPUMIPSState *env,
+                                     target_ulong value, unsigned bits)
+{
+    unsigned mask = (bits / BITS_PER_BYTE) - 1;
+
+    value &= mask;
+
+    if (!cpu_is_bigendian(env)) {
+        value ^= mask;
+    }
+
+    return value;
+}
 
 void helper_swl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
+    target_ulong lmask = get_lmask(env, arg2, 32);
+    int dir = cpu_is_bigendian(env) ? 1 : -1;
+
     cpu_stb_mmuidx_ra(env, arg2, (uint8_t)(arg1 >> 24), mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) <= 2) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16),
+    if (lmask <= 2) {
+        cpu_stb_mmuidx_ra(env, arg2 + 1 * dir, (uint8_t)(arg1 >> 16),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK(arg2) <= 1) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8),
+    if (lmask <= 1) {
+        cpu_stb_mmuidx_ra(env, arg2 + 2 * dir, (uint8_t)(arg1 >> 8),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK(arg2) == 0) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 3), (uint8_t)arg1,
+    if (lmask == 0) {
+        cpu_stb_mmuidx_ra(env, arg2 + 3 * dir, (uint8_t)arg1,
                           mem_idx, GETPC());
     }
 }
@@ -355,20 +369,23 @@ void helper_swl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
 void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
+    target_ulong lmask = get_lmask(env, arg2, 32);
+    int dir = cpu_is_bigendian(env) ? 1 : -1;
+
     cpu_stb_mmuidx_ra(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) >= 1) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8),
+    if (lmask >= 1) {
+        cpu_stb_mmuidx_ra(env, arg2 - 1 * dir, (uint8_t)(arg1 >> 8),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK(arg2) >= 2) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16),
+    if (lmask >= 2) {
+        cpu_stb_mmuidx_ra(env, arg2 - 2 * dir, (uint8_t)(arg1 >> 16),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK(arg2) == 3) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24),
+    if (lmask == 3) {
+        cpu_stb_mmuidx_ra(env, arg2 - 3 * dir, (uint8_t)(arg1 >> 24),
                           mem_idx, GETPC());
     }
 }
@@ -378,49 +395,46 @@ void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
  * "half" load and stores.  We must do the memory access inline,
  * or fault handling won't work.
  */
-#ifdef TARGET_WORDS_BIGENDIAN
-#define GET_LMASK64(v) ((v) & 7)
-#else
-#define GET_LMASK64(v) (((v) & 7) ^ 7)
-#endif
-
 void helper_sdl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
+    target_ulong lmask = get_lmask(env, arg2, 64);
+    int dir = cpu_is_bigendian(env) ? 1 : -1;
+
     cpu_stb_mmuidx_ra(env, arg2, (uint8_t)(arg1 >> 56), mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) <= 6) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48),
+    if (lmask <= 6) {
+        cpu_stb_mmuidx_ra(env, arg2 + 1 * dir, (uint8_t)(arg1 >> 48),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) <= 5) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40),
+    if (lmask <= 5) {
+        cpu_stb_mmuidx_ra(env, arg2 + 2 * dir, (uint8_t)(arg1 >> 40),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) <= 4) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32),
+    if (lmask <= 4) {
+        cpu_stb_mmuidx_ra(env, arg2 + 3 * dir, (uint8_t)(arg1 >> 32),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) <= 3) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24),
+    if (lmask <= 3) {
+        cpu_stb_mmuidx_ra(env, arg2 + 4 * dir, (uint8_t)(arg1 >> 24),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) <= 2) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16),
+    if (lmask <= 2) {
+        cpu_stb_mmuidx_ra(env, arg2 + 5 * dir, (uint8_t)(arg1 >> 16),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) <= 1) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8),
+    if (lmask <= 1) {
+        cpu_stb_mmuidx_ra(env, arg2 + 6 * dir, (uint8_t)(arg1 >> 8),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) <= 0) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, 7), (uint8_t)arg1,
+    if (lmask <= 0) {
+        cpu_stb_mmuidx_ra(env, arg2 + 7 * dir, (uint8_t)arg1,
                           mem_idx, GETPC());
     }
 }
@@ -428,40 +442,43 @@ void helper_sdl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
 void helper_sdr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
+    target_ulong lmask = get_lmask(env, arg2, 64);
+    int dir = cpu_is_bigendian(env) ? 1 : -1;
+
     cpu_stb_mmuidx_ra(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) >= 1) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8),
+    if (lmask >= 1) {
+        cpu_stb_mmuidx_ra(env, arg2 - 1 * dir, (uint8_t)(arg1 >> 8),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) >= 2) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16),
+    if (lmask >= 2) {
+        cpu_stb_mmuidx_ra(env, arg2 - 2 * dir, (uint8_t)(arg1 >> 16),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) >= 3) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24),
+    if (lmask >= 3) {
+        cpu_stb_mmuidx_ra(env, arg2 - 3 * dir, (uint8_t)(arg1 >> 24),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) >= 4) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32),
+    if (lmask >= 4) {
+        cpu_stb_mmuidx_ra(env, arg2 - 4 * dir, (uint8_t)(arg1 >> 32),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) >= 5) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40),
+    if (lmask >= 5) {
+        cpu_stb_mmuidx_ra(env, arg2 - 5 * dir, (uint8_t)(arg1 >> 40),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) >= 6) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48),
+    if (lmask >= 6) {
+        cpu_stb_mmuidx_ra(env, arg2 - 6 * dir, (uint8_t)(arg1 >> 48),
                           mem_idx, GETPC());
     }
 
-    if (GET_LMASK64(arg2) == 7) {
-        cpu_stb_mmuidx_ra(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56),
+    if (lmask == 7) {
+        cpu_stb_mmuidx_ra(env, arg2 - 7 * dir, (uint8_t)(arg1 >> 56),
                           mem_idx, GETPC());
     }
 }
diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index 277740e466..7739f7bd02 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -78,9 +78,12 @@ static const int tcg_target_call_oarg_regs[1] = {
 #define TCG_REG_GUEST_BASE TCG_REG_X28
 #endif
 
-static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc26(tcg_insn_unit *code_ptr,
+                              const tcg_insn_unit *target)
 {
-    ptrdiff_t offset = target - code_ptr;
+    const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(code_ptr);
+    ptrdiff_t offset = target - src_rx;
+
     if (offset == sextract64(offset, 0, 26)) {
         /* read instruction, mask away previous PC_REL26 parameter contents,
            set the proper offset, then write back the instruction. */
@@ -90,9 +93,12 @@ static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
     return false;
 }
 
-static inline bool reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc19(tcg_insn_unit *code_ptr,
+                              const tcg_insn_unit *target)
 {
-    ptrdiff_t offset = target - code_ptr;
+    const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(code_ptr);
+    ptrdiff_t offset = target - src_rx;
+
     if (offset == sextract64(offset, 0, 19)) {
         *code_ptr = deposit32(*code_ptr, 5, 19, offset);
         return true;
@@ -107,9 +113,9 @@ static inline bool patch_reloc(tcg_insn_unit *code_ptr, int type,
     switch (type) {
     case R_AARCH64_JUMP26:
     case R_AARCH64_CALL26:
-        return reloc_pc26(code_ptr, (tcg_insn_unit *)value);
+        return reloc_pc26(code_ptr, (const tcg_insn_unit *)value);
     case R_AARCH64_CONDBR19:
-        return reloc_pc19(code_ptr, (tcg_insn_unit *)value);
+        return reloc_pc19(code_ptr, (const tcg_insn_unit *)value);
     default:
         g_assert_not_reached();
     }
@@ -1052,12 +1058,14 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
     /* Look for host pointer values within 4G of the PC.  This happens
        often when loading pointers to QEMU's own data structures.  */
     if (type == TCG_TYPE_I64) {
-        tcg_target_long disp = value - (intptr_t)s->code_ptr;
+        intptr_t src_rx = (intptr_t)tcg_splitwx_to_rx(s->code_ptr);
+        tcg_target_long disp = value - src_rx;
+
         if (disp == sextract64(disp, 0, 21)) {
             tcg_out_insn(s, 3406, ADR, rd, disp);
             return;
         }
-        disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
+        disp = (value >> 12) - (src_rx >> 12);
         if (disp == sextract64(disp, 0, 21)) {
             tcg_out_insn(s, 3406, ADRP, rd, disp);
             if (value & 0xfff) {
@@ -1583,7 +1591,8 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     MemOp size = opc & MO_SIZE;
 
     const int type = tcg_uc_has_hookmem(s) ? R_AARCH64_JUMP26 : R_AARCH64_CONDBR19;
-    if (!patch_reloc(lb->label_ptr[0], type, (intptr_t)s->code_ptr, 0)) {
+    if (!patch_reloc(lb->label_ptr[0], type,
+                     (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 0)) {
         return false;
     }
 
@@ -1609,7 +1618,8 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     MemOp size = opc & MO_SIZE;
 
     const int type = tcg_uc_has_hookmem(s) ? R_AARCH64_JUMP26 : R_AARCH64_CONDBR19;
-    if (!patch_reloc(lb->label_ptr[0], type, (intptr_t)s->code_ptr, 0)) {
+    if (!patch_reloc(lb->label_ptr[0], type,
+                     (intptr_t)tcg_splitwx_to_rx(s->code_ptr), 0)) {
         return false;
     }
 
@@ -1634,7 +1644,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
     label->type = ext;
     label->datalo_reg = data_reg;
     label->addrlo_reg = addr_reg;
-    label->raddr = raddr;
+    label->raddr = (tcg_insn_unit *)tcg_splitwx_to_rx(raddr);
     label->label_ptr[0] = label_ptr;
 }
 
@@ -2854,11 +2864,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
      * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
      * and fall through to the rest of the epilogue.
      */
-    s->code_gen_epilogue = s->code_ptr;
+    s->code_gen_epilogue = (void *)tcg_splitwx_to_rx(s->code_ptr);
     tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_X0, 0);
 
     /* TB epilogue */
-    s->tb_ret_addr = s->code_ptr;
+    s->tb_ret_addr = (void *)tcg_splitwx_to_rx(s->code_ptr);
 
     /* Remove TCG locals stack space.  */
     tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
diff --git a/qemu/tcg/ppc/tcg-target.h b/qemu/tcg/ppc/tcg-target.h
index 9bd91da97b..f43c7432c0 100644
--- a/qemu/tcg/ppc/tcg-target.h
+++ b/qemu/tcg/ppc/tcg-target.h
@@ -181,6 +181,7 @@ extern bool have_vsx;
 #define TCG_TARGET_HAS_cmpsel_vec       0
 
 void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+void flush_icache_range(uintptr_t start, uintptr_t stop);
 
 #define TCG_TARGET_HAS_goto_ptr         0
 #define TCG_TARGET_DEFAULT_MO (0)
diff --git a/qemu/tcg/tcg.c b/qemu/tcg/tcg.c
index 17f4de7cc0..72f8088ce6 100644
--- a/qemu/tcg/tcg.c
+++ b/qemu/tcg/tcg.c
@@ -63,6 +63,8 @@
 
 #include <uc_priv.h>
 
+uintptr_t tcg_splitwx_diff;
+
 /* Forward declarations for functions declared in tcg-target.inc.c and
    used here. */
 static void tcg_target_init(TCGContext *s);
@@ -251,7 +253,7 @@ static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
 {
     tcg_debug_assert(!l->has_value);
     l->has_value = 1;
-    l->u.value_ptr = ptr;
+    l->u.value_ptr = (tcg_insn_unit *)tcg_splitwx_to_rx(ptr);
 }
 
 TCGLabel *gen_new_label(TCGContext *s)
@@ -835,7 +837,7 @@ void tcg_prologue_init(TCGContext *s)
     s->code_ptr = buf0;
     s->code_buf = buf0;
     s->data_gen_ptr = NULL;
-    s->code_gen_prologue = buf0;
+    s->code_gen_prologue = (void *)tcg_splitwx_to_rx(buf0);
 
     /* Compute a high-water mark, at which we voluntarily flush the buffer
        and start over.  The size here is arbitrary, significantly larger
@@ -858,7 +860,7 @@ void tcg_prologue_init(TCGContext *s)
 #endif
 
     buf1 = s->code_ptr;
-    flush_idcache_range((uintptr_t)buf0, (uintptr_t)buf0,
+    flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(buf0), (uintptr_t)buf0,
                         (uintptr_t)buf1 - (uintptr_t)buf0);
 
     /* Deduct the prologue from the buffer.  */
@@ -869,7 +871,8 @@ void tcg_prologue_init(TCGContext *s)
     total_size -= prologue_size;
     s->code_gen_buffer_size = total_size;
 
-    tcg_register_jit(s, s->code_gen_buffer, total_size);
+    tcg_register_jit(s, (void *)tcg_splitwx_to_rx(s->code_gen_buffer),
+                     total_size);
 
     /* Assert that goto_ptr is implemented completely.  */
     if (TCG_TARGET_HAS_goto_ptr) {
@@ -3765,8 +3768,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
     //tcg_dump_ops(s, false, "after opt4:");
     tcg_reg_alloc_start(s);
 
-    s->code_buf = tb->tc.ptr;
-    s->code_ptr = tb->tc.ptr;
+    s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
+    s->code_ptr = s->code_buf;
 
 #ifdef TCG_TARGET_NEED_LDST_LABELS
     QSIMPLEQ_INIT(&s->ldst_labels);
@@ -3872,7 +3875,8 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
     }
 
     /* flush instruction cache */
-    flush_idcache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_buf,
+    flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
+                        (uintptr_t)s->code_buf,
                         (uintptr_t)s->code_ptr - (uintptr_t)s->code_buf);
 
     return tcg_current_code_size(s);

From e55e1611e85fde4bb80f6edca218086e09494cf1 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 15:20:18 +0300
Subject: [PATCH 22/32] fix qemu 7.2 splitwx link

Keep split-WX conversion inline in the reduced single-mapping TCG runtime so per-arch archives do not define duplicate data symbols during GNU ld links.
---
 qemu/include/tcg/tcg.h | 6 ++----
 qemu/tcg/tcg.c         | 2 --
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/qemu/include/tcg/tcg.h b/qemu/include/tcg/tcg.h
index 56e09be3ef..888117a59d 100644
--- a/qemu/include/tcg/tcg.h
+++ b/qemu/include/tcg/tcg.h
@@ -821,16 +821,14 @@ struct TCGContext {
     bool use_lsx_instructions;
 };
 
-extern uintptr_t tcg_splitwx_diff;
-
 static inline const void *tcg_splitwx_to_rx(void *rw)
 {
-    return rw ? (void *)((uintptr_t)rw + tcg_splitwx_diff) : NULL;
+    return rw;
 }
 
 static inline void *tcg_splitwx_to_rw(const void *rx)
 {
-    return rx ? (void *)((uintptr_t)rx - tcg_splitwx_diff) : NULL;
+    return (void *)rx;
 }
 
 static inline size_t temp_idx(TCGContext *tcg_ctx, TCGTemp *ts)
diff --git a/qemu/tcg/tcg.c b/qemu/tcg/tcg.c
index 72f8088ce6..4482fd5e44 100644
--- a/qemu/tcg/tcg.c
+++ b/qemu/tcg/tcg.c
@@ -63,8 +63,6 @@
 
 #include <uc_priv.h>
 
-uintptr_t tcg_splitwx_diff;
-
 /* Forward declarations for functions declared in tcg-target.inc.c and
    used here. */
 static void tcg_target_init(TCGContext *s);

From 45f548ffbc9fe936dd53816961a957b42e58298f Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 15:25:13 +0300
Subject: [PATCH 23/32] fix zig macos runner

Pin the Zig macOS workflow to macos-14 so Zig 0.14 does not run on an unsupported macOS 26 image.
---
 .github/workflows/zigbuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/zigbuild.yml b/.github/workflows/zigbuild.yml
index 6b39211da1..77e0a94e50 100644
--- a/.github/workflows/zigbuild.yml
+++ b/.github/workflows/zigbuild.yml
@@ -47,7 +47,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runs-on: [macos-latest]
+        runs-on: [macos-14]
     runs-on: ${{ matrix.runs-on }}
     steps:
       - uses: actions/checkout@v4

From abee67e8860763750ccccd7622d5c9c7236020b0 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 15:38:27 +0300
Subject: [PATCH 24/32] fix macos apple jit state

Use pthread JIT write-protection transitions directly and avoid private SPRR state probes on Apple Silicon hosts.
---
 qemu/include/tcg/tcg-apple-jit.h | 53 ++++++++------------------------
 1 file changed, 13 insertions(+), 40 deletions(-)

diff --git a/qemu/include/tcg/tcg-apple-jit.h b/qemu/include/tcg/tcg-apple-jit.h
index 2854f20a1b..017643d41a 100644
--- a/qemu/include/tcg/tcg-apple-jit.h
+++ b/qemu/include/tcg/tcg-apple-jit.h
@@ -32,59 +32,29 @@
 
 #if defined(__APPLE__) && defined(HAVE_PTHREAD_JIT_PROTECT) && (defined(__arm__) || defined(__aarch64__))
 
-// Returns the S3_6_c15_c1_5 register's value
-// Taken from
-// https://stackoverflow.com/questions/70019553/lldb-how-to-read-the-permissions-of-a-memory-region-for-a-thread
-// https://blog.svenpeter.dev/posts/m1_sprr_gxf/
-// On Github Action (Virtualized environment), this shall always returns 0
-static inline uint64_t read_sprr_perm(void)
+QEMU_UNUSED_FUNC static inline uint8_t thread_mask()
 {
-    uint64_t v;
-    __asm__ __volatile__("isb sy\n"
-                         "mrs %0, S3_6_c15_c1_5\n"
-                         : "=r"(v)::"memory");
-    return v;
-}
-
-QEMU_UNUSED_FUNC static inline uint8_t thread_mask() 
-{
-    if (!pthread_jit_write_protect_supported_np()) {
-        return 0;
-    }
-    uint64_t v = read_sprr_perm();
-
-    if (v == 0) {
-        return 0;
-    } else {
-        return (v >> 20) & 3;
-    }
+    return 0;
 }
 
 QEMU_UNUSED_FUNC static inline bool thread_writeable()
 {
-    return thread_mask() == 3;
+    return false;
 }
 
 QEMU_UNUSED_FUNC static inline bool thread_executable()
 {
-    return thread_mask() == 1;
+    return true;
 }
 
-static inline void assert_executable(bool executable) {
-    if (!pthread_jit_write_protect_supported_np()) {
-        return;
-    }
-    uint64_t v = read_sprr_perm();
-
-    if (!v) {
-        return;
-    }
-    assert(executable == thread_executable());
+static inline void assert_executable(bool executable)
+{
+    (void)executable;
 }
 
 #else
 
-QEMU_UNUSED_FUNC static inline uint8_t thread_mask() 
+QEMU_UNUSED_FUNC static inline uint8_t thread_mask()
 {
     return 0;
 }
@@ -99,7 +69,8 @@ QEMU_UNUSED_FUNC static inline bool thread_executable()
     return false;
 }
 
-static inline void assert_executable(bool executable) {
+static inline void assert_executable(bool executable)
+{
 }
 
 #endif
@@ -110,7 +81,9 @@ static inline void assert_executable(bool executable) {
 /* write protect enable = write disable */
 static inline void jit_write_protect(int enabled)
 {
-    return pthread_jit_write_protect_np(enabled);
+    if (pthread_jit_write_protect_supported_np()) {
+        pthread_jit_write_protect_np(enabled);
+    }
 }
 
 #define JIT_CALLBACK_GUARD(x)                       \

From d7784e15eaf85b669bbe188ac762e5aa230237fe Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 19:42:21 +0300
Subject: [PATCH 25/32] fix ubuntu mips opcode mask

---
 qemu/target/mips/translate.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/qemu/target/mips/translate.c b/qemu/target/mips/translate.c
index 1994b9767d..a41cb49d61 100644
--- a/qemu/target/mips/translate.c
+++ b/qemu/target/mips/translate.c
@@ -37,7 +37,7 @@
 #define MIPS_DEBUG_DISAS 0
 
 /* MIPS major opcodes */
-#define MASK_OP_MAJOR(op)       (op & (0x3FUL << 26))
+#define MASK_OP_MAJOR(op)       (op & (0x3Fu << 26))
 
 enum {
     /* indirect opcode tables */
@@ -5953,12 +5953,14 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 static void gen_loongson_lswc2(DisasContext *ctx, int rt, int rs)
 {
     TCGContext *tcg_ctx = ctx->uc->tcg_ctx;
-    int rt1 = ctx->opcode & 0x1f;
-    int lsq_offset = sextract32(ctx->opcode, 6, 9) << 4;
     int shf_offset = sextract32(ctx->opcode, 6, 8);
     uint32_t opc = MASK_LOONGSON_GSLSQ(ctx->opcode);
     TCGv t0, t1, t2;
     TCGv_i32 fp0;
+#if defined(TARGET_MIPS64)
+    int rt1 = ctx->opcode & 0x1f;
+    int lsq_offset = sextract32(ctx->opcode, 6, 9) << 4;
+#endif
 
     switch (opc) {
 #if defined(TARGET_MIPS64)

From 6a360809a365fbe6e71ff178dc2d5afbd6cf65eb Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 19:55:45 +0300
Subject: [PATCH 26/32] fix aarch64 tcg branch offsets

---
 qemu/tcg/aarch64/tcg-target.inc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index 7739f7bd02..0796f1e516 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -1402,7 +1402,7 @@ static void tcg_out_brcond(TCGContext *s, TCGType ext, TCGCond c, TCGArg a,
         tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
         offset = tcg_in32(s) >> 5;
     } else {
-        offset = l->u.value_ptr - s->code_ptr;
+        offset = tcg_pcrel_diff(s, l->u.value_ptr) >> 2;
         tcg_debug_assert(offset == sextract64(offset, 0, 19));
     }
 

From 7afe90e77abd9229f836d608079efa5a95e468c5 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 20:22:20 +0300
Subject: [PATCH 27/32] fix aarch64 tcg sli opcode

---
 qemu/tcg/aarch64/tcg-target.inc.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index 0796f1e516..e9821541bb 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -563,6 +563,7 @@ typedef enum {
     I3614_SSHR      = 0x0f000400,
     I3614_SSRA      = 0x0f001400,
     I3614_SHL       = 0x0f005400,
+    I3614_SLI       = 0x2f005400,
     I3614_USHR      = 0x2f000400,
     I3614_USRA      = 0x2f001400,
 
@@ -2427,6 +2428,10 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_sari_vec:
         tcg_out_insn(s, 3614, SSHR, is_q, a0, a1, (16 << vece) - a2);
         break;
+    case INDEX_op_aa64_sli_vec:
+        tcg_out_insn(s, 3614, SLI, is_q, a0, a2,
+                     args[3] + (8 << vece));
+        break;
     case INDEX_op_shlv_vec:
         tcg_out_insn(s, 3616, USHL, is_q, vece, a0, a1, a2);
         break;
@@ -2576,6 +2581,7 @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
     static const TCGTargetOpDef w_w_wO = { .args_ct_str = { "w", "w", "wO" } };
     static const TCGTargetOpDef w_w_wN = { .args_ct_str = { "w", "w", "wN" } };
     static const TCGTargetOpDef w_w_wZ = { .args_ct_str = { "w", "w", "wZ" } };
+    static const TCGTargetOpDef w_0_w = { .args_ct_str = { "w", "0", "w" } };
     static const TCGTargetOpDef r_r_ri = { .args_ct_str = { "r", "r", "ri" } };
     static const TCGTargetOpDef r_r_rA = { .args_ct_str = { "r", "r", "rA" } };
     static const TCGTargetOpDef r_r_rL = { .args_ct_str = { "r", "r", "rL" } };
@@ -2767,6 +2773,8 @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
         return &w_w_wZ;
     case INDEX_op_bitsel_vec:
         return &w_w_w_w;
+    case INDEX_op_aa64_sli_vec:
+        return &w_0_w;
 
     default:
         return NULL;

From 49b145a182245f88ecf372dd533a7db7b0e23452 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sat, 27 Jun 2026 21:33:47 +0300
Subject: [PATCH 28/32] fix msvc artifact staging

---
 .github/workflows/build-uc2.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-uc2.yml b/.github/workflows/build-uc2.yml
index 2ee8a2eed5..fe2d4ef05d 100644
--- a/.github/workflows/build-uc2.yml
+++ b/.github/workflows/build-uc2.yml
@@ -169,7 +169,7 @@ jobs:
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip --config ${{ env.BUILD_TYPE }}
           ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
-          mv ${{ env.BUILD_TYPE }} instdir
+          cmake -E copy_directory "${{ env.BUILD_TYPE }}" "instdir/${{ env.BUILD_TYPE }}"
 
       - name: '🛠️ Win MSVC 32 setup'
         if: contains(matrix.config.name, 'MSVC 32')
@@ -196,7 +196,7 @@ jobs:
           cmake --build . --config ${{ env.BUILD_TYPE }}
           cmake --install . --strip --config ${{ env.BUILD_TYPE }}
           ctest --parallel ${{ env.CTEST_PARALLEL_LEVEL }} --output-on-failure -C ${{ env.BUILD_TYPE }}
-          mv ${{ env.BUILD_TYPE }} instdir
+          cmake -E copy_directory "${{ env.BUILD_TYPE }}" "instdir/${{ env.BUILD_TYPE }}"
 
       - name: '🚧 Win MINGW build'
         if: contains(matrix.config.mingw, 'MINGW')

From 95204c92d7ecbb0872687bbc392c410eae959b0f Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sun, 28 Jun 2026 00:13:53 +0300
Subject: [PATCH 29/32] fix macos aarch64 tcg vectors

---
 qemu/tcg/aarch64/tcg-target.inc.c | 218 +++++++++++++++++++++++++-----
 1 file changed, 187 insertions(+), 31 deletions(-)

diff --git a/qemu/tcg/aarch64/tcg-target.inc.c b/qemu/tcg/aarch64/tcg-target.inc.c
index e9821541bb..28a5ce2da0 100644
--- a/qemu/tcg/aarch64/tcg-target.inc.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -559,6 +559,39 @@ typedef enum {
     I3606_BIC       = 0x2f001400,
     I3606_ORR       = 0x0f001400,
 
+    /* AdvSIMD scalar shift by immediate */
+    I3609_SSHR      = 0x5f000400,
+    I3609_SSRA      = 0x5f001400,
+    I3609_SHL       = 0x5f005400,
+    I3609_USHR      = 0x7f000400,
+    I3609_USRA      = 0x7f001400,
+    I3609_SLI       = 0x7f005400,
+
+    /* AdvSIMD scalar three same */
+    I3611_SQADD     = 0x5e200c00,
+    I3611_SQSUB     = 0x5e202c00,
+    I3611_CMGT      = 0x5e203400,
+    I3611_CMGE      = 0x5e203c00,
+    I3611_SSHL      = 0x5e204400,
+    I3611_ADD       = 0x5e208400,
+    I3611_CMTST     = 0x5e208c00,
+    I3611_UQADD     = 0x7e200c00,
+    I3611_UQSUB     = 0x7e202c00,
+    I3611_CMHI      = 0x7e203400,
+    I3611_CMHS      = 0x7e203c00,
+    I3611_USHL      = 0x7e204400,
+    I3611_SUB       = 0x7e208400,
+    I3611_CMEQ      = 0x7e208c00,
+
+    /* AdvSIMD scalar two-reg misc */
+    I3612_CMGT0     = 0x5e208800,
+    I3612_CMEQ0     = 0x5e209800,
+    I3612_CMLT0     = 0x5e20a800,
+    I3612_ABS       = 0x5e20b800,
+    I3612_CMGE0     = 0x7e208800,
+    I3612_CMLE0     = 0x7e209800,
+    I3612_NEG       = 0x7e20b800,
+
     /* AdvSIMD shift by immediate */
     I3614_SSHR      = 0x0f000400,
     I3614_SSRA      = 0x0f001400,
@@ -775,6 +808,25 @@ static void tcg_out_insn_3606(TCGContext *s, AArch64Insn insn, bool q,
               | (imm8 & 0xe0) << (16 - 5) | (imm8 & 0x1f) << 5);
 }
 
+static void tcg_out_insn_3609(TCGContext *s, AArch64Insn insn,
+                              TCGReg rd, TCGReg rn, unsigned immhb)
+{
+    tcg_out32(s, insn | immhb << 16 | (rn & 0x1f) << 5 | (rd & 0x1f));
+}
+
+static void tcg_out_insn_3611(TCGContext *s, AArch64Insn insn,
+                              unsigned size, TCGReg rd, TCGReg rn, TCGReg rm)
+{
+    tcg_out32(s, insn | (size << 22) | (rm & 0x1f) << 16
+              | (rn & 0x1f) << 5 | (rd & 0x1f));
+}
+
+static void tcg_out_insn_3612(TCGContext *s, AArch64Insn insn,
+                              unsigned size, TCGReg rd, TCGReg rn)
+{
+    tcg_out32(s, insn | (size << 22) | (rn & 0x1f) << 5 | (rd & 0x1f));
+}
+
 static void tcg_out_insn_3614(TCGContext *s, AArch64Insn insn, bool q,
                               TCGReg rd, TCGReg rn, unsigned immhb)
 {
@@ -2292,23 +2344,38 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
                            unsigned vecl, unsigned vece,
                            const TCGArg *args, const int *const_args)
 {
-    static const AArch64Insn cmp_insn[16] = {
+    static const AArch64Insn cmp_vec_insn[16] = {
         [TCG_COND_EQ] = I3616_CMEQ,
         [TCG_COND_GT] = I3616_CMGT,
         [TCG_COND_GE] = I3616_CMGE,
         [TCG_COND_GTU] = I3616_CMHI,
         [TCG_COND_GEU] = I3616_CMHS,
     };
-    static const AArch64Insn cmp0_insn[16] = {
+    static const AArch64Insn cmp_scalar_insn[16] = {
+        [TCG_COND_EQ] = I3611_CMEQ,
+        [TCG_COND_GT] = I3611_CMGT,
+        [TCG_COND_GE] = I3611_CMGE,
+        [TCG_COND_GTU] = I3611_CMHI,
+        [TCG_COND_GEU] = I3611_CMHS,
+    };
+    static const AArch64Insn cmp0_vec_insn[16] = {
         [TCG_COND_EQ] = I3617_CMEQ0,
         [TCG_COND_GT] = I3617_CMGT0,
         [TCG_COND_GE] = I3617_CMGE0,
         [TCG_COND_LT] = I3617_CMLT0,
         [TCG_COND_LE] = I3617_CMLE0,
     };
+    static const AArch64Insn cmp0_scalar_insn[16] = {
+        [TCG_COND_EQ] = I3612_CMEQ0,
+        [TCG_COND_GT] = I3612_CMGT0,
+        [TCG_COND_GE] = I3612_CMGE0,
+        [TCG_COND_LT] = I3612_CMLT0,
+        [TCG_COND_LE] = I3612_CMLE0,
+    };
 
     TCGType type = vecl + TCG_TYPE_V64;
     unsigned is_q = vecl;
+    bool is_scalar = !is_q && vece == MO_64;
     TCGArg a0, a1, a2, a3;
     int cmode, imm8;
 
@@ -2327,19 +2394,35 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
         tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
         break;
     case INDEX_op_add_vec:
-        tcg_out_insn(s, 3616, ADD, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, ADD, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, ADD, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_sub_vec:
-        tcg_out_insn(s, 3616, SUB, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, SUB, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, SUB, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_mul_vec:
         tcg_out_insn(s, 3616, MUL, is_q, vece, a0, a1, a2);
         break;
     case INDEX_op_neg_vec:
-        tcg_out_insn(s, 3617, NEG, is_q, vece, a0, a1);
+        if (is_scalar) {
+            tcg_out_insn(s, 3612, NEG, vece, a0, a1);
+        } else {
+            tcg_out_insn(s, 3617, NEG, is_q, vece, a0, a1);
+        }
         break;
     case INDEX_op_abs_vec:
-        tcg_out_insn(s, 3617, ABS, is_q, vece, a0, a1);
+        if (is_scalar) {
+            tcg_out_insn(s, 3612, ABS, vece, a0, a1);
+        } else {
+            tcg_out_insn(s, 3617, ABS, is_q, vece, a0, a1);
+        }
         break;
     case INDEX_op_and_vec:
         if (const_args[2]) {
@@ -2393,16 +2476,32 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
         tcg_out_insn(s, 3616, EOR, is_q, 0, a0, a1, a2);
         break;
     case INDEX_op_ssadd_vec:
-        tcg_out_insn(s, 3616, SQADD, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, SQADD, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, SQADD, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_sssub_vec:
-        tcg_out_insn(s, 3616, SQSUB, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, SQSUB, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, SQSUB, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_usadd_vec:
-        tcg_out_insn(s, 3616, UQADD, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, UQADD, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, UQADD, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_ussub_vec:
-        tcg_out_insn(s, 3616, UQSUB, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, UQSUB, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, UQSUB, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_smax_vec:
         tcg_out_insn(s, 3616, SMAX, is_q, vece, a0, a1, a2);
@@ -2420,23 +2519,50 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
         tcg_out_insn(s, 3617, NOT, is_q, 0, a0, a1);
         break;
     case INDEX_op_shli_vec:
-        tcg_out_insn(s, 3614, SHL, is_q, a0, a1, a2 + (8 << vece));
+        if (is_scalar) {
+            tcg_out_insn(s, 3609, SHL, a0, a1, a2 + (8 << vece));
+        } else {
+            tcg_out_insn(s, 3614, SHL, is_q, a0, a1,
+                         a2 + (8 << vece));
+        }
         break;
     case INDEX_op_shri_vec:
-        tcg_out_insn(s, 3614, USHR, is_q, a0, a1, (16 << vece) - a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3609, USHR, a0, a1, (16 << vece) - a2);
+        } else {
+            tcg_out_insn(s, 3614, USHR, is_q, a0, a1,
+                         (16 << vece) - a2);
+        }
         break;
     case INDEX_op_sari_vec:
-        tcg_out_insn(s, 3614, SSHR, is_q, a0, a1, (16 << vece) - a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3609, SSHR, a0, a1, (16 << vece) - a2);
+        } else {
+            tcg_out_insn(s, 3614, SSHR, is_q, a0, a1,
+                         (16 << vece) - a2);
+        }
         break;
     case INDEX_op_aa64_sli_vec:
-        tcg_out_insn(s, 3614, SLI, is_q, a0, a2,
-                     args[3] + (8 << vece));
+        if (is_scalar) {
+            tcg_out_insn(s, 3609, SLI, a0, a2, args[3] + (8 << vece));
+        } else {
+            tcg_out_insn(s, 3614, SLI, is_q, a0, a2,
+                         args[3] + (8 << vece));
+        }
         break;
     case INDEX_op_shlv_vec:
-        tcg_out_insn(s, 3616, USHL, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, USHL, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, USHL, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_aa64_sshl_vec:
-        tcg_out_insn(s, 3616, SSHL, is_q, vece, a0, a1, a2);
+        if (is_scalar) {
+            tcg_out_insn(s, 3611, SSHL, vece, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3616, SSHL, is_q, vece, a0, a1, a2);
+        }
         break;
     case INDEX_op_cmp_vec:
         {
@@ -2445,30 +2571,60 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
 
             if (cond == TCG_COND_NE) {
                 if (const_args[2]) {
-                    tcg_out_insn(s, 3616, CMTST, is_q, vece, a0, a1, a1);
+                    if (is_scalar) {
+                        tcg_out_insn(s, 3611, CMTST, vece, a0, a1, a1);
+                    } else {
+                        tcg_out_insn(s, 3616, CMTST, is_q, vece,
+                                     a0, a1, a1);
+                    }
                 } else {
-                    tcg_out_insn(s, 3616, CMEQ, is_q, vece, a0, a1, a2);
+                    if (is_scalar) {
+                        tcg_out_insn(s, 3611, CMEQ, vece, a0, a1, a2);
+                    } else {
+                        tcg_out_insn(s, 3616, CMEQ, is_q, vece,
+                                     a0, a1, a2);
+                    }
                     tcg_out_insn(s, 3617, NOT, is_q, 0, a0, a0);
                 }
             } else {
                 if (const_args[2]) {
-                    insn = cmp0_insn[cond];
-                    if (insn) {
-                        tcg_out_insn_3617(s, insn, is_q, vece, a0, a1);
-                        break;
+                    if (is_scalar) {
+                        insn = cmp0_scalar_insn[cond];
+                        if (insn) {
+                            tcg_out_insn_3612(s, insn, vece, a0, a1);
+                            break;
+                        }
+                    } else {
+                        insn = cmp0_vec_insn[cond];
+                        if (insn) {
+                            tcg_out_insn_3617(s, insn, is_q, vece, a0, a1);
+                            break;
+                        }
                     }
                     tcg_out_dupi_vec(s, type, TCG_VEC_TMP, 0);
                     a2 = TCG_VEC_TMP;
                 }
-                insn = cmp_insn[cond];
-                if (insn == 0) {
-                    TCGArg t;
-                    t = a1, a1 = a2, a2 = t;
-                    cond = tcg_swap_cond(cond);
-                    insn = cmp_insn[cond];
-                    tcg_debug_assert(insn != 0);
+                if (is_scalar) {
+                    insn = cmp_scalar_insn[cond];
+                    if (insn == 0) {
+                        TCGArg t;
+                        t = a1, a1 = a2, a2 = t;
+                        cond = tcg_swap_cond(cond);
+                        insn = cmp_scalar_insn[cond];
+                        tcg_debug_assert(insn != 0);
+                    }
+                    tcg_out_insn_3611(s, insn, vece, a0, a1, a2);
+                } else {
+                    insn = cmp_vec_insn[cond];
+                    if (insn == 0) {
+                        TCGArg t;
+                        t = a1, a1 = a2, a2 = t;
+                        cond = tcg_swap_cond(cond);
+                        insn = cmp_vec_insn[cond];
+                        tcg_debug_assert(insn != 0);
+                    }
+                    tcg_out_insn_3616(s, insn, is_q, vece, a0, a1, a2);
                 }
-                tcg_out_insn_3616(s, insn, is_q, vece, a0, a1, a2);
             }
         }
         break;

From 378830071e35b09bafd24458f410e4f105ad5f9d Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sun, 28 Jun 2026 00:51:37 +0300
Subject: [PATCH 30/32] fix macos aarch64 tcg asserts

Use helper typemasks when extending TCG call arguments so pointer operands are not treated as 32-bit values on aarch64 hosts. Restore s390x instruction-start metadata emission for early Unicorn exit TBs.
---
 qemu/include/exec/helper-tcg.h |  34 ++++-----
 qemu/target/m68k/helper.h      |   1 +
 qemu/target/ppc/helper.h       |   4 +
 qemu/target/s390x/translate.c  |  11 ++-
 qemu/tcg/tcg.c                 | 131 +++++++++++----------------------
 5 files changed, 71 insertions(+), 110 deletions(-)

diff --git a/qemu/include/exec/helper-tcg.h b/qemu/include/exec/helper-tcg.h
index 42d145b809..1720cbe466 100644
--- a/qemu/include/exec/helper-tcg.h
+++ b/qemu/include/exec/helper-tcg.h
@@ -13,50 +13,50 @@
 #define DEF_HELPER_FLAGS_0(NAME, FLAGS, ret) \
   { .func = HELPER(NAME), .name = str(NAME), \
     .flags = FLAGS | dh_callflag(ret), \
-    .sizemask = dh_sizemask(ret, 0) },
+    .typemask = dh_typemask(ret, 0) },
 
 #define DEF_HELPER_FLAGS_1(NAME, FLAGS, ret, t1) \
   { .func = HELPER(NAME), .name = str(NAME), \
     .flags = FLAGS | dh_callflag(ret), \
-    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) },
+    .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) },
 
 #define DEF_HELPER_FLAGS_2(NAME, FLAGS, ret, t1, t2) \
   { .func = HELPER(NAME), .name = str(NAME), \
     .flags = FLAGS | dh_callflag(ret), \
-    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
-    | dh_sizemask(t2, 2) },
+    .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \
+    | dh_typemask(t2, 2) },
 
 #define DEF_HELPER_FLAGS_3(NAME, FLAGS, ret, t1, t2, t3) \
   { .func = HELPER(NAME), .name = str(NAME), \
     .flags = FLAGS | dh_callflag(ret), \
-    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
-    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) },
+    .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \
+    | dh_typemask(t2, 2) | dh_typemask(t3, 3) },
 
 #define DEF_HELPER_FLAGS_4(NAME, FLAGS, ret, t1, t2, t3, t4) \
   { .func = HELPER(NAME), .name = str(NAME), \
     .flags = FLAGS | dh_callflag(ret), \
-    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
-    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) },
+    .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \
+    | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) },
 
 #define DEF_HELPER_FLAGS_5(NAME, FLAGS, ret, t1, t2, t3, t4, t5) \
   { .func = HELPER(NAME), .name = str(NAME), \
     .flags = FLAGS | dh_callflag(ret), \
-    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
-    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \
-    | dh_sizemask(t5, 5) },
+    .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \
+    | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \
+    | dh_typemask(t5, 5) },
 
 #define DEF_HELPER_FLAGS_6(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6) \
   { .func = HELPER(NAME), .name = str(NAME), \
     .flags = FLAGS | dh_callflag(ret), \
-    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
-    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \
-    | dh_sizemask(t5, 5) | dh_sizemask(t6, 6) },
+    .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \
+    | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \
+    | dh_typemask(t5, 5) | dh_typemask(t6, 6) },
 
 #define DEF_HELPER_FLAGS_7(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6, t7) \
   { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \
-    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
-    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \
-    | dh_sizemask(t5, 5) | dh_sizemask(t6, 6) | dh_sizemask(t7, 7) },
+    .typemask = dh_typemask(ret, 0) | dh_typemask(t1, 1) \
+    | dh_typemask(t2, 2) | dh_typemask(t3, 3) | dh_typemask(t4, 4) \
+    | dh_typemask(t5, 5) | dh_typemask(t6, 6) | dh_typemask(t7, 7) },
 
 #include "helper.h"
 #include "accel/tcg/tcg-runtime.h"
diff --git a/qemu/target/m68k/helper.h b/qemu/target/m68k/helper.h
index fea07f143b..1880e0bfd0 100644
--- a/qemu/target/m68k/helper.h
+++ b/qemu/target/m68k/helper.h
@@ -21,6 +21,7 @@ DEF_HELPER_4(cas2l_parallel, void, env, i32, i32, i32)
 #define dh_alias_fp ptr
 #define dh_ctype_fp FPReg *
 #define dh_is_signed_fp dh_is_signed_ptr
+#define dh_typecode_fp dh_typecode_ptr
 
 DEF_HELPER_3(exts32, void, env, fp, s32)
 DEF_HELPER_3(extf32, void, env, fp, f32)
diff --git a/qemu/target/ppc/helper.h b/qemu/target/ppc/helper.h
index c62d9d26fc..d4a0a2f5a5 100644
--- a/qemu/target/ppc/helper.h
+++ b/qemu/target/ppc/helper.h
@@ -118,14 +118,17 @@ DEF_HELPER_FLAGS_1(ftsqrt, TCG_CALL_NO_RWG_SE, i32, i64)
 #define dh_alias_avr ptr
 #define dh_ctype_avr ppc_avr_t *
 #define dh_is_signed_avr dh_is_signed_ptr
+#define dh_typecode_avr dh_typecode_ptr
 
 #define dh_alias_vsr ptr
 #define dh_ctype_vsr ppc_vsr_t *
 #define dh_is_signed_vsr dh_is_signed_ptr
+#define dh_typecode_vsr dh_typecode_ptr
 
 #define dh_alias_acc ptr
 #define dh_ctype_acc ppc_acc_t *
 #define dh_is_signed_acc dh_is_signed_ptr
+#define dh_typecode_acc dh_typecode_ptr
 
 DEF_HELPER_3(vavgub, void, avr, avr, avr)
 DEF_HELPER_3(vavguh, void, avr, avr, avr)
@@ -792,6 +795,7 @@ DEF_HELPER_3(store_601_batu, void, env, i32, tl)
 #define dh_alias_fprp ptr
 #define dh_ctype_fprp ppc_fprp_t *
 #define dh_is_signed_fprp dh_is_signed_ptr
+#define dh_typecode_fprp dh_typecode_ptr
 
 DEF_HELPER_4(dadd, void, env, fprp, fprp, fprp)
 DEF_HELPER_4(daddq, void, env, fprp, fprp, fprp)
diff --git a/qemu/target/s390x/translate.c b/qemu/target/s390x/translate.c
index 7f08bf4bcb..4d237400ff 100644
--- a/qemu/target/s390x/translate.c
+++ b/qemu/target/s390x/translate.c
@@ -138,6 +138,7 @@ struct DisasFields {
 struct DisasContext {
     DisasContextBase base;
     const DisasInsn *insn;
+    TCGOp *insn_start;
     DisasFields fields;
     uint64_t ex_value;
     /*
@@ -6823,8 +6824,8 @@ static DisasJumpType translate_one(CPUS390XState *env, DisasContext *s)
     /* Search for the insn in the table.  */
     insn = extract_insn(env, s);
 
-    /* Emit insn_start now that we know the ILEN.  */
-    tcg_gen_insn_start(tcg_ctx, s->base.pc_next, s->cc_op, s->ilen);
+    /* Update insn_start now that we know the ILEN.  */
+    tcg_set_insn_start_param(s->insn_start, 2, s->ilen);
 
     // Unicorn: trace this instruction on request
     if (HOOK_EXISTS_BOUNDED(s->uc, UC_HOOK_CODE, s->base.pc_next)) {
@@ -6985,6 +6986,11 @@ static void s390x_tr_tb_start(DisasContextBase *db, CPUState *cs)
 
 static void s390x_tr_insn_start(DisasContextBase *dcbase, CPUState *cs)
 {
+    DisasContext *dc = container_of(dcbase, DisasContext, base);
+    TCGContext *tcg_ctx = dc->uc->tcg_ctx;
+
+    tcg_gen_insn_start(tcg_ctx, dc->base.pc_next, dc->cc_op, 0);
+    dc->insn_start = tcg_last_op(tcg_ctx);
 }
 
 static bool s390x_tr_breakpoint_check(DisasContextBase *dcbase, CPUState *cs,
@@ -7040,7 +7046,6 @@ static void s390x_tr_tb_stop(DisasContextBase *dcbase, CPUState *cs)
 
     switch (dc->base.is_jmp) {
     case DISAS_UNICORN_HALT:
-        tcg_gen_insn_start(tcg_ctx, dc->base.pc_next, 0, 0);
         update_psw_addr(dc);
         update_cc_op(dc);
         gen_helper_uc_s390x_exit(tcg_ctx, tcg_ctx->cpu_env);
diff --git a/qemu/tcg/tcg.c b/qemu/tcg/tcg.c
index 4482fd5e44..458c85c418 100644
--- a/qemu/tcg/tcg.c
+++ b/qemu/tcg/tcg.c
@@ -652,7 +652,7 @@ typedef struct TCGHelperInfo {
     void *func;
     const char *name;
     unsigned flags;
-    unsigned sizemask;
+    unsigned typemask;
 } TCGHelperInfo;
 
 #include "exec/helper-proto.h"
@@ -676,7 +676,7 @@ void uc_add_inline_hook(uc_engine *uc, struct hook *hk, void** args, int args_le
 {
     TCGHelperInfo* info = g_malloc(sizeof(TCGHelperInfo));
     char *name = g_malloc(64);
-    unsigned sizemask = 0xFFFFFFFF;
+    unsigned typemask = 0xFFFFFFFF;
     TCGContext *tcg_ctx = uc->tcg_ctx;
     GHashTable *helper_table = uc->tcg_ctx->helper_table;
 
@@ -689,7 +689,9 @@ void uc_add_inline_hook(uc_engine *uc, struct hook *hk, void** args, int args_le
     case UC_HOOK_BLOCK:
     case UC_HOOK_CODE:
         // (*uc_cb_hookcode_t)(uc_engine *uc, uint64_t address, uint32_t size, void *user_data);
-        sizemask = dh_sizemask(void, 0) | dh_sizemask(ptr, 1) | dh_sizemask(i64, 2) | dh_sizemask(i32, 3) | dh_sizemask(ptr, 4);
+        typemask = dh_typemask(void, 0) | dh_typemask(ptr, 1) |
+                   dh_typemask(i64, 2) | dh_typemask(i32, 3) |
+                   dh_typemask(ptr, 4);
         snprintf(name, 63, "hookcode_%d_%" PRIxPTR , hk->type, (uintptr_t)hk->callback);
         break;
 
@@ -699,7 +701,7 @@ void uc_add_inline_hook(uc_engine *uc, struct hook *hk, void** args, int args_le
 
     name[63] = 0;
     info->name = name;
-    info->sizemask = sizemask;
+    info->typemask = typemask;
 
     g_hash_table_insert(helper_table, (gpointer)info->func, (gpointer)info);
     g_hash_table_insert(uc->tcg_ctx->custom_helper_infos, (gpointer)info->func, (gpointer)info);
@@ -1441,53 +1443,27 @@ bool tcg_op_supported(TCGOpcode op)
 void tcg_gen_callN(TCGContext *tcg_ctx, void *func, TCGTemp *ret, int nargs, TCGTemp **args)
 {
     int i, real_args, nb_rets, pi;
-    unsigned sizemask, flags;
+    unsigned typemask, flags;
     TCGHelperInfo *info;
     TCGOp *op;
 
     info = g_hash_table_lookup(tcg_ctx->helper_table, (gpointer)func);
     flags = info->flags;
-    sizemask = info->sizemask;
-
-#if defined(__sparc__) && !defined(__arch64__)
-    /* We have 64-bit values in one register, but need to pass as two
-       separate parameters.  Split them.  */
-    int orig_sizemask = sizemask;
-    int orig_nargs = nargs;
-    TCGv_i64 retl, reth;
-    TCGTemp *split_args[MAX_OPC_PARAM];
-
-    retl = NULL;
-    reth = NULL;
-    if (sizemask != 0) {
-        for (i = real_args = 0; i < nargs; ++i) {
-            int is_64bit = sizemask & (1 << (i+1)*2);
-            if (is_64bit) {
-                TCGv_i64 orig = temp_tcgv_i64(args[i]);
-                TCGv_i32 h = tcg_temp_new_i32();
-                TCGv_i32 l = tcg_temp_new_i32();
-                tcg_gen_extr_i64_i32(l, h, orig);
-                split_args[real_args++] = tcgv_i32_temp(h);
-                split_args[real_args++] = tcgv_i32_temp(l);
-            } else {
-                split_args[real_args++] = args[i];
-            }
-        }
-        nargs = real_args;
-        args = split_args;
-        sizemask = 0;
-    }
-#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
+    typemask = info->typemask;
+
+#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
     for (i = 0; i < nargs; ++i) {
-        int is_64bit = sizemask & (1 << (i+1)*2);
-        int is_signed = sizemask & (2 << (i+1)*2);
-        if (!is_64bit) {
+        int argtype = extract32(typemask, (i + 1) * 3, 3);
+        bool is_32bit = (argtype & ~1) == dh_typecode_i32;
+        bool is_signed = argtype & 1;
+
+        if (is_32bit) {
             TCGv_i64 temp = tcg_temp_new_i64(tcg_ctx);
-            TCGv_i64 orig = temp_tcgv_i64(tcg_ctx, args[i]);
+            TCGv_i32 orig = temp_tcgv_i32(tcg_ctx, args[i]);
             if (is_signed) {
-                tcg_gen_ext32s_i64(tcg_ctx, temp, orig);
+                tcg_gen_ext_i32_i64(tcg_ctx, temp, orig);
             } else {
-                tcg_gen_ext32u_i64(tcg_ctx, temp, orig);
+                tcg_gen_extu_i32_i64(tcg_ctx, temp, orig);
             }
             args[i] = tcgv_i64_temp(tcg_ctx, temp);
         }
@@ -1498,22 +1474,8 @@ void tcg_gen_callN(TCGContext *tcg_ctx, void *func, TCGTemp *ret, int nargs, TCG
 
     pi = 0;
     if (ret != NULL) {
-#if defined(__sparc__) && !defined(__arch64__)
-        if (orig_sizemask & 1) {
-            /* The 32-bit ABI is going to return the 64-bit value in
-               the %o0/%o1 register pair.  Prepare for this by using
-               two return temporaries, and reassemble below.  */
-            retl = tcg_temp_new_i64();
-            reth = tcg_temp_new_i64();
-            op->args[pi++] = tcgv_i64_arg(reth);
-            op->args[pi++] = tcgv_i64_arg(retl);
-            nb_rets = 2;
-        } else {
-            op->args[pi++] = temp_arg(ret);
-            nb_rets = 1;
-        }
-#else
-        if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) {
+        if (TCG_TARGET_REG_BITS < 64 &&
+            (typemask & 6) == dh_typecode_i64) {
 #ifdef HOST_WORDS_BIGENDIAN
             op->args[pi++] = temp_arg(ret + 1);
             op->args[pi++] = temp_arg(ret);
@@ -1526,7 +1488,6 @@ void tcg_gen_callN(TCGContext *tcg_ctx, void *func, TCGTemp *ret, int nargs, TCG
             op->args[pi++] = temp_arg(ret);
             nb_rets = 1;
         }
-#endif
     } else {
         nb_rets = 0;
     }
@@ -1534,16 +1495,23 @@ void tcg_gen_callN(TCGContext *tcg_ctx, void *func, TCGTemp *ret, int nargs, TCG
 
     real_args = 0;
     for (i = 0; i < nargs; i++) {
-        int is_64bit = sizemask & (1 << (i+1)*2);
-        if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
-#ifdef TCG_TARGET_CALL_ALIGN_ARGS
-            /* some targets want aligned 64 bit args */
-            if (real_args & 1) {
-                op->args[pi++] = TCG_CALL_DUMMY_ARG;
-                real_args++;
-            }
+        int argtype = extract32(typemask, (i + 1) * 3, 3);
+        bool is_64bit = (argtype & ~1) == dh_typecode_i64;
+        bool want_align = false;
+
+#if defined(CONFIG_TCG_INTERPRETER)
+        want_align = true;
+#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
+        want_align = is_64bit;
 #endif
-           /* If stack grows up, then we will be placing successive
+
+        if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
+            op->args[pi++] = TCG_CALL_DUMMY_ARG;
+            real_args++;
+        }
+
+        if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
+            /* If stack grows up, then we will be placing successive
               arguments at lower addresses, which means we need to
               reverse the order compared to how we would normally
               treat either big or little-endian.  For those arguments
@@ -1575,29 +1543,12 @@ void tcg_gen_callN(TCGContext *tcg_ctx, void *func, TCGTemp *ret, int nargs, TCG
     tcg_debug_assert(TCGOP_CALLI(op) == real_args);
     tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
 
-#if defined(__sparc__) && !defined(__arch64__)
-    /* Free all of the parts we allocated above.  */
-    for (i = real_args = 0; i < orig_nargs; ++i) {
-        int is_64bit = orig_sizemask & (1 << (i+1)*2);
-        if (is_64bit) {
-            tcg_temp_free_internal(args[real_args++]);
-            tcg_temp_free_internal(args[real_args++]);
-        } else {
-            real_args++;
-        }
-    }
-    if (orig_sizemask & 1) {
-        /* The 32-bit ABI returned two 32-bit pieces.  Re-assemble them.
-           Note that describing these as TCGv_i64 eliminates an unnecessary
-           zero-extension that tcg_gen_concat_i32_i64 would create.  */
-        tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
-        tcg_temp_free_i64(retl);
-        tcg_temp_free_i64(reth);
-    }
-#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
+#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
     for (i = 0; i < nargs; ++i) {
-        int is_64bit = sizemask & (1 << (i+1)*2);
-        if (!is_64bit) {
+        int argtype = extract32(typemask, (i + 1) * 3, 3);
+        bool is_32bit = (argtype & ~1) == dh_typecode_i32;
+
+        if (is_32bit) {
             tcg_temp_free_internal(tcg_ctx, args[i]);
         }
     }

From 3d71dd4b88e11cbdf5b6a195a28c182d8407d50f Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sun, 28 Jun 2026 01:08:19 +0300
Subject: [PATCH 31/32] fix macos arm bf16 test

Avoid freeing BF16 VCVT temporaries after neon_store_reg consumes them.
---
 qemu/target/arm/translate.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/qemu/target/arm/translate.c b/qemu/target/arm/translate.c
index 61cf376aba..3671b6fb1c 100644
--- a/qemu/target/arm/translate.c
+++ b/qemu/target/arm/translate.c
@@ -6501,8 +6501,6 @@ static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
                     neon_store_reg(tcg_ctx, rd, 1, tmp2);
 
                     tcg_temp_free_i64(tcg_ctx, tmp64);
-                    tcg_temp_free_i32(tcg_ctx, tmp2);
-                    tcg_temp_free_i32(tcg_ctx, tmp);
                     tcg_temp_free_ptr(tcg_ctx, fpst);
                     return 0;
                 }

From 749c854dfd92b9a1224e56a4013cc6c4a612f804 Mon Sep 17 00:00:00 2001
From: Nitr0-G <120374383+Nitr0-G@users.noreply.github.com>
Date: Sun, 28 Jun 2026 02:33:11 +0300
Subject: [PATCH 32/32] fix ppc64le and alpine ci

Restore QEMU 7.2 ppc modulo lowering and long goto_tb reset handling.

Revalidate TLB entries after Unicorn memory callbacks that can flush or resize TLB state.
---
 qemu/accel/tcg/cputlb.c       | 126 +++++++++++++++++++---------------
 qemu/tcg/ppc/tcg-target.inc.c |  31 +++++++--
 2 files changed, 98 insertions(+), 59 deletions(-)

diff --git a/qemu/accel/tcg/cputlb.c b/qemu/accel/tcg/cputlb.c
index 14267b4f6c..25cd63a493 100644
--- a/qemu/accel/tcg/cputlb.c
+++ b/qemu/accel/tcg/cputlb.c
@@ -1529,6 +1529,49 @@ static inline void tlb_hook_state_restore(CPUArchState *env,
 #endif
 }
 
+static inline target_ulong tlb_addr_for_access(const CPUTLBEntry *entry,
+                                               MMUAccessType access_type)
+{
+    switch (access_type) {
+    case MMU_DATA_STORE:
+        return tlb_addr_write(entry);
+    case MMU_INST_FETCH:
+        return entry->addr_code;
+    case MMU_DATA_LOAD:
+    default:
+        return entry->addr_read;
+    }
+}
+
+static inline void tlb_revalidate_entry(CPUArchState *env, uintptr_t mmu_idx,
+                                        target_ulong addr, size_t size,
+                                        MMUAccessType access_type,
+                                        size_t tlb_off, uintptr_t retaddr,
+                                        uintptr_t *index,
+                                        CPUTLBEntry **entry,
+                                        target_ulong *tlb_addr)
+{
+    struct uc_struct *uc = env->uc;
+    target_ulong page = addr & TARGET_PAGE_MASK;
+
+    (void)uc;
+
+    *index = tlb_index(env, mmu_idx, addr);
+    *entry = tlb_entry(env, mmu_idx, addr);
+    *tlb_addr = tlb_addr_for_access(*entry, access_type);
+
+    if (!tlb_hit(env->uc, *tlb_addr, addr)) {
+        if (!victim_tlb_hit(env, mmu_idx, *index, tlb_off, page)) {
+            tlb_fill(env_cpu(env), addr, size, access_type, mmu_idx, retaddr);
+            *index = tlb_index(env, mmu_idx, addr);
+            *entry = tlb_entry(env, mmu_idx, addr);
+        }
+        *tlb_addr = tlb_addr_for_access(*entry, access_type);
+    }
+
+    *tlb_addr &= ~TLB_INVALID_MASK;
+}
+
 static inline uint64_t
 load_memop(const void *haddr, MemOp op)
 {
@@ -1651,18 +1694,9 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
 
         if (handled) {
             uc->invalid_error = UC_ERR_OK;
-            /* If the TLB entry is for a different page, reload and try again.  */
-            if (!tlb_hit(env->uc, tlb_addr, addr)) {
-                if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
-                                    addr & TARGET_PAGE_MASK)) {
-                    tlb_fill(env_cpu(env), addr, size,
-                             access_type, mmu_idx, retaddr);
-                    index = tlb_index(env, mmu_idx, addr);
-                    entry = tlb_entry(env, mmu_idx, addr);
-                }
-                tlb_addr = code_read ? entry->addr_code : entry->addr_read;
-                tlb_addr &= ~TLB_INVALID_MASK;
-            }
+            tlb_revalidate_entry(env, mmu_idx, addr, size, access_type,
+                                 tlb_off, retaddr, &index, &entry,
+                                 &tlb_addr);
             paddr = entry->paddr | (addr & ~TARGET_PAGE_MASK);
             mr = uc->memory_mapping(uc, paddr);
             if (mr == NULL) {
@@ -1713,14 +1747,10 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
                     A better approach is not always invalidating tlb but this
                     might cause more chaos regarding re-entry (nested uc_emu_start).
         */
-        if (tlb_entry_is_empty(entry)) {
-            tlb_fill(env_cpu(env), addr, size,
-                        access_type, mmu_idx, retaddr);
-            index = tlb_index(env, mmu_idx, addr);
-            entry = tlb_entry(env, mmu_idx, addr);
-            tlb_addr = code_read ? entry->addr_code : entry->addr_read;
-            tlb_addr &= ~TLB_INVALID_MASK;
-        }
+        tlb_revalidate_entry(env, mmu_idx, addr, size, access_type, tlb_off,
+                             retaddr, &index, &entry, &tlb_addr);
+        paddr = entry->paddr | (addr & ~TARGET_PAGE_MASK);
+        mr = uc->memory_mapping(uc, paddr);
 
         // callback on non-readable memory
         if (mr != NULL && !(mr->perms & UC_PROT_READ)) {  //non-readable
@@ -1743,18 +1773,11 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
 
             if (handled) {
                 uc->invalid_error = UC_ERR_OK;
-                /* If the TLB entry is for a different page, reload and try again.  */
-                if (!tlb_hit(env->uc, tlb_addr, addr)) {
-                    if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
-                                        addr & TARGET_PAGE_MASK)) {
-                        tlb_fill(env_cpu(env), addr, size,
-                                 access_type, mmu_idx, retaddr);
-                        index = tlb_index(env, mmu_idx, addr);
-                        entry = tlb_entry(env, mmu_idx, addr);
-                    }
-                    tlb_addr = code_read ? entry->addr_code : entry->addr_read;
-                    tlb_addr &= ~TLB_INVALID_MASK;
-                }
+                tlb_revalidate_entry(env, mmu_idx, addr, size, access_type,
+                                     tlb_off, retaddr, &index, &entry,
+                                     &tlb_addr);
+                paddr = entry->paddr | (addr & ~TARGET_PAGE_MASK);
+                mr = uc->memory_mapping(uc, paddr);
                 tlb_hook_state_restore(env, &hook_state);
             } else {
                 uc->invalid_addr = paddr;
@@ -1791,6 +1814,11 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
 
             if (handled) {
                 uc->invalid_error = UC_ERR_OK;
+                tlb_revalidate_entry(env, mmu_idx, addr, size, access_type,
+                                     tlb_off, retaddr, &index, &entry,
+                                     &tlb_addr);
+                paddr = entry->paddr | (addr & ~TARGET_PAGE_MASK);
+                mr = uc->memory_mapping(uc, paddr);
                 tlb_hook_state_restore(env, &hook_state);
             } else {
                 uc->invalid_addr = paddr;
@@ -2245,6 +2273,10 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
                 break;
         }
         tlb_hook_state_restore(env, &hook_state);
+        tlb_revalidate_entry(env, mmu_idx, addr, size, MMU_DATA_STORE,
+                             tlb_off, retaddr, &index, &entry, &tlb_addr);
+        paddr = entry->paddr | (addr & ~TARGET_PAGE_MASK);
+        mr = uc->memory_mapping(uc, paddr);
     }
 
     // Unicorn: callback on invalid memory
@@ -2275,17 +2307,9 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
             return;
         } else {
             uc->invalid_error = UC_ERR_OK;
-            /* If the TLB entry is for a different page, reload and try again.  */
-            if (!tlb_hit(env->uc, tlb_addr, addr)) {
-                if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
-                    addr & TARGET_PAGE_MASK)) {
-                    tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE,
-                             mmu_idx, retaddr);
-                    index = tlb_index(env, mmu_idx, addr);
-                    entry = tlb_entry(env, mmu_idx, addr);
-                }
-                tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
-            }
+            tlb_revalidate_entry(env, mmu_idx, addr, size, MMU_DATA_STORE,
+                                 tlb_off, retaddr, &index, &entry,
+                                 &tlb_addr);
             paddr = entry->paddr | (addr & ~TARGET_PAGE_MASK);
             mr = uc->memory_mapping(uc, paddr);
             if (mr == NULL) {
@@ -2318,17 +2342,11 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
         }
 
         if (handled) {
-            /* If the TLB entry is for a different page, reload and try again.  */
-            if (!tlb_hit(env->uc, tlb_addr, addr)) {
-                if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
-                    addr & TARGET_PAGE_MASK)) {
-                    tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE,
-                             mmu_idx, retaddr);
-                    index = tlb_index(env, mmu_idx, addr);
-                    entry = tlb_entry(env, mmu_idx, addr);
-                }
-                tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
-            }
+            tlb_revalidate_entry(env, mmu_idx, addr, size, MMU_DATA_STORE,
+                                 tlb_off, retaddr, &index, &entry,
+                                 &tlb_addr);
+            paddr = entry->paddr | (addr & ~TARGET_PAGE_MASK);
+            mr = uc->memory_mapping(uc, paddr);
             uc->invalid_error = UC_ERR_OK;
             tlb_hook_state_restore(env, &hook_state);
         } else {
diff --git a/qemu/tcg/ppc/tcg-target.inc.c b/qemu/tcg/ppc/tcg-target.inc.c
index d3d09547d1..41af409df4 100644
--- a/qemu/tcg/ppc/tcg-target.inc.c
+++ b/qemu/tcg/ppc/tcg-target.inc.c
@@ -401,6 +401,8 @@ static int tcg_target_const_match(tcg_target_long val, TCGType type,
 #define MULHWU XO31( 11)
 #define DIVW   XO31(491)
 #define DIVWU  XO31(459)
+#define MODSW  XO31(779)
+#define MODUW  XO31(267)
 #define CMP    XO31(  0)
 #define CMPL   XO31( 32)
 #define LHBRX  XO31(790)
@@ -433,6 +435,8 @@ static int tcg_target_const_match(tcg_target_long val, TCGType type,
 #define MULHDU XO31(  9)
 #define DIVD   XO31(489)
 #define DIVDU  XO31(457)
+#define MODSD  XO31(777)
+#define MODUD  XO31(265)
 
 #define LBZX   XO31( 87)
 #define LHZX   XO31(279)
@@ -2378,8 +2382,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         if (s->tb_jmp_insn_offset) {
             /* Direct jump. */
             if (TCG_TARGET_REG_BITS == 64) {
-                /* Ensure the next insns are 8-byte aligned. */
-                if ((uintptr_t)s->code_ptr & 7) {
+                /* Ensure the next insns are 8 or 16-byte aligned. */
+                while ((uintptr_t)s->code_ptr & (have_isa_2_07 ? 15 : 7)) {
                     tcg_out32(s, NOP);
                 }
                 s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
@@ -2402,9 +2406,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         set_jmp_reset_offset(s, args[0]);
         if (USE_REG_TB) {
             /* For the unlinked case, need to reset TCG_REG_TB.  */
-            c = -tcg_current_code_size(s);
-            assert(c == (int16_t)c);
-            tcg_out32(s, ADDI | TAI(TCG_REG_TB, TCG_REG_TB, c));
+            tcg_out_mem_long(s, ADDI, ADD, TCG_REG_TB, TCG_REG_TB,
+                             -tcg_current_code_size(s));
         }
         break;
     case INDEX_op_goto_ptr:
@@ -2614,6 +2617,14 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2]));
         break;
 
+    case INDEX_op_rem_i32:
+        tcg_out32(s, MODSW | TAB(args[0], args[1], args[2]));
+        break;
+
+    case INDEX_op_remu_i32:
+        tcg_out32(s, MODUW | TAB(args[0], args[1], args[2]));
+        break;
+
     case INDEX_op_shl_i32:
         if (const_args[2]) {
             tcg_out_shli32(s, args[0], args[1], args[2]);
@@ -2752,6 +2763,12 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_divu_i64:
         tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2]));
         break;
+    case INDEX_op_rem_i64:
+        tcg_out32(s, MODSD | TAB(args[0], args[1], args[2]));
+        break;
+    case INDEX_op_remu_i64:
+        tcg_out32(s, MODUD | TAB(args[0], args[1], args[2]));
+        break;
 
     case INDEX_op_qemu_ld_i32:
         tcg_out_qemu_ld(s, args, false);
@@ -3588,6 +3605,8 @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
         return &r_r_rI;
     case INDEX_op_div_i32:
     case INDEX_op_divu_i32:
+    case INDEX_op_rem_i32:
+    case INDEX_op_remu_i32:
     case INDEX_op_nand_i32:
     case INDEX_op_nor_i32:
     case INDEX_op_muluh_i32:
@@ -3598,6 +3617,8 @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op)
     case INDEX_op_nor_i64:
     case INDEX_op_div_i64:
     case INDEX_op_divu_i64:
+    case INDEX_op_rem_i64:
+    case INDEX_op_remu_i64:
     case INDEX_op_mulsh_i64:
     case INDEX_op_muluh_i64:
         return &r_r_r;